mihari 1.2.0 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 901c334bf0485bbb82a422a1900347e77e476143afaef3036c177ceadbb7e6c6
-  data.tar.gz: 3fd3663d4d05518b46f9d1a53d9d742b150a84ffba93aaf4d9ebdcbd93505342
+  metadata.gz: ab3906ec64d1f2fe33db26d91da0459b3509a8a30b7ad68bd5613fb04f6b788b
+  data.tar.gz: 6824cc1b248e17828f57fd5c39089a04ac49ac5e8e92b7b30491c2e32d2eefe7
 SHA512:
-  metadata.gz: b97df59e99c969940ffe54a1ecf1e655f582ed4f2372c4e08feb3572fd7f38e767303911a4ec36151325da78415738a0f952f35dfbd010bd9dc6a1832635c78a
-  data.tar.gz: b092fdfa627a2ab2d2e4a71c4e070e1788fa496b3be6f64e038b332632adbdd4d70c2cae239e14814a31f03def374eb215f3cc580695af38d9562e1b2e1da4e1
+  metadata.gz: 6449ac095213ed065d8a00f98d34666b3824acf77a1183e17b65a24d1cb29088284677b7169e5832755daa20a8db16df474b241df74f4ee06e556bf6cebaf7ae
+  data.tar.gz: 6816de8e51d95352265678bc6cba347b462f1d4a100896a3a48a848c92516bb13e8d106964a13d44082d55e2be88447e92bac6abee799ef50b2b935543a588e9

data/README.md

@@ -71,6 +71,7 @@ Mihari supports the following services by default.
 - [PassiveTotal](https://community.riskiq.com/)
 - [SecurityTrails](https://securitytrails.com/)
 - [Shodan](https://shodan.io)
+- [Spyse](https://spyse.com)
 - [urlscan.io](https://urlscan.io)
 - [VirusTotal](http://virustotal.com)
 - [ZoomEye](https://zoomeye.org)
@@ -99,6 +100,7 @@ Commands:
   mihari securitytrails [IP|DOMAIN|EMAIL]     # SecurityTrails lookup by an ip, domain or email
   mihari securitytrails_domain_feed [REGEXP]  # SecurityTrails new domain feed search by a regexp
   mihari shodan [QUERY]                       # Shodan host search by a query
+  mihari spyse [QUERY]                        # Spyse search by a query
   mihari ssh_fingerprint [FINGERPRINT]        # Cross search with search engines by an SSH fingerprint (e.g. dc:14:de:8e:d7:c1:15:43:23:82:25:81:d2:59:e8:c0)
   mihari status                               # Show the current configuration status
   mihari urlscan [QUERY]                      # urlscan search by a given query
@@ -221,8 +223,10 @@ Configuration can be done via environment variables or a YAML file.
 | SHODAN_API_KEY         | Shodan API key                                                                                  |             |
 | SLACK_CHANNEL          | Slack channel name                                                                              | `#general`  |
 | SLACK_WEBHOOK_URL      | Slack Webhook URL                                                                               |             |
+| SPYSE_API_KEY          | Spyse API key                                                                                   |             |
 | THEHIVE_API_ENDPOINT   | TheHive URL                                                                                     |             |
 | THEHIVE_API_KEY        | TheHive API key                                                                                 |             |
+| URLSCAN_API_KEY        | urlscan.io API key                                                                              |             |
 | VIRUSTOTAL_API_KEY     | VirusTotal API key                                                                              |             |
 | ZOOMEYE_PASSWORD       | ZoomEye password                                                                                |             |
 | ZOOMEYE_USERNAMME      | ZoomEye username                                                                                |             |

data/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM ruby:2.6-alpine3.10
+FROM ruby:2.7-alpine3.10
 RUN apk --no-cache add git build-base ruby-dev sqlite-dev postgresql-dev \
   && cd /tmp/ \
   && git clone https://github.com/ninoseki/mihari.git \

data/lib/mihari.rb

@@ -56,6 +56,7 @@ require "mihari/analyzers/pulsedive"
 require "mihari/analyzers/securitytrails_domain_feed"
 require "mihari/analyzers/securitytrails"
 require "mihari/analyzers/shodan"
+require "mihari/analyzers/spyse"
 require "mihari/analyzers/urlscan"
 require "mihari/analyzers/virustotal"
 require "mihari/analyzers/zoomeye"

data/lib/mihari/analyzers/binaryedge.rb

@@ -37,6 +37,10 @@ module Mihari
 
       def search_with_page(query, page: 1)
         api.host.search(query, page: page)
+      rescue ::BinaryEdge::Error => e
+        raise RetryableError, e if e.message.include?("Request time limit exceeded")
+
+        raise e
       end
 
       def search

data/lib/mihari/analyzers/shodan.rb

@@ -45,6 +45,10 @@ module Mihari
 
       def search_with_page(query, page: 1)
         api.host.search(query, page: page)
+      rescue ::Shodan::Error => e
+        raise RetryableError, e if e.message.include?("request timed out")
+
+        raise e
       end
 
       def search

data/lib/mihari/analyzers/spyse.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "spyse"
+require "json"
+
+module Mihari
+  module Analyzers
+    class Spyse < Base
+      attr_reader :query
+      attr_reader :type
+
+      attr_reader :title
+      attr_reader :description
+      attr_reader :tags
+
+      def initialize(query, title: nil, description: nil, tags: [], type: "domain")
+        super()
+
+        @query = query
+
+        @title = title || "Spyse lookup"
+        @description = description || "query = #{query}"
+        @tags = tags
+        @type = type
+      end
+
+      def artifacts
+        lookup || []
+      end
+
+      private
+
+      def search_params
+        @search_params ||= JSON.parse(query)
+      end
+
+      def config_keys
+        %w(spyse_api_key)
+      end
+
+      def api
+        @api ||= ::Spyse::API.new(Mihari.config.spyse_api_key)
+      end
+
+      def valid_type?
+        %w(ip domain cert).include? type
+      end
+
+      def domain_lookup
+        res = api.domain.search(search_params, limit: 100)
+        items = res.dig("data", "items") || []
+        items.map do |item|
+          item.dig("name")
+        end.uniq.compact
+      end
+
+      def ip_lookup
+        res = api.ip.search(search_params, limit: 100)
+        items = res.dig("data", "items") || []
+        items.map do |item|
+          item.dig("ip")
+        end.uniq.compact
+      end
+
+      def lookup
+        case type
+        when "domain"
+          domain_lookup
+        when "ip"
+          ip_lookup
+        else
+          raise InvalidInputError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
+        end
+      end
+    end
+  end
+end

data/lib/mihari/analyzers/urlscan.rb

@@ -9,16 +9,33 @@ module Mihari
       attr_reader :description
       attr_reader :query
       attr_reader :tags
+
+      attr_reader :filter
       attr_reader :target_type
+      attr_reader :use_pro
+      attr_reader :use_similarity
 
-      def initialize(query, title: nil, description: nil, tags: [], target_type: "url")
+      def initialize(
+        query,
+        description: nil,
+        filter: nil,
+        tags: [],
+        target_type: "url",
+        title: nil,
+        use_pro: false,
+        use_similarity: false
+      )
         super()
 
         @query = query
         @title = title || "urlscan lookup"
         @description = description || "query = #{query}"
         @tags = tags
+
+        @filter = filter
         @target_type = target_type
+        @use_pro = use_pro
+        @use_similarity = use_similarity
 
         raise InvalidInputError, "type should be url, domain or ip." unless valid_target_type?
       end
@@ -35,11 +52,18 @@ module Mihari
 
       private
 
+      def config_keys
+        %w(urlscan_api_key)
+      end
+
       def api
-        @api ||= ::UrlScan::API.new
+        @api ||= ::UrlScan::API.new(Mihari.config.urlscan_api_key)
       end
 
       def search
+        return api.pro.similar(query) if use_similarity
+        return api.pro.search(query: query, filter: filter, size: 10_000) if use_pro
+
         api.search(query, size: 10_000)
       end
 

data/lib/mihari/cli.rb

@@ -7,6 +7,10 @@ module Mihari
   class CLI < Thor
     class_option :config, type: :string, desc: "path to config file"
 
+    def self.exit_on_failure?
+      true
+    end
+
     desc "censys [QUERY]", "Censys IPv4 search by a query"
     method_option :title, type: :string, desc: "title"
     method_option :description, type: :string, desc: "description"
@@ -42,7 +46,10 @@ module Mihari
     method_option :title, type: :string, desc: "title"
     method_option :description, type: :string, desc: "description"
     method_option :tags, type: :array, desc: "tags"
+    method_option :filter, type: :string, desc: "filter for urlscan pro search"
     method_option :target_type, type: :string, default: "url", desc: "target type to fetch from lookup results (target type should be 'url', 'domain' or 'ip')"
+    method_option :use_pro, type: :boolean, default: false, desc: "use pro search API or not"
+    method_option :use_similarity, type: :boolean, default: false, desc: "use similarity API or not"
     def urlscan(query)
       with_error_handling do
         run_analyzer Analyzers::Urlscan, query: query, options: options
@@ -174,6 +181,17 @@ module Mihari
       end
     end
 
+    desc "spyse [QUERY]", "Spyse search by a query"
+    method_option :title, type: :string, desc: "title"
+    method_option :description, type: :string, desc: "description"
+    method_option :tags, type: :array, desc: "tags"
+    method_option :type, type: :string, desc: "type to search (ip or domain)", default: "doamin"
+    def spyse(query)
+      with_error_handling do
+        run_analyzer Analyzers::Spyse, query: query, options: options
+      end
+    end
+
     desc "passive_dns [IP|DOMAIN]", "Cross search with passive DNS services by an ip or domain"
     method_option :title, type: :string, desc: "title"
     method_option :description, type: :string, desc: "description"

data/lib/mihari/config.rb

@@ -20,8 +20,10 @@ module Mihari
     attr_accessor :shodan_api_key
     attr_accessor :slack_channel
     attr_accessor :slack_webhook_url
+    attr_accessor :spyse_api_key
     attr_accessor :thehive_api_endpoint
     attr_accessor :thehive_api_key
+    attr_accessor :urlscan_api_key
     attr_accessor :virustotal_api_key
     attr_accessor :zoomeye_password
     attr_accessor :zoomeye_username
@@ -49,8 +51,10 @@ module Mihari
       @shodan_api_key = ENV["SHODAN_API_KEY"]
       @slack_channel = ENV["SLACK_CHANNEL"]
       @slack_webhook_url = ENV["SLACK_WEBHOOK_URL"]
+      @spyse_api_key = ENV["SPYSE_API_KEY"]
       @thehive_api_endpoint = ENV["THEHIVE_API_ENDPOINT"]
       @thehive_api_key = ENV["THEHIVE_API_KEY"]
+      @urlscan_api_key = ENV["URLSCAN_API_KEY"]
       @virustotal_api_key = ENV["VIRUSTOTAL_API_KEY"]
       @zoomeye_password = ENV["ZOOMEYE_PASSWORD"]
       @zoomeye_username = ENV["ZOOMEYE_USERNAME"]

data/lib/mihari/errors.rb

@@ -3,4 +3,5 @@
 module Mihari
   class Error < StandardError; end
   class InvalidInputError < Error; end
+  class RetryableError < Error; end
 end

data/lib/mihari/retriable.rb

@@ -7,10 +7,10 @@ module Mihari
       begin
         try += 1
         yield
-      rescue Errno::ECONNRESET, Errno::ECONNABORTED, Errno::EPIPE, OpenSSL::SSL::SSLError, Timeout::Error => _e
+      rescue Errno::ECONNRESET, Errno::ECONNABORTED, Errno::EPIPE, OpenSSL::SSL::SSLError, Timeout::Error, RetryableError => e
         sleep interval
         retry if try < times
-        raise
+        raise e
       end
     end
   end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "1.2.0"
+  VERSION = "1.4.0"
 end

data/mihari.gemspec

@@ -31,11 +31,11 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "pre-commit", "~> 0.39"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.9"
-  spec.add_development_dependency "rubocop", "~> 0.88"
-  spec.add_development_dependency "rubocop-performance", "~> 1.7"
+  spec.add_development_dependency "rubocop", "~> 0.90"
+  spec.add_development_dependency "rubocop-performance", "~> 1.8"
   spec.add_development_dependency "timecop", "~> 0.9"
   spec.add_development_dependency "vcr", "~> 6.0"
-  spec.add_development_dependency "webmock", "~> 3.8"
+  spec.add_development_dependency "webmock", "~> 3.9"
 
   spec.add_dependency "active_model_serializers", "~> 0.10"
   spec.add_dependency "activerecord", "~> 6.0"
@@ -62,9 +62,10 @@ Gem::Specification.new do |spec|
   spec.add_dependency "securitytrails", "~> 1.0"
   spec.add_dependency "shodanx", "~> 0.2"
   spec.add_dependency "slack-notifier", "~> 2.3"
+  spec.add_dependency "spysex", "~> 0.1"
   spec.add_dependency "sqlite3", "~> 1.4"
   spec.add_dependency "thor", "~> 1.0"
-  spec.add_dependency "urlscan", "~> 0.5"
+  spec.add_dependency "urlscan", "~> 0.6"
   spec.add_dependency "virustotalx", "~> 1.1"
   spec.add_dependency "zoomeye-rb", "~> 0.1"
 end

data/renovate.json

@@ -0,0 +1,5 @@
+{
+  "extends": [
+    "config:base"
+  ]
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mihari
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.4.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-08-01 00:00:00.000000000 Z
+date: 2020-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -114,28 +114,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.88'
+        version: '0.90'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.88'
+        version: '0.90'
 - !ruby/object:Gem::Dependency
   name: rubocop-performance
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '1.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '1.8'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -170,14 +170,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.8'
+        version: '3.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.8'
+        version: '3.9'
 - !ruby/object:Gem::Dependency
   name: active_model_serializers
   requirement: !ruby/object:Gem::Requirement
@@ -528,6 +528,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.3'
+- !ruby/object:Gem::Dependency
+  name: spysex
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -562,14 +576,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.6'
 - !ruby/object:Gem::Dependency
   name: virustotalx
   requirement: !ruby/object:Gem::Requirement
@@ -642,6 +656,7 @@ files:
 - lib/mihari/analyzers/securitytrails.rb
 - lib/mihari/analyzers/securitytrails_domain_feed.rb
 - lib/mihari/analyzers/shodan.rb
+- lib/mihari/analyzers/spyse.rb
 - lib/mihari/analyzers/ssh_fingerprint.rb
 - lib/mihari/analyzers/urlscan.rb
 - lib/mihari/analyzers/virustotal.rb
@@ -674,6 +689,7 @@ files:
 - lib/mihari/type_checker.rb
 - lib/mihari/version.rb
 - mihari.gemspec
+- renovate.json
 - screenshots/alert.png
 - screenshots/eyecatch.png
 - screenshots/misp.png