mihari 1.2.1 → 1.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5bd4fc32291966114c9c687f44c930d1974348cc61a36cf682f33efcc617118f
-  data.tar.gz: a7ed78b49a8b3fe3e9ea398d1e41d13c3ed9f3dd6c23ac2de4e1d8cd8328e59e
+  metadata.gz: 3739df12ff7fe35c98ad1dbd181fd8b300486443e36b7ed50bbbd8507a00eccf
+  data.tar.gz: 43e048cf9eb60f8c0e32cbd43b729e25fa86da2e09784dc24d52163260653988
 SHA512:
-  metadata.gz: f752ce54d8cccc4b6f8c81a87939dabf707629ca1ce5c3244f7ae7a595e808b0a326dc068eac58f705b48b65c7d378aa22f7ee062bcce3d967a6e3518ea29974
-  data.tar.gz: 00110a28edd487a1f4f200f3f0fd93f28b8fd358de936651453b2caa75420a90b330f1605f3a8c2a3e36b362fc269a178b693f1752249768591dbb413dcaf555
+  metadata.gz: f62b84ba24b7405b0977414c6c91c9d97219a77f5d166c46c83682613a99f19a17dfedd8768f560eed7c0e6d28d7ea343a332400512748551b518b8e09e038fd
+  data.tar.gz: 794a4b44504e01cbb8b772fb7d7f9e389b67895352e4552549cf02bb70827695d44424ba583059020394ea0eb620a58ef05c4e34bed78b9dabc842abad996516

data/README.md

@@ -71,6 +71,7 @@ Mihari supports the following services by default.
 - [PassiveTotal](https://community.riskiq.com/)
 - [SecurityTrails](https://securitytrails.com/)
 - [Shodan](https://shodan.io)
+- [Spyse](https://spyse.com)
 - [urlscan.io](https://urlscan.io)
 - [VirusTotal](http://virustotal.com)
 - [ZoomEye](https://zoomeye.org)
@@ -99,6 +100,7 @@ Commands:
   mihari securitytrails [IP|DOMAIN|EMAIL]     # SecurityTrails lookup by an ip, domain or email
   mihari securitytrails_domain_feed [REGEXP]  # SecurityTrails new domain feed search by a regexp
   mihari shodan [QUERY]                       # Shodan host search by a query
+  mihari spyse [QUERY]                        # Spyse search by a query
   mihari ssh_fingerprint [FINGERPRINT]        # Cross search with search engines by an SSH fingerprint (e.g. dc:14:de:8e:d7:c1:15:43:23:82:25:81:d2:59:e8:c0)
   mihari status                               # Show the current configuration status
   mihari urlscan [QUERY]                      # urlscan search by a given query
@@ -221,8 +223,10 @@ Configuration can be done via environment variables or a YAML file.
 | SHODAN_API_KEY         | Shodan API key                                                                                  |             |
 | SLACK_CHANNEL          | Slack channel name                                                                              | `#general`  |
 | SLACK_WEBHOOK_URL      | Slack Webhook URL                                                                               |             |
+| SPYSE_API_KEY          | Spyse API key                                                                                   |             |
 | THEHIVE_API_ENDPOINT   | TheHive URL                                                                                     |             |
 | THEHIVE_API_KEY        | TheHive API key                                                                                 |             |
+| URLSCAN_API_KEY        | urlscan.io API key                                                                              |             |
 | VIRUSTOTAL_API_KEY     | VirusTotal API key                                                                              |             |
 | ZOOMEYE_PASSWORD       | ZoomEye password                                                                                |             |
 | ZOOMEYE_USERNAMME      | ZoomEye username                                                                                |             |

data/lib/mihari.rb

@@ -56,6 +56,7 @@ require "mihari/analyzers/pulsedive"
 require "mihari/analyzers/securitytrails_domain_feed"
 require "mihari/analyzers/securitytrails"
 require "mihari/analyzers/shodan"
+require "mihari/analyzers/spyse"
 require "mihari/analyzers/urlscan"
 require "mihari/analyzers/virustotal"
 require "mihari/analyzers/zoomeye"

data/lib/mihari/analyzers/binaryedge.rb

@@ -37,6 +37,10 @@ module Mihari
 
       def search_with_page(query, page: 1)
         api.host.search(query, page: page)
+      rescue ::BinaryEdge::Error => e
+        raise RetryableError, e if e.message.include?("Request time limit exceeded")
+
+        raise e
       end
 
       def search

data/lib/mihari/analyzers/shodan.rb

@@ -45,6 +45,10 @@ module Mihari
 
       def search_with_page(query, page: 1)
         api.host.search(query, page: page)
+      rescue ::Shodan::Error => e
+        raise RetryableError, e if e.message.include?("request timed out")
+
+        raise e
       end
 
       def search

data/lib/mihari/analyzers/spyse.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "spyse"
+require "json"
+
+module Mihari
+  module Analyzers
+    class Spyse < Base
+      attr_reader :query
+      attr_reader :type
+
+      attr_reader :title
+      attr_reader :description
+      attr_reader :tags
+
+      def initialize(query, title: nil, description: nil, tags: [], type: "domain")
+        super()
+
+        @query = query
+
+        @title = title || "Spyse lookup"
+        @description = description || "query = #{query}"
+        @tags = tags
+        @type = type
+      end
+
+      def artifacts
+        lookup || []
+      end
+
+      private
+
+      def search_params
+        @search_params ||= JSON.parse(query)
+      end
+
+      def config_keys
+        %w(spyse_api_key)
+      end
+
+      def api
+        @api ||= ::Spyse::API.new(Mihari.config.spyse_api_key)
+      end
+
+      def valid_type?
+        %w(ip domain cert).include? type
+      end
+
+      def domain_lookup
+        res = api.domain.search(search_params, limit: 100)
+        items = res.dig("data", "items") || []
+        items.map do |item|
+          item.dig("name")
+        end.uniq.compact
+      end
+
+      def ip_lookup
+        res = api.ip.search(search_params, limit: 100)
+        items = res.dig("data", "items") || []
+        items.map do |item|
+          item.dig("ip")
+        end.uniq.compact
+      end
+
+      def lookup
+        case type
+        when "domain"
+          domain_lookup
+        when "ip"
+          ip_lookup
+        else
+          raise InvalidInputError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
+        end
+      end
+    end
+  end
+end

data/lib/mihari/analyzers/urlscan.rb

@@ -9,16 +9,33 @@ module Mihari
       attr_reader :description
       attr_reader :query
       attr_reader :tags
+
+      attr_reader :filter
       attr_reader :target_type
+      attr_reader :use_pro
+      attr_reader :use_similarity
 
-      def initialize(query, title: nil, description: nil, tags: [], target_type: "url")
+      def initialize(
+        query,
+        description: nil,
+        filter: nil,
+        tags: [],
+        target_type: "url",
+        title: nil,
+        use_pro: false,
+        use_similarity: false
+      )
         super()
 
         @query = query
         @title = title || "urlscan lookup"
         @description = description || "query = #{query}"
         @tags = tags
+
+        @filter = filter
         @target_type = target_type
+        @use_pro = use_pro
+        @use_similarity = use_similarity
 
         raise InvalidInputError, "type should be url, domain or ip." unless valid_target_type?
       end
@@ -35,11 +52,18 @@ module Mihari
 
       private
 
+      def config_keys
+        %w(urlscan_api_key)
+      end
+
       def api
-        @api ||= ::UrlScan::API.new
+        @api ||= ::UrlScan::API.new(Mihari.config.urlscan_api_key)
       end
 
       def search
+        return api.pro.similar(query) if use_similarity
+        return api.pro.search(query: query, filter: filter, size: 10_000) if use_pro
+
         api.search(query, size: 10_000)
       end
 

data/lib/mihari/cli.rb

@@ -7,6 +7,10 @@ module Mihari
   class CLI < Thor
     class_option :config, type: :string, desc: "path to config file"
 
+    def self.exit_on_failure?
+      true
+    end
+
     desc "censys [QUERY]", "Censys IPv4 search by a query"
     method_option :title, type: :string, desc: "title"
     method_option :description, type: :string, desc: "description"
@@ -42,7 +46,10 @@ module Mihari
     method_option :title, type: :string, desc: "title"
     method_option :description, type: :string, desc: "description"
     method_option :tags, type: :array, desc: "tags"
+    method_option :filter, type: :string, desc: "filter for urlscan pro search"
     method_option :target_type, type: :string, default: "url", desc: "target type to fetch from lookup results (target type should be 'url', 'domain' or 'ip')"
+    method_option :use_pro, type: :boolean, default: false, desc: "use pro search API or not"
+    method_option :use_similarity, type: :boolean, default: false, desc: "use similarity API or not"
     def urlscan(query)
       with_error_handling do
         run_analyzer Analyzers::Urlscan, query: query, options: options
@@ -174,6 +181,17 @@ module Mihari
       end
     end
 
+    desc "spyse [QUERY]", "Spyse search by a query"
+    method_option :title, type: :string, desc: "title"
+    method_option :description, type: :string, desc: "description"
+    method_option :tags, type: :array, desc: "tags"
+    method_option :type, type: :string, desc: "type to search (ip or domain)", default: "doamin"
+    def spyse(query)
+      with_error_handling do
+        run_analyzer Analyzers::Spyse, query: query, options: options
+      end
+    end
+
     desc "passive_dns [IP|DOMAIN]", "Cross search with passive DNS services by an ip or domain"
     method_option :title, type: :string, desc: "title"
     method_option :description, type: :string, desc: "description"

data/lib/mihari/config.rb

@@ -20,8 +20,10 @@ module Mihari
     attr_accessor :shodan_api_key
     attr_accessor :slack_channel
     attr_accessor :slack_webhook_url
+    attr_accessor :spyse_api_key
     attr_accessor :thehive_api_endpoint
     attr_accessor :thehive_api_key
+    attr_accessor :urlscan_api_key
     attr_accessor :virustotal_api_key
     attr_accessor :zoomeye_password
     attr_accessor :zoomeye_username
@@ -49,8 +51,10 @@ module Mihari
       @shodan_api_key = ENV["SHODAN_API_KEY"]
       @slack_channel = ENV["SLACK_CHANNEL"]
       @slack_webhook_url = ENV["SLACK_WEBHOOK_URL"]
+      @spyse_api_key = ENV["SPYSE_API_KEY"]
       @thehive_api_endpoint = ENV["THEHIVE_API_ENDPOINT"]
       @thehive_api_key = ENV["THEHIVE_API_KEY"]
+      @urlscan_api_key = ENV["URLSCAN_API_KEY"]
       @virustotal_api_key = ENV["VIRUSTOTAL_API_KEY"]
       @zoomeye_password = ENV["ZOOMEYE_PASSWORD"]
       @zoomeye_username = ENV["ZOOMEYE_USERNAME"]

data/lib/mihari/errors.rb

@@ -3,4 +3,5 @@
 module Mihari
   class Error < StandardError; end
   class InvalidInputError < Error; end
+  class RetryableError < Error; end
 end

data/lib/mihari/retriable.rb

@@ -7,7 +7,7 @@ module Mihari
       begin
         try += 1
         yield
-      rescue Errno::ECONNRESET, Errno::ECONNABORTED, Errno::EPIPE, OpenSSL::SSL::SSLError, Timeout::Error, ::Shodan::Error => e
+      rescue Errno::ECONNRESET, Errno::ECONNABORTED, Errno::EPIPE, OpenSSL::SSL::SSLError, Timeout::Error, RetryableError => e
         sleep interval
         retry if try < times
         raise e

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "1.2.1"
+  VERSION = "1.4.1"
 end

data/mihari.gemspec

@@ -30,15 +30,15 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "fakefs", "~> 1.2"
   spec.add_development_dependency "pre-commit", "~> 0.39"
   spec.add_development_dependency "rake", "~> 13.0"
-  spec.add_development_dependency "rspec", "~> 3.9"
-  spec.add_development_dependency "rubocop", "~> 0.88"
-  spec.add_development_dependency "rubocop-performance", "~> 1.7"
+  spec.add_development_dependency "rspec", "~> 3.10"
+  spec.add_development_dependency "rubocop", "~> 1.6.0"
+  spec.add_development_dependency "rubocop-performance", "~> 1.9"
   spec.add_development_dependency "timecop", "~> 0.9"
   spec.add_development_dependency "vcr", "~> 6.0"
-  spec.add_development_dependency "webmock", "~> 3.8"
+  spec.add_development_dependency "webmock", "~> 3.10"
 
   spec.add_dependency "active_model_serializers", "~> 0.10"
-  spec.add_dependency "activerecord", "~> 6.0"
+  spec.add_dependency "activerecord", "~> 6.1"
   spec.add_dependency "addressable", "~> 2.7"
   spec.add_dependency "binaryedge", "~> 0.1"
   spec.add_dependency "censu", "~> 0.2"
@@ -53,7 +53,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "net-ping", "~> 2.0"
   spec.add_dependency "onyphe", "~> 2.0"
   spec.add_dependency "otx_ruby", "~> 0.9"
-  spec.add_dependency "parallel", "~> 1.19"
+  spec.add_dependency "parallel", "~> 1.20"
   spec.add_dependency "passive_circl", "~> 0.1"
   spec.add_dependency "passivetotalx", "~> 0.1"
   spec.add_dependency "pg", "~> 1.2"
@@ -62,9 +62,11 @@ Gem::Specification.new do |spec|
   spec.add_dependency "securitytrails", "~> 1.0"
   spec.add_dependency "shodanx", "~> 0.2"
   spec.add_dependency "slack-notifier", "~> 2.3"
+  spec.add_dependency "spysex", "~> 0.1"
   spec.add_dependency "sqlite3", "~> 1.4"
   spec.add_dependency "thor", "~> 1.0"
-  spec.add_dependency "urlscan", "~> 0.5"
+  spec.add_dependency "thread_safe", "~> 0.3"
+  spec.add_dependency "urlscan", "~> 0.6"
   spec.add_dependency "virustotalx", "~> 1.1"
   spec.add_dependency "zoomeye-rb", "~> 0.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mihari
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.4.1
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-08-10 00:00:00.000000000 Z
+date: 2020-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -100,42 +100,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.9'
+        version: '3.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.9'
+        version: '3.10'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.88'
+        version: 1.6.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.88'
+        version: 1.6.0
 - !ruby/object:Gem::Dependency
   name: rubocop-performance
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '1.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '1.9'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -170,14 +170,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.8'
+        version: '3.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.8'
+        version: '3.10'
 - !ruby/object:Gem::Dependency
   name: active_model_serializers
   requirement: !ruby/object:Gem::Requirement
@@ -198,14 +198,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '6.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '6.1'
 - !ruby/object:Gem::Dependency
   name: addressable
   requirement: !ruby/object:Gem::Requirement
@@ -408,14 +408,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.19'
+        version: '1.20'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.19'
+        version: '1.20'
 - !ruby/object:Gem::Dependency
   name: passive_circl
   requirement: !ruby/object:Gem::Requirement
@@ -528,6 +528,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.3'
+- !ruby/object:Gem::Dependency
+  name: spysex
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -556,20 +570,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: thread_safe
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
 - !ruby/object:Gem::Dependency
   name: urlscan
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.6'
 - !ruby/object:Gem::Dependency
   name: virustotalx
   requirement: !ruby/object:Gem::Requirement
@@ -642,6 +670,7 @@ files:
 - lib/mihari/analyzers/securitytrails.rb
 - lib/mihari/analyzers/securitytrails_domain_feed.rb
 - lib/mihari/analyzers/shodan.rb
+- lib/mihari/analyzers/spyse.rb
 - lib/mihari/analyzers/ssh_fingerprint.rb
 - lib/mihari/analyzers/urlscan.rb
 - lib/mihari/analyzers/virustotal.rb
@@ -698,7 +727,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.4
 signing_key:
 specification_version: 4
 summary: A framework for continuous malicious hosts monitoring.