mihari 1.1.1 → 1.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c43df888d661331830b74ecbf097f2bf7f0450850fca0ad8bfe86d58fe5d32f
-  data.tar.gz: 123d3a13867550f57557472e63819b1bc8a70fd80bb0ef3fa0543d9414fc84a8
+  metadata.gz: 512d3ee8179279b931bd9510c652693ead1108ead99c823e26880e2a75234b24
+  data.tar.gz: 81946c213ef30712644637e8ea5e01bea36956aad077ed37bdac60d0adf71f19
 SHA512:
-  metadata.gz: 96693d5a7ca81b7a1f6834ffedb7ad9897dba6fe510dc632c204a2497595ae67c6bf7c70895bccc5738d6d83369cafc7f309962e6e29a941a2ccb5f5fc84b68a
-  data.tar.gz: 9b4d2ccb878b2aec9b82ac158b5da5bea76653b443c3d53c5d4146a4e3900f400680d690cb0ab6becb526f827d1cb482d663719625f10ee6011e598411c29b67
+  metadata.gz: d6e8b1e9a8791aebfe042de31b1f895fce4bc20d26a417672eaf4dafa83319f2cfb407ee635fe004f70a9d78441c43398c0af79cde01b3bb39bdcfb1dfd9e0c3
+  data.tar.gz: '09ad98242f96474358908d68ef24f2f8711206b698499d18419346004a853ed5cace40df63e1055e82444e43ac9160fa02fc40d0a662d7a17d290a0dee13fb50'

data/README.md

@@ -67,9 +67,11 @@ Mihari supports the following services by default.
 - [DN Pedia](https://dnpedia.com/)
 - [dnstwister](https://dnstwister.report/)
 - [Onyphe](https://onyphe.io)
+- [OTX](https://otx.alienvault.com/)
 - [PassiveTotal](https://community.riskiq.com/)
 - [SecurityTrails](https://securitytrails.com/)
 - [Shodan](https://shodan.io)
+- [Spyse](https://spyse.com)
 - [urlscan.io](https://urlscan.io)
 - [VirusTotal](http://virustotal.com)
 - [ZoomEye](https://zoomeye.org)
@@ -89,6 +91,7 @@ Commands:
   mihari http_hash                            # Cross search with search engines by a hash of an HTTP response (SHA256, MD5 and MurmurHash3)
   mihari import_from_json                     # Give a JSON input via STDIN
   mihari onyphe [QUERY]                       # Onyphe datascan search by a query
+  mihari otx [IP|DOMAIN]                      # OTX lookup by an IP or domain
   mihari passive_dns [IP|DOMAIN]              # Cross search with passive DNS services by an ip or domain
   mihari passive_ssl [SHA1]                   # Cross search with passive SSL services by an SHA1 certificate fingerprint
   mihari passivetotal [IP|DOMAIN|EMAIL|SHA1]  # PassiveTotal lookup by an ip, domain, email or SHA1 certificate fingerprint
@@ -97,6 +100,7 @@ Commands:
   mihari securitytrails [IP|DOMAIN|EMAIL]     # SecurityTrails lookup by an ip, domain or email
   mihari securitytrails_domain_feed [REGEXP]  # SecurityTrails new domain feed search by a regexp
   mihari shodan [QUERY]                       # Shodan host search by a query
+  mihari spyse [QUERY]                        # Spyse search by a query
   mihari ssh_fingerprint [FINGERPRINT]        # Cross search with search engines by an SSH fingerprint (e.g. dc:14:de:8e:d7:c1:15:43:23:82:25:81:d2:59:e8:c0)
   mihari status                               # Show the current configuration status
   mihari urlscan [QUERY]                      # urlscan search by a given query
@@ -116,7 +120,7 @@ You can get aggregated results by using the following commands.
 
 | Command         | Desc.                                                                                                   |
 |-----------------|---------------------------------------------------------------------------------------------------------|
-| passive_dns     | Passive DNS lookup with CIRCL passive DNS, PassiveTotal, Pulsedive, SecurityTrails and VirusTotal       |
+| passive_dns     | Passive DNS lookup with CIRCL passive DNS, OTX, PassiveTotal, Pulsedive, SecurityTrails and VirusTotal  |
 | passive_ssl     | Passive SSL lookup with CIRCL passive SSL and PassiveTotal                                              |
 | reverse_whois   | Revese Whois lookup with PassiveTotal and SecurityTrails                                                |
 | http_hash       | HTTP response hash lookup with BinaryEdge(SHA256), Censys(SHA256), Onyphpe(MD5) and Shodan(MurmurHash3) |
@@ -211,6 +215,7 @@ Configuration can be done via environment variables or a YAML file.
 | MISP_API_ENDPOINT      | MISP URL                                                                                        |             |
 | MISP_API_KEY           | MISP API key                                                                                    |             |
 | ONYPHE_API_KEY         | Onyphe API key                                                                                  |             |
+| OTX_API_KEY            | OTX API key                                                                                     |             |
 | PASSIVETOTAL_API_KEY   | PassiveTotal API key                                                                            |             |
 | PASSIVETOTAL_USERNAME  | PassiveTotal username                                                                           |             |
 | PULSEDIVE_API_KEY      | Pulsedive API key                                                                               |             |
@@ -218,8 +223,10 @@ Configuration can be done via environment variables or a YAML file.
 | SHODAN_API_KEY         | Shodan API key                                                                                  |             |
 | SLACK_CHANNEL          | Slack channel name                                                                              | `#general`  |
 | SLACK_WEBHOOK_URL      | Slack Webhook URL                                                                               |             |
+| SPYSE_API_KEY          | Spyse API key                                                                                   |             |
 | THEHIVE_API_ENDPOINT   | TheHive URL                                                                                     |             |
 | THEHIVE_API_KEY        | TheHive API key                                                                                 |             |
+| URLSCAN_API_KEY        | urlscan.io API key                                                                              |             |
 | VIRUSTOTAL_API_KEY     | VirusTotal API key                                                                              |             |
 | ZOOMEYE_PASSWORD       | ZoomEye password                                                                                |             |
 | ZOOMEYE_USERNAMME      | ZoomEye username                                                                                |             |

data/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM ruby:2.6-alpine3.10
+FROM ruby:2.7-alpine3.10
 RUN apk --no-cache add git build-base ruby-dev sqlite-dev postgresql-dev \
   && cd /tmp/ \
   && git clone https://github.com/ninoseki/mihari.git \

data/lib/mihari.rb

@@ -50,11 +50,13 @@ require "mihari/analyzers/crtsh"
 require "mihari/analyzers/dnpedia"
 require "mihari/analyzers/dnstwister"
 require "mihari/analyzers/onyphe"
+require "mihari/analyzers/otx"
 require "mihari/analyzers/passivetotal"
 require "mihari/analyzers/pulsedive"
 require "mihari/analyzers/securitytrails_domain_feed"
 require "mihari/analyzers/securitytrails"
 require "mihari/analyzers/shodan"
+require "mihari/analyzers/spyse"
 require "mihari/analyzers/urlscan"
 require "mihari/analyzers/virustotal"
 require "mihari/analyzers/zoomeye"

data/lib/mihari/analyzers/binaryedge.rb

@@ -37,6 +37,10 @@ module Mihari
 
       def search_with_page(query, page: 1)
         api.host.search(query, page: page)
+      rescue ::BinaryEdge::Error => e
+        raise RetryableError, e if e.message.include?("Request time limit exceeded")
+
+        raise e
       end
 
       def search

data/lib/mihari/analyzers/otx.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require "otx_ruby"
+
+module Mihari
+  module Analyzers
+    class OTX < Base
+      attr_reader :query
+      attr_reader :type
+
+      attr_reader :title
+      attr_reader :description
+      attr_reader :tags
+
+      def initialize(query, title: nil, description: nil, tags: [])
+        super()
+
+        @query = query
+        @type = TypeChecker.type(query)
+
+        @title = title || "OTX lookup"
+        @description = description || "query = #{query}"
+        @tags = tags
+      end
+
+      def artifacts
+        lookup || []
+      end
+
+      private
+
+      def config_keys
+        %w(otx_api_key)
+      end
+
+      def domain_client
+        @domain_client ||= ::OTX::Domain.new(Mihari.config.otx_api_key)
+      end
+
+      def ip_client
+        @ip_client ||= ::OTX::IP.new(Mihari.config.otx_api_key)
+      end
+
+      def valid_type?
+        %w(ip domain).include? type
+      end
+
+      def lookup
+        case type
+        when "domain"
+          domain_lookup
+        when "ip"
+          ip_lookup
+        else
+          raise InvalidInputError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
+        end
+      end
+
+      def domain_lookup
+        records = domain_client.get_passive_dns(query)
+        records.map do |record|
+          record.address if record.record_type == "A"
+        end.compact.uniq
+      end
+
+      def ip_lookup
+        records = ip_client.get_passive_dns(query)
+        records.map do |record|
+          record.hostname if record.record_type == "A"
+        end.compact.uniq
+      end
+    end
+  end
+end

data/lib/mihari/analyzers/passive_dns.rb

@@ -14,6 +14,7 @@ module Mihari
 
       ANALYZERS = [
         Mihari::Analyzers::CIRCL,
+        Mihari::Analyzers::OTX,
         Mihari::Analyzers::PassiveTotal,
         Mihari::Analyzers::Pulsedive,
         Mihari::Analyzers::SecurityTrails,
@@ -55,7 +56,7 @@ module Mihari
         analyzer.artifacts
       rescue ArgumentError, InvalidInputError => _e
         nil
-      rescue ::PassiveCIRCL::Error, ::PassiveTotal::Error, ::Pulsedive::ResponseError, ::SecurityTrails::Error, ::VirusTotal::Error => _e
+      rescue Faraday::Error, ::PassiveCIRCL::Error, ::PassiveTotal::Error, ::Pulsedive::ResponseError, ::SecurityTrails::Error, ::VirusTotal::Error => _e
         nil
       end
     end

data/lib/mihari/analyzers/shodan.rb

@@ -45,6 +45,10 @@ module Mihari
 
       def search_with_page(query, page: 1)
         api.host.search(query, page: page)
+      rescue ::Shodan::Error => e
+        raise RetryableError, e if e.message.include?("request timed out")
+
+        raise e
       end
 
       def search

data/lib/mihari/analyzers/spyse.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "spyse"
+require "json"
+
+module Mihari
+  module Analyzers
+    class Spyse < Base
+      attr_reader :query
+      attr_reader :type
+
+      attr_reader :title
+      attr_reader :description
+      attr_reader :tags
+
+      def initialize(query, title: nil, description: nil, tags: [], type: "domain")
+        super()
+
+        @query = query
+
+        @title = title || "Spyse lookup"
+        @description = description || "query = #{query}"
+        @tags = tags
+        @type = type
+      end
+
+      def artifacts
+        lookup || []
+      end
+
+      private
+
+      def search_params
+        @search_params ||= JSON.parse(query)
+      end
+
+      def config_keys
+        %w(spyse_api_key)
+      end
+
+      def api
+        @api ||= ::Spyse::API.new(Mihari.config.spyse_api_key)
+      end
+
+      def valid_type?
+        %w(ip domain cert).include? type
+      end
+
+      def domain_lookup
+        res = api.domain.search(search_params, limit: 100)
+        items = res.dig("data", "items") || []
+        items.map do |item|
+          item.dig("name")
+        end.uniq.compact
+      end
+
+      def ip_lookup
+        res = api.ip.search(search_params, limit: 100)
+        items = res.dig("data", "items") || []
+        items.map do |item|
+          item.dig("ip")
+        end.uniq.compact
+      end
+
+      def lookup
+        case type
+        when "domain"
+          domain_lookup
+        when "ip"
+          ip_lookup
+        else
+          raise InvalidInputError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
+        end
+      end
+    end
+  end
+end

data/lib/mihari/analyzers/urlscan.rb

@@ -35,8 +35,12 @@ module Mihari
 
       private
 
+      def config_keys
+        %w(urlscan_api_key)
+      end
+
       def api
-        @api ||= ::UrlScan::API.new
+        @api ||= ::UrlScan::API.new(Mihari.config.urlscan_api_key)
       end
 
       def search

data/lib/mihari/cli.rb

@@ -7,6 +7,10 @@ module Mihari
   class CLI < Thor
     class_option :config, type: :string, desc: "path to config file"
 
+    def self.exit_on_failure?
+      true
+    end
+
     desc "censys [QUERY]", "Censys IPv4 search by a query"
     method_option :title, type: :string, desc: "title"
     method_option :description, type: :string, desc: "description"
@@ -164,6 +168,27 @@ module Mihari
       end
     end
 
+    desc "otx [IP|DOMAIN]", "OTX lookup by an IP or domain"
+    method_option :title, type: :string, desc: "title"
+    method_option :description, type: :string, desc: "description"
+    method_option :tags, type: :array, desc: "tags"
+    def otx(domain)
+      with_error_handling do
+        run_analyzer Analyzers::OTX, query: refang(domain), options: options
+      end
+    end
+
+    desc "spyse [QUERY]", "Spyse search by a query"
+    method_option :title, type: :string, desc: "title"
+    method_option :description, type: :string, desc: "description"
+    method_option :tags, type: :array, desc: "tags"
+    method_option :type, type: :string, desc: "type to search (ip or domain)", default: "doamin"
+    def spyse(query)
+      with_error_handling do
+        run_analyzer Analyzers::Spyse, query: query, options: options
+      end
+    end
+
     desc "passive_dns [IP|DOMAIN]", "Cross search with passive DNS services by an ip or domain"
     method_option :title, type: :string, desc: "title"
     method_option :description, type: :string, desc: "description"

data/lib/mihari/config.rb

@@ -12,6 +12,7 @@ module Mihari
     attr_accessor :misp_api_endpoint
     attr_accessor :misp_api_key
     attr_accessor :onyphe_api_key
+    attr_accessor :otx_api_key
     attr_accessor :passivetotal_api_key
     attr_accessor :passivetotal_username
     attr_accessor :pulsedive_api_key
@@ -19,8 +20,10 @@ module Mihari
     attr_accessor :shodan_api_key
     attr_accessor :slack_channel
     attr_accessor :slack_webhook_url
+    attr_accessor :spyse_api_key
     attr_accessor :thehive_api_endpoint
     attr_accessor :thehive_api_key
+    attr_accessor :urlscan_api_key
     attr_accessor :virustotal_api_key
     attr_accessor :zoomeye_password
     attr_accessor :zoomeye_username
@@ -40,6 +43,7 @@ module Mihari
       @misp_api_endpoint = ENV["MISP_API_ENDPOINT"]
       @misp_api_key = ENV["MISP_API_KEY"]
       @onyphe_api_key = ENV["ONYPHE_API_KEY"]
+      @otx_api_key = ENV["OTX_API_KEY"]
       @passivetotal_api_key = ENV["PASSIVETOTAL_API_KEY"]
       @passivetotal_username = ENV["PASSIVETOTAL_USERNAME"]
       @pulsedive_api_key = ENV["PULSEDIVE_API_KEY"]
@@ -47,8 +51,10 @@ module Mihari
       @shodan_api_key = ENV["SHODAN_API_KEY"]
       @slack_channel = ENV["SLACK_CHANNEL"]
       @slack_webhook_url = ENV["SLACK_WEBHOOK_URL"]
+      @spyse_api_key = ENV["SPYSE_API_KEY"]
       @thehive_api_endpoint = ENV["THEHIVE_API_ENDPOINT"]
       @thehive_api_key = ENV["THEHIVE_API_KEY"]
+      @urlscan_api_key = ENV["URLSCAN_API_KEY"]
       @virustotal_api_key = ENV["VIRUSTOTAL_API_KEY"]
       @zoomeye_password = ENV["ZOOMEYE_PASSWORD"]
       @zoomeye_username = ENV["ZOOMEYE_USERNAME"]

data/lib/mihari/errors.rb

@@ -3,4 +3,5 @@
 module Mihari
   class Error < StandardError; end
   class InvalidInputError < Error; end
+  class RetryableError < Error; end
 end

data/lib/mihari/retriable.rb

@@ -7,10 +7,10 @@ module Mihari
       begin
         try += 1
         yield
-      rescue Errno::ECONNRESET, Errno::ECONNABORTED, Errno::EPIPE, OpenSSL::SSL::SSLError, Timeout::Error => _e
+      rescue Errno::ECONNRESET, Errno::ECONNABORTED, Errno::EPIPE, OpenSSL::SSL::SSLError, Timeout::Error, RetryableError => e
         sleep interval
         retry if try < times
-        raise
+        raise e
       end
     end
   end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "1.1.1"
+  VERSION = "1.3.2"
 end

data/mihari.gemspec

@@ -31,8 +31,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "pre-commit", "~> 0.39"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.9"
-  spec.add_development_dependency "rubocop", "~> 0.88"
-  spec.add_development_dependency "rubocop-performance", "~> 1.7"
+  spec.add_development_dependency "rubocop", "~> 0.90"
+  spec.add_development_dependency "rubocop-performance", "~> 1.8"
   spec.add_development_dependency "timecop", "~> 0.9"
   spec.add_development_dependency "vcr", "~> 6.0"
   spec.add_development_dependency "webmock", "~> 3.8"
@@ -52,6 +52,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "murmurhash3", "~> 0.1"
   spec.add_dependency "net-ping", "~> 2.0"
   spec.add_dependency "onyphe", "~> 2.0"
+  spec.add_dependency "otx_ruby", "~> 0.9"
   spec.add_dependency "parallel", "~> 1.19"
   spec.add_dependency "passive_circl", "~> 0.1"
   spec.add_dependency "passivetotalx", "~> 0.1"
@@ -61,6 +62,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "securitytrails", "~> 1.0"
   spec.add_dependency "shodanx", "~> 0.2"
   spec.add_dependency "slack-notifier", "~> 2.3"
+  spec.add_dependency "spysex", "~> 0.1"
   spec.add_dependency "sqlite3", "~> 1.4"
   spec.add_dependency "thor", "~> 1.0"
   spec.add_dependency "urlscan", "~> 0.5"

data/renovate.json

@@ -0,0 +1,5 @@
+{
+  "extends": [
+    "config:base"
+  ]
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mihari
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.3.2
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-07-19 00:00:00.000000000 Z
+date: 2020-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -114,28 +114,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.88'
+        version: '0.90'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.88'
+        version: '0.90'
 - !ruby/object:Gem::Dependency
   name: rubocop-performance
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '1.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '1.8'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -388,6 +388,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: otx_ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
@@ -514,6 +528,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.3'
+- !ruby/object:Gem::Dependency
+  name: spysex
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -619,6 +647,7 @@ files:
 - lib/mihari/analyzers/free_text.rb
 - lib/mihari/analyzers/http_hash.rb
 - lib/mihari/analyzers/onyphe.rb
+- lib/mihari/analyzers/otx.rb
 - lib/mihari/analyzers/passive_dns.rb
 - lib/mihari/analyzers/passive_ssl.rb
 - lib/mihari/analyzers/passivetotal.rb
@@ -627,6 +656,7 @@ files:
 - lib/mihari/analyzers/securitytrails.rb
 - lib/mihari/analyzers/securitytrails_domain_feed.rb
 - lib/mihari/analyzers/shodan.rb
+- lib/mihari/analyzers/spyse.rb
 - lib/mihari/analyzers/ssh_fingerprint.rb
 - lib/mihari/analyzers/urlscan.rb
 - lib/mihari/analyzers/virustotal.rb
@@ -659,6 +689,7 @@ files:
 - lib/mihari/type_checker.rb
 - lib/mihari/version.rb
 - mihari.gemspec
+- renovate.json
 - screenshots/alert.png
 - screenshots/eyecatch.png
 - screenshots/misp.png