mihari 0.15.0 → 0.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 720fd6a91aeb9dad3ad3ad37129b6f810da8dd66f6e007638471c52c7050313b
-  data.tar.gz: 276123ca8645e124110c5f7cf9012481846344188ac0173792f9f2d16c5ea351
+  metadata.gz: b70407115db8ca6adbc35ecc24971e2b62d7726e5f226b56dc9323679634aa85
+  data.tar.gz: 681bb8f07a46cd3dcb6205a3699d4b2b205f3ac3e22b527af6ac79bddb1a4f70
 SHA512:
-  metadata.gz: f4cdeaf7a7040a72ab30795c0ac2d446c13c1ceee3352384382c4faa912b6f8293db9ee3d69a08b72c84b206fc8a99e98df9253cc050196ea48d188497e3df9c
-  data.tar.gz: da6e803d6c250be258cb0cb542cbb2255f413c6199a43dfd728f4f347129087b27857d36b8039d23ad475603c73dbe1e2b36ccd4f110eb02d44f0a7f02bc1f7a
+  metadata.gz: 3d5e0327c338fe14398a570fb4b028613a3ffd374bbd88046fc5993fd61071ddd73ab2b8e3535ea7613a3a0f10b43f0552799ab66cc0532296d4962247784582
+  data.tar.gz: 986e547c7c821ccd47cb0634eab643a908dba8a76a302564228c3da74cefe6c223fa6f79f6911f12b012aa9da19d68691d5025b5fe72b4888a8bee6809f7c9b6

data/README.md

@@ -17,7 +17,7 @@ mihari(`見張り`) is a sidekick tool for [TheHive](https://github.com/TheHive-
     - mihari sends a notification to Slack. (Optional)
     - mihari creates an event on MISP. (Optional)
 
-![img](https://github.com/ninoseki/mihari/blob/master/screenshots/eyecatch.png)
+![img](https://github.com/ninoseki/mihari/raw/master/screenshots/eyecatch.png)
 
 Check this blog post for more details: [Continuous C2 hunting with Censys, Shodan, Onyphe and TheHive](https://hackmd.io/s/SkUaSrqoE).
 
@@ -71,7 +71,7 @@ Commands:
   mihari alerts                               # Show the alerts on TheHive
   mihari binaryedge [QUERY]                   # BinaryEdge host search by a query
   mihari censys [QUERY]                       # Censys IPv4 search by a query
-  mihari circl [DOMAIN|SHA1]                  # CIRCL passive DNS/SSL lookup by a domain / SHA1 certificate fingerprint
+  mihari circl [DOMAIN|SHA1]                  # CIRCL passive DNS/SSL lookup by a domain or SHA1 certificate fingerprint
   mihari crtsh [QUERY]                        # crt.sh search by a query
   mihari dnpedia [QUERY]                      # DNPedia domain search by a query
   mihari free_text [TEXT]                     # Cross search with search engines by a free text
@@ -79,14 +79,15 @@ Commands:
   mihari http_hash                            # Cross search with search engines by a hash of an HTTP response (SHA256, MD5 and MurmurHash3)
   mihari import_from_json                     # Give a JSON input via STDIN
   mihari onyphe [QUERY]                       # Onyphe datascan search by a query
-  mihari passive_dns [IP|Domain]              # Cross search with passive DNS services by an ip / domain
+  mihari passive_dns [IP|DOMAIN]              # Cross search with passive DNS services by an ip or domain
   mihari passive_ssl [SHA1]                   # Cross search with passive SSL services by an SHA1 certificate fingerprint
-  mihari passivetotal [IP|DOMAIN|EMAIL|SHA1]  # PassiveTotal lookup by an ip / domain / email / SHA1 certificate fingerprint
-  mihari reverse_whois [email]                # Cross search with reverse whois services by an email
+  mihari passivetotal [IP|DOMAIN|EMAIL|SHA1]  # PassiveTotal lookup by an ip, domain, email or SHA1 certificate fingerprint
+  mihari pulsedive [IP|DOMAIN]                # Pulsedive lookup by an ip or domain
+  mihari reverse_whois [EMAIL]                # Cross search with reverse whois services by an email
   mihari securitytrails [IP|DOMAIN|EMAIL]     # SecurityTrails lookup by an ip, domain or email
   mihari securitytrails_domain_feed [REGEXP]  # SecurityTrails new domain feed search by a regexp
   mihari shodan [QUERY]                       # Shodan host search by a query
-  mihari ssh_fingerprint [FINGERPRINT]        # Cross search with search engines by an SSH fingerprint
+  mihari ssh_fingerprint [FINGERPRINT]        # Cross search with search engines by an SSH fingerprint (e.g. dc:14:de:8e:d7:c1:15:43:23:82:25:81:d2:59:e8:c0)
   mihari status                               # Show the current configuration status
   mihari urlscan [QUERY]                      # urlscan search by a given query
   mihari virustotal [IP|DOMAIN]               # VirusTotal resolutions lookup by an ip or domain
@@ -102,13 +103,49 @@ You can get aggregated results by using the following commands.
 
 | Command         | Desc.                                                                                                   |
 | --------------- | ------------------------------------------------------------------------------------------------------- |
-| passive_dns     | Passive DNS lookup with CIRCL passive DNS, PassiveTotal, SecurityTrails and VirusTotal                  |
+| passive_dns     | Passive DNS lookup with CIRCL passive DNS, PassiveTotal, Pulsedive, SecurityTrails and VirusTotal       |
 | passive_ssl     | Passive SSL lookup with CIRCL passive SSL and PassiveTotal                                              |
 | reverse_whois   | Revese Whois lookup with PassiveTotal and SecurityTrails                                                |
 | http_hash       | HTTP response hash lookup with BinaryEdge(SHA256), Censys(SHA256), Onyphpe(MD5) and Shodan(MurmurHash3) |
 | free_text       | Free text lookup with BinaryEdge and Censys                                                             |
 | ssh_fingerprint | SSH fingerprint lookup with BinaryEdge and Shodan                                                       |
 
+#### http_hash command
+
+The usage of `http_hash` command is a little bit tricky.
+
+```bash
+$ mihari help http_hash
+Usage:
+  mihari http_hash
+
+Options:
+  [--title=TITLE]              # title
+  [--description=DESCRIPTION]  # description
+  [--tags=one two three]       # tags
+  [--md5=MD5]                  # MD5 hash
+  [--sha256=SHA256]            # SHA256 hash
+  [--mmh3=N]                   # MurmurHash3 hash
+
+Cross search with search engines by a hash of an HTTP response (SHA256, MD5 and MurmurHash3)
+
+```
+
+There are 2 ways to use this command.
+
+First one is passing `--md5`, `--sha256` and `--mmh3` parameters.
+
+```bash
+mihari http_hash --md5=881191f7736b5b8cfad5959ca99d2a51 --sha256=b064187ebdc51721708ad98cd89dacc346017cb0fb0457d530032d387f1ff20e --mmh3=-1467534799
+```
+
+Another one is passing `--html` parameter. In this case, hashes of an HTML file are automatically calculated.
+
+```bash
+wget http://example.com -O /tmp/index.html
+mihari http_hash --html /tmp/index.html
+```
+
 ### Example usages
 
 ```bash
@@ -202,6 +239,7 @@ All configuration is done via ENV variables.
 | ONYPHE_API_KEY         | Onyphe API key                 | Optional                       |
 | PASSIVETOTAL_API_KEY   | PassiveTotal API key           | Optional                       |
 | PASSIVETOTAL_USERNAME  | PassiveTotal username          | Optional                       |
+| PULSEDIVE_API_KEY      | Pulsedive API key              | Optional                       |
 | SECURITYTRAILS_API_KEY | SecurityTrails API key         | Optional                       |
 | SHODAN_API_KEY         | Shodan API key                 | Optional                       |
 | VIRUSTOTAL_API_KEY     | VirusTotal API key             | Optional                       |

data/lib/mihari.rb

@@ -26,6 +26,8 @@ require "mihari/artifact"
 require "mihari/cache"
 require "mihari/type_checker"
 
+require "mihari/html"
+
 require "mihari/configurable"
 require "mihari/retriable"
 
@@ -44,6 +46,7 @@ require "mihari/analyzers/crtsh"
 require "mihari/analyzers/dnpedia"
 require "mihari/analyzers/onyphe"
 require "mihari/analyzers/passivetotal"
+require "mihari/analyzers/pulsedive"
 require "mihari/analyzers/securitytrails_domain_feed"
 require "mihari/analyzers/securitytrails"
 require "mihari/analyzers/shodan"

data/lib/mihari/analyzers/http_hash.rb

@@ -9,17 +9,22 @@ module Mihari
       attr_reader :sha256
       attr_reader :mmh3
 
+      attr_reader :html
+
       attr_reader :title
       attr_reader :description
       attr_reader :tags
 
-      def initialize(_query, md5: nil, sha256: nil, mmh3: nil, title: nil, description: nil, tags: [])
+      def initialize(_query, md5: nil, sha256: nil, mmh3: nil, html: nil, title: nil, description: nil, tags: [])
         super()
 
         @md5 = md5
         @sha256 = sha256
         @mmh3 = mmh3
 
+        @html = html
+        load_from_html
+
         @title = title || "HTTP hash cross search"
         @description = description || "query = #{query}"
         @tags = tags
@@ -37,6 +42,17 @@ module Mihari
         [md5, sha256, mmh3].compact.any?
       end
 
+      def load_from_html
+        return unless html
+
+        html_file = HTML.new(html)
+        return unless html_file.exists?
+
+        @md5 = html_file.md5
+        @sha256 = html_file.sha256
+        @mmh3 = html_file.mmh3
+      end
+
       def query
         [
           md5 ? "md5:#{md5}" : nil,

data/lib/mihari/analyzers/passive_dns.rb

@@ -15,6 +15,7 @@ module Mihari
       ANALYZERS = [
         Mihari::Analyzers::CIRCL,
         Mihari::Analyzers::PassiveTotal,
+        Mihari::Analyzers::Pulsedive,
         Mihari::Analyzers::SecurityTrails,
         Mihari::Analyzers::VirusTotal,
       ].freeze
@@ -54,7 +55,7 @@ module Mihari
         analyzer.artifacts
       rescue ArgumentError, InvalidInputError => _e
         nil
-      rescue ::PassiveCIRCL::Error, ::PassiveTotal::Error, ::SecurityTrails::Error, ::VirusTotal::Error => _e
+      rescue ::PassiveCIRCL::Error, ::PassiveTotal::Error, ::Pulsedive::ResponseError, ::SecurityTrails::Error, ::VirusTotal::Error => _e
         nil
       end
     end

data/lib/mihari/analyzers/pulsedive.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "pulsedive"
+
+module Mihari
+  module Analyzers
+    class Pulsedive < Base
+      attr_reader :query
+      attr_reader :type
+
+      attr_reader :title
+      attr_reader :description
+      attr_reader :tags
+
+      def initialize(query, title: nil, description: nil, tags: [])
+        super()
+
+        @query = query
+        @type = TypeChecker.type(query)
+
+        @title = title || "Pulsedive lookup"
+        @description = description || "query = #{query}"
+        @tags = tags
+      end
+
+      def artifacts
+        lookup || []
+      end
+
+      private
+
+      def config_keys
+        %w(PULSEDIVE_API_KEY)
+      end
+
+      def api
+        @api ||= ::Pulsedive::API.new
+      end
+
+      def valid_type?
+        %w(ip domain).include? type
+      end
+
+      def lookup
+        raise InvalidInputError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
+
+        indicator = api.indicator.get_by_value(query)
+        iid = indicator.dig("iid")
+
+        properties = api.indicator.get_properties_by_id(iid)
+        (properties.dig("dns") || []).map do |property|
+          property.dig("value") if ["A", "PTR"].include?(property.dig("name"))
+        end.compact
+      rescue ::Pulsedive::ResponseError => _e
+        nil
+      end
+    end
+  end
+end

data/lib/mihari/cli.rb

@@ -100,7 +100,7 @@ module Mihari
       end
     end
 
-    desc "circl [DOMAIN|SHA1]", "CIRCL passive DNS/SSL lookup by a domain / SHA1 certificate fingerprint"
+    desc "circl [DOMAIN|SHA1]", "CIRCL passive DNS/SSL lookup by a domain or SHA1 certificate fingerprint"
     method_option :title, type: :string, desc: "title"
     method_option :description, type: :string, desc: "description"
     method_option :tags, type: :array, desc: "tags"
@@ -110,7 +110,7 @@ module Mihari
       end
     end
 
-    desc "passivetotal [IP|DOMAIN|EMAIL|SHA1]", "PassiveTotal lookup by an ip / domain / email / SHA1 certificate fingerprint"
+    desc "passivetotal [IP|DOMAIN|EMAIL|SHA1]", "PassiveTotal lookup by an ip, domain, email or SHA1 certificate fingerprint"
     method_option :title, type: :string, desc: "title"
     method_option :description, type: :string, desc: "description"
     method_option :tags, type: :array, desc: "tags"
@@ -141,13 +141,23 @@ module Mihari
       end
     end
 
-    desc "passive_dns [IP|Domain]", "Cross search with passive DNS services by an ip / domain"
+    desc "pulsedive [IP|DOMAIN]", "Pulsedive lookup by an ip or domain"
+    method_option :title, type: :string, desc: "title"
+    method_option :description, type: :string, desc: "description"
+    method_option :tags, type: :array, desc: "tags"
+    def pulsedive(indiactor)
+      with_error_handling do
+        run_analyzer Analyzers::Pulsedive, query: refang(indiactor), options: options
+      end
+    end
+
+    desc "passive_dns [IP|DOMAIN]", "Cross search with passive DNS services by an ip or domain"
     method_option :title, type: :string, desc: "title"
     method_option :description, type: :string, desc: "description"
     method_option :tags, type: :array, desc: "tags"
     def passive_dns(query)
       with_error_handling do
-        run_analyzer Analyzers::PassiveDNS, query: query, options: options
+        run_analyzer Analyzers::PassiveDNS, query: refang(query), options: options
       end
     end
 
@@ -161,7 +171,7 @@ module Mihari
       end
     end
 
-    desc "reverse_whois [email]", "Cross search with reverse whois services by an email"
+    desc "reverse_whois [EMAIL]", "Cross search with reverse whois services by an email"
     method_option :title, type: :string, desc: "title"
     method_option :description, type: :string, desc: "description"
     method_option :tags, type: :array, desc: "tags"
@@ -178,6 +188,7 @@ module Mihari
     method_option :md5, type: :string, desc: "MD5 hash"
     method_option :sha256, type: :string, desc: "SHA256 hash"
     method_option :mmh3, type: :numeric, desc: "MurmurHash3 hash"
+    method_option :html, type: :string, desc: "path to an HTML file"
     def http_hash
       with_error_handling do
         run_analyzer Analyzers::HTTPHash, query: nil, options: options
@@ -194,7 +205,7 @@ module Mihari
       end
     end
 
-    desc "ssh_fingerprint [FINGERPRINT]", "Cross search with search engines by an SSH fingerprint"
+    desc "ssh_fingerprint [FINGERPRINT]", "Cross search with search engines by an SSH fingerprint (e.g. dc:14:de:8e:d7:c1:15:43:23:82:25:81:d2:59:e8:c0)"
     method_option :title, type: :string, desc: "title"
     method_option :description, type: :string, desc: "description"
     method_option :tags, type: :array, desc: "tags"

data/lib/mihari/html.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "digest/sha2"
+require "murmurhash3"
+
+module Mihari
+  class HTML
+    attr_reader :path
+
+    def initialize(path)
+      @path = path
+    end
+
+    def exists?
+      return false unless path
+
+      File.exist? path
+    end
+
+    def sha256
+      Digest::SHA256.hexdigest data
+    end
+
+    def md5
+      Digest::MD5.hexdigest data
+    end
+
+    def mmh3
+      hash = MurmurHash3::V32.str_hash(data)
+      if (hash & 0x80000000).zero?
+        hash
+      else
+        -((hash ^ 0xFFFFFFFF) + 1)
+      end
+    end
+
+    private
+
+    def data
+      File.read path
+    end
+  end
+end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "0.15.0"
+  VERSION = "0.16.0"
 end

data/mihari.gemspec

@@ -43,12 +43,14 @@ Gem::Specification.new do |spec|
   spec.add_dependency "lightly", "~> 0.3"
   spec.add_dependency "mem", "~> 0.1"
   spec.add_dependency "misp", "~> 0.1"
+  spec.add_dependency "murmurhash3", "~> 0.1"
   spec.add_dependency "net-ping", "~> 2.0"
   spec.add_dependency "onyphe", "~> 1.0"
   spec.add_dependency "parallel", "~> 1.19"
   spec.add_dependency "passive_circl", "~> 0.1"
   spec.add_dependency "passivetotalx", "~> 0.1"
   spec.add_dependency "public_suffix", "~> 4.0"
+  spec.add_dependency "pulsedive", "~> 0.1"
   spec.add_dependency "securitytrails", "~> 1.0"
   spec.add_dependency "shodanx", "~> 0.2"
   spec.add_dependency "slack-notifier", "~> 2.3"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mihari
 version: !ruby/object:Gem::Version
-  version: 0.15.0
+  version: 0.16.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-11-15 00:00:00.000000000 Z
+date: 2019-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -262,6 +262,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: murmurhash3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: net-ping
   requirement: !ruby/object:Gem::Requirement
@@ -346,6 +360,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: pulsedive
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: securitytrails
   requirement: !ruby/object:Gem::Requirement
@@ -479,6 +507,7 @@ files:
 - lib/mihari/analyzers/passive_dns.rb
 - lib/mihari/analyzers/passive_ssl.rb
 - lib/mihari/analyzers/passivetotal.rb
+- lib/mihari/analyzers/pulsedive.rb
 - lib/mihari/analyzers/reverse_whois.rb
 - lib/mihari/analyzers/securitytrails.rb
 - lib/mihari/analyzers/securitytrails_domain_feed.rb
@@ -497,6 +526,7 @@ files:
 - lib/mihari/emitters/stdout.rb
 - lib/mihari/emitters/the_hive.rb
 - lib/mihari/errors.rb
+- lib/mihari/html.rb
 - lib/mihari/notifiers/base.rb
 - lib/mihari/notifiers/exception_notifier.rb
 - lib/mihari/notifiers/slack.rb