mihari 0.14.0 → 0.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 429387bdb6483e260f1631661721fce1a5abc284711a2c763b50452140388a0c
-  data.tar.gz: 7ba6ad5c62da25d668cf975a4ffecb3e70a01c7c5c6961e646412b0227886fc0
+  metadata.gz: 720fd6a91aeb9dad3ad3ad37129b6f810da8dd66f6e007638471c52c7050313b
+  data.tar.gz: 276123ca8645e124110c5f7cf9012481846344188ac0173792f9f2d16c5ea351
 SHA512:
-  metadata.gz: c64e408f6e8d11c27285f1bceccd504cbffa360978864cf5c4a45b7841a1718166158b6818af2d2a3f83087af21cda7c1808e948335270ac742a0c0094c467c4
-  data.tar.gz: c36b4d97ae530b19c5c9f06b22438032eeca23aeea941a39a12b56a0236565de1a09c7f977b8d04b7f2a5c62fbcfe4e2abf06f553d497d05660982879fa9edba
+  metadata.gz: f4cdeaf7a7040a72ab30795c0ac2d446c13c1ceee3352384382c4faa912b6f8293db9ee3d69a08b72c84b206fc8a99e98df9253cc050196ea48d188497e3df9c
+  data.tar.gz: da6e803d6c250be258cb0cb542cbb2255f413c6199a43dfd728f4f347129087b27857d36b8039d23ad475603c73dbe1e2b36ccd4f110eb02d44f0a7f02bc1f7a

data/README.md

@@ -17,7 +17,7 @@ mihari(`見張り`) is a sidekick tool for [TheHive](https://github.com/TheHive-
     - mihari sends a notification to Slack. (Optional)
     - mihari creates an event on MISP. (Optional)
 
-![img](./screenshots/eyecatch.png)
+![img](https://github.com/ninoseki/mihari/blob/master/screenshots/eyecatch.png)
 
 Check this blog post for more details: [Continuous C2 hunting with Censys, Shodan, Onyphe and TheHive](https://hackmd.io/s/SkUaSrqoE).
 
@@ -74,7 +74,9 @@ Commands:
   mihari circl [DOMAIN|SHA1]                  # CIRCL passive DNS/SSL lookup by a domain / SHA1 certificate fingerprint
   mihari crtsh [QUERY]                        # crt.sh search by a query
   mihari dnpedia [QUERY]                      # DNPedia domain search by a query
+  mihari free_text [TEXT]                     # Cross search with search engines by a free text
   mihari help [COMMAND]                       # Describe available commands or one specific command
+  mihari http_hash                            # Cross search with search engines by a hash of an HTTP response (SHA256, MD5 and MurmurHash3)
   mihari import_from_json                     # Give a JSON input via STDIN
   mihari onyphe [QUERY]                       # Onyphe datascan search by a query
   mihari passive_dns [IP|Domain]              # Cross search with passive DNS services by an ip / domain
@@ -83,8 +85,8 @@ Commands:
   mihari reverse_whois [email]                # Cross search with reverse whois services by an email
   mihari securitytrails [IP|DOMAIN|EMAIL]     # SecurityTrails lookup by an ip, domain or email
   mihari securitytrails_domain_feed [REGEXP]  # SecurityTrails new domain feed search by a regexp
-  mihari sha256 [SHA256]                      # Cross search with search engines by an SHA256 hash
   mihari shodan [QUERY]                       # Shodan host search by a query
+  mihari ssh_fingerprint [FINGERPRINT]        # Cross search with search engines by an SSH fingerprint
   mihari status                               # Show the current configuration status
   mihari urlscan [QUERY]                      # urlscan search by a given query
   mihari virustotal [IP|DOMAIN]               # VirusTotal resolutions lookup by an ip or domain
@@ -98,12 +100,14 @@ mihari has cross search features. A cross search is a search across a number of
 
 You can get aggregated results by using the following commands.
 
-| Command       | Desc.                                                                                  |
-|---------------|----------------------------------------------------------------------------------------|
-| passive_dns   | Passive DNS lookup with CIRCL passive DNS, PassiveTotal, SecurityTrails and VirusTotal |
-| passive_ssl   | Passive SSL lookup with CIRCL passive SSL and PassiveTotal                             |
-| reverse_whois | Revese Whois lookup with PassiveTotal and SecurityTrails                               |
-| sha256        | SHA256 hash search with BinaryEdge and Censys                                          |
+| Command         | Desc.                                                                                                   |
+| --------------- | ------------------------------------------------------------------------------------------------------- |
+| passive_dns     | Passive DNS lookup with CIRCL passive DNS, PassiveTotal, SecurityTrails and VirusTotal                  |
+| passive_ssl     | Passive SSL lookup with CIRCL passive SSL and PassiveTotal                                              |
+| reverse_whois   | Revese Whois lookup with PassiveTotal and SecurityTrails                                                |
+| http_hash       | HTTP response hash lookup with BinaryEdge(SHA256), Censys(SHA256), Onyphpe(MD5) and Shodan(MurmurHash3) |
+| free_text       | Free text lookup with BinaryEdge and Censys                                                             |
+| ssh_fingerprint | SSH fingerprint lookup with BinaryEdge and Shodan                                                       |
 
 ### Example usages
 
@@ -140,7 +144,7 @@ $ mihari virustotal "jppost-hi.top" --title "FAKESPY host passive DNS results"
 $ mihari virustotal "jppost-hi[.]top" --title "FAKESPY host passive DNS results"
 
 # SecurityTrails domain feed lookup for finding (possibly) Apple phishing websites
-mihari securitytrails_domain_feed "apple-" --type new
+$ mihari securitytrails_domain_feed "apple-" --type new
 {
   "title": "SecurityTrails domain feed lookup",
   "description": "Regexp = /apple-/",
@@ -172,7 +176,7 @@ The input is a JSON data should have `title`, `description` and `artifacts` key.
 ```
 
 | Key         | Desc.                                                                      | Required or optional |
-|-------------|----------------------------------------------------------------------------|----------------------|
+| ----------- | -------------------------------------------------------------------------- | -------------------- |
 | title       | A title of an alert                                                        | Required             |
 | description | A description of an alert                                                  | Required             |
 | artifacts   | An array of artifacts (supported data types: ip, domain, url, email, hash) | Required             |
@@ -183,7 +187,7 @@ The input is a JSON data should have `title`, `description` and `artifacts` key.
 All configuration is done via ENV variables.
 
 | Key                    | Desc.                          | Required or optional           |
-|------------------------|--------------------------------|--------------------------------|
+| ---------------------- | ------------------------------ | ------------------------------ |
 | THEHIVE_API_ENDPOINT   | TheHive URL                    | Required                       |
 | THEHIVE_API_KEY        | TheHive API key                | Required                       |
 | MISP_API_ENDPOINT      | MISP URL                       | Optional                       |
@@ -215,7 +219,7 @@ mihari status
 Create a class which extends `Mihari::Analyzers::Base` and implements the following methods.
 
 | Name           | Desc.                                                                      | @return       | Required or optional |
-|----------------|----------------------------------------------------------------------------|---------------|----------------------|
+| -------------- | -------------------------------------------------------------------------- | ------------- | -------------------- |
 | `#title`       | A title of an alert                                                        | String        | Required             |
 | `#description` | A description of an alert                                                  | String        | Required             |
 | `#artifacts`   | An array of artifacts (supported data types: ip, domain, url, email, hash) | Array<String> | Required             |

data/lib/mihari.rb

@@ -51,10 +51,12 @@ require "mihari/analyzers/urlscan"
 require "mihari/analyzers/virustotal"
 require "mihari/analyzers/zoomeye"
 
+require "mihari/analyzers/free_text"
+require "mihari/analyzers/http_hash"
 require "mihari/analyzers/passive_dns"
 require "mihari/analyzers/passive_ssl"
 require "mihari/analyzers/reverse_whois"
-require "mihari/analyzers/sha256"
+require "mihari/analyzers/ssh_fingerprint"
 
 require "mihari/notifiers/base"
 require "mihari/notifiers/slack"

data/lib/mihari/analyzers/{sha256.rb → free_text.rb}

@@ -4,9 +4,8 @@ require "parallel"
 
 module Mihari
   module Analyzers
-    class SHA256 < Base
+    class FreeText < Base
       attr_reader :query
-      attr_reader :type
 
       attr_reader :title
       attr_reader :description
@@ -21,9 +20,8 @@ module Mihari
         super()
 
         @query = query
-        @type = TypeChecker.detailed_type(query)
 
-        @title = title || "SHA256 hash cross search"
+        @title = title || "Free text cross search"
         @description = description || "query = #{query}"
         @tags = tags
       end
@@ -36,13 +34,7 @@ module Mihari
 
       private
 
-      def valid_type?
-        %w(sha256).include? type
-      end
-
       def analyzers
-        raise InvalidInputError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
-
         ANALYZERS.map do |klass|
           klass.new(query)
         end
@@ -52,7 +44,7 @@ module Mihari
         analyzer.artifacts
       rescue ArgumentError, InvalidInputError => _e
         nil
-      rescue ::BinaryEdge::Error, ::Censys::ResponseError => _e
+      rescue ::BinaryEdge::Error, ::Censys::Error => _e
         nil
       end
     end

data/lib/mihari/analyzers/http_hash.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require "parallel"
+
+module Mihari
+  module Analyzers
+    class HTTPHash < Base
+      attr_reader :md5
+      attr_reader :sha256
+      attr_reader :mmh3
+
+      attr_reader :title
+      attr_reader :description
+      attr_reader :tags
+
+      def initialize(_query, md5: nil, sha256: nil, mmh3: nil, title: nil, description: nil, tags: [])
+        super()
+
+        @md5 = md5
+        @sha256 = sha256
+        @mmh3 = mmh3
+
+        @title = title || "HTTP hash cross search"
+        @description = description || "query = #{query}"
+        @tags = tags
+      end
+
+      def artifacts
+        Parallel.map(analyzers) do |analyzer|
+          run_analyzer analyzer
+        end.flatten
+      end
+
+      private
+
+      def valid_query?
+        [md5, sha256, mmh3].compact.any?
+      end
+
+      def query
+        [
+          md5 ? "md5:#{md5}" : nil,
+          sha256 ? "sha256:#{sha256}" : nil,
+          mmh3 ? "mmh3:#{mmh3}" : nil,
+        ].compact.join(",")
+      end
+
+      def binary_edge
+        return nil unless sha256
+
+        BinaryEdge.new sha256
+      end
+
+      def censys
+        return nil unless sha256
+
+        Censys.new sha256
+      end
+
+      def onyphe
+        return nil unless md5
+
+        Onyphe.new "app.http.bodymd5:#{md5}"
+      end
+
+      def shodan
+        return nil unless mmh3
+
+        Shodan.new "http.html_hash:#{mmh3}"
+      end
+
+      def analyzers
+        raise InvalidInputError, "No hashes are given." unless valid_query?
+
+        [
+          binary_edge,
+          censys,
+          onyphe,
+          shodan,
+        ].compact
+      end
+
+      def run_analyzer(analyzer)
+        analyzer.artifacts
+      rescue ArgumentError, InvalidInputError => _e
+        nil
+      rescue ::BinaryEdge::Error, ::Censys::ResponseError, ::Onyphe::Error, ::Shodan::Error => _e
+        nil
+      end
+    end
+  end
+end

data/lib/mihari/analyzers/ssh_fingerprint.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "parallel"
+
+module Mihari
+  module Analyzers
+    class SSHFingerprint < Base
+      attr_reader :fingerprint
+
+      attr_reader :title
+      attr_reader :description
+      attr_reader :tags
+
+      def initialize(fingerprint, title: nil, description: nil, tags: [])
+        super()
+
+        @fingerprint = fingerprint
+
+        @title = title || "SSH fingerprint cross search"
+        @description = description || "fingerprint = #{fingerprint}"
+        @tags = tags
+      end
+
+      def artifacts
+        Parallel.map(analyzers) do |analyzer|
+          run_analyzer analyzer
+        end.flatten
+      end
+
+      private
+
+      def valid_fingerprint?
+        /^([0-9a-f]{2}:){15}[0-9a-f]{2}$/.match? fingerprint
+      end
+
+      def binary_edge
+        BinaryEdge.new "ssh.fingerprint:\"#{fingerprint}\""
+      end
+
+      def shodan
+        Shodan.new fingerprint
+      end
+
+      def analyzers
+        raise InvalidInputError, "Invalid fingerprint is given." unless valid_fingerprint?
+
+        [
+          binary_edge,
+          shodan,
+        ].compact
+      end
+
+      def run_analyzer(analyzer)
+        analyzer.artifacts
+      rescue ArgumentError, InvalidInputError => _e
+        nil
+      rescue ::BinaryEdge::Error, ::Shodan::Error => _e
+        nil
+      end
+    end
+  end
+end

data/lib/mihari/cli.rb

@@ -171,13 +171,36 @@ module Mihari
       end
     end
 
-    desc "sha256 [SHA256]", "Cross search with search engines by an SHA256 hash"
+    desc "http_hash", "Cross search with search engines by a hash of an HTTP response (SHA256, MD5 and MurmurHash3)"
     method_option :title, type: :string, desc: "title"
     method_option :description, type: :string, desc: "description"
     method_option :tags, type: :array, desc: "tags"
-    def sha256(query)
+    method_option :md5, type: :string, desc: "MD5 hash"
+    method_option :sha256, type: :string, desc: "SHA256 hash"
+    method_option :mmh3, type: :numeric, desc: "MurmurHash3 hash"
+    def http_hash
       with_error_handling do
-        run_analyzer Analyzers::SHA256, query: query, options: options
+        run_analyzer Analyzers::HTTPHash, query: nil, options: options
+      end
+    end
+
+    desc "free_text [TEXT]", "Cross search with search engines by a free text"
+    method_option :title, type: :string, desc: "title"
+    method_option :description, type: :string, desc: "description"
+    method_option :tags, type: :array, desc: "tags"
+    def free_text(text)
+      with_error_handling do
+        run_analyzer Analyzers::FreeText, query: text, options: options
+      end
+    end
+
+    desc "ssh_fingerprint [FINGERPRINT]", "Cross search with search engines by an SSH fingerprint"
+    method_option :title, type: :string, desc: "title"
+    method_option :description, type: :string, desc: "description"
+    method_option :tags, type: :array, desc: "tags"
+    def ssh_fingerprint(fingerprint)
+      with_error_handling do
+        run_analyzer Analyzers::SSHFingerprint, query: fingerprint, options: options
       end
     end
 

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "0.14.0"
+  VERSION = "0.15.0"
 end

data/mihari.gemspec

@@ -45,7 +45,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "misp", "~> 0.1"
   spec.add_dependency "net-ping", "~> 2.0"
   spec.add_dependency "onyphe", "~> 1.0"
-  spec.add_dependency "parallel", "~> 1.18"
+  spec.add_dependency "parallel", "~> 1.19"
   spec.add_dependency "passive_circl", "~> 0.1"
   spec.add_dependency "passivetotalx", "~> 0.1"
   spec.add_dependency "public_suffix", "~> 4.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mihari
 version: !ruby/object:Gem::Version
-  version: 0.14.0
+  version: 0.15.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-11-10 00:00:00.000000000 Z
+date: 2019-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -296,14 +296,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.18'
+        version: '1.19'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.18'
+        version: '1.19'
 - !ruby/object:Gem::Dependency
   name: passive_circl
   requirement: !ruby/object:Gem::Requirement
@@ -473,6 +473,8 @@ files:
 - lib/mihari/analyzers/circl.rb
 - lib/mihari/analyzers/crtsh.rb
 - lib/mihari/analyzers/dnpedia.rb
+- lib/mihari/analyzers/free_text.rb
+- lib/mihari/analyzers/http_hash.rb
 - lib/mihari/analyzers/onyphe.rb
 - lib/mihari/analyzers/passive_dns.rb
 - lib/mihari/analyzers/passive_ssl.rb
@@ -480,8 +482,8 @@ files:
 - lib/mihari/analyzers/reverse_whois.rb
 - lib/mihari/analyzers/securitytrails.rb
 - lib/mihari/analyzers/securitytrails_domain_feed.rb
-- lib/mihari/analyzers/sha256.rb
 - lib/mihari/analyzers/shodan.rb
+- lib/mihari/analyzers/ssh_fingerprint.rb
 - lib/mihari/analyzers/urlscan.rb
 - lib/mihari/analyzers/virustotal.rb
 - lib/mihari/analyzers/zoomeye.rb