mihari 0.11.0 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4a0fd2207c7c92a24cbeda2fb328ecca19db331aa00132b3f6d361d6f8b4b6fd
-  data.tar.gz: 77b42d18d1509f6a33be27d38064298b726fa3244364d468714926d2878c3e89
+  metadata.gz: f8c53264f15a01502a5a872c013db461b718655608941c88af53c0424261de76
+  data.tar.gz: 1f6146bce1c0eaacfcd688cb33794f974433f8418d7f8eb38513ef338b5914ac
 SHA512:
-  metadata.gz: 14bbf6da4dffb24e0fb3c696bc664970a33784298b7fd09d24c9e792262b27a8eebd229fa1b0d17218e1116590acda49b56d4afff0bbc147c8c661dc545df811
-  data.tar.gz: 308620e95164284b5370985899ed524705b64808a2ceb073af404e619901a0dc00277653e2e0eb41e83d5a9db4678577436fe28a78c5e8accb2534dd5362ae22
+  metadata.gz: f8e79477f488c81f84a9c594bbcfec1e50aa60b66f3e8c0f8a7e71672efc22cceb0ddbd62330b9962e091878f364c9a0475356b54b1924465cd433579530a4f9
+  data.tar.gz: 966b523e74b2769501a613c2b4adeda2d521791a8b6376fb6e47dc8d60fe1b41316e077cb1bc4a95a630397f398b345c5f12d56298581128714c60b2076a7de3

data/README.md

@@ -19,7 +19,7 @@ mihari(`見張り`) is a sidekick tool for [TheHive](https://github.com/TheHive-
 
 ![img](./screenshots/eyecatch.png)
 
-Check this blog post for more detail: [Continuous C2 hunting with Censys, Shodan, Onyphe and TheHive](https://hackmd.io/s/SkUaSrqoE).
+Check this blog post for more details: [Continuous C2 hunting with Censys, Shodan, Onyphe and TheHive](https://hackmd.io/s/SkUaSrqoE).
 
 You can use mihari without TheHive. But note that mihari depends on TheHive to manage artifacts. It means mihari might make duplications when without TheHive.
 
@@ -51,7 +51,7 @@ docker pull ninoseki/mihari
 
 ## Basic usage
 
-mihari supports Censys, Shodan, Onyphe, urlscan, SecurityTrails, crt.sh, CIRCL passive DNS/SSL, PassiveTotal and VirusTotal by default.
+mihari supports Censys, Shodan, Onyphe, urlscan, SecurityTrails, crt.sh, CIRCL passive DNS/SSL, PassiveTotal, VirusTotal and ZoomEye by default.
 
 ```bash
 $ mihari
@@ -71,6 +71,7 @@ Commands:
   mihari status                               # Show the current configuration status
   mihari urlscan [QUERY]                      # urlscan lookup by a given query
   mihari virustotal [IP|DOMAIN]               # VirusTotal resolutions lookup by a given ip or domain
+  mihari zoommeye [QUERY]                     # ZoomEye lookup by a given query
 
 ```
 
@@ -91,7 +92,7 @@ $ mihari censys '("PANDA" AND "SMAdmin" AND "layui")' --title "PANDA C2"
   "tags": []
 }
 
-# VirusTotal passive DNS lookup for a FAKESPY host
+# VirusTotal passive DNS lookup of a FAKESPY host
 $ mihari virustotal "jppost-hi.top" --title "FAKESPY host passive DNS results"
 {
   "title": "FAKESPY host passive DNS results",
@@ -105,6 +106,9 @@ $ mihari virustotal "jppost-hi.top" --title "FAKESPY host passive DNS results"
   "tags": []
 }
 
+# You can pass a "defanged" indicator as an input
+$ mihari virustotal "jppost-hi[.]top" --title "FAKESPY host passive DNS results"
+
 # SecurityTrails domain feed lookup for finding (possibly) Apple phishing websites
 mihari securitytrails_domain_feed "apple-" --type new
 {
@@ -158,7 +162,7 @@ All configuration is done via ENV variables.
 | SLACK_CHANNEL          | Slack channel name             | Optional (default: `#general`) |
 | CENSYS_ID              | Censys API ID                  | Optional                       |
 | CENSYS_SECRET          | Censys secret                  | Optional                       |
-| CIRCL_PASSIVE_PASSWORD | CIRC_ passive DNS/SSL password | Optional                       |
+| CIRCL_PASSIVE_PASSWORD | CIRCL passive DNS/SSL password | Optional                       |
 | CIRCL_PASSIVE_USERNAME | CIRCL passive DNS/SSL username | Optional                       |
 | ONYPHE_API_KEY         | Onyphe API key                 | Optional                       |
 | PASSIVETOTAL_API_KEY   | PassiveTotal API key           | Optional                       |
@@ -166,6 +170,8 @@ All configuration is done via ENV variables.
 | SECURITYTRAILS_API_KEY | SecurityTrails API key         | Optional                       |
 | SHODAN_API_KEY         | Shodan API key                 | Optional                       |
 | VIRUSTOTAL_API_KEY     | VirusTotal API key             | Optional                       |
+| ZOOMEYE_USERNAMME      | ZoomEye username               | Optional                       |
+| ZOOMEYE_PASSWORD       | ZoomEye password               | Optional                       |
 
 You can check the configuration status via `status` command.
 

data/lib/mihari.rb

@@ -46,6 +46,7 @@ require "mihari/analyzers/securitytrails"
 require "mihari/analyzers/shodan"
 require "mihari/analyzers/urlscan"
 require "mihari/analyzers/virustotal"
+require "mihari/analyzers/zoomeye"
 
 require "mihari/notifiers/base"
 require "mihari/notifiers/slack"

data/lib/mihari/analyzers/censys.rb

@@ -9,17 +9,44 @@ module Mihari
       attr_reader :description
       attr_reader :query
       attr_reader :tags
+      attr_reader :type
 
-      def initialize(query, title: nil, description: nil, tags: [])
+      def initialize(query, title: nil, description: nil, tags: [], type: "ipv4")
         super()
 
         @query = query
         @title = title || "Censys lookup"
         @description = description || "query = #{query}"
         @tags = tags
+        @type = type
       end
 
       def artifacts
+        case type
+        when "ipv4"
+          ipv4_lookup
+        when "websites"
+          websites_lookup
+        when "certificates"
+          certificates_lookup
+        else
+          raise TypeError, "#{type} type is not supported." unless valid_type?
+        end
+      end
+
+      private
+
+      def valid_type?
+        %w(ipv4 websites certificates).include? type
+      end
+
+      def normalize(domain)
+        return domain unless domain.start_with?("*.")
+
+        domain.sub("*.", "")
+      end
+
+      def ipv4_lookup
         ipv4s = []
 
         res = api.ipv4.search(query: query)
@@ -30,7 +57,33 @@ module Mihari
         ipv4s.flatten
       end
 
-      private
+      def websites_lookup
+        domains = []
+
+        res = api.websites.search(query: query)
+        res.each_page do |page|
+          domains << page.map(&:domain)
+        end
+
+        domains.flatten
+      end
+
+      def certificates_lookup
+        domains = []
+
+        res = api.certificates.search(query: query)
+        res.each_page do |page|
+          page.each do |result|
+            subject_dn = result.subject_dn
+            names = subject_dn.scan(/CN=(.+)/).flatten.first
+            next unless names
+
+            domains << names.split(",").map { |domain| normalize(domain) }
+          end
+        end
+
+        domains.flatten
+      end
 
       def config_keys
         %w(CENSYS_ID CENSYS_SECRET)

data/lib/mihari/analyzers/circl.rb

@@ -41,7 +41,7 @@ module Mihari
         when "hash"
           passive_ssl_lookup
         else
-          raise ArgumentError, "#{@query}(type: #{@type || 'unknown'}) is not supported."
+          raise TypeError, "#{@query}(type: #{@type || 'unknown'}) is not supported."
         end
       rescue ::PassiveCIRCL::Error => _e
         nil

data/lib/mihari/analyzers/passivetotal.rb

@@ -52,7 +52,7 @@ module Mihari
         when "hash"
           ssl_lookup
         else
-          raise ArgumentError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
+          raise TypeError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
         end
       rescue ::PassiveTotal::Error => _e
         nil

data/lib/mihari/analyzers/securitytrails.rb

@@ -50,7 +50,7 @@ module Mihari
         when "mail"
           mail_lookup
         else
-          raise ArgumentError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
+          raise TypeError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
         end
       rescue ::SecurityTrails::Error => _e
         nil

data/lib/mihari/analyzers/securitytrails_domain_feed.rb

@@ -17,8 +17,8 @@ module Mihari
         @_regexp = regexp
         @type = type
 
-        raise ArgumentError, "#{@_regexp} is not a valid regexp" unless regexp
-        raise ArgumentError, "#{type} is not a valid type" unless valid_type?
+        raise TypeError, "#{@_regexp} is not a valid regexp" unless regexp
+        raise TypeError, "#{type} is not a valid type" unless valid_type?
 
         @title = title || "SecurityTrails domain feed lookup"
         @description = description || "Regexp = /#{@_regexp}/"

data/lib/mihari/analyzers/shodan.rb

@@ -20,13 +20,15 @@ module Mihari
       end
 
       def artifacts
-        result = search
-        return [] unless result
-
-        matches = result.dig("matches") || []
-        matches.map do |match|
-          match.dig "ip_str"
-        end.compact
+        results = search
+        return [] unless results || results.empty?
+
+        results.map do |result|
+          matches = result.dig("matches") || []
+          matches.map do |match|
+            match.dig "ip_str"
+          end.compact
+        end.flatten.compact.uniq
       end
 
       private
@@ -39,11 +41,23 @@ module Mihari
         @api ||= ::Shodan::API.new
       end
 
-      def search
-        api.host.search(query)
+      def search_with_page(query, page: 1)
+        api.host.search(query, page: page)
       rescue ::Shodan::Error => _e
         nil
       end
+
+      def search
+        responses = []
+        (1..Float::INFINITY).each do |page|
+          res = search_with_page(query, page: page)
+          break unless res
+
+          responses << res
+          break if res.dig("total").to_i <= page * 100
+        end
+        responses
+      end
     end
   end
 end

data/lib/mihari/analyzers/urlscan.rb

@@ -20,7 +20,7 @@ module Mihari
         @tags = tags
         @target_type = target_type
 
-        raise ArgumentError, "type should be url, domain or ip." unless valid_target_type?
+        raise TypeError, "type should be url, domain or ip." unless valid_target_type?
       end
 
       def artifacts

data/lib/mihari/analyzers/virustotal.rb

@@ -48,7 +48,7 @@ module Mihari
         when "ip"
           ip_lookup
         else
-          raise ArgumentError, "#{indicator}(type: #{type || 'unknown'}) is not supported." unless valid_type?
+          raise TypeError, "#{indicator}(type: #{type || 'unknown'}) is not supported." unless valid_type?
         end
       rescue ::VirusTotal::Error => _e
         nil

data/lib/mihari/analyzers/zoomeye.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require "zoomeye"
+
+module Mihari
+  module Analyzers
+    class ZoomEye < Base
+      attr_reader :title
+      attr_reader :description
+      attr_reader :query
+      attr_reader :tags
+      attr_reader :type
+
+      def initialize(query, title: nil, description: nil, tags: [], type: "host")
+        super()
+
+        @query = query
+        @title = title || "ZoomEye lookup"
+        @description = description || "query = #{query}"
+        @tags = tags
+        @type = type
+      end
+
+      def artifacts
+        case type
+        when "host"
+          host_lookup
+        when "web"
+          web_lookup
+        else
+          raise TypeError, "#{type} type is not supported." unless valid_type?
+        end
+      end
+
+      private
+
+      def valid_type?
+        %w(host web).include? type
+      end
+
+      def config_keys
+        %w(ZOOMEYE_USERNAME ZOOMEYE_PASSWORD)
+      end
+
+      def api
+        @api ||= ::ZoomEye::API.new
+      end
+
+      def convert_responses(responses)
+        responses.map do |res|
+          matches = res.dig("matches") || []
+          matches.map do |match|
+            match.dig "ip"
+          end
+        end.flatten.compact.uniq
+      end
+
+      def _host_lookup(query, page: 1)
+        api.host.search(query, page: page)
+      rescue ::ZoomEye::Error => _e
+        nil
+      end
+
+      def host_lookup
+        responses = []
+
+        res = _host_lookup(query)
+        total = res.dig("total").to_f
+        max = (total / 10.0).ceil
+        responses << res
+
+        (2..max).each do |page|
+          responses << _host_lookup(query, page: page)
+        end
+        convert_responses responses.compact
+      end
+
+      def _web_lookup(query, page: 1)
+        api.web.search(query, page: page)
+      rescue ::ZoomEye::Error => _e
+        nil
+      end
+
+      def web_lookup
+        responses = []
+
+        res = _web_lookup(query)
+        total = res.dig("total").to_f
+        max = (total / 10.0).ceil
+        responses << res
+
+        (2..max).each do |page|
+          responses << _web_lookup(query, page: page)
+        end
+        convert_responses responses.compact
+      end
+    end
+  end
+end

data/lib/mihari/cli.rb

@@ -9,6 +9,7 @@ module Mihari
     method_option :title, type: :string, desc: "title"
     method_option :description, type: :string, desc: "description"
     method_option :tags, type: :array, desc: "tags"
+    method_option :type, type: :string, desc: "type to search (ipv4 / websites / certificates)", default: "ipv4"
     def censys(query)
       with_error_handling do
         run_analyzer Analyzers::Censys, query: query, options: options
@@ -52,7 +53,7 @@ module Mihari
     method_option :tags, type: :array, desc: "tags"
     def virustotal(indiactor)
       with_error_handling do
-        run_analyzer Analyzers::VirusTotal, query: indiactor, options: options
+        run_analyzer Analyzers::VirusTotal, query: refang(indiactor), options: options
       end
     end
 
@@ -62,7 +63,7 @@ module Mihari
     method_option :tags, type: :array, desc: "tags"
     def securitytrails(indiactor)
       with_error_handling do
-        run_analyzer Analyzers::SecurityTrails, query: indiactor, options: options
+        run_analyzer Analyzers::SecurityTrails, query: refang(indiactor), options: options
       end
     end
     map "st" => :securitytrails
@@ -113,9 +114,20 @@ module Mihari
     method_option :title, type: :string, desc: "title"
     method_option :description, type: :string, desc: "description"
     method_option :tags, type: :array, desc: "tags"
-    def passivetotal(query)
+    def passivetotal(indicator)
       with_error_handling do
-        run_analyzer Analyzers::PassiveTotal, query: query, options: options
+        run_analyzer Analyzers::PassiveTotal, query: refang(indicator), options: options
+      end
+    end
+
+    desc "zoommeye [QUERY]", "ZoomEye lookup by a given query"
+    method_option :title, type: :string, desc: "title"
+    method_option :description, type: :string, desc: "description"
+    method_option :tags, type: :array, desc: "tags"
+    method_option :type, type: :string, desc: "type to search(host / web)", default: "host"
+    def zoomeye(query)
+      with_error_handling do
+        run_analyzer Analyzers::ZoomEye, query: query, options: options
       end
     end
 
@@ -184,6 +196,10 @@ module Mihari
       def symbolize_hash_keys(hash)
         hash.map{ |k, v| [k.to_sym, v] }.to_h
       end
+
+      def refang(indicator)
+        indicator.gsub("[.]", ".").gsub("(.)", ".")
+      end
     end
   end
 end

data/lib/mihari/emitters/slack.rb

@@ -113,12 +113,21 @@ module Mihari
         end.flatten
       end
 
+      def to_text(title:, description:, tags: [])
+        tags = ["N/A"] if tags.empty?
+
+        [
+          "*#{title}*",
+          "*Desc.*: #{description}",
+          "*Tags*: #{tags.join(', ')}",
+        ].join("\n")
+      end
+
       def emit(title:, description:, artifacts:, tags: [])
         return if artifacts.empty?
 
         attachments = to_attachments(artifacts)
-        tags = ["N/A"] if tags.empty?
-        text = "#{title} (desc.: #{description} / tags: #{tags.join(', ')})"
+        text = to_text(title: title, description: description, tags: tags)
 
         notifier.notify(text: text, attachments: attachments)
       end

data/lib/mihari/notifiers/slack.rb

@@ -5,6 +5,7 @@ module Mihari
     class Slack < Base
       SLACK_WEBHOOK_URL_KEY = "SLACK_WEBHOOK_URL"
       SLACK_CHANNEL_KEY = "SLACK_CHANNEL"
+      DEFAULT_USERNAME = "mihari"
 
       def slack_channel
         ENV.fetch SLACK_CHANNEL_KEY, "#general"
@@ -22,9 +23,9 @@ module Mihari
         slack_webhook_url?
       end
 
-      def notify(text:, attachments: [])
-        notifier = ::Slack::Notifier.new(slack_webhook_url, channel: slack_channel)
-        notifier.post(text: text, attachments: attachments)
+      def notify(text:, attachments: [], mrkdwn: true)
+        notifier = ::Slack::Notifier.new(slack_webhook_url, channel: slack_channel, username: DEFAULT_USERNAME)
+        notifier.post(text: text, attachments: attachments, mrkdwn: true)
       end
     end
   end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "0.11.0"
+  VERSION = "0.12.0"
 end

data/mihari.gemspec

@@ -28,7 +28,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "coveralls", "~> 0.8"
   spec.add_development_dependency "fakefs", "~> 0.20"
   spec.add_development_dependency "rake", "~> 13.0"
-  spec.add_development_dependency "rspec", "~> 3.8"
+  spec.add_development_dependency "rspec", "~> 3.9"
   spec.add_development_dependency "timecop", "~> 0.9"
   spec.add_development_dependency "vcr", "~> 5.0"
   spec.add_development_dependency "webmock", "~> 3.7"
@@ -54,4 +54,5 @@ Gem::Specification.new do |spec|
   spec.add_dependency "thor", "~> 0.20"
   spec.add_dependency "urlscan", "~> 0.4"
   spec.add_dependency "virustotalx", "~> 1.0"
+  spec.add_dependency "zoomeye-rb", "~> 0.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mihari
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.12.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-10-07 00:00:00.000000000 Z
+date: 2019-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -72,14 +72,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.8'
+        version: '3.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.8'
+        version: '3.9'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -416,6 +416,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: zoomeye-rb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 description: A framework for continuous malicious hosts monitoring.
 email:
 - manabu.niseki@gmail.com
@@ -451,6 +465,7 @@ files:
 - lib/mihari/analyzers/shodan.rb
 - lib/mihari/analyzers/urlscan.rb
 - lib/mihari/analyzers/virustotal.rb
+- lib/mihari/analyzers/zoomeye.rb
 - lib/mihari/artifact.rb
 - lib/mihari/cache.rb
 - lib/mihari/cli.rb