mihari 0.12.0 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f8c53264f15a01502a5a872c013db461b718655608941c88af53c0424261de76
-  data.tar.gz: 1f6146bce1c0eaacfcd688cb33794f974433f8418d7f8eb38513ef338b5914ac
+  metadata.gz: c45276dc9bbc108475c4db517d7bf0e7e809c85de3ee3e7f6141df444e1890db
+  data.tar.gz: 90f9bc7cfa1d25a1186b98b3418069b0c8e30bde28bdb2ebd3ded6930e48b015
 SHA512:
-  metadata.gz: f8e79477f488c81f84a9c594bbcfec1e50aa60b66f3e8c0f8a7e71672efc22cceb0ddbd62330b9962e091878f364c9a0475356b54b1924465cd433579530a4f9
-  data.tar.gz: 966b523e74b2769501a613c2b4adeda2d521791a8b6376fb6e47dc8d60fe1b41316e077cb1bc4a95a630397f398b345c5f12d56298581128714c60b2076a7de3
+  metadata.gz: fa740d55a2fad831f1bf18aeae5aa2695b00c8cc92f09512ebd1c178eaa66083cb65309c4ccd6b2e2e81db450fb21c213d556ad6d5974f7e9037eb814fe75a20
+  data.tar.gz: 03edf15c5fe58fe9b237829510eda124fc91507a6f9903a5f2ecccef275e29ceb94d30a73a4cfc9485a3ca1905c406b58fd87655c4d0ec4d05514755ca768e22

data/.travis.yml

@@ -3,5 +3,5 @@ sudo: false
 language: ruby
 cache: bundler
 rvm:
-  - 2.6.1
+  - 2.6
 before_install: gem install bundler -v 2.0.1

data/README.md

@@ -51,12 +51,25 @@ docker pull ninoseki/mihari
 
 ## Basic usage
 
-mihari supports Censys, Shodan, Onyphe, urlscan, SecurityTrails, crt.sh, CIRCL passive DNS/SSL, PassiveTotal, VirusTotal and ZoomEye by default.
+mihari supports the following services by default.
+
+- [BinaryEdge](https://www.binaryedge.io/)
+- [Censys](http://censys.io)
+- [CIRCL passive DNS](https://www.circl.lu/services/passive-dns/) / [passive SSL](https://www.circl.lu/services/passive-ssl/)
+- [crt.sh](https://crt.sh/)
+- [Onyphe](https://onyphe.io)
+- [PassiveTotal](https://community.riskiq.com/)
+- [SecurityTrails](https://securitytrails.com/)
+- [Shodan](https://shodan.io)
+- [urlscan.io](https://urlscan.io)
+- [VirusTotal](http://virustotal.com)
+- [ZoomEye](https://zoomeye.org)
 
 ```bash
 $ mihari
 Commands:
   mihari alerts                               # Show the alerts on TheHive
+  mihari binaryedge [QUERY]                   # BinaryEdge lookup by a given query
   mihari censys [QUERY]                       # Censys IPv4 lookup by a given query
   mihari circl [DOMAIN|SHA1]                  # CIRCL passive DNS/SSL lookup by a given domain / SHA1 certificate fingerprint
   mihari crtsh [QUERY]                        # crt.sh lookup by a given query
@@ -71,7 +84,7 @@ Commands:
   mihari status                               # Show the current configuration status
   mihari urlscan [QUERY]                      # urlscan lookup by a given query
   mihari virustotal [IP|DOMAIN]               # VirusTotal resolutions lookup by a given ip or domain
-  mihari zoommeye [QUERY]                     # ZoomEye lookup by a given query
+  mihari zoomeye [QUERY]                      # ZoomEye lookup by a given query
 
 ```
 
@@ -160,6 +173,7 @@ All configuration is done via ENV variables.
 | MISP_API_KEY           | MISP API key                   | Optional                       |
 | SLACK_WEBHOOK_URL      | Slack Webhook URL              | Optional                       |
 | SLACK_CHANNEL          | Slack channel name             | Optional (default: `#general`) |
+| BINARYEDGE_API_KEY     | BinaryEdge API key             | Optional                       |
 | CENSYS_ID              | Censys API ID                  | Optional                       |
 | CENSYS_SECRET          | Censys secret                  | Optional                       |
 | CIRCL_PASSIVE_PASSWORD | CIRCL passive DNS/SSL password | Optional                       |

data/lib/mihari.rb

@@ -35,6 +35,8 @@ require "mihari/the_hive"
 
 require "mihari/analyzers/base"
 require "mihari/analyzers/basic"
+
+require "mihari/analyzers/binaryedge"
 require "mihari/analyzers/censys"
 require "mihari/analyzers/circl"
 require "mihari/analyzers/crtsh"

data/lib/mihari/analyzers/base.rb

@@ -60,7 +60,7 @@ module Mihari
 
       # @return [Array<Mihari::Artifact>]
       def normalized_artifacts
-        @normalized_artifacts ||= artifacts.map do |artifact|
+        @normalized_artifacts ||= artifacts.compact.uniq.map do |artifact|
           artifact.is_a?(Artifact) ? artifact : Artifact.new(artifact)
         end.select(&:valid?)
       end

data/lib/mihari/analyzers/binaryedge.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require "binaryedge"
+
+module Mihari
+  module Analyzers
+    class BinaryEdge < Base
+      attr_reader :title
+      attr_reader :description
+      attr_reader :query
+      attr_reader :tags
+
+      def initialize(query, title: nil, description: nil, tags: [])
+        super()
+
+        @query = query
+        @title = title || "BinaryEdge lookup"
+        @description = description || "query = #{query}"
+        @tags = tags
+      end
+
+      def artifacts
+        results = search
+        return [] unless results || results.empty?
+
+        results.map do |result|
+          events = result.dig("events") || []
+          events.map do |event|
+            event.dig "origin", "ip"
+          end.compact
+        end.flatten.compact.uniq
+      end
+
+      private
+
+      PAGE_SIZE = 20
+
+      def search_with_page(query, page: 1)
+        api.host.search(query, page: page)
+      end
+
+      def search
+        responses = []
+        (1..Float::INFINITY).each do |page|
+          res = search_with_page(query, page: page)
+          total = res.dig("total").to_i
+
+          responses << res
+          break if total <= page * PAGE_SIZE
+        end
+        responses
+      end
+
+      def config_keys
+        %w(BINARYEDGE_API_KEY)
+      end
+
+      def api
+        @api ||= ::BinaryEdge::API.new
+      end
+    end
+  end
+end

data/lib/mihari/analyzers/onyphe.rb

@@ -20,15 +20,20 @@ module Mihari
       end
 
       def artifacts
-        result = search
-        return [] unless result
+        results = search
+        return [] unless results
 
-        results = result.dig("results") || []
-        results.map { |e| e.dig("ip") }.compact
+        flat_results = results.map do |result|
+          result.dig("results")
+        end.flatten.compact
+
+        flat_results.map { |result| result.dig("ip") }.compact.uniq
       end
 
       private
 
+      PAGE_SIZE = 10
+
       def config_keys
         %w(ONYPHE_API_KEY)
       end
@@ -37,8 +42,19 @@ module Mihari
         @api ||= ::Onyphe::API.new
       end
 
+      def search_with_page(query, page: 1)
+        api.datascan(query, page: page)
+      end
+
       def search
-        api.datascan(query)
+        responses = []
+        (1..Float::INFINITY).each do |page|
+          res = search_with_page(query, page: page)
+          responses << res
+          total = res.dig("total").to_i
+          break if total <= page * PAGE_SIZE
+        end
+        responses
       end
     end
   end

data/lib/mihari/analyzers/securitytrails.rb

@@ -57,23 +57,23 @@ module Mihari
       end
 
       def domain_lookup
-        result = api.history.get_all_dns_history(query, "a")
-        records = result.records || []
+        result = api.history.get_all_dns_history(query, type: "a")
+        records = result.dig("records") || []
         records.map do |record|
-          (record.values || []).map(&:ip)
+          (record.dig("values") || []).map { |value| value.dig("ip") }
         end.flatten.compact.uniq
       end
 
       def ip_lookup
         result = api.domains.search( filter: { ipv4: query })
-        records = result.records || []
-        records.map(&:hostname).compact.uniq
+        records = result.dig("records") || []
+        records.map { |record| record.dig("hostname") }.compact.uniq
       end
 
       def mail_lookup
         result = api.domains.search( filter: { whois_email: query })
-        records = result.records || []
-        records.map(&:hostname).compact.uniq
+        records = result.dig("records") || []
+        records.map { |record| record.dig("hostname") }.compact.uniq
       end
     end
   end

data/lib/mihari/analyzers/shodan.rb

@@ -33,6 +33,8 @@ module Mihari
 
       private
 
+      PAGE_SIZE = 100
+
       def config_keys
         %w(SHODAN_API_KEY)
       end
@@ -54,7 +56,7 @@ module Mihari
           break unless res
 
           responses << res
-          break if res.dig("total").to_i <= page * 100
+          break if res.dig("total").to_i <= page * PAGE_SIZE
         end
         responses
       end

data/lib/mihari/analyzers/zoomeye.rb

@@ -34,6 +34,8 @@ module Mihari
 
       private
 
+      PAGE_SIZE = 10
+
       def valid_type?
         %w(host web).include? type
       end
@@ -63,14 +65,13 @@ module Mihari
 
       def host_lookup
         responses = []
+        (1..Float::INFINITY).each do |page|
+          res = _host_lookup(query, page: page)
+          break unless res
 
-        res = _host_lookup(query)
-        total = res.dig("total").to_f
-        max = (total / 10.0).ceil
-        responses << res
-
-        (2..max).each do |page|
-          responses << _host_lookup(query, page: page)
+          total = res.dig("total").to_i
+          responses << res
+          break if total <= page * PAGE_SIZE
         end
         convert_responses responses.compact
       end
@@ -83,14 +84,13 @@ module Mihari
 
       def web_lookup
         responses = []
+        (1..Float::INFINITY).each do |page|
+          res = _web_lookup(query, page: page)
+          break unless res
 
-        res = _web_lookup(query)
-        total = res.dig("total").to_f
-        max = (total / 10.0).ceil
-        responses << res
-
-        (2..max).each do |page|
-          responses << _web_lookup(query, page: page)
+          total = res.dig("total").to_i
+          responses << res
+          break if total <= page * PAGE_SIZE
         end
         convert_responses responses.compact
       end

data/lib/mihari/cli.rb

@@ -120,7 +120,7 @@ module Mihari
       end
     end
 
-    desc "zoommeye [QUERY]", "ZoomEye lookup by a given query"
+    desc "zoomeye [QUERY]", "ZoomEye lookup by a given query"
     method_option :title, type: :string, desc: "title"
     method_option :description, type: :string, desc: "description"
     method_option :tags, type: :array, desc: "tags"
@@ -131,6 +131,16 @@ module Mihari
       end
     end
 
+    desc "binaryedge [QUERY]", "BinaryEdge lookup by a given query"
+    method_option :title, type: :string, desc: "title"
+    method_option :description, type: :string, desc: "description"
+    method_option :tags, type: :array, desc: "tags"
+    def binaryedge(query)
+      with_error_handling do
+        run_analyzer Analyzers::BinaryEdge, query: query, options: options
+      end
+    end
+
     desc "import_from_json", "Give a JSON input via STDIN"
     def import_from_json(input = nil)
       with_error_handling do

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "0.12.0"
+  VERSION = "0.13.0"
 end

data/mihari.gemspec

@@ -34,6 +34,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "webmock", "~> 3.7"
 
   spec.add_dependency "addressable", "~> 2.7"
+  spec.add_dependency "binaryedge", "~> 0.1"
   spec.add_dependency "censu", "~> 0.2"
   spec.add_dependency "crtsh-rb", "~> 0.1"
   spec.add_dependency "dnpedia", "~> 0.1"
@@ -43,16 +44,16 @@ Gem::Specification.new do |spec|
   spec.add_dependency "mem", "~> 0.1"
   spec.add_dependency "misp", "~> 0.1"
   spec.add_dependency "net-ping", "~> 2.0"
-  spec.add_dependency "onyphe", "~> 0.2"
+  spec.add_dependency "onyphe", "~> 1.0"
   spec.add_dependency "parallel", "~> 1.18"
   spec.add_dependency "passive_circl", "~> 0.1"
   spec.add_dependency "passivetotalx", "~> 0.1"
   spec.add_dependency "public_suffix", "~> 4.0"
-  spec.add_dependency "securitytrails", "~> 0.2"
+  spec.add_dependency "securitytrails", "~> 1.0"
   spec.add_dependency "shodanx", "~> 0.2"
   spec.add_dependency "slack-notifier", "~> 2.3"
   spec.add_dependency "thor", "~> 0.20"
   spec.add_dependency "urlscan", "~> 0.4"
-  spec.add_dependency "virustotalx", "~> 1.0"
+  spec.add_dependency "virustotalx", "~> 1.1"
   spec.add_dependency "zoomeye-rb", "~> 0.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mihari
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.13.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-10-17 00:00:00.000000000 Z
+date: 2019-10-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -136,6 +136,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.7'
+- !ruby/object:Gem::Dependency
+  name: binaryedge
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: censu
   requirement: !ruby/object:Gem::Requirement
@@ -268,14 +282,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
@@ -338,14 +352,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: shodanx
   requirement: !ruby/object:Gem::Requirement
@@ -408,14 +422,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: zoomeye-rb
   requirement: !ruby/object:Gem::Requirement
@@ -454,6 +468,7 @@ files:
 - lib/mihari/alert_viewer.rb
 - lib/mihari/analyzers/base.rb
 - lib/mihari/analyzers/basic.rb
+- lib/mihari/analyzers/binaryedge.rb
 - lib/mihari/analyzers/censys.rb
 - lib/mihari/analyzers/circl.rb
 - lib/mihari/analyzers/crtsh.rb
@@ -510,7 +525,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: A framework for continuous malicious hosts monitoring.