mihari 3.4.0 → 3.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 142df147927aee93e6c653b2eb29cca50f4ba11606d68f1302af21780d6f0dc5
-  data.tar.gz: 594f762c94e361cc53cab08b39abad5e3503555b51d93add3cbce744fcbff711
+  metadata.gz: 14d0c74e85fbf6ef624afefe7e948595586d6c00fa8bc32f211e60caee581fc3
+  data.tar.gz: 9d5fde6d69f664efac0d6c56e6a0ba60adcad0edcfe45c69f285ffcaba8d11f0
 SHA512:
-  metadata.gz: 8483d2d125e30f04bdaf74243877dd9a261511d7d857f6dba42c6ea54af5de95f63c23759d1937badfdb5ae75819b99e70270edb0fb2f466601b8eaf6912dc8e
-  data.tar.gz: 8ffca15aadc0bc783086dd11df9fd051933af31e6778fcd4a67ddf51af5532b452603526a0303eb7c1dfdb530636a6e91df8440210189af3dc0f2806a4e64218
+  metadata.gz: 30597743e91f124388fbdf426199d97a00f92db9e525fe5725643d529d319ca670d1e9aa6eccff86c0844802157ca3d5c7db9b9afd925d6f0ac4d8b881c44949
+  data.tar.gz: 9d199a3c2f6c7794214730de7c8db812e939c5ddd11ab06d1c6f28c3b8764b2881958b0684adb59e8516d0ac4b6a1dc66b19c5a593efaac9695cb6e317ce8105

data/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "vendor/rbs/gem_rbs_collection"]
+	path = vendor/rbs/gem_rbs_collection
+	url = https://github.com/ruby/gem_rbs_collection.git

data/README.md

@@ -64,3 +64,5 @@ The gem is available as open source under the terms of the [MIT License](https:/
 ## Acknowledgement
 
 Mihari is proudly supported by [Tines.io](https://tines.io?utm_source=github&utm_medium=sponsorship&utm_campaign=ninoseki), The SOAR Platform for Enterprise Security Teams.
+
+$ bundle exec rbs -rpathname  --repo=gem_rbs/gems -ractivesupport -ractionpack -ractivejob -ractivemodel -ractionview -ractiverecord -rrailties -I sig validate

data/Steepfile

@@ -0,0 +1,32 @@
+target :lib do
+  signature "sig"
+  check "lib"
+
+  repo_path "vendor/rbs/gem_rbs_collection/gems"
+
+  library "date"
+  library "json"
+  library "logger"
+  library "monitor"
+  library "mutex_m"
+  library "pathname"
+  library "securerandom"
+  library "singleton"
+  library "time"
+  library "tsort"
+  library "uri"
+  library "resolv"
+  library "timeout"
+  library "socket"
+
+  library "rack"
+
+  library "actionpack"
+  library "actionview"
+  library "activejob"
+  library "activemodel"
+  library "activerecord"
+  library "activesupport"
+  library "parallel"
+  library "railties"
+end

data/config.ru

@@ -1,3 +1,4 @@
+# bundle exec rerun -- rackup config.ru
 require "./lib/mihari"
 
 # set rack env as development

data/lib/mihari/analyzers/base.rb

@@ -27,7 +27,7 @@ module Mihari
 
       # @return [String]
       def title
-        self.class.to_s.split("::").last
+        self.class.to_s.split("::").last.to_s
       end
 
       # @return [String]
@@ -37,7 +37,7 @@ module Mihari
 
       # @return [String]
       def source
-        self.class.to_s.split("::").last
+        self.class.to_s.split("::").last.to_s
       end
 
       # @return [Array<String>]
@@ -51,7 +51,7 @@ module Mihari
       # @return [nil]
       #
       def run
-        set_unique_artifacts
+        set_enriched_artifacts
 
         Parallel.each(valid_emitters) do |emitter|
           run_emitter emitter
@@ -66,7 +66,7 @@ module Mihari
       # @return [nil]
       #
       def run_emitter(emitter)
-        emitter.run(title: title, description: description, artifacts: unique_artifacts, source: source, tags: tags)
+        emitter.run(title: title, description: description, artifacts: enriched_artifacts, source: source, tags: tags)
       rescue StandardError => e
         puts "Emission by #{emitter.class} is failed: #{e}"
       end
@@ -88,7 +88,7 @@ module Mihari
           # No need to set data_type manually
           # It is set automatically in #initialize
           artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact, source: source)
-        end.select(&:valid?)
+        end.select(&:valid?).uniq(&:data)
       end
 
       private
@@ -105,15 +105,29 @@ module Mihari
       end
 
       #
-      # Set unique artifacts
+      # Enriched artifacts
+      #
+      # @return [Array<Mihari::Artifact>]
+      #
+      def enriched_artifacts
+        @enriched_artifacts ||= unique_artifacts.map do |artifact|
+          artifact.enrich_whois
+          artifact.enrich_dns
+          artifact.enrich_reverse_dns
+          artifact
+        end
+      end
+
+      #
+      # Set enriched artifacts
       #
       # @return [nil]
       #
-      def set_unique_artifacts
-        retry_on_error { unique_artifacts }
-      rescue ArgumentError => _e
+      def set_enriched_artifacts
+        retry_on_error { enriched_artifacts }
+      rescue ArgumentError => e
         klass = self.class.to_s.split("::").last.to_s
-        raise Error, "Please configure #{klass} API settings properly"
+        raise Error, "Please configure #{klass} settings properly. (#{e})"
       end
 
       #
@@ -125,7 +139,21 @@ module Mihari
         @valid_emitters ||= Mihari.emitters.filter_map do |klass|
           emitter = klass.new
           emitter.valid? ? emitter : nil
-        end
+        end.compact
+      end
+
+      #
+      # Normalize ASN value
+      #
+      # @param [String, Integer] asn
+      #
+      # @return [Integer]
+      #
+      def normalize_asn(asn)
+        return asn if asn.is_a?(Integer)
+        return asn.to_i unless asn.start_with?("AS")
+
+        asn.delete_prefix("AS").to_i
       end
     end
   end

data/lib/mihari/analyzers/binaryedge.rb

@@ -26,6 +26,14 @@ module Mihari
 
       PAGE_SIZE = 20
 
+      #
+      # Search with pagination
+      #
+      # @param [String] query
+      # @param [Integer] page
+      #
+      # @return [Hash]
+      #
       def search_with_page(query, page: 1)
         api.host.search(query, page: page)
       rescue ::BinaryEdge::Error => e
@@ -34,6 +42,11 @@ module Mihari
         raise e
       end
 
+      #
+      # Search
+      #
+      # @return [Array<Hash>]
+      #
       def search
         responses = []
         (1..Float::INFINITY).each do |page|

data/lib/mihari/analyzers/censys.rb

@@ -16,32 +16,65 @@ module Mihari
 
       private
 
+      #
+      # Search
+      #
+      # @return [Array<String>]
+      #
       def search
-        ipv4s = []
+        artifacts = []
 
         cursor = nil
         loop do
           response = api.search(query, cursor: cursor)
-          ipv4s << response_to_ipv4s(response)
+          response = Structs::Censys::Response.from_dynamic!(response)
 
-          links = response.dig("result", "links")
-          cursor = links["next"]
+          artifacts << response_to_artifacts(response)
+
+          cursor = response.result.links.next
           break if cursor == ""
         end
 
-        ipv4s.flatten
+        artifacts.flatten.uniq(&:data)
       end
 
       #
       # Extract IPv4s from Censys search API response
       #
-      # @param [Hash] response
+      # @param [Structs::Censys::Response] response
       #
       # @return [Array<String>]
       #
-      def response_to_ipv4s(response)
-        hits = response.dig("result", "hits") || []
-        hits.map { |hit| hit["ip"] }
+      def response_to_artifacts(response)
+        response.result.hits.map { |hit| build_artifact(hit) }
+      end
+
+      #
+      # Build an artifact from a Shodan search API response
+      #
+      # @param [Structs::Censys::Hit] hit
+      #
+      # @return [Artifact]
+      #
+      def build_artifact(hit)
+        as = AutonomousSystem.new(asn: normalize_asn(hit.autonomous_system.asn))
+
+        # sometimes Censys overlooks country
+        # then set geolocation as nil
+        geolocation = nil
+        unless hit.location.country.nil?
+          geolocation = Geolocation.new(
+            country: hit.location.country,
+            country_code: hit.location.country_code
+          )
+        end
+
+        Artifact.new(
+          data: hit.ip,
+          source: source,
+          autonomous_system: as,
+          geolocation: geolocation
+        )
       end
 
       def configuration_keys

data/lib/mihari/analyzers/circl.rb

@@ -35,6 +35,11 @@ module Mihari
         @api ||= ::PassiveCIRCL::API.new(username: Mihari.config.circl_passive_username, password: Mihari.config.circl_passive_password)
       end
 
+      #
+      # Passive DNS/SSL search
+      #
+      # @return [Array<String>]
+      #
       def search
         case @type
         when "domain"
@@ -46,6 +51,11 @@ module Mihari
         end
       end
 
+      #
+      # Passive DNS search
+      #
+      # @return [Array<String>]
+      #
       def passive_dns_search
         results = api.dns.query(@query)
         results.filter_map do |result|
@@ -54,6 +64,11 @@ module Mihari
         end.uniq
       end
 
+      #
+      # Passive SSL search
+      #
+      # @return [Array<String>]
+      #
       def passive_ssl_search
         result = api.ssl.cquery(@query)
         seen = result["seen"] || []

data/lib/mihari/analyzers/crtsh.rb

@@ -23,6 +23,11 @@ module Mihari
         @api ||= ::Crtsh::API.new
       end
 
+      #
+      # Search
+      #
+      # @return [Array<Hash>]
+      #
       def search
         exclude = exclude_expired ? "expired" : nil
         api.search(query, exclude: exclude)

data/lib/mihari/analyzers/dnpedia.rb

@@ -20,6 +20,11 @@ module Mihari
         @api ||= ::DNPedia::API.new
       end
 
+      #
+      # Search
+      #
+      # @return [Array<String>]
+      #
       def search
         res = api.search(query)
         rows = res["rows"] || []

data/lib/mihari/analyzers/dnstwister.rb

@@ -29,6 +29,11 @@ module Mihari
 
       private
 
+      #
+      # Check whether a type is valid or not
+      #
+      # @return [Boolean]
+      #
       def valid_type?
         type == "domain"
       end
@@ -37,6 +42,13 @@ module Mihari
         @api ||= ::DNSTwister::API.new
       end
 
+      #
+      # Check whether a domain is resolvable or not
+      #
+      # @param [String] domain
+      #
+      # @return [Boolean]
+      #
       def resolvable?(domain)
         Resolv.getaddress domain
         true
@@ -44,6 +56,11 @@ module Mihari
         false
       end
 
+      #
+      # Search
+      #
+      # @return [Array<String>]
+      #
       def search
         raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
 

data/lib/mihari/analyzers/onyphe.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "onyphe"
+require "normalize_country"
 
 module Mihari
   module Analyzers
@@ -11,14 +12,13 @@ module Mihari
       option :tags, default: proc { [] }
 
       def artifacts
-        results = search
-        return [] unless results
+        responses = search
+        return [] unless responses
 
-        flat_results = results.map do |result|
-          result["results"]
-        end.flatten.compact
-
-        flat_results.filter_map { |result| result["ip"] }.uniq
+        results = responses.map(&:results).flatten
+        results.map do |result|
+          build_artifact result
+        end
       end
 
       private
@@ -33,20 +33,61 @@ module Mihari
         @api ||= ::Onyphe::API.new(Mihari.config.onyphe_api_key)
       end
 
+      #
+      # Search with pagination
+      #
+      # @param [String] query
+      # @param [Integer] page
+      #
+      # @return [Structs::Onyphe::Response]
+      #
       def search_with_page(query, page: 1)
-        api.simple.datascan(query, page: page)
+        res = api.simple.datascan(query, page: page)
+        Structs::Onyphe::Response.from_dynamic!(res)
       end
 
+      #
+      # Search
+      #
+      # @return [Array<Structs::Onyphe::Response>]
+      #
       def search
         responses = []
         (1..Float::INFINITY).each do |page|
           res = search_with_page(query, page: page)
           responses << res
-          total = res["total"].to_i
+
+          total = res.total
           break if total <= page * PAGE_SIZE
         end
         responses
       end
+
+      #
+      # Build an artifact from an Onyphe search API result
+      #
+      # @param [Structs::Onyphe::Result] result
+      #
+      # @return [Artifact]
+      #
+      def build_artifact(result)
+        as = AutonomousSystem.new(asn: normalize_asn(result.asn))
+
+        geolocation = nil
+        unless result.country_code.nil?
+          geolocation = Geolocation.new(
+            country: NormalizeCountry(result.country_code, to: :short),
+            country_code: result.country_code
+          )
+        end
+
+        Artifact.new(
+          data: result.ip,
+          source: source,
+          autonomous_system: as,
+          geolocation: geolocation
+        )
+      end
     end
   end
 end