mihari 3.4.1 → 3.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1d3afbbcde503f0bf4c63e2ed78441c92eb287e1d820bc075ff86587b0d0616d
-  data.tar.gz: 0bb88ca8b8c58d962c2e45b765935ac8ad8e9804c72edc39df082d5f019dce74
+  metadata.gz: f6e9001e10ac0891e2e10d90bccbd165afbdfc3bcd2724a3d5adf521b2be843b
+  data.tar.gz: 98119fae9302eb4251ceda56c1ddb0ab615fbf36d67505fd8af2ffb08f1dcfaf
 SHA512:
-  metadata.gz: 298000884be755ef975bb13a5963203d693429f72c5fe0ffe8f63d7bc407d876ffacda068aae1d837d0fff5a71ea8b278f7409f068560df885e4178b4d70905d
-  data.tar.gz: c98fa5a38aacf7949f40d317484a86b245a5b82ede6eaf1d8ed2a1e361623712b08e464a1f8d9b2a89f77e63fbc5a0e3ab56488494134222fbc7fb0e70399cac
+  metadata.gz: 92ae37318ffb97ab4746cb44c266e63fba5c505a6c81c141be362c11ecf4e399850f1921b2907bf511f6b0a4ea5884642282f319962af1628fc8bf9e69503fa8
+  data.tar.gz: 45f2a2a748bb362a0daa7f9c3c7da5329cb7bab002f1ddcec4a56d111102d311471b3904b2d77f50f406e05f43deeacbd8f235e5ff4ae2dd2a0096a4dd98074d

data/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "vendor/rbs/gem_rbs_collection"]
+	path = vendor/rbs/gem_rbs_collection
+	url = https://github.com/ruby/gem_rbs_collection.git

data/README.md

@@ -64,3 +64,5 @@ The gem is available as open source under the terms of the [MIT License](https:/
 ## Acknowledgement
 
 Mihari is proudly supported by [Tines.io](https://tines.io?utm_source=github&utm_medium=sponsorship&utm_campaign=ninoseki), The SOAR Platform for Enterprise Security Teams.
+
+$ bundle exec rbs -rpathname  --repo=gem_rbs/gems -ractivesupport -ractionpack -ractivejob -ractivemodel -ractionview -ractiverecord -rrailties -I sig validate

data/Steepfile

@@ -0,0 +1,32 @@
+target :lib do
+  signature "sig"
+  check "lib"
+
+  repo_path "vendor/rbs/gem_rbs_collection/gems"
+
+  library "date"
+  library "json"
+  library "logger"
+  library "monitor"
+  library "mutex_m"
+  library "pathname"
+  library "securerandom"
+  library "singleton"
+  library "time"
+  library "tsort"
+  library "uri"
+  library "resolv"
+  library "timeout"
+  library "socket"
+
+  library "rack"
+
+  library "actionpack"
+  library "actionview"
+  library "activejob"
+  library "activemodel"
+  library "activerecord"
+  library "activesupport"
+  library "parallel"
+  library "railties"
+end

data/config.ru

@@ -1,3 +1,4 @@
+# bundle exec rerun -- rackup config.ru
 require "./lib/mihari"
 
 # set rack env as development

data/lib/mihari/analyzers/base.rb

@@ -8,6 +8,7 @@ module Mihari
     class Base
       extend Dry::Initializer
 
+      include Mixins::AutonomousSystem
       include Mixins::Configurable
       include Mixins::Retriable
 
@@ -27,7 +28,7 @@ module Mihari
 
       # @return [String]
       def title
-        self.class.to_s.split("::").last
+        self.class.to_s.split("::").last.to_s
       end
 
       # @return [String]
@@ -37,7 +38,7 @@ module Mihari
 
       # @return [String]
       def source
-        self.class.to_s.split("::").last
+        self.class.to_s.split("::").last.to_s
       end
 
       # @return [Array<String>]
@@ -51,7 +52,7 @@ module Mihari
       # @return [nil]
       #
       def run
-        set_unique_artifacts
+        set_enriched_artifacts
 
         Parallel.each(valid_emitters) do |emitter|
           run_emitter emitter
@@ -66,7 +67,7 @@ module Mihari
       # @return [nil]
       #
       def run_emitter(emitter)
-        emitter.run(title: title, description: description, artifacts: unique_artifacts, source: source, tags: tags)
+        emitter.run(title: title, description: description, artifacts: enriched_artifacts, source: source, tags: tags)
       rescue StandardError => e
         puts "Emission by #{emitter.class} is failed: #{e}"
       end
@@ -88,7 +89,7 @@ module Mihari
           # No need to set data_type manually
           # It is set automatically in #initialize
           artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact, source: source)
-        end.select(&:valid?)
+        end.select(&:valid?).uniq(&:data)
       end
 
       private
@@ -105,15 +106,27 @@ module Mihari
       end
 
       #
-      # Set unique artifacts
+      # Enriched artifacts
+      #
+      # @return [Array<Mihari::Artifact>]
+      #
+      def enriched_artifacts
+        @enriched_artifacts ||= unique_artifacts.map do |artifact|
+          artifact.enrich_all
+          artifact
+        end
+      end
+
+      #
+      # Set enriched artifacts
       #
       # @return [nil]
       #
-      def set_unique_artifacts
-        retry_on_error { unique_artifacts }
-      rescue ArgumentError => _e
+      def set_enriched_artifacts
+        retry_on_error { enriched_artifacts }
+      rescue ArgumentError => e
         klass = self.class.to_s.split("::").last.to_s
-        raise Error, "Please configure #{klass} API settings properly"
+        raise Error, "Please configure #{klass} settings properly. (#{e})"
       end
 
       #
@@ -125,7 +138,7 @@ module Mihari
         @valid_emitters ||= Mihari.emitters.filter_map do |klass|
           emitter = klass.new
           emitter.valid? ? emitter : nil
-        end
+        end.compact
       end
     end
   end

data/lib/mihari/analyzers/binaryedge.rb

@@ -26,6 +26,14 @@ module Mihari
 
       PAGE_SIZE = 20
 
+      #
+      # Search with pagination
+      #
+      # @param [String] query
+      # @param [Integer] page
+      #
+      # @return [Hash]
+      #
       def search_with_page(query, page: 1)
         api.host.search(query, page: page)
       rescue ::BinaryEdge::Error => e
@@ -34,6 +42,11 @@ module Mihari
         raise e
       end
 
+      #
+      # Search
+      #
+      # @return [Array<Hash>]
+      #
       def search
         responses = []
         (1..Float::INFINITY).each do |page|

data/lib/mihari/analyzers/censys.rb

@@ -16,32 +16,65 @@ module Mihari
 
       private
 
+      #
+      # Search
+      #
+      # @return [Array<String>]
+      #
       def search
-        ipv4s = []
+        artifacts = []
 
         cursor = nil
         loop do
           response = api.search(query, cursor: cursor)
-          ipv4s << response_to_ipv4s(response)
+          response = Structs::Censys::Response.from_dynamic!(response)
 
-          links = response.dig("result", "links")
-          cursor = links["next"]
+          artifacts << response_to_artifacts(response)
+
+          cursor = response.result.links.next
           break if cursor == ""
         end
 
-        ipv4s.flatten
+        artifacts.flatten.uniq(&:data)
       end
 
       #
       # Extract IPv4s from Censys search API response
       #
-      # @param [Hash] response
+      # @param [Structs::Censys::Response] response
       #
       # @return [Array<String>]
       #
-      def response_to_ipv4s(response)
-        hits = response.dig("result", "hits") || []
-        hits.map { |hit| hit["ip"] }
+      def response_to_artifacts(response)
+        response.result.hits.map { |hit| build_artifact(hit) }
+      end
+
+      #
+      # Build an artifact from a Shodan search API response
+      #
+      # @param [Structs::Censys::Hit] hit
+      #
+      # @return [Artifact]
+      #
+      def build_artifact(hit)
+        as = AutonomousSystem.new(asn: normalize_asn(hit.autonomous_system.asn))
+
+        # sometimes Censys overlooks country
+        # then set geolocation as nil
+        geolocation = nil
+        unless hit.location.country.nil?
+          geolocation = Geolocation.new(
+            country: hit.location.country,
+            country_code: hit.location.country_code
+          )
+        end
+
+        Artifact.new(
+          data: hit.ip,
+          source: source,
+          autonomous_system: as,
+          geolocation: geolocation
+        )
       end
 
       def configuration_keys

data/lib/mihari/analyzers/circl.rb

@@ -35,6 +35,11 @@ module Mihari
         @api ||= ::PassiveCIRCL::API.new(username: Mihari.config.circl_passive_username, password: Mihari.config.circl_passive_password)
       end
 
+      #
+      # Passive DNS/SSL search
+      #
+      # @return [Array<String>]
+      #
       def search
         case @type
         when "domain"
@@ -46,6 +51,11 @@ module Mihari
         end
       end
 
+      #
+      # Passive DNS search
+      #
+      # @return [Array<String>]
+      #
       def passive_dns_search
         results = api.dns.query(@query)
         results.filter_map do |result|
@@ -54,6 +64,11 @@ module Mihari
         end.uniq
       end
 
+      #
+      # Passive SSL search
+      #
+      # @return [Array<String>]
+      #
       def passive_ssl_search
         result = api.ssl.cquery(@query)
         seen = result["seen"] || []

data/lib/mihari/analyzers/crtsh.rb

@@ -23,6 +23,11 @@ module Mihari
         @api ||= ::Crtsh::API.new
       end
 
+      #
+      # Search
+      #
+      # @return [Array<Hash>]
+      #
       def search
         exclude = exclude_expired ? "expired" : nil
         api.search(query, exclude: exclude)

data/lib/mihari/analyzers/dnpedia.rb

@@ -20,6 +20,11 @@ module Mihari
         @api ||= ::DNPedia::API.new
       end
 
+      #
+      # Search
+      #
+      # @return [Array<String>]
+      #
       def search
         res = api.search(query)
         rows = res["rows"] || []

data/lib/mihari/analyzers/dnstwister.rb

@@ -29,6 +29,11 @@ module Mihari
 
       private
 
+      #
+      # Check whether a type is valid or not
+      #
+      # @return [Boolean]
+      #
       def valid_type?
         type == "domain"
       end
@@ -37,6 +42,13 @@ module Mihari
         @api ||= ::DNSTwister::API.new
       end
 
+      #
+      # Check whether a domain is resolvable or not
+      #
+      # @param [String] domain
+      #
+      # @return [Boolean]
+      #
       def resolvable?(domain)
         Resolv.getaddress domain
         true
@@ -44,6 +56,11 @@ module Mihari
         false
       end
 
+      #
+      # Search
+      #
+      # @return [Array<String>]
+      #
       def search
         raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
 

data/lib/mihari/analyzers/onyphe.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "onyphe"
+require "normalize_country"
 
 module Mihari
   module Analyzers
@@ -11,14 +12,13 @@ module Mihari
       option :tags, default: proc { [] }
 
       def artifacts
-        results = search
-        return [] unless results
+        responses = search
+        return [] unless responses
 
-        flat_results = results.map do |result|
-          result["results"]
-        end.flatten.compact
-
-        flat_results.filter_map { |result| result["ip"] }.uniq
+        results = responses.map(&:results).flatten
+        results.map do |result|
+          build_artifact result
+        end
       end
 
       private
@@ -33,20 +33,61 @@ module Mihari
         @api ||= ::Onyphe::API.new(Mihari.config.onyphe_api_key)
       end
 
+      #
+      # Search with pagination
+      #
+      # @param [String] query
+      # @param [Integer] page
+      #
+      # @return [Structs::Onyphe::Response]
+      #
       def search_with_page(query, page: 1)
-        api.simple.datascan(query, page: page)
+        res = api.simple.datascan(query, page: page)
+        Structs::Onyphe::Response.from_dynamic!(res)
       end
 
+      #
+      # Search
+      #
+      # @return [Array<Structs::Onyphe::Response>]
+      #
       def search
         responses = []
         (1..Float::INFINITY).each do |page|
           res = search_with_page(query, page: page)
           responses << res
-          total = res["total"].to_i
+
+          total = res.total
           break if total <= page * PAGE_SIZE
         end
         responses
       end
+
+      #
+      # Build an artifact from an Onyphe search API result
+      #
+      # @param [Structs::Onyphe::Result] result
+      #
+      # @return [Artifact]
+      #
+      def build_artifact(result)
+        as = AutonomousSystem.new(asn: normalize_asn(result.asn))
+
+        geolocation = nil
+        unless result.country_code.nil?
+          geolocation = Geolocation.new(
+            country: NormalizeCountry(result.country_code, to: :short),
+            country_code: result.country_code
+          )
+        end
+
+        Artifact.new(
+          data: result.ip,
+          source: source,
+          autonomous_system: as,
+          geolocation: geolocation
+        )
+      end
     end
   end
 end