mihari 6.0.0 → 6.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2846f5154fcdde4cda4f0237c79e047fc498b96b5c21a6152287ac9ab11faac9
-  data.tar.gz: f43b4e20a59b1274b62c5e9768f153429540067882508b34f0c36932cc9cb5ef
+  metadata.gz: d49d8a079c765cb4b7ad7a08825a9d0bd4e9e21fbaf2c0f185c4f27e8f2f2ca6
+  data.tar.gz: aa693115eb1dacc09d13ca0942d859e24566c09a4cd6d702840ec9df6c3cc87a
 SHA512:
-  metadata.gz: b54da0da25e57531c1efef94bdc42df715569b9e88842ca330d1d0893f4c20990bb2ded22a001f31816cbd72f00f43473f0254f222a9849308be746576128e3c
-  data.tar.gz: 7167afd356c4f945e66631bb3c6cefc06678961b480511d85939e6ede9eccec9a5f83a948a169be9ee7ba5cc79af918a447b8f01e5280faff3138071e7808beb
+  metadata.gz: e50a254b0c5565c3691cc34df0413258a1903f8864f0ced141fab63cb673f02998451723036917f2adf21f759ec3f0086b772e98a02ee0436869dd5846bf3427
+  data.tar.gz: 63738367f8edd23331518eb839c188dde8fb67257b83b5cd6a11e79dc725aa89b151753e7429515ff1b2f7c689da77adf4eac998241b7bfd04f88302aa52c923

data/.rubocop.yml

@@ -13,6 +13,10 @@ Metrics/MethodLength:
   Max: 50
 Metrics/AbcSize:
   Max: 50
+RSpec/MultipleMemoizedHelpers:
+  Max: 10
+RSpec/ExampleLength:
+  Max: 20
 require:
   - rubocop-rspec
   - rubocop-yard

data/lib/mihari/actor.rb

@@ -65,11 +65,9 @@ module Mihari
 
     def result(...)
       Try[StandardError] do
-        retry_on_error(
-          times: retry_times,
-          interval: retry_interval,
-          exponential_backoff: retry_exponential_backoff
-        ) { call(...) }
+        retry_on_error(times: retry_times, interval: retry_interval, exponential_backoff: retry_exponential_backoff) do
+          call(...)
+        end
       end.to_result
     end
 
@@ -78,7 +76,7 @@ module Mihari
       # @return [String]
       #
       def class_key
-        to_s.split("::").last
+        to_s.split("::").last.downcase
       end
 
       #

data/lib/mihari/analyzers/base.rb

@@ -37,10 +37,14 @@ module Mihari
       # @return [Boolean]
       #
       def ignore_error?
-        ignore_error = options[:ignore_error]
-        return ignore_error unless ignore_error.nil?
+        options[:ignore_error] || Mihari.config.ignore_error
+      end
 
-        Mihari.config.ignore_error
+      #
+      # @return [Boolean]
+      #
+      def parallel?
+        options[:parallel] || Mihari.config.parallel
       end
 
       # @return [Array<String>, Array<Mihari::Models::Artifact>]
@@ -60,7 +64,10 @@ module Mihari
           # No need to set data_type manually
           # It is set automatically in #initialize
           artifact = artifact.is_a?(Models::Artifact) ? artifact : Models::Artifact.new(data: artifact)
+
           artifact.source = self.class.class_key
+          artifact.query = query
+
           artifact
         end.select(&:valid?).uniq(&:data)
       end

data/lib/mihari/analyzers/circl.rb

@@ -26,7 +26,7 @@ module Mihari
       def initialize(query, options: nil, username: nil, password: nil)
         super(refang(query), options: options)
 
-        @type = TypeChecker.type(query)
+        @type = DataType.type(query)
 
         @username = username || Mihari.config.circl_passive_username
         @password = password || Mihari.config.circl_passive_password

data/lib/mihari/analyzers/dnstwister.rb

@@ -18,7 +18,7 @@ module Mihari
       def initialize(query, options: nil)
         super(refang(query), options: options)
 
-        @type = TypeChecker.type(query)
+        @type = DataType.type(query)
       end
 
       def artifacts

data/lib/mihari/analyzers/hunterhow.rb

@@ -22,7 +22,7 @@ module Mihari
       # @param [Hash, nil] options
       # @param [String, nil] api_key
       #
-      def initialize(query, start_time:, end_time:, options: nil, api_key: nil)
+      def initialize(query, start_time: nil, end_time: nil, options: nil, api_key: nil)
         super(query, options: options)
 
         @api_key = api_key || Mihari.config.hunterhow_api_key

data/lib/mihari/analyzers/otx.rb

@@ -22,7 +22,7 @@ module Mihari
       def initialize(query, options: nil, api_key: nil)
         super(refang(query), options: options)
 
-        @type = TypeChecker.type(query)
+        @type = DataType.type(query)
 
         @api_key = api_key || Mihari.config.otx_api_key
       end

data/lib/mihari/analyzers/passivetotal.rb

@@ -26,7 +26,7 @@ module Mihari
       def initialize(query, options: nil, api_key: nil, username: nil)
         super(refang(query), options: options)
 
-        @type = TypeChecker.type(query)
+        @type = DataType.type(query)
 
         @username = username || Mihari.config.passivetotal_username
         @api_key = api_key || Mihari.config.passivetotal_api_key

data/lib/mihari/analyzers/pulsedive.rb

@@ -22,7 +22,7 @@ module Mihari
       def initialize(query, options: nil, api_key: nil)
         super(refang(query), options: options)
 
-        @type = TypeChecker.type(query)
+        @type = DataType.type(query)
 
         @api_key = api_key || Mihari.config.pulsedive_api_key
       end

data/lib/mihari/analyzers/securitytrails.rb

@@ -25,7 +25,7 @@ module Mihari
       def initialize(query, options: nil, api_key: nil)
         super(refang(query), options: options)
 
-        @type = TypeChecker.type(query)
+        @type = DataType.type(query)
 
         @api_key = api_key || Mihari.config.securitytrails_api_key
       end

data/lib/mihari/analyzers/virustotal.rb

@@ -22,7 +22,7 @@ module Mihari
       def initialize(query, options: nil, api_key: nil)
         super(refang(query), options: options)
 
-        @type = TypeChecker.type(query)
+        @type = DataType.type(query)
 
         @api_key = api_key || Mihari.config.virustotal_api_key
       end

data/lib/mihari/clients/google_public_dns.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Clients
+    #
+    # Google Public DNS enricher
+    #
+    class GooglePublicDNS < Base
+      #
+      # @param [String] base_url
+      # @param [Hash] headers
+      # @param [Integer, nil] timeout
+      #
+      def initialize(base_url = "https://dns.google", headers: {}, timeout: nil)
+        super(base_url, headers: headers, timeout: timeout)
+      end
+
+      #
+      # Query Google Public DNS by resource type
+      #
+      # @param [String] name
+      #
+      # @return [Mihari::Structs::GooglePublicDNS::Response, nil]
+      #
+      def query_all(name)
+        Structs::GooglePublicDNS::Response.from_dynamic! get_json("/resolve",
+          params: { name: name, type: "ALL" })
+      end
+    end
+  end
+end

data/lib/mihari/config.rb

@@ -42,10 +42,12 @@ module Mihari
       ignore_error: false,
       pagination_interval: 0,
       pagination_limit: 100,
+      parallel: false,
       retry_exponential_backoff: true,
       retry_interval: 5,
       retry_times: 3,
-      sentry_dsn: nil
+      sentry_dsn: nil,
+      sentry_trace_sample_rate: 0.25
     )
 
     # @!attribute [r] binaryedge_api_key
@@ -61,7 +63,7 @@ module Mihari
     #   @return [String, nil]
 
     # @!attribute [r] database_url
-    #   @return [String, nil]
+    #   @return [URI, nil]
 
     # @!attribute [r] fofa_api_key
     #   @return [String, nil]
@@ -132,6 +134,9 @@ module Mihari
     # @!attribute [r] sentry_dsn
     #   @return [String, nil]
 
+    # @!attribute [r] sentry_trace_sample_rate
+    #   @return [Float]
+
     # @!attribute [r] retry_interval
     #   @return [Integer]
 
@@ -147,6 +152,9 @@ module Mihari
     # @!attribute [r] pagination_limit
     #   @return [Integer]
 
+    # @!attribute [r] parallel
+    #   @return [Boolean]
+
     # @!attribute [r] ignore_error
     #   @return [Boolean]
 

data/lib/mihari/{type_checker.rb → data_type.rb}

@@ -2,9 +2,11 @@
 
 module Mihari
   #
-  # Artifact type checker
+  # (Artifact) Data Type
   #
-  class TypeChecker
+  class DataType
+    include Dry::Monads[:try]
+
     # @return [String]
     attr_reader :data
 
@@ -24,26 +26,25 @@ module Mihari
 
     # @return [Boolean]
     def ip?
-      IPAddr.new data
-      true
-    rescue IPAddr::InvalidAddressError => _e
-      false
+      Try[IPAddr::InvalidAddressError] do
+        IPAddr.new(data).to_s == data
+      end.to_result.value_or(false)
     end
 
     # @return [Boolean]
     def domain?
-      uri = Addressable::URI.parse("http://#{data}")
-      uri.host == data && PublicSuffix.valid?(uri.host)
-    rescue Addressable::URI::InvalidURIError => _e
-      false
+      Try[Addressable::URI::InvalidURIError] do
+        uri = Addressable::URI.parse("http://#{data}")
+        uri.host == data && PublicSuffix.valid?(uri.host)
+      end.to_result.value_or(false)
     end
 
     # @return [Boolean]
     def url?
-      uri = Addressable::URI.parse(data)
-      uri.scheme && uri.host && uri.path && PublicSuffix.valid?(uri.host)
-    rescue Addressable::URI::InvalidURIError => _e
-      false
+      Try[Addressable::URI::InvalidURIError] do
+        uri = Addressable::URI.parse(data)
+        uri.scheme && uri.host && uri.path && PublicSuffix.valid?(uri.host)
+      end.to_result.value_or(false)
     end
 
     # @return [Boolean]
@@ -53,38 +54,20 @@ module Mihari
 
     # @return [String, nil]
     def type
-      return "hash" if hash?
-      return "ip" if ip?
-      return "domain" if domain?
-      return "url" if url?
+      found = %i[hash? ip? domain? url? mail?].find { |method| send(method) if respond_to?(method) }
+      return nil if found.nil?
 
-      "mail" if mail?
+      found[...-1].to_s
     end
 
     # @return [String, nil]
     def detailed_type
-      return "md5" if md5?
-      return "sha1" if sha1?
-      return "sha256" if sha256?
-      return "sha512" if sha512?
+      found = %i[md5? sha1? sha256? sha512?].find { |method| send(method) if respond_to?(method) }
+      return found[...-1].to_s unless found.nil?
 
       type
     end
 
-    class << self
-      # @return [String, nil]
-      def type(data)
-        new(data).type
-      end
-
-      # @return [String, nil]
-      def detailed_type(data)
-        new(data).detailed_type
-      end
-    end
-
-    private
-
     # @return [Boolean]
     def md5?
       data.match?(/^[A-Fa-f0-9]{32}$/)
@@ -104,5 +87,17 @@ module Mihari
     def sha512?
       data.match?(/^[A-Fa-f0-9]{128}$/)
     end
+
+    class << self
+      # @return [String, nil]
+      def type(data)
+        new(data).type
+      end
+
+      # @return [String, nil]
+      def detailed_type(data)
+        new(data).detailed_type
+      end
+    end
   end
 end

data/lib/mihari/database.rb

@@ -111,6 +111,12 @@ class V5Schema < ActiveRecord::Migration[7.1]
   end
 end
 
+class V61Schema < ActiveRecord::Migration[7.1]
+  def change
+    add_column :artifacts, :query, :string
+  end
+end
+
 def adapter
   return "postgresql" if %w[postgresql postgres].include?(Mihari.config.database_url.scheme)
   return "mysql2" if Mihari.config.database_url.scheme == "mysql2"
@@ -122,7 +128,7 @@ end
 # @return [Array<ActiveRecord::Migration>] schemas
 #
 def schemas
-  [V5Schema]
+  [V5Schema, V61Schema]
 end
 
 module Mihari
@@ -148,7 +154,7 @@ module Mihari
 
         case adapter
         when "postgresql", "mysql2"
-          ActiveRecord::Base.establish_connection(Mihari.config.database_url.to_s)
+          ActiveRecord::Base.establish_connection Mihari.config.database_url.to_s
         else
           ActiveRecord::Base.establish_connection(
             adapter: adapter,
@@ -156,8 +162,6 @@ module Mihari
           )
         end
         ActiveRecord::Base.logger = Logger.new($stdout) if development_env?
-      rescue StandardError => e
-        Mihari.logger.error e
       end
 
       #

data/lib/mihari/emitters/misp.rb

@@ -56,12 +56,12 @@ module Mihari
         })
       end
 
-      private
-
       def configuration_keys
         %w[misp_url misp_api_key]
       end
 
+      private
+
       def client
         @client ||= Clients::MISP.new(url, api_key: api_key, timeout: timeout)
       end

data/lib/mihari/emitters/slack.rb

@@ -6,7 +6,7 @@ require "slack-notifier"
 module Mihari
   module Emitters
     class Attachment
-      include Memist::Memoizable
+      prepend MemoWise
 
       # @return [String]
       attr_reader :data
@@ -76,7 +76,7 @@ module Mihari
           "https://urlscan.io/domain/#{uri.hostname}"
         end
       end
-      memoize :_urlscan_link
+      memo_wise :_urlscan_link
 
       # @return [String, nil]
       def _vt_link
@@ -93,19 +93,19 @@ module Mihari
           "https://www.virustotal.com/#/search/#{data}"
         end
       end
-      memoize :_vt_link
+      memo_wise :_vt_link
 
       # @return [String, nil]
       def _censys_link
         (data_type == "ip") ? "https://search.censys.io/hosts/#{data}" : nil
       end
-      memoize :_censys_link
+      memo_wise :_censys_link
 
       # @return [String, nil]
       def _shodan_link
         (data_type == "ip") ? "https://www.shodan.io/host/#{data}" : nil
       end
-      memoize :_shodan_link
+      memo_wise :_shodan_link
 
       # @return [String]
       def sha256

data/lib/mihari/emitters/the_hive.rb

@@ -66,12 +66,12 @@ module Mihari
         end.first
       end
 
-      private
-
       def configuration_keys
         %w[thehive_url thehive_api_key]
       end
 
+      private
+
       def client
         @client ||= Clients::TheHive.new(url, api_key: api_key, api_version: normalized_api_version, timeout: timeout)
       end

data/lib/mihari/enrichers/base.rb

@@ -6,6 +6,8 @@ module Mihari
     # Base class for enrichers
     #
     class Base < Actor
+      prepend MemoWise
+
       def initialize(options: nil)
         super(options: options)
       end

data/lib/mihari/enrichers/google_public_dns.rb

@@ -11,27 +11,10 @@ module Mihari
       #
       # @param [String] name
       #
-      # @return [Array<Mihari::Structs::GooglePublicDNS::Response>]
+      # @return [Mihari::Structs::GooglePublicDNS::Response]
       #
       def call(name)
-        %w[A AAAA CNAME TXT NS].filter_map { |resource_type| query_by_type(name, resource_type) }
-      end
-
-      #
-      # Query Google Public DNS by resource type
-      #
-      # @param [String] name
-      # @param [String] resource_type
-      #
-      # @return [Mihari::Structs::GooglePublicDNS::Response, nil]
-      #
-      def query_by_type(name, resource_type)
-        url = "https://dns.google/resolve"
-        params = { name: name, type: resource_type }
-        res = http.get(url, params: params)
-        Structs::GooglePublicDNS::Response.from_dynamic! JSON.parse(res.body.to_s)
-      rescue HTTPError
-        nil
+        client.query_all name
       end
 
       class << self
@@ -45,8 +28,8 @@ module Mihari
 
       private
 
-      def http
-        HTTP::Factory.build timeout: timeout
+      def client
+        Clients::GooglePublicDNS.new
       end
     end
   end

data/lib/mihari/enrichers/ipinfo.rb

@@ -35,6 +35,7 @@ module Mihari
         res = http.get(url)
         Structs::IPInfo::Response.from_dynamic! JSON.parse(res.body.to_s)
       end
+      memo_wise :call
 
       private
 

data/lib/mihari/enrichers/shodan.rb

@@ -18,6 +18,7 @@ module Mihari
         res = http.get(url)
         Structs::Shodan::InternetDBResponse.from_dynamic! JSON.parse(res.body.to_s)
       end
+      memo_wise :call
 
       private
 

data/lib/mihari/enrichers/whois.rb

@@ -8,16 +8,11 @@ module Mihari
     # Whois enricher
     #
     class Whois < Base
-      # @return [Hash]
-      attr_accessor :memo
-
       #
       # @param [Hash, nil] options
       #
       def initialize(options: nil)
         super(options: options)
-
-        @memo = {}
       end
 
       #
@@ -28,16 +23,22 @@ module Mihari
       # @return [Mihari::Models::WhoisRecord, nil]
       #
       def call(domain)
-        domain = PublicSuffix.domain(domain)
+        _call PublicSuffix.domain(domain)
+      end
 
-        # check memo
-        return memo[domain].dup if memo.key?(domain)
+      private
 
+      #
+      # @param [String] domain
+      #
+      # @return [Mihari::Models::WhoisRecord, nil]
+      #
+      def _call(domain)
         record = whois.lookup(domain)
         parser = record.parser
         return nil if parser.available?
 
-        whois_record = Models::WhoisRecord.new(
+        Models::WhoisRecord.new(
           domain: domain,
           created_on: get_created_on(parser),
           updated_on: get_updated_on(parser),
@@ -45,14 +46,8 @@ module Mihari
           registrar: get_registrar(parser),
           contacts: get_contacts(parser)
         )
-
-        # set memo
-        memo[domain] = whois_record
-
-        whois_record
       end
-
-      private
+      memo_wise :_call
 
       #
       # @return [::Whois::Client]

data/lib/mihari/entities/artifact.rb

@@ -11,6 +11,7 @@ module Mihari
       expose :data, documentation: { type: String, required: true }
       expose :data_type, documentation: { type: String, required: true }, as: :dataType
       expose :source, documentation: { type: String, required: true }
+      expose :query, documentation: { type: String, required: false }
       expose :tags, documentation: { type: String, is_array: true }
     end
 

data/lib/mihari/mixins/falsepositive.rb

@@ -6,7 +6,7 @@ module Mihari
     # False positive mixins
     #
     module FalsePositive
-      include Memist::Memoizable
+      prepend MemoWise
 
       #
       # Normalize a falsepositive value
@@ -22,7 +22,7 @@ module Mihari
         value_without_slashes = value[1..-2]
         Regexp.compile value_without_slashes.to_s
       end
-      memoize :normalize_falsepositive
+      memo_wise :normalize_falsepositive
 
       #
       # Check whether a value is valid format as a disallowed data value

data/lib/mihari/models/artifact.rb

@@ -46,7 +46,7 @@ module Mihari
 
         super(*args, **kwargs)
 
-        self.data_type = TypeChecker.type(data)
+        self.data_type = DataType.type(data)
 
         @tags = []
         @rule_id = ""
@@ -158,13 +158,13 @@ module Mihari
       # Enrich all the enrichable relationships of the artifact
       #
       def enrich_all
-        enrich_autonomous_system
+        enrich_autonomous_system ipinfo
         enrich_dns
-        enrich_geolocation
-        enrich_reverse_dns
+        enrich_geolocation ipinfo
+        enrich_reverse_dns shodan
         enrich_whois
-        enrich_ports
-        enrich_cpes
+        enrich_ports shodan
+        enrich_cpes shodan
       end
 
       ENRICH_METHODS_BY_ENRICHER = {
@@ -197,6 +197,14 @@ module Mihari
 
       private
 
+      def ipinfo
+        @ipinfo ||= Enrichers::IPInfo.new
+      end
+
+      def shodan
+        @shodan ||= Enrichers::Shodan.new
+      end
+
       def normalize_as_domain(url_or_domain)
         return url_or_domain if data_type == "domain"