mihari 5.5.0 → 5.6.1

data/lib/mihari/analyzers/base.rb

@@ -2,7 +2,7 @@
 
 module Mihari
   module Analyzers
-    class Base
+    class Base < Mihari::Base
       include Dry::Monads[:result, :try]
 
       include Mixins::Configurable
@@ -11,37 +11,14 @@ module Mihari
       # @return [String]
       attr_reader :query
 
-      # @return [Hash]
-      attr_reader :options
-
       #
       # @param [String] query
       # @param [Hash, nil] options
       #
       def initialize(query, options: nil)
-        @query = query
-        @options = options || {}
-      end
-
-      #
-      # @return [Integer]
-      #
-      def retry_interval
-        options[:retry_interval] || Mihari.config.retry_interval
-      end
-
-      #
-      # @return [Boolean]
-      #
-      def retry_exponential_backoff
-        options[:retry_exponential_backoff] || Mihari.config.retry_exponential_backoff
-      end
+        super(options: options)
 
-      #
-      # @return [Integer]
-      #
-      def retry_times
-        options[:retry_times] || Mihari.config.retry_times
+        @query = query
       end
 
       #
@@ -68,13 +45,6 @@ module Mihari
         Mihari.config.ignore_error
       end
 
-      #
-      # @return [Integer, nil]
-      #
-      def timeout
-        options[:timeout]
-      end
-
       # @return [Array<String>, Array<Mihari::Artifact>]
       def artifacts
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
@@ -93,7 +63,7 @@ module Mihari
             # No need to set data_type manually
             # It is set automatically in #initialize
             artifact = artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact)
-            artifact.source = source
+            artifact.source = self.class.class_key
             artifact
           end.select(&:valid?).uniq(&:data)
         end
@@ -106,13 +76,6 @@ module Mihari
         Try[StandardError] { normalized_artifacts }.to_result
       end
 
-      # @return [String]
-      def class_name
-        self.class.to_s.split("::").last
-      end
-
-      alias_method :source, :class_name
-
       class << self
         #
         # Initialize an analyzer by query params

data/lib/mihari/analyzers/circl.rb

@@ -51,7 +51,7 @@ module Mihari
       private
 
       def client
-        @client ||= Clients::CIRCL.new(username: username, password: password)
+        @client ||= Clients::CIRCL.new(username: username, password: password, timeout: timeout)
       end
 
       def username?

data/lib/mihari/analyzers/crtsh.rb

@@ -28,7 +28,7 @@ module Mihari
       # @return [Mihari::Clients::Crtsh]
       #
       def client
-        @client ||= Mihari::Clients::Crtsh.new
+        @client ||= Mihari::Clients::Crtsh.new(timeout: timeout)
       end
     end
   end

data/lib/mihari/analyzers/dnstwister.rb

@@ -39,7 +39,7 @@ module Mihari
       end
 
       def client
-        @client ||= Clients::DNSTwister.new
+        @client ||= Clients::DNSTwister.new(timeout: timeout)
       end
 
       #

data/lib/mihari/analyzers/otx.rb

@@ -42,7 +42,7 @@ module Mihari
       private
 
       def client
-        @client ||= Mihari::Clients::OTX.new(api_key: api_key)
+        @client ||= Mihari::Clients::OTX.new(api_key: api_key, timeout: timeout)
       end
 
       #

data/lib/mihari/analyzers/passivetotal.rb

@@ -50,10 +50,19 @@ module Mihari
         %w[passivetotal_username passivetotal_api_key]
       end
 
+      class << self
+        #
+        # @return [Array<String>, nil]
+        #
+        def key_aliases
+          ["pt"]
+        end
+      end
+
       private
 
       def client
-        @client ||= Clients::PassiveTotal.new(username: username, api_key: api_key)
+        @client ||= Clients::PassiveTotal.new(username: username, api_key: api_key, timeout: timeout)
       end
 
       #

data/lib/mihari/analyzers/pulsedive.rb

@@ -35,7 +35,7 @@ module Mihari
             nil
           else
             data = property["value"]
-            Artifact.new(data: data, source: source, metadata: property)
+            Artifact.new(data: data, metadata: property)
           end
         end
       end
@@ -47,7 +47,7 @@ module Mihari
       private
 
       def client
-        @client ||= Clients::PulseDive.new(api_key: api_key)
+        @client ||= Clients::PulseDive.new(api_key: api_key, timeout: timeout)
       end
 
       #

data/lib/mihari/analyzers/rule.rb

@@ -2,46 +2,6 @@
 
 module Mihari
   module Analyzers
-    ANALYZER_TO_CLASS = {
-      "binaryedge" => BinaryEdge,
-      "censys" => Censys,
-      "circl" => CIRCL,
-      "crtsh" => Crtsh,
-      "dnstwister" => DNSTwister,
-      "feed" => Feed,
-      "greynoise" => GreyNoise,
-      "hunterhow" => HunterHow,
-      "onyphe" => Onyphe,
-      "otx" => OTX,
-      "passivetotal" => PassiveTotal,
-      "pt" => PassiveTotal,
-      "pulsedive" => Pulsedive,
-      "securitytrails" => SecurityTrails,
-      "shodan" => Shodan,
-      "st" => SecurityTrails,
-      "urlscan" => Urlscan,
-      "virustotal_intelligence" => VirusTotalIntelligence,
-      "virustotal" => VirusTotal,
-      "vt_intel" => VirusTotalIntelligence,
-      "vt" => VirusTotal,
-      "zoomeye" => ZoomEye
-    }.freeze
-
-    EMITTER_TO_CLASS = {
-      "database" => Emitters::Database,
-      "misp" => Emitters::MISP,
-      "slack" => Emitters::Slack,
-      "the_hive" => Emitters::TheHive,
-      "webhook" => Emitters::Webhook
-    }.freeze
-
-    ENRICHER_TO_CLASS = {
-      "whois" => Enrichers::Whois,
-      "ipinfo" => Enrichers::IPInfo,
-      "shodan" => Enrichers::Shodan,
-      "google_public_dns" => Enrichers::GooglePublicDNS
-    }.freeze
-
     class Rule
       include Mixins::FalsePositive
 
@@ -126,8 +86,14 @@ module Mihari
       def bulk_emit
         return [] if enriched_artifacts.empty?
 
-        Parallel.map(valid_emitters) do |emitter|
-          result = emitter.result
+        # NOTE: separate parallel execution and logging
+        #       because the logger does not work along with Parallel
+        results = Parallel.map(valid_emitters) do |emitter|
+          emitter.result
+        end
+
+        results.zip(valid_emitters).map do |result_and_emitter|
+          result, emitter = result_and_emitter
 
           Mihari.logger.info "Emission by #{emitter.class} is failed: #{result.failure}" if result.failure?
           Mihari.logger.info "Emission by #{emitter.class} is succeeded" if result.success?
@@ -164,15 +130,14 @@ module Mihari
       #
       # Get analyzer class
       #
-      # @param [String] analyzer_name
+      # @param [String] key
       #
       # @return [Class<Mihari::Analyzers::Base>] analyzer class
       #
-      def get_analyzer_class(analyzer_name)
-        analyzer = ANALYZER_TO_CLASS[analyzer_name]
-        return analyzer if analyzer
+      def get_analyzer_class(key)
+        raise ArgumentError, "#{key} is not supported" unless Mihari.analyzer_to_class.key?(key)
 
-        raise ArgumentError, "#{analyzer_name} is not supported"
+        Mihari.analyzer_to_class[key]
       end
 
       #
@@ -189,15 +154,14 @@ module Mihari
       #
       # Get emitter class
       #
-      # @param [String] emitter_name
+      # @param [String] key
       #
       # @return [Class<Mihari::Emitters::Base>] emitter class
       #
-      def get_emitter_class(emitter_name)
-        emitter = EMITTER_TO_CLASS[emitter_name]
-        return emitter if emitter
+      def get_emitter_class(key)
+        raise ArgumentError, "#{key} is not supported" unless Mihari.emitter_to_class.key?(key)
 
-        raise ArgumentError, "#{emitter_name} is not supported"
+        Mihari.emitter_to_class[key]
       end
 
       #
@@ -219,21 +183,20 @@ module Mihari
       # @return [Array<Mihari::Emitters::Base>]
       #
       def valid_emitters
-        emitters.select(&:valid?)
+        @valid_emitters ||= emitters.select(&:valid?)
       end
 
       #
       # Get enricher class
       #
-      # @param [String] enricher_name
+      # @param [String] key
       #
       # @return [Class<Mihari::Enrichers::Base>] enricher class
       #
-      def get_enricher_class(enricher_name)
-        enricher = ENRICHER_TO_CLASS[enricher_name]
-        return enricher if enricher
+      def get_enricher_class(key)
+        raise ArgumentError, "#{key} is not supported" unless Mihari.enricher_to_class.key?(key)
 
-        raise ArgumentError, "#{enricher_name} is not supported"
+        Mihari.enricher_to_class[key]
       end
 
       #
@@ -258,7 +221,9 @@ module Mihari
         analyzers.map do |analyzer|
           next if analyzer.configured?
 
-          message = "#{analyzer.source} is not configured correctly. #{analyzer.configuration_keys.join(", ")} is/are missing."
+          joined = analyzer.configuration_keys.join(", ")
+          be = (analyzer.configuration_keys.length > 1) ? "are" : "is"
+          message = "#{analyzer.class.class_key} is not configured correctly. #{joined} #{be} missing."
           raise ConfigurationError, message
         end
       end

data/lib/mihari/analyzers/securitytrails.rb

@@ -44,10 +44,19 @@ module Mihari
         %w[securitytrails_api_key]
       end
 
+      class << self
+        #
+        # @return [Array<String>, nil]
+        #
+        def key_aliases
+          ["st"]
+        end
+      end
+
       private
 
       def client
-        @client ||= Clients::SecurityTrails.new(api_key: api_key)
+        @client ||= Clients::SecurityTrails.new(api_key: api_key, timeout: timeout)
       end
 
       #

data/lib/mihari/analyzers/virustotal.rb

@@ -39,6 +39,15 @@ module Mihari
         %w[virustotal_api_key]
       end
 
+      class << self
+        #
+        # @return [Array<String>, nil]
+        #
+        def key_aliases
+          ["vt"]
+        end
+      end
+
       private
 
       def client
@@ -65,7 +74,7 @@ module Mihari
         data = res["data"] || []
         data.filter_map do |item|
           data = item.dig("attributes", "ip_address")
-          data.nil? ? nil : Artifact.new(data: data, source: source, metadata: item)
+          data.nil? ? nil : Artifact.new(data: data, metadata: item)
         end
       end
 
@@ -80,7 +89,7 @@ module Mihari
         data = res["data"] || []
         data.filter_map do |item|
           data = item.dig("attributes", "host_name")
-          Artifact.new(data: data, source: source, metadata: item)
+          Artifact.new(data: data, metadata: item)
         end.uniq
       end
     end

data/lib/mihari/analyzers/virustotal_intelligence.rb

@@ -25,6 +25,22 @@ module Mihari
         %w[virustotal_api_key]
       end
 
+      class << self
+        #
+        # @return [String]
+        #
+        def class_key
+          "virustotal_intelligence"
+        end
+
+        #
+        # @return [Array<String>, nil]
+        #
+        def class_key_aliases
+          ["vt_intel"]
+        end
+      end
+
       private
 
       #

data/lib/mihari/analyzers/zoomeye.rb

@@ -73,9 +73,9 @@ module Mihari
           data = match["ip"]
 
           if data.is_a?(Array)
-            data.map { |d| Artifact.new(data: d, source: source, metadata: match) }
+            data.map { |d| Artifact.new(data: d, metadata: match) }
           else
-            Artifact.new(data: data, source: source, metadata: match)
+            Artifact.new(data: data, metadata: match)
           end
         end.flatten
       end

data/lib/mihari/base.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Mihari
+  #
+  # Base class for Analyzer, Emitter and Enricher
+  #
+  class Base
+    # @return [Hash]
+    attr_reader :options
+
+    #
+    # @param [Hash, nil] options
+    #
+    def initialize(*_args, options: nil, **_kwargs)
+      @options = options || {}
+    end
+
+    #
+    # @return [Integer]
+    #
+    def retry_interval
+      options[:retry_interval] || Mihari.config.retry_interval
+    end
+
+    #
+    # @return [Boolean]
+    #
+    def retry_exponential_backoff
+      options[:retry_exponential_backoff] || Mihari.config.retry_exponential_backoff
+    end
+
+    #
+    # @return [Integer]
+    #
+    def retry_times
+      options[:retry_times] || Mihari.config.retry_times
+    end
+
+    #
+    # @return [Integer, nil]
+    #
+    def timeout
+      options[:timeout]
+    end
+
+    class << self
+      #
+      # @return [String]
+      #
+      def class_key
+        to_s.split("::").last
+      end
+
+      #
+      # @return [Array<String>, nil]
+      #
+      def class_key_aliases
+        nil
+      end
+
+      #
+      # @return [Array<String>]
+      #
+      def class_keys
+        ([class_key] + [class_key_aliases]).flatten.compact
+      end
+    end
+  end
+end

data/lib/mihari/cli/main.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "thor"
+require "thor/hollaback"
 
 # Commands
 require "mihari/commands/alert"
@@ -19,10 +20,45 @@ require "mihari/cli/rule"
 module Mihari
   module CLI
     class Main < Base
+      class_option :debug, desc: "Sets up debug mode", aliases: ["-d"], type: :boolean
+      class_around :safe_execute
+
       include Mihari::Commands::Search
       include Mihari::Commands::Version
       include Mihari::Commands::Web
 
+      no_commands do
+        def unwrap_error(err)
+          return err unless err.is_a?(Dry::Monads::UnwrapError)
+
+          # NOTE: UnwrapError's receiver can be either of:
+          #       - Dry::Monads::Try::Error
+          #       - Dry::Monads::Result::Failure
+          receiver = err.receiver
+          return receiver.exception if receiver.is_a?(Dry::Monads::Try::Error)
+
+          receiver.failure
+        end
+
+        def safe_execute
+          yield
+        rescue StandardError => e
+          err = unwrap_error(e)
+
+          raise err if options["debug"]
+
+          case err
+          when ValidationError
+            warn JSON.pretty_generate(err.errors.to_h)
+          when StandardError
+            Sentry.capture_exception(err) if Sentry.initialized?
+            warn err
+          end
+
+          exit 1
+        end
+      end
+
       desc "db", "Sub commands for DB"
       subcommand "db", Database
 

data/lib/mihari/clients/base.rb

@@ -18,10 +18,10 @@ module Mihari
       #
       # @param [String] base_url
       # @param [Hash] headers
-      # @param [Integer] interval
+      # @param [Integer] pagination_interval
       # @param [Integer, nil] timeout
       #
-      def initialize(base_url, headers: {}, pagination_interval: 0, timeout: nil)
+      def initialize(base_url, headers: {}, pagination_interval: Mihari.config.pagination_interval, timeout: nil)
         @base_url = base_url
         @headers = headers || {}
         @pagination_interval = pagination_interval

data/lib/mihari/clients/binaryedge.rb

@@ -7,21 +7,19 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
-      # @param [Integer] pagnation_interval
       # @param [Integer, nil] timeout
+      # @param [Integer] pagination_interval
       #
       def initialize(
         base_url = "https://api.binaryedge.io/v2",
         api_key:,
         headers: {},
-        pagination_interval: 0,
+        pagination_interval: Mihari.config.pagination_interval,
         timeout: nil
       )
-        raise(ArgumentError, "'api_key' argument is required") unless api_key
-
         headers["x-key"] = api_key
 
-        super(base_url, headers: headers, pagination_interval: pagination_interval, timeout: timeout)
+        super(base_url, headers: headers, timeout: timeout, pagination_interval: pagination_interval)
       end
 
       #

data/lib/mihari/clients/censys.rb

@@ -18,11 +18,11 @@ module Mihari
         id:,
         secret:,
         headers: {},
-        pagination_interval: 0,
+        pagination_interval: Mihari.config.pagination_interval,
         timeout: nil
       )
-        raise(ArgumentError, "'id' argument is required") if id.nil?
-        raise(ArgumentError, "'secret' argument is required") if secret.nil?
+        raise(ArgumentError, "id is required") if id.nil?
+        raise(ArgumentError, "secret is required") if secret.nil?
 
         headers["authorization"] = "Basic #{Base64.strict_encode64("#{id}:#{secret}")}"
 

data/lib/mihari/clients/circl.rb

@@ -10,14 +10,15 @@ module Mihari
       # @param [String, nil] username
       # @param [String, nil] password
       # @param [Hash] headers
+      # @param [Integer, nil] timeout
       #
-      def initialize(base_url = "https://www.circl.lu", username:, password:, headers: {})
-        raise(ArgumentError, "'username' argument is required") if username.nil?
-        raise(ArgumentError, "'password' argument is required") if password.nil?
+      def initialize(base_url = "https://www.circl.lu", username:, password:, headers: {}, timeout: nil)
+        raise(ArgumentError, "username is required") if username.nil?
+        raise(ArgumentError, "password is required") if password.nil?
 
         headers["authorization"] = "Basic #{Base64.strict_encode64("#{username}:#{password}")}"
 
-        super(base_url, headers: headers)
+        super(base_url, headers: headers, timeout: timeout)
       end
 
       #

data/lib/mihari/clients/crtsh.rb

@@ -6,9 +6,10 @@ module Mihari
       #
       # @param [String] base_url
       # @param [Hash] headers
+      # @param [Integer, nil] timeout
       #
-      def initialize(base_url = "https://crt.sh", headers: {})
-        super(base_url, headers: headers)
+      def initialize(base_url = "https://crt.sh", headers: {}, timeout: nil)
+        super(base_url, headers: headers, timeout: timeout)
       end
 
       #

data/lib/mihari/clients/dnstwister.rb

@@ -6,9 +6,10 @@ module Mihari
       #
       # @param [String] base_url
       # @param [Hash] headers
+      # @param [Integer, nil] timeout
       #
-      def initialize(base_url = "https://dnstwister.report", headers: {})
-        super(base_url, headers: headers)
+      def initialize(base_url = "https://dnstwister.report", headers: {}, timeout: nil)
+        super(base_url, headers: headers, timeout: timeout)
       end
 
       #

data/lib/mihari/clients/greynoise.rb

@@ -16,10 +16,10 @@ module Mihari
         base_url = "https://api.greynoise.io",
         api_key:,
         headers: {},
-        pagination_interval: 0,
+        pagination_interval: Mihari.config.pagination_interval,
         timeout: nil
       )
-        raise(ArgumentError, "'api_key' argument is required") unless api_key
+        raise(ArgumentError, "api_key is required") unless api_key
 
         headers["key"] = api_key
         super(base_url, headers: headers, pagination_interval: pagination_interval, timeout: timeout)

data/lib/mihari/clients/hunterhow.rb

@@ -21,10 +21,10 @@ module Mihari
         base_url = "https://api.hunter.how/",
         api_key:,
         headers: {},
-        pagination_interval: 0,
+        pagination_interval: Mihari.config.pagination_interval,
         timeout: nil
       )
-        raise(ArgumentError, "'api_key' argument is required") unless api_key
+        raise(ArgumentError, "api_key is required") unless api_key
 
         super(base_url, headers: headers, pagination_interval: pagination_interval, timeout: timeout)
 

data/lib/mihari/clients/misp.rb

@@ -10,7 +10,7 @@ module Mihari
       # @param [Integer, nil] timeout
       #
       def initialize(base_url, api_key:, headers: {}, timeout: nil)
-        raise(ArgumentError, "'api_key' argument is required") unless api_key
+        raise(ArgumentError, "api_key is required") unless api_key
 
         headers["authorization"] = api_key
         headers["accept"] = "application/json"

data/lib/mihari/clients/onyphe.rb

@@ -19,10 +19,10 @@ module Mihari
         base_url = "https://www.onyphe.io",
         api_key:,
         headers: {},
-        pagination_interval: 0,
+        pagination_interval: Mihari.config.pagination_interval,
         timeout: nil
       )
-        raise(ArgumentError, "'api_key' argument is required") if api_key.nil?
+        raise(ArgumentError, "api_key is required") if api_key.nil?
 
         super(base_url, headers: headers, pagination_interval: pagination_interval, timeout: timeout)