mihari 5.4.9 → 5.6.0

data/lib/mihari/analyzers/base.rb

@@ -2,7 +2,7 @@
 
 module Mihari
   module Analyzers
-    class Base
+    class Base < Mihari::Base
       include Dry::Monads[:result, :try]
 
       include Mixins::Configurable
@@ -11,37 +11,21 @@ module Mihari
       # @return [String]
       attr_reader :query
 
-      # @return [Hash]
-      attr_reader :options
-
       #
       # @param [String] query
       # @param [Hash, nil] options
       #
       def initialize(query, options: nil)
-        @query = query
-        @options = options || {}
-      end
-
-      #
-      # @return [Integer, nil]
-      #
-      def interval
-        options[:interval]
-      end
+        super(options: options)
 
-      #
-      # @return [Integer]
-      #
-      def retry_interval
-        options[:retry_interval] || Mihari.config.retry_interval
+        @query = query
       end
 
       #
       # @return [Integer]
       #
-      def retry_times
-        options[:retry_times] || Mihari.config.retry_times
+      def pagination_interval
+        options[:pagination_interval] || Mihari.config.pagination_interval
       end
 
       #
@@ -61,13 +45,6 @@ module Mihari
         Mihari.config.ignore_error
       end
 
-      #
-      # @return [Integer, nil]
-      #
-      def timeout
-        options[:timeout]
-      end
-
       # @return [Array<String>, Array<Mihari::Artifact>]
       def artifacts
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
@@ -81,12 +58,12 @@ module Mihari
       # @return [Array<Mihari::Artifact>]
       #
       def normalized_artifacts
-        retry_on_error(times: retry_times, interval: retry_interval) do
+        retry_on_error(times: retry_times, interval: retry_interval, exponential_backoff: retry_exponential_backoff) do
           artifacts.compact.sort.map do |artifact|
             # No need to set data_type manually
             # It is set automatically in #initialize
             artifact = artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact)
-            artifact.source = source
+            artifact.source = self.class.class_key
             artifact
           end.select(&:valid?).uniq(&:data)
         end
@@ -99,13 +76,6 @@ module Mihari
         Try[StandardError] { normalized_artifacts }.to_result
       end
 
-      # @return [String]
-      def class_name
-        self.class.to_s.split("::").last
-      end
-
-      alias source class_name
-
       class << self
         #
         # Initialize an analyzer by query params

data/lib/mihari/analyzers/binaryedge.rb

@@ -32,7 +32,11 @@ module Mihari
       # @return [Mihari::Clients::BinaryEdge]
       #
       def client
-        @client ||= Clients::BinaryEdge.new(api_key: api_key, interval: interval, timeout: timeout)
+        @client ||= Clients::BinaryEdge.new(
+          api_key: api_key,
+          pagination_interval: pagination_interval,
+          timeout: timeout
+        )
       end
     end
   end

data/lib/mihari/analyzers/censys.rb

@@ -52,7 +52,12 @@ module Mihari
       # @return [Mihari::Clients::Censys]
       #
       def client
-        @client ||= Clients::Censys.new(id: id, secret: secret, interval: interval, timeout: timeout)
+        @client ||= Clients::Censys.new(
+          id: id,
+          secret: secret,
+          pagination_interval: pagination_interval,
+          timeout: timeout
+        )
       end
 
       #

data/lib/mihari/analyzers/greynoise.rb

@@ -31,7 +31,11 @@ module Mihari
       private
 
       def client
-        @client ||= Clients::GreyNoise.new(api_key: api_key, interval: interval, timeout: timeout)
+        @client ||= Clients::GreyNoise.new(
+          api_key: api_key,
+          pagination_interval: pagination_interval,
+          timeout: timeout
+        )
       end
     end
   end

data/lib/mihari/analyzers/hunterhow.rb

@@ -46,7 +46,11 @@ module Mihari
       private
 
       def client
-        @client ||= Clients::HunterHow.new(api_key: api_key, interval: interval, timeout: timeout)
+        @client ||= Clients::HunterHow.new(
+          api_key: api_key,
+          pagination_interval: pagination_interval,
+          timeout: timeout
+        )
       end
     end
   end

data/lib/mihari/analyzers/onyphe.rb

@@ -33,7 +33,11 @@ module Mihari
       private
 
       def client
-        @client ||= Clients::Onyphe.new(api_key: api_key, interval: interval, timeout: timeout)
+        @client ||= Clients::Onyphe.new(
+          api_key: api_key,
+          pagination_interval: pagination_interval,
+          timeout: timeout
+        )
       end
     end
   end

data/lib/mihari/analyzers/passivetotal.rb

@@ -50,6 +50,15 @@ module Mihari
         %w[passivetotal_username passivetotal_api_key]
       end
 
+      class << self
+        #
+        # @return [Array<String>, nil]
+        #
+        def key_aliases
+          ["pt"]
+        end
+      end
+
       private
 
       def client

data/lib/mihari/analyzers/pulsedive.rb

@@ -35,7 +35,7 @@ module Mihari
             nil
           else
             data = property["value"]
-            Artifact.new(data: data, source: source, metadata: property)
+            Artifact.new(data: data, metadata: property)
           end
         end
       end

data/lib/mihari/analyzers/rule.rb

@@ -2,50 +2,17 @@
 
 module Mihari
   module Analyzers
-    ANALYZER_TO_CLASS = {
-      "binaryedge" => BinaryEdge,
-      "censys" => Censys,
-      "circl" => CIRCL,
-      "crtsh" => Crtsh,
-      "dnstwister" => DNSTwister,
-      "feed" => Feed,
-      "greynoise" => GreyNoise,
-      "hunterhow" => HunterHow,
-      "onyphe" => Onyphe,
-      "otx" => OTX,
-      "passivetotal" => PassiveTotal,
-      "pt" => PassiveTotal,
-      "pulsedive" => Pulsedive,
-      "securitytrails" => SecurityTrails,
-      "shodan" => Shodan,
-      "st" => SecurityTrails,
-      "urlscan" => Urlscan,
-      "virustotal_intelligence" => VirusTotalIntelligence,
-      "virustotal" => VirusTotal,
-      "vt_intel" => VirusTotalIntelligence,
-      "vt" => VirusTotal,
-      "zoomeye" => ZoomEye
-    }.freeze
-
-    EMITTER_TO_CLASS = {
-      "database" => Emitters::Database,
-      "misp" => Emitters::MISP,
-      "slack" => Emitters::Slack,
-      "the_hive" => Emitters::TheHive,
-      "webhook" => Emitters::Webhook
-    }.freeze
-
     class Rule
       include Mixins::FalsePositive
 
-      # @return [Mihari::Services::Rule]
+      # @return [Mihari::Services::RuleProxy]
       attr_reader :rule
 
       # @return [Time]
       attr_reader :base_time
 
       #
-      # @param [Mihari::Services::Rule] rule
+      # @param [Mihari::Services::RuleProxy] rule
       #
       def initialize(rule)
         @rule = rule
@@ -106,7 +73,7 @@ module Mihari
       #
       def enriched_artifacts
         @enriched_artifacts ||= Parallel.map(unique_artifacts) do |artifact|
-          rule.enrichers.each { |enricher| artifact.enrich_by_enricher enricher[:enricher] }
+          enrichers.each { |enricher| artifact.enrich_by_enricher enricher }
           artifact
         end
       end
@@ -119,8 +86,14 @@ module Mihari
       def bulk_emit
         return [] if enriched_artifacts.empty?
 
-        Parallel.map(valid_emitters) do |emitter|
-          result = emitter.result
+        # NOTE: separate parallel execution and logging
+        #       because the logger does not work along with Parallel
+        results = Parallel.map(valid_emitters) do |emitter|
+          emitter.result
+        end
+
+        results.zip(valid_emitters).map do |result_and_emitter|
+          result, emitter = result_and_emitter
 
           Mihari.logger.info "Emission by #{emitter.class} is failed: #{result.failure}" if result.failure?
           Mihari.logger.info "Emission by #{emitter.class} is succeeded" if result.success?
@@ -157,15 +130,14 @@ module Mihari
       #
       # Get analyzer class
       #
-      # @param [String] analyzer_name
+      # @param [String] key
       #
       # @return [Class<Mihari::Analyzers::Base>] analyzer class
       #
-      def get_analyzer_class(analyzer_name)
-        analyzer = ANALYZER_TO_CLASS[analyzer_name]
-        return analyzer if analyzer
+      def get_analyzer_class(key)
+        raise ArgumentError, "#{key} is not supported" unless Mihari.analyzer_to_class.key?(key)
 
-        raise ArgumentError, "#{analyzer_name} is not supported"
+        Mihari.analyzer_to_class[key]
       end
 
       #
@@ -182,29 +154,28 @@ module Mihari
       #
       # Get emitter class
       #
-      # @param [String] emitter_name
+      # @param [String] key
       #
       # @return [Class<Mihari::Emitters::Base>] emitter class
       #
-      def get_emitter_class(emitter_name)
-        emitter = EMITTER_TO_CLASS[emitter_name]
-        return emitter if emitter
+      def get_emitter_class(key)
+        raise ArgumentError, "#{key} is not supported" unless Mihari.emitter_to_class.key?(key)
 
-        raise ArgumentError, "#{emitter_name} is not supported"
+        Mihari.emitter_to_class[key]
       end
 
-      #
-      # Deep copied emitters
       #
       # @return [Array<Mihari::Emitters::Base>]
       #
       def emitters
         rule.emitters.map(&:deep_dup).map do |params|
           name = params[:emitter]
-          params.delete(:emitter)
+          options = params[:options]
+
+          %i[emitter options].each { |key| params.delete key }
 
           klass = get_emitter_class(name)
-          klass.new(artifacts: enriched_artifacts, rule: rule, **params)
+          klass.new(artifacts: enriched_artifacts, rule: rule, options: options, **params)
         end
       end
 
@@ -212,7 +183,35 @@ module Mihari
       # @return [Array<Mihari::Emitters::Base>]
       #
       def valid_emitters
-        emitters.select(&:valid?)
+        @valid_emitters ||= emitters.select(&:valid?)
+      end
+
+      #
+      # Get enricher class
+      #
+      # @param [String] key
+      #
+      # @return [Class<Mihari::Enrichers::Base>] enricher class
+      #
+      def get_enricher_class(key)
+        raise ArgumentError, "#{key} is not supported" unless Mihari.enricher_to_class.key?(key)
+
+        Mihari.enricher_to_class[key]
+      end
+
+      #
+      # @return [Array<Mihari::Enrichers::Base>] enrichers
+      #
+      def enrichers
+        @enrichers ||= rule.enrichers.map(&:deep_dup).map do |params|
+          name = params[:enricher]
+          options = params[:options]
+
+          %i[enricher options].each { |key| params.delete key }
+
+          klass = get_enricher_class(name)
+          klass.new(options: options, **params)
+        end
       end
 
       #
@@ -222,7 +221,9 @@ module Mihari
         analyzers.map do |analyzer|
           next if analyzer.configured?
 
-          message = "#{analyzer.source} is not configured correctly. #{analyzer.configuration_keys.join(", ")} is/are missing."
+          joined = analyzer.configuration_keys.join(", ")
+          be = (analyzer.configuration_keys.length > 1) ? "are" : "is"
+          message = "#{analyzer.class.class_key} is not configured correctly. #{joined} #{be} missing."
           raise ConfigurationError, message
         end
       end

data/lib/mihari/analyzers/securitytrails.rb

@@ -44,6 +44,15 @@ module Mihari
         %w[securitytrails_api_key]
       end
 
+      class << self
+        #
+        # @return [Array<String>, nil]
+        #
+        def key_aliases
+          ["st"]
+        end
+      end
+
       private
 
       def client

data/lib/mihari/analyzers/shodan.rb

@@ -34,7 +34,11 @@ module Mihari
       # @return [Clients::Shodan]
       #
       def client
-        @client ||= Clients::Shodan.new(api_key: api_key, interval: interval, timeout: timeout)
+        @client ||= Clients::Shodan.new(
+          api_key: api_key,
+          pagination_interval: pagination_interval,
+          timeout: timeout
+        )
       end
     end
   end

data/lib/mihari/analyzers/urlscan.rb

@@ -44,7 +44,11 @@ module Mihari
       private
 
       def client
-        @client ||= Clients::UrlScan.new(api_key: api_key, interval: interval, timeout: timeout)
+        @client ||= Clients::UrlScan.new(
+          api_key: api_key,
+          pagination_interval: pagination_interval,
+          timeout: timeout
+        )
       end
 
       #

data/lib/mihari/analyzers/virustotal.rb

@@ -39,6 +39,15 @@ module Mihari
         %w[virustotal_api_key]
       end
 
+      class << self
+        #
+        # @return [Array<String>, nil]
+        #
+        def key_aliases
+          ["vt"]
+        end
+      end
+
       private
 
       def client
@@ -65,7 +74,7 @@ module Mihari
         data = res["data"] || []
         data.filter_map do |item|
           data = item.dig("attributes", "ip_address")
-          data.nil? ? nil : Artifact.new(data: data, source: source, metadata: item)
+          data.nil? ? nil : Artifact.new(data: data, metadata: item)
         end
       end
 
@@ -80,7 +89,7 @@ module Mihari
         data = res["data"] || []
         data.filter_map do |item|
           data = item.dig("attributes", "host_name")
-          Artifact.new(data: data, source: source, metadata: item)
+          Artifact.new(data: data, metadata: item)
         end.uniq
       end
     end

data/lib/mihari/analyzers/virustotal_intelligence.rb

@@ -25,6 +25,22 @@ module Mihari
         %w[virustotal_api_key]
       end
 
+      class << self
+        #
+        # @return [String]
+        #
+        def class_key
+          "virustotal_intelligence"
+        end
+
+        #
+        # @return [Array<String>, nil]
+        #
+        def class_key_aliases
+          ["vt_intel"]
+        end
+      end
+
       private
 
       #
@@ -33,7 +49,11 @@ module Mihari
       # @return [::VirusTotal::API]
       #
       def client
-        @client = Clients::VirusTotal.new(api_key: api_key, interval: interval, timeout: timeout)
+        @client = Clients::VirusTotal.new(
+          api_key: api_key,
+          pagination_interval: pagination_interval,
+          timeout: timeout
+        )
       end
     end
   end

data/lib/mihari/analyzers/zoomeye.rb

@@ -53,7 +53,11 @@ module Mihari
       end
 
       def client
-        @client ||= Clients::ZoomEye.new(api_key: api_key, interval: interval, timeout: timeout)
+        @client ||= Clients::ZoomEye.new(
+          api_key: api_key,
+          pagination_interval: pagination_interval,
+          timeout: timeout
+        )
       end
 
       #
@@ -69,9 +73,9 @@ module Mihari
           data = match["ip"]
 
           if data.is_a?(Array)
-            data.map { |d| Artifact.new(data: d, source: source, metadata: match) }
+            data.map { |d| Artifact.new(data: d, metadata: match) }
           else
-            Artifact.new(data: data, source: source, metadata: match)
+            Artifact.new(data: data, metadata: match)
           end
         end.flatten
       end

data/lib/mihari/base.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Mihari
+  #
+  # Base class for Analyzer, Emitter and Enricher
+  #
+  class Base
+    # @return [Hash]
+    attr_reader :options
+
+    #
+    # @param [Hash, nil] options
+    #
+    def initialize(*_args, options: nil, **_kwargs)
+      @options = options || {}
+    end
+
+    #
+    # @return [Integer]
+    #
+    def retry_interval
+      options[:retry_interval] || Mihari.config.retry_interval
+    end
+
+    #
+    # @return [Boolean]
+    #
+    def retry_exponential_backoff
+      options[:retry_exponential_backoff] || Mihari.config.retry_exponential_backoff
+    end
+
+    #
+    # @return [Integer]
+    #
+    def retry_times
+      options[:retry_times] || Mihari.config.retry_times
+    end
+
+    #
+    # @return [Integer, nil]
+    #
+    def timeout
+      options[:timeout]
+    end
+
+    class << self
+      #
+      # @return [String]
+      #
+      def class_key
+        to_s.split("::").last
+      end
+
+      #
+      # @return [Array<String>, nil]
+      #
+      def class_key_aliases
+        nil
+      end
+
+      #
+      # @return [Array<String>]
+      #
+      def class_keys
+        ([class_key] + [class_key_aliases]).flatten.compact
+      end
+    end
+  end
+end

data/lib/mihari/cli/main.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "thor"
+require "thor/hollaback"
 
 # Commands
 require "mihari/commands/alert"
@@ -19,10 +20,45 @@ require "mihari/cli/rule"
 module Mihari
   module CLI
     class Main < Base
+      class_option :debug, desc: "Sets up debug mode", aliases: ["-d"], type: :boolean
+      class_around :safe_execute
+
       include Mihari::Commands::Search
       include Mihari::Commands::Version
       include Mihari::Commands::Web
 
+      no_commands do
+        def unwrap_error(err)
+          return err unless err.is_a?(Dry::Monads::UnwrapError)
+
+          # NOTE: UnwrapError's receiver can be either of:
+          #       - Dry::Monads::Try::Error
+          #       - Dry::Monads::Result::Failure
+          receiver = err.receiver
+          return receiver.exception if receiver.is_a?(Dry::Monads::Try::Error)
+
+          receiver.failure
+        end
+
+        def safe_execute
+          yield
+        rescue StandardError => e
+          err = unwrap_error(e)
+
+          raise err if options["debug"]
+
+          case err
+          when ValidationError
+            warn JSON.pretty_generate(err.errors.to_h)
+          when StandardError
+            Sentry.capture_exception(err) if Sentry.initialized?
+            warn err
+          end
+
+          exit 1
+        end
+      end
+
       desc "db", "Sub commands for DB"
       subcommand "db", Database
 

data/lib/mihari/clients/base.rb

@@ -9,8 +9,8 @@ module Mihari
       # @return [Hash]
       attr_reader :headers
 
-      # @return [Integer, nil]
-      attr_reader :interval
+      # @return [Integer]
+      attr_reader :pagination_interval
 
       # @return [Integer, nil]
       attr_reader :timeout
@@ -18,20 +18,20 @@ module Mihari
       #
       # @param [String] base_url
       # @param [Hash] headers
-      # @param [Integer, nil] interval
+      # @param [Integer] interval
       # @param [Integer, nil] timeout
       #
-      def initialize(base_url, headers: {}, interval: nil, timeout: nil)
+      def initialize(base_url, headers: {}, pagination_interval: 0, timeout: nil)
         @base_url = base_url
         @headers = headers || {}
-        @interval = interval
+        @pagination_interval = pagination_interval
         @timeout = timeout
       end
 
       private
 
-      def sleep_interval
-        sleep(interval) if interval
+      def sleep_pagination_interval
+        sleep pagination_interval
       end
 
       #