mihari 5.5.0 → 5.6.0

data/lib/mihari/analyzers/base.rb

@@ -2,7 +2,7 @@
 
 module Mihari
   module Analyzers
-    class Base
+    class Base < Mihari::Base
       include Dry::Monads[:result, :try]
 
       include Mixins::Configurable
@@ -11,37 +11,14 @@ module Mihari
       # @return [String]
       attr_reader :query
 
-      # @return [Hash]
-      attr_reader :options
-
       #
       # @param [String] query
       # @param [Hash, nil] options
       #
       def initialize(query, options: nil)
-        @query = query
-        @options = options || {}
-      end
-
-      #
-      # @return [Integer]
-      #
-      def retry_interval
-        options[:retry_interval] || Mihari.config.retry_interval
-      end
-
-      #
-      # @return [Boolean]
-      #
-      def retry_exponential_backoff
-        options[:retry_exponential_backoff] || Mihari.config.retry_exponential_backoff
-      end
+        super(options: options)
 
-      #
-      # @return [Integer]
-      #
-      def retry_times
-        options[:retry_times] || Mihari.config.retry_times
+        @query = query
       end
 
       #
@@ -68,13 +45,6 @@ module Mihari
         Mihari.config.ignore_error
       end
 
-      #
-      # @return [Integer, nil]
-      #
-      def timeout
-        options[:timeout]
-      end
-
       # @return [Array<String>, Array<Mihari::Artifact>]
       def artifacts
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
@@ -93,7 +63,7 @@ module Mihari
             # No need to set data_type manually
             # It is set automatically in #initialize
             artifact = artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact)
-            artifact.source = source
+            artifact.source = self.class.class_key
             artifact
           end.select(&:valid?).uniq(&:data)
         end
@@ -106,13 +76,6 @@ module Mihari
         Try[StandardError] { normalized_artifacts }.to_result
       end
 
-      # @return [String]
-      def class_name
-        self.class.to_s.split("::").last
-      end
-
-      alias_method :source, :class_name
-
       class << self
         #
         # Initialize an analyzer by query params

data/lib/mihari/analyzers/passivetotal.rb

@@ -50,6 +50,15 @@ module Mihari
         %w[passivetotal_username passivetotal_api_key]
       end
 
+      class << self
+        #
+        # @return [Array<String>, nil]
+        #
+        def key_aliases
+          ["pt"]
+        end
+      end
+
       private
 
       def client

data/lib/mihari/analyzers/pulsedive.rb

@@ -35,7 +35,7 @@ module Mihari
             nil
           else
             data = property["value"]
-            Artifact.new(data: data, source: source, metadata: property)
+            Artifact.new(data: data, metadata: property)
           end
         end
       end

data/lib/mihari/analyzers/rule.rb

@@ -2,46 +2,6 @@
 
 module Mihari
   module Analyzers
-    ANALYZER_TO_CLASS = {
-      "binaryedge" => BinaryEdge,
-      "censys" => Censys,
-      "circl" => CIRCL,
-      "crtsh" => Crtsh,
-      "dnstwister" => DNSTwister,
-      "feed" => Feed,
-      "greynoise" => GreyNoise,
-      "hunterhow" => HunterHow,
-      "onyphe" => Onyphe,
-      "otx" => OTX,
-      "passivetotal" => PassiveTotal,
-      "pt" => PassiveTotal,
-      "pulsedive" => Pulsedive,
-      "securitytrails" => SecurityTrails,
-      "shodan" => Shodan,
-      "st" => SecurityTrails,
-      "urlscan" => Urlscan,
-      "virustotal_intelligence" => VirusTotalIntelligence,
-      "virustotal" => VirusTotal,
-      "vt_intel" => VirusTotalIntelligence,
-      "vt" => VirusTotal,
-      "zoomeye" => ZoomEye
-    }.freeze
-
-    EMITTER_TO_CLASS = {
-      "database" => Emitters::Database,
-      "misp" => Emitters::MISP,
-      "slack" => Emitters::Slack,
-      "the_hive" => Emitters::TheHive,
-      "webhook" => Emitters::Webhook
-    }.freeze
-
-    ENRICHER_TO_CLASS = {
-      "whois" => Enrichers::Whois,
-      "ipinfo" => Enrichers::IPInfo,
-      "shodan" => Enrichers::Shodan,
-      "google_public_dns" => Enrichers::GooglePublicDNS
-    }.freeze
-
     class Rule
       include Mixins::FalsePositive
 
@@ -126,8 +86,14 @@ module Mihari
       def bulk_emit
         return [] if enriched_artifacts.empty?
 
-        Parallel.map(valid_emitters) do |emitter|
-          result = emitter.result
+        # NOTE: separate parallel execution and logging
+        #       because the logger does not work along with Parallel
+        results = Parallel.map(valid_emitters) do |emitter|
+          emitter.result
+        end
+
+        results.zip(valid_emitters).map do |result_and_emitter|
+          result, emitter = result_and_emitter
 
           Mihari.logger.info "Emission by #{emitter.class} is failed: #{result.failure}" if result.failure?
           Mihari.logger.info "Emission by #{emitter.class} is succeeded" if result.success?
@@ -164,15 +130,14 @@ module Mihari
       #
       # Get analyzer class
       #
-      # @param [String] analyzer_name
+      # @param [String] key
       #
       # @return [Class<Mihari::Analyzers::Base>] analyzer class
       #
-      def get_analyzer_class(analyzer_name)
-        analyzer = ANALYZER_TO_CLASS[analyzer_name]
-        return analyzer if analyzer
+      def get_analyzer_class(key)
+        raise ArgumentError, "#{key} is not supported" unless Mihari.analyzer_to_class.key?(key)
 
-        raise ArgumentError, "#{analyzer_name} is not supported"
+        Mihari.analyzer_to_class[key]
       end
 
       #
@@ -189,15 +154,14 @@ module Mihari
       #
       # Get emitter class
       #
-      # @param [String] emitter_name
+      # @param [String] key
       #
       # @return [Class<Mihari::Emitters::Base>] emitter class
       #
-      def get_emitter_class(emitter_name)
-        emitter = EMITTER_TO_CLASS[emitter_name]
-        return emitter if emitter
+      def get_emitter_class(key)
+        raise ArgumentError, "#{key} is not supported" unless Mihari.emitter_to_class.key?(key)
 
-        raise ArgumentError, "#{emitter_name} is not supported"
+        Mihari.emitter_to_class[key]
       end
 
       #
@@ -219,21 +183,20 @@ module Mihari
       # @return [Array<Mihari::Emitters::Base>]
       #
       def valid_emitters
-        emitters.select(&:valid?)
+        @valid_emitters ||= emitters.select(&:valid?)
       end
 
       #
       # Get enricher class
       #
-      # @param [String] enricher_name
+      # @param [String] key
       #
       # @return [Class<Mihari::Enrichers::Base>] enricher class
       #
-      def get_enricher_class(enricher_name)
-        enricher = ENRICHER_TO_CLASS[enricher_name]
-        return enricher if enricher
+      def get_enricher_class(key)
+        raise ArgumentError, "#{key} is not supported" unless Mihari.enricher_to_class.key?(key)
 
-        raise ArgumentError, "#{enricher_name} is not supported"
+        Mihari.enricher_to_class[key]
       end
 
       #
@@ -258,7 +221,9 @@ module Mihari
         analyzers.map do |analyzer|
           next if analyzer.configured?
 
-          message = "#{analyzer.source} is not configured correctly. #{analyzer.configuration_keys.join(", ")} is/are missing."
+          joined = analyzer.configuration_keys.join(", ")
+          be = (analyzer.configuration_keys.length > 1) ? "are" : "is"
+          message = "#{analyzer.class.class_key} is not configured correctly. #{joined} #{be} missing."
           raise ConfigurationError, message
         end
       end

data/lib/mihari/analyzers/securitytrails.rb

@@ -44,6 +44,15 @@ module Mihari
         %w[securitytrails_api_key]
       end
 
+      class << self
+        #
+        # @return [Array<String>, nil]
+        #
+        def key_aliases
+          ["st"]
+        end
+      end
+
       private
 
       def client

data/lib/mihari/analyzers/virustotal.rb

@@ -39,6 +39,15 @@ module Mihari
         %w[virustotal_api_key]
       end
 
+      class << self
+        #
+        # @return [Array<String>, nil]
+        #
+        def key_aliases
+          ["vt"]
+        end
+      end
+
       private
 
       def client
@@ -65,7 +74,7 @@ module Mihari
         data = res["data"] || []
         data.filter_map do |item|
           data = item.dig("attributes", "ip_address")
-          data.nil? ? nil : Artifact.new(data: data, source: source, metadata: item)
+          data.nil? ? nil : Artifact.new(data: data, metadata: item)
         end
       end
 
@@ -80,7 +89,7 @@ module Mihari
         data = res["data"] || []
         data.filter_map do |item|
           data = item.dig("attributes", "host_name")
-          Artifact.new(data: data, source: source, metadata: item)
+          Artifact.new(data: data, metadata: item)
         end.uniq
       end
     end

data/lib/mihari/analyzers/virustotal_intelligence.rb

@@ -25,6 +25,22 @@ module Mihari
         %w[virustotal_api_key]
       end
 
+      class << self
+        #
+        # @return [String]
+        #
+        def class_key
+          "virustotal_intelligence"
+        end
+
+        #
+        # @return [Array<String>, nil]
+        #
+        def class_key_aliases
+          ["vt_intel"]
+        end
+      end
+
       private
 
       #

data/lib/mihari/analyzers/zoomeye.rb

@@ -73,9 +73,9 @@ module Mihari
           data = match["ip"]
 
           if data.is_a?(Array)
-            data.map { |d| Artifact.new(data: d, source: source, metadata: match) }
+            data.map { |d| Artifact.new(data: d, metadata: match) }
           else
-            Artifact.new(data: data, source: source, metadata: match)
+            Artifact.new(data: data, metadata: match)
           end
         end.flatten
       end

data/lib/mihari/base.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Mihari
+  #
+  # Base class for Analyzer, Emitter and Enricher
+  #
+  class Base
+    # @return [Hash]
+    attr_reader :options
+
+    #
+    # @param [Hash, nil] options
+    #
+    def initialize(*_args, options: nil, **_kwargs)
+      @options = options || {}
+    end
+
+    #
+    # @return [Integer]
+    #
+    def retry_interval
+      options[:retry_interval] || Mihari.config.retry_interval
+    end
+
+    #
+    # @return [Boolean]
+    #
+    def retry_exponential_backoff
+      options[:retry_exponential_backoff] || Mihari.config.retry_exponential_backoff
+    end
+
+    #
+    # @return [Integer]
+    #
+    def retry_times
+      options[:retry_times] || Mihari.config.retry_times
+    end
+
+    #
+    # @return [Integer, nil]
+    #
+    def timeout
+      options[:timeout]
+    end
+
+    class << self
+      #
+      # @return [String]
+      #
+      def class_key
+        to_s.split("::").last
+      end
+
+      #
+      # @return [Array<String>, nil]
+      #
+      def class_key_aliases
+        nil
+      end
+
+      #
+      # @return [Array<String>]
+      #
+      def class_keys
+        ([class_key] + [class_key_aliases]).flatten.compact
+      end
+    end
+  end
+end

data/lib/mihari/cli/main.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "thor"
+require "thor/hollaback"
 
 # Commands
 require "mihari/commands/alert"
@@ -19,10 +20,45 @@ require "mihari/cli/rule"
 module Mihari
   module CLI
     class Main < Base
+      class_option :debug, desc: "Sets up debug mode", aliases: ["-d"], type: :boolean
+      class_around :safe_execute
+
       include Mihari::Commands::Search
       include Mihari::Commands::Version
       include Mihari::Commands::Web
 
+      no_commands do
+        def unwrap_error(err)
+          return err unless err.is_a?(Dry::Monads::UnwrapError)
+
+          # NOTE: UnwrapError's receiver can be either of:
+          #       - Dry::Monads::Try::Error
+          #       - Dry::Monads::Result::Failure
+          receiver = err.receiver
+          return receiver.exception if receiver.is_a?(Dry::Monads::Try::Error)
+
+          receiver.failure
+        end
+
+        def safe_execute
+          yield
+        rescue StandardError => e
+          err = unwrap_error(e)
+
+          raise err if options["debug"]
+
+          case err
+          when ValidationError
+            warn JSON.pretty_generate(err.errors.to_h)
+          when StandardError
+            Sentry.capture_exception(err) if Sentry.initialized?
+            warn err
+          end
+
+          exit 1
+        end
+      end
+
       desc "db", "Sub commands for DB"
       subcommand "db", Database
 

data/lib/mihari/commands/alert.rb

@@ -14,30 +14,14 @@ module Mihari
             #
             def add(path)
               Mihari::Database.with_db_connection do
-                builder = Mihari::Services::AlertBuilder.new(path)
+                builder = Services::AlertBuilder.new(path)
 
                 run_proxy_l = ->(proxy) { run_proxy proxy }
-                check_nil_l = ->(alert_or_nil) { check_nil alert_or_nil }
+                result = builder.result.bind(run_proxy_l)
 
-                result = builder.result.bind(run_proxy_l).bind(check_nil_l)
-
-                if result.success?
-                  alert = result.value!
-                  data = Mihari::Entities::Alert.represent(alert)
-                  puts JSON.pretty_generate(data.as_json)
-                  return
-                end
-
-                failure = result.failure
-                case failure
-                when ValidationError
-                  Mihari.logger.error "Failed to parse the input as an alert:"
-                  Mihari.logger.error JSON.pretty_generate(failure.errors.to_h)
-                when StandardError
-                  raise failure
-                else
-                  Mihari.logger.info failure
-                end
+                alert = result.value!
+                data = Entities::Alert.represent(alert)
+                puts JSON.pretty_generate(data.as_json)
               end
             end
 
@@ -47,21 +31,10 @@ module Mihari
               #
               def run_proxy(proxy)
                 Dry::Monads::Try[StandardError] do
-                  runner = Mihari::Services::AlertRunner.new(proxy)
+                  runner = Services::AlertRunner.new(proxy)
                   runner.run
                 end.to_result
               end
-
-              #
-              # @param [Mihari::Alert, nil] alert_or_nil
-              #
-              def check_nil(alert_or_nil)
-                if alert_or_nil.nil?
-                  Failure "There is no new artifact found"
-                else
-                  Success alert_or_nil
-                end
-              end
             end
           end
         end

data/lib/mihari/commands/rule.rb

@@ -8,7 +8,7 @@ module Mihari
       class << self
         def included(thor)
           thor.class_eval do
-            include Dry::Monads[:result, :try]
+            include Dry::Monads[:try, :result]
 
             desc "validate [PATH]", "Validate a rule file"
             #
@@ -18,16 +18,11 @@ module Mihari
             #
             def validate(path)
               res = Dry::Monads::Try[ValidationError] do
-                Services::RuleProxy.from_yaml(File.read(path))
-              end.fmap do |rule|
-                Mihari.logger.info "Valid format. The input is parsed as the following:"
-                Mihari.logger.info rule.data.to_yaml
+                Services::RuleProxy.from_yaml File.read(path)
               end
 
-              return unless res.error?
-
-              Mihari.logger.error "Failed to parse the input as a rule:"
-              Mihari.logger.error JSON.pretty_generate(res.exception.errors.to_h)
+              rule = res.value!
+              puts rule.data.to_yaml
             end
 
             desc "init [PATH]", "Initialize a new rule file"
@@ -43,14 +38,14 @@ module Mihari
 
               initialize_rule path
 
-              Mihari.logger.info "A new rule file has been initialized: #{path}."
+              puts "A new rule file has been initialized: #{path}."
             end
 
             no_commands do
               #
               # @return [Mihari::Services::Rule]
               #
-              def rule_template
+              def rule
                 Services::RuleProxy.from_yaml File.read(File.expand_path("../templates/rule.yml.erb", __dir__))
               end
 
@@ -63,7 +58,7 @@ module Mihari
               # @return [nil]
               #
               def initialize_rule(path, files = Dry::Files.new)
-                files.write(path, rule_template.yaml)
+                files.write(path, rule.yaml)
               end
             end
           end

data/lib/mihari/commands/search.rb

@@ -6,10 +6,10 @@ module Mihari
       class << self
         def included(thor)
           thor.class_eval do
-            include Dry::Monads[:result, :try]
+            include Dry::Monads[:try, :result]
 
-            desc "search [PATH_OR_ID]", "Search by a rule"
-            method_option :force_overwrite, type: :boolean, aliases: "-f", desc: "Force an overwrite the rule"
+            desc "search [PATH_OR_ID]", "Search by a rule (Outputs null if there is no new finding)"
+            method_option :force_overwrite, type: :boolean, aliases: "-f", desc: "Force overwriting a rule"
             #
             # Search by a rule
             #
@@ -21,27 +21,12 @@ module Mihari
 
                 check_diff_l = ->(rule) { check_diff rule }
                 update_and_run_l = ->(runner) { update_and_run runner }
-                check_nil_l = ->(alert_or_nil) { check_nil alert_or_nil }
 
-                result = builder.result.bind(check_diff_l).bind(update_and_run_l).bind(check_nil_l)
+                result = builder.result.bind(check_diff_l).bind(update_and_run_l)
 
-                if result.success?
-                  alert = result.value!
-                  data = Mihari::Entities::Alert.represent(alert)
-                  puts JSON.pretty_generate(data.as_json)
-                  return
-                end
-
-                failure = result.failure
-                case failure
-                when ValidationError
-                  Mihari.logger.error "Failed to parse the input as a rule:"
-                  Mihari.logger.error JSON.pretty_generate(failure.errors.to_h)
-                when StandardError
-                  raise failure
-                else
-                  Mihari.logger.info failure
-                end
+                alert = result.value!
+                data = Entities::Alert.represent(alert)
+                puts JSON.pretty_generate(data.as_json)
               end
             end
 
@@ -51,27 +36,14 @@ module Mihari
               #
               def check_diff(rule)
                 force_overwrite = options["force_overwrite"] || false
-                runner = Services::RuleRunner.new(rule, force_overwrite: force_overwrite)
-                message = "There is a diff in the rule (#{rule.id}). Are you sure you want to overwrite the rule? (y/n)"
+                message = "There is a diff in the rule. Are you sure you want to overwrite the rule? (y/n)"
+                runner = Services::RuleRunner.new(rule)
 
-                if runner.diff? && !force_overwrite && !yes?(message)
-                  return Failure("Stop overwriting the rule (#{rule.id})")
-                end
+                exit 0 if runner.diff? && !force_overwrite && !yes?(message)
 
                 Success runner
               end
 
-              #
-              # @param [Mihari::Alert, nil] alert_or_nil
-              #
-              def check_nil(alert_or_nil)
-                if alert_or_nil.nil?
-                  Failure "There is no new artifact found"
-                else
-                  Success alert_or_nil
-                end
-              end
-
               #
               # @param [Mihari::Services::RuleRunner] runner
               #

data/lib/mihari/constants.rb

@@ -2,11 +2,11 @@
 
 module Mihari
   # @return [Array<String>]
-  DEFAULT_DATA_TYPES = %w[hash ip domain url mail].freeze
+  DEFAULT_DATA_TYPES = Types::DataTypes.values.freeze
 
   # @return [Array<Hash>]
-  DEFAULT_EMITTERS = %w[database misp slack the_hive].map { |name| { emitter: name } }.freeze
+  DEFAULT_EMITTERS = %w[database].map { |name| { emitter: name } }.freeze
 
   # @return [Array<Hash>]
-  DEFAULT_ENRICHERS = %w[whois ipinfo shodan google_public_dns].map { |name| { enricher: name } }.freeze
+  DEFAULT_ENRICHERS = Mihari.enricher_to_class.keys.map { |name| { enricher: name.downcase } }.freeze
 end