mihari 5.4.9 → 5.6.0

data/lib/mihari/clients/binaryedge.rb

@@ -7,15 +7,21 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
-      # @param [Integer, nil] interval
+      # @param [Integer] pagnation_interval
       # @param [Integer, nil] timeout
       #
-      def initialize(base_url = "https://api.binaryedge.io/v2", api_key:, headers: {}, interval: nil, timeout: nil)
+      def initialize(
+        base_url = "https://api.binaryedge.io/v2",
+        api_key:,
+        headers: {},
+        pagination_interval: 0,
+        timeout: nil
+      )
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
         headers["x-key"] = api_key
 
-        super(base_url, headers: headers, interval: interval, timeout: timeout)
+        super(base_url, headers: headers, pagination_interval: pagination_interval, timeout: timeout)
       end
 
       #
@@ -52,7 +58,7 @@ module Mihari
 
             break if res.events.length < res.pagesize
 
-            sleep_interval
+            sleep_pagination_interval
           end
         end
       end

data/lib/mihari/clients/censys.rb

@@ -10,16 +10,23 @@ module Mihari
       # @param [String, nil] id
       # @param [String, nil] secret
       # @param [Hash] headers
-      # @param [Integer, nil] interval
+      # @param [Integer] pagination_interval
       # @param [Integer, nil] timeout
       #
-      def initialize(base_url = "https://search.censys.io", id:, secret:, headers: {}, interval: nil, timeout: nil)
+      def initialize(
+        base_url = "https://search.censys.io",
+        id:,
+        secret:,
+        headers: {},
+        pagination_interval: 0,
+        timeout: nil
+      )
         raise(ArgumentError, "'id' argument is required") if id.nil?
         raise(ArgumentError, "'secret' argument is required") if secret.nil?
 
         headers["authorization"] = "Basic #{Base64.strict_encode64("#{id}:#{secret}")}"
 
-        super(base_url, headers: headers, interval: interval, timeout: timeout)
+        super(base_url, headers: headers, pagination_interval: pagination_interval, timeout: timeout)
       end
 
       #
@@ -64,7 +71,7 @@ module Mihari
             # So it needs to check both cases
             break if cursor.nil? || cursor.empty?
 
-            sleep_interval
+            sleep_pagination_interval
           end
         end
       end

data/lib/mihari/clients/greynoise.rb

@@ -9,14 +9,20 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
-      # @param [Integer, nil] interval
+      # @param [Integer] pagination_interval
       # @param [Integer, nil] timeout
       #
-      def initialize(base_url = "https://api.greynoise.io", api_key:, headers: {}, interval: nil, timeout: nil)
+      def initialize(
+        base_url = "https://api.greynoise.io",
+        api_key:,
+        headers: {},
+        pagination_interval: 0,
+        timeout: nil
+      )
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
         headers["key"] = api_key
-        super(base_url, headers: headers, interval: interval, timeout: timeout)
+        super(base_url, headers: headers, pagination_interval: pagination_interval, timeout: timeout)
       end
 
       #
@@ -53,7 +59,7 @@ module Mihari
             scroll = res.scroll
             break if scroll.nil?
 
-            sleep_interval
+            sleep_pagination_interval
           end
         end
       end

data/lib/mihari/clients/hunterhow.rb

@@ -14,13 +14,19 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
-      # @param [Integer, nil] interval
+      # @param [Integer] pagination_interval
       # @param [Integer, nil] timeout
       #
-      def initialize(base_url = "https://api.hunter.how/", api_key:, headers: {}, interval: nil, timeout: nil)
+      def initialize(
+        base_url = "https://api.hunter.how/",
+        api_key:,
+        headers: {},
+        pagination_interval: 0,
+        timeout: nil
+      )
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
-        super(base_url, headers: headers, interval: interval, timeout: timeout)
+        super(base_url, headers: headers, pagination_interval: pagination_interval, timeout: timeout)
 
         @api_key = api_key
       end
@@ -77,7 +83,7 @@ module Mihari
 
             break if res.data.list.length < page_size
 
-            sleep_interval
+            sleep_pagination_interval
           end
         end
       end

data/lib/mihari/clients/misp.rb

@@ -7,14 +7,15 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
+      # @param [Integer, nil] timeout
       #
-      def initialize(base_url, api_key:, headers: {})
+      def initialize(base_url, api_key:, headers: {}, timeout: nil)
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
         headers["authorization"] = api_key
         headers["accept"] = "application/json"
 
-        super(base_url, headers: headers)
+        super(base_url, headers: headers, timeout: timeout)
       end
 
       #

data/lib/mihari/clients/onyphe.rb

@@ -12,13 +12,19 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
-      # @param [Integer, nil] interval
+      # @param [Integer] pagination_interval
       # @param [Integer, nil] timeout
       #
-      def initialize(base_url = "https://www.onyphe.io", api_key:, headers: {}, interval: nil, timeout: nil)
+      def initialize(
+        base_url = "https://www.onyphe.io",
+        api_key:,
+        headers: {},
+        pagination_interval: 0,
+        timeout: nil
+      )
         raise(ArgumentError, "'api_key' argument is required") if api_key.nil?
 
-        super(base_url, headers: headers, interval: interval, timeout: timeout)
+        super(base_url, headers: headers, pagination_interval: pagination_interval, timeout: timeout)
 
         @api_key = api_key
       end
@@ -50,7 +56,7 @@ module Mihari
 
             break if res.total <= page * PAGE_SIZE
 
-            sleep_interval
+            sleep_pagination_interval
           end
         end
       end

data/lib/mihari/clients/shodan.rb

@@ -12,13 +12,19 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
-      # @param [Integer, nil] interval
+      # @param [Integer] pagination_interval
       # @param [Integer, nil] timeout
       #
-      def initialize(base_url = "https://api.shodan.io", api_key:, headers: {}, interval: nil, timeout: nil)
+      def initialize(
+        base_url = "https://api.shodan.io",
+        api_key:,
+        headers: {},
+        pagination_interval: 0,
+        timeout: nil
+      )
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
-        super(base_url, headers: headers, interval: interval, timeout: timeout)
+        super(base_url, headers: headers, pagination_interval: pagination_interval, timeout: timeout)
 
         @api_key = api_key
       end
@@ -57,7 +63,7 @@ module Mihari
 
             break if res.total <= page * PAGE_SIZE
 
-            sleep_interval
+            sleep_pagination_interval
           rescue JSON::ParserError
             # ignore JSON::ParserError
             # ref. https://github.com/ninoseki/mihari/issues/197

data/lib/mihari/clients/the_hive.rb

@@ -8,14 +8,15 @@ module Mihari
       # @param [String, nil] api_key
       # @param [String, nil] api_version
       # @param [Hash] headers
+      # @param [Integer, nil] timeout
       #
-      def initialize(base_url, api_key:, api_version:, headers: {})
+      def initialize(base_url, api_key:, api_version:, headers: {}, timeout: nil)
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
         base_url += "/#{api_version}" unless api_version.nil?
         headers["authorization"] = "Bearer #{api_key}"
 
-        super(base_url, headers: headers)
+        super(base_url, headers: headers, timeout: timeout)
       end
 
       #

data/lib/mihari/clients/urlscan.rb

@@ -10,12 +10,18 @@ module Mihari
       # @param [Interval, nil] interval
       # @param [Interval, nil] timeout
       #
-      def initialize(base_url = "https://urlscan.io", api_key:, headers: {}, interval: nil, timeout: nil)
+      def initialize(
+        base_url = "https://urlscan.io",
+        api_key:,
+        headers: {},
+        pagination_interval: 0,
+        timeout: nil
+      )
         raise(ArgumentError, "'api_key' argument is required") if api_key.nil?
 
         headers["api-key"] = api_key
 
-        super(base_url, headers: headers, interval: interval, timeout: timeout)
+        super(base_url, headers: headers, pagination_interval: pagination_interval, timeout: timeout)
       end
 
       #
@@ -51,7 +57,7 @@ module Mihari
 
             search_after = res.results.last.sort.join(",")
 
-            sleep_interval
+            sleep_pagination_interval
           end
         end
       end

data/lib/mihari/clients/virustotal.rb

@@ -7,15 +7,21 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
-      # @param [Integer, nil] interval
+      # @param [Integer] pagination_interval
       # @param [Integer, nil] timeout
       #
-      def initialize(base_url = "https://www.virustotal.com", api_key:, headers: {}, interval: nil, timeout: nil)
+      def initialize(
+        base_url = "https://www.virustotal.com",
+        api_key:,
+        headers: {},
+        pagination_interval: 0,
+        timeout: nil
+      )
         raise(ArgumentError, "'api_key' argument is required") if api_key.nil?
 
         headers["x-apikey"] = api_key
 
-        super(base_url, headers: headers, interval: interval, timeout: timeout)
+        super(base_url, headers: headers, pagination_interval: pagination_interval, timeout: timeout)
       end
 
       #
@@ -66,7 +72,7 @@ module Mihari
             cursor = res.meta.cursor
             break if cursor.nil?
 
-            sleep_interval
+            sleep_pagination_interval
           end
         end
       end

data/lib/mihari/clients/zoomeye.rb

@@ -11,14 +11,20 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
-      # @param [Integer, nil] interval
+      # @param [Integer] pagination_interval
       # @param [Integer, nil] timeout
       #
-      def initialize(base_url = "https://api.zoomeye.org", api_key:, headers: {}, interval: nil, timeout: nil)
+      def initialize(
+        base_url = "https://api.zoomeye.org",
+        api_key:,
+        headers: {},
+        pagination_interval: 0,
+        timeout: nil
+      )
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
         headers["api-key"] = api_key
-        super(base_url, headers: headers, interval: interval, timeout: timeout)
+        super(base_url, headers: headers, pagination_interval: pagination_interval, timeout: timeout)
       end
 
       #
@@ -59,7 +65,7 @@ module Mihari
             total = res["total"].to_i
             break if total <= page * PAGE_SIZE
 
-            sleep_interval
+            sleep_pagination_interval
           end
         end
       end
@@ -102,7 +108,7 @@ module Mihari
             total = res["total"].to_i
             break if total <= page * PAGE_SIZE
 
-            sleep_interval
+            sleep_pagination_interval
           end
         end
       end

data/lib/mihari/commands/alert.rb

@@ -14,30 +14,14 @@ module Mihari
             #
             def add(path)
               Mihari::Database.with_db_connection do
-                builder = Mihari::Services::AlertBuilder.new(path)
+                builder = Services::AlertBuilder.new(path)
 
                 run_proxy_l = ->(proxy) { run_proxy proxy }
-                check_nil_l = ->(alert_or_nil) { check_nil alert_or_nil }
+                result = builder.result.bind(run_proxy_l)
 
-                result = builder.result.bind(run_proxy_l).bind(check_nil_l)
-
-                if result.success?
-                  alert = result.value!
-                  data = Mihari::Entities::Alert.represent(alert)
-                  puts JSON.pretty_generate(data.as_json)
-                  return
-                end
-
-                failure = result.failure
-                case failure
-                when ValidationError
-                  Mihari.logger.error "Failed to parse the input as an alert:"
-                  Mihari.logger.error JSON.pretty_generate(failure.errors.to_h)
-                when StandardError
-                  raise failure
-                else
-                  Mihari.logger.info failure
-                end
+                alert = result.value!
+                data = Entities::Alert.represent(alert)
+                puts JSON.pretty_generate(data.as_json)
               end
             end
 
@@ -47,21 +31,10 @@ module Mihari
               #
               def run_proxy(proxy)
                 Dry::Monads::Try[StandardError] do
-                  runner = Mihari::Services::AlertRunner.new(proxy)
+                  runner = Services::AlertRunner.new(proxy)
                   runner.run
                 end.to_result
               end
-
-              #
-              # @param [Mihari::Alert, nil] alert_or_nil
-              #
-              def check_nil(alert_or_nil)
-                if alert_or_nil.nil?
-                  Failure "There is no new artifact found"
-                else
-                  Success alert_or_nil
-                end
-              end
             end
           end
         end

data/lib/mihari/commands/rule.rb

@@ -8,7 +8,7 @@ module Mihari
       class << self
         def included(thor)
           thor.class_eval do
-            include Dry::Monads[:result, :try]
+            include Dry::Monads[:try, :result]
 
             desc "validate [PATH]", "Validate a rule file"
             #
@@ -18,16 +18,11 @@ module Mihari
             #
             def validate(path)
               res = Dry::Monads::Try[ValidationError] do
-                Services::RuleProxy.from_yaml(File.read(path))
-              end.fmap do |rule|
-                Mihari.logger.info "Valid format. The input is parsed as the following:"
-                Mihari.logger.info rule.data.to_yaml
+                Services::RuleProxy.from_yaml File.read(path)
               end
 
-              return unless res.error?
-
-              Mihari.logger.error "Failed to parse the input as a rule:"
-              Mihari.logger.error JSON.pretty_generate(res.exception.errors.to_h)
+              rule = res.value!
+              puts rule.data.to_yaml
             end
 
             desc "init [PATH]", "Initialize a new rule file"
@@ -43,14 +38,14 @@ module Mihari
 
               initialize_rule path
 
-              Mihari.logger.info "A new rule file has been initialized: #{path}."
+              puts "A new rule file has been initialized: #{path}."
             end
 
             no_commands do
               #
               # @return [Mihari::Services::Rule]
               #
-              def rule_template
+              def rule
                 Services::RuleProxy.from_yaml File.read(File.expand_path("../templates/rule.yml.erb", __dir__))
               end
 
@@ -63,7 +58,7 @@ module Mihari
               # @return [nil]
               #
               def initialize_rule(path, files = Dry::Files.new)
-                files.write(path, rule_template.yaml)
+                files.write(path, rule.yaml)
               end
             end
           end

data/lib/mihari/commands/search.rb

@@ -6,10 +6,10 @@ module Mihari
       class << self
         def included(thor)
           thor.class_eval do
-            include Dry::Monads[:result, :try]
+            include Dry::Monads[:try, :result]
 
-            desc "search [PATH_OR_ID]", "Search by a rule"
-            method_option :force_overwrite, type: :boolean, aliases: "-f", desc: "Force an overwrite the rule"
+            desc "search [PATH_OR_ID]", "Search by a rule (Outputs null if there is no new finding)"
+            method_option :force_overwrite, type: :boolean, aliases: "-f", desc: "Force overwriting a rule"
             #
             # Search by a rule
             #
@@ -21,27 +21,12 @@ module Mihari
 
                 check_diff_l = ->(rule) { check_diff rule }
                 update_and_run_l = ->(runner) { update_and_run runner }
-                check_nil_l = ->(alert_or_nil) { check_nil alert_or_nil }
 
-                result = builder.result.bind(check_diff_l).bind(update_and_run_l).bind(check_nil_l)
+                result = builder.result.bind(check_diff_l).bind(update_and_run_l)
 
-                if result.success?
-                  alert = result.value!
-                  data = Mihari::Entities::Alert.represent(alert)
-                  puts JSON.pretty_generate(data.as_json)
-                  return
-                end
-
-                failure = result.failure
-                case failure
-                when ValidationError
-                  Mihari.logger.error "Failed to parse the input as a rule:"
-                  Mihari.logger.error JSON.pretty_generate(failure.errors.to_h)
-                when StandardError
-                  raise failure
-                else
-                  Mihari.logger.info failure
-                end
+                alert = result.value!
+                data = Entities::Alert.represent(alert)
+                puts JSON.pretty_generate(data.as_json)
               end
             end
 
@@ -51,27 +36,14 @@ module Mihari
               #
               def check_diff(rule)
                 force_overwrite = options["force_overwrite"] || false
-                runner = Services::RuleRunner.new(rule, force_overwrite: force_overwrite)
-                message = "There is a diff in the rule (#{rule.id}). Are you sure you want to overwrite the rule? (y/n)"
+                message = "There is a diff in the rule. Are you sure you want to overwrite the rule? (y/n)"
+                runner = Services::RuleRunner.new(rule)
 
-                if runner.diff? && !force_overwrite && !yes?(message)
-                  return Failure("Stop overwriting the rule (#{rule.id})")
-                end
+                exit 0 if runner.diff? && !force_overwrite && !yes?(message)
 
                 Success runner
               end
 
-              #
-              # @param [Mihari::Alert, nil] alert_or_nil
-              #
-              def check_nil(alert_or_nil)
-                if alert_or_nil.nil?
-                  Failure "There is no new artifact found"
-                else
-                  Success alert_or_nil
-                end
-              end
-
               #
               # @param [Mihari::Services::RuleRunner] runner
               #

data/lib/mihari/config.rb

@@ -90,6 +90,12 @@ module Mihari
     # @return [Integer]
     attr_reader :retry_times
 
+    # @return [Boolean]
+    attr_reader :retry_exponential_backoff
+
+    # @return [Integer]
+    attr_reader :pagination_interval
+
     # @return [Integer]
     attr_reader :pagination_limit
 
@@ -152,7 +158,9 @@ module Mihari
 
       @retry_times = ENV.fetch("RETRY_TIMES", 3).to_i
       @retry_interval = ENV.fetch("RETRY_INTERVAL", 5).to_i
+      @retry_exponential_backoff = ENV.fetch("RETRY_EXPONENTIAL_BACKOFF", true).to_s.downcase == "true"
 
+      @pagination_interval = ENV.fetch("PAGINATION_INTERVAL", 0).to_i
       @pagination_limit = ENV.fetch("PAGINATION_LIMIT", 100).to_i
 
       @ignore_error = ENV.fetch("IGNORE_ERROR", false)

data/lib/mihari/constants.rb

@@ -2,11 +2,11 @@
 
 module Mihari
   # @return [Array<String>]
-  DEFAULT_DATA_TYPES = %w[hash ip domain url mail].freeze
+  DEFAULT_DATA_TYPES = Types::DataTypes.values.freeze
 
   # @return [Array<Hash>]
-  DEFAULT_EMITTERS = %w[database misp slack the_hive].map { |name| { emitter: name } }.freeze
+  DEFAULT_EMITTERS = %w[database].map { |name| { emitter: name } }.freeze
 
   # @return [Array<Hash>]
-  DEFAULT_ENRICHERS = %w[whois ipinfo shodan google_public_dns].map { |name| { enricher: name } }.freeze
+  DEFAULT_ENRICHERS = Mihari.enricher_to_class.keys.map { |name| { enricher: name.downcase } }.freeze
 end

data/lib/mihari/emitters/base.rb

@@ -2,7 +2,7 @@
 
 module Mihari
   module Emitters
-    class Base
+    class Base < Mihari::Base
       include Dry::Monads[:result, :try]
 
       include Mixins::Configurable
@@ -17,36 +17,43 @@ module Mihari
       #
       # @param [Array<Mihari::Artifact>] artifacts
       # @param [Mihari::Services::RuleProxy] rule
-      # @param [Hash] **_options
+      # @param [Hash, nil] options
+      # @param [Hash] **_params
       #
-      def initialize(artifacts:, rule:, **_options)
+      def initialize(artifacts:, rule:, options: nil, **_params)
+        super(options: options)
+
         @artifacts = artifacts
         @rule = rule
       end
 
-      class << self
-        def inherited(child)
-          super
-          Mihari.emitters << child
-        end
-      end
-
       # @return [Boolean]
       def valid?
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
       end
 
-      def run
-        retry_on_error { emit }
-      end
-
       def result
-        Try[StandardError] { run }.to_result
+        Try[StandardError] do
+          retry_on_error(
+            times: retry_times,
+            interval: retry_interval,
+            exponential_backoff: retry_exponential_backoff
+          ) do
+            emit
+          end
+        end.to_result
       end
 
       def emit
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
       end
+
+      class << self
+        def inherited(child)
+          super
+          Mihari.emitters << child
+        end
+      end
     end
   end
 end

data/lib/mihari/emitters/database.rb

@@ -13,7 +13,7 @@ module Mihari
       # @return [Mihari::Alert, nil]
       #
       def emit
-        return nil if artifacts.empty?
+        return if artifacts.empty?
 
         tags = rule.tags.filter_map { |name| Tag.find_or_create_by(name: name) }.uniq
         taggings = tags.map { |tag| Tagging.new(tag_id: tag.id) }