mihari 5.4.9 → 5.5.0

data/lib/mihari/clients/zoomeye.rb

@@ -11,14 +11,20 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
-      # @param [Integer, nil] interval
+      # @param [Integer] pagination_interval
       # @param [Integer, nil] timeout
       #
-      def initialize(base_url = "https://api.zoomeye.org", api_key:, headers: {}, interval: nil, timeout: nil)
+      def initialize(
+        base_url = "https://api.zoomeye.org",
+        api_key:,
+        headers: {},
+        pagination_interval: 0,
+        timeout: nil
+      )
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
         headers["api-key"] = api_key
-        super(base_url, headers: headers, interval: interval, timeout: timeout)
+        super(base_url, headers: headers, pagination_interval: pagination_interval, timeout: timeout)
       end
 
       #
@@ -59,7 +65,7 @@ module Mihari
             total = res["total"].to_i
             break if total <= page * PAGE_SIZE
 
-            sleep_interval
+            sleep_pagination_interval
           end
         end
       end
@@ -102,7 +108,7 @@ module Mihari
             total = res["total"].to_i
             break if total <= page * PAGE_SIZE
 
-            sleep_interval
+            sleep_pagination_interval
           end
         end
       end

data/lib/mihari/config.rb

@@ -90,6 +90,12 @@ module Mihari
     # @return [Integer]
     attr_reader :retry_times
 
+    # @return [Boolean]
+    attr_reader :retry_exponential_backoff
+
+    # @return [Integer]
+    attr_reader :pagination_interval
+
     # @return [Integer]
     attr_reader :pagination_limit
 
@@ -152,7 +158,9 @@ module Mihari
 
       @retry_times = ENV.fetch("RETRY_TIMES", 3).to_i
       @retry_interval = ENV.fetch("RETRY_INTERVAL", 5).to_i
+      @retry_exponential_backoff = ENV.fetch("RETRY_EXPONENTIAL_BACKOFF", true).to_s.downcase == "true"
 
+      @pagination_interval = ENV.fetch("PAGINATION_INTERVAL", 0).to_i
       @pagination_limit = ENV.fetch("PAGINATION_LIMIT", 100).to_i
 
       @ignore_error = ENV.fetch("IGNORE_ERROR", false)

data/lib/mihari/emitters/base.rb

@@ -14,39 +14,76 @@ module Mihari
       # @return [Mihari::Services::Rule]
       attr_reader :rule
 
+      # @return [Hash]
+      attr_reader :options
+
       #
       # @param [Array<Mihari::Artifact>] artifacts
       # @param [Mihari::Services::RuleProxy] rule
-      # @param [Hash] **_options
+      # @param [Hash, nil] options
+      # @param [Hash] **_params
       #
-      def initialize(artifacts:, rule:, **_options)
+      def initialize(artifacts:, rule:, options: nil, **_params)
         @artifacts = artifacts
         @rule = rule
+        @options = options || {}
       end
 
-      class << self
-        def inherited(child)
-          super
-          Mihari.emitters << child
-        end
+      #
+      # @return [Integer]
+      #
+      def retry_interval
+        options[:retry_interval] || Mihari.config.retry_interval
       end
 
+      #
       # @return [Boolean]
-      def valid?
-        raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+      #
+      def retry_exponential_backoff
+        options[:retry_exponential_backoff] || Mihari.config.retry_exponential_backoff
+      end
+
+      #
+      # @return [Integer]
+      #
+      def retry_times
+        options[:retry_times] || Mihari.config.retry_times
+      end
+
+      #
+      # @return [Integer, nil]
+      #
+      def timeout
+        options[:timeout]
       end
 
-      def run
-        retry_on_error { emit }
+      # @return [Boolean]
+      def valid?
+        raise NotImplementedError, "You must implement #{self.class}##{__method__}"
       end
 
       def result
-        Try[StandardError] { run }.to_result
+        Try[StandardError] do
+          retry_on_error(
+            times: retry_times,
+            interval: retry_interval,
+            exponential_backoff: retry_exponential_backoff
+          ) do
+            emit
+          end
+        end.to_result
       end
 
       def emit
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
       end
+
+      class << self
+        def inherited(child)
+          super
+          Mihari.emitters << child
+        end
+      end
     end
   end
 end

data/lib/mihari/emitters/misp.rb

@@ -18,13 +18,14 @@ module Mihari
       #
       # @param [Array<Mihari::Artifact>] artifacts
       # @param [Mihari::Services::Rule] rule
-      # @param [Hash] **options
+      # @param [Hash, nil] options
+      # @param [Hash] **params
       #
-      def initialize(artifacts:, rule:, **options)
-        super(artifacts: artifacts, rule: rule, **options)
+      def initialize(artifacts:, rule:, options: nil, **params)
+        super(artifacts: artifacts, rule: rule, options: options)
 
-        @url = options[:url] || Mihari.config.misp_url
-        @api_key = options[:api_key] || Mihari.config.misp_api_key
+        @url = params[:url] || Mihari.config.misp_url
+        @api_key = params[:api_key] || Mihari.config.misp_api_key
       end
 
       # @return [Boolean]
@@ -70,7 +71,7 @@ module Mihari
       end
 
       def client
-        @client ||= Clients::MISP.new(url, api_key: api_key)
+        @client ||= Clients::MISP.new(url, api_key: api_key, timeout: timeout)
       end
 
       #

data/lib/mihari/emitters/slack.rb

@@ -134,13 +134,14 @@ module Mihari
       #
       # @param [Array<Mihari::Artifact>] artifacts
       # @param [Mihari::Services::Rule] rule
-      # @param [Hash] **_options
+      # @param [Hash, nil] options
+      # @param [Hash] **params
       #
-      def initialize(artifacts:, rule:, **options)
-        super(artifacts: artifacts, rule: rule, **options)
+      def initialize(artifacts:, rule:, options: nil, **params)
+        super(artifacts: artifacts, rule: rule, options: options)
 
-        @webhook_url = options[:webhook_url] || Mihari.config.slack_webhook_url
-        @channel = options[:channel] || Mihari.config.slack_channel || DEFAULT_CHANNEL
+        @webhook_url = params[:webhook_url] || Mihari.config.slack_webhook_url
+        @channel = params[:channel] || Mihari.config.slack_channel || DEFAULT_CHANNEL
         @username = DEFAULT_USERNAME
       end
 
@@ -162,8 +163,25 @@ module Mihari
         webhook_url?
       end
 
+      #
+      # @return [::Slack::Notifier]
+      #
       def notifier
-        @notifier ||= ::Slack::Notifier.new(webhook_url, channel: channel, username: username)
+        @notifier ||= [].tap do |out|
+          out << if timeout.nil?
+            ::Slack::Notifier.new(
+              webhook_url,
+              channel: channel, username: username
+            )
+          else
+            ::Slack::Notifier.new(
+              webhook_url,
+              channel: channel,
+              username: username,
+              http_options: { timeout: timeout }
+            )
+          end
+        end.first
       end
 
       #

data/lib/mihari/emitters/the_hive.rb

@@ -15,14 +15,15 @@ module Mihari
       #
       # @param [Array<Mihari::Artifact>] artifacts
       # @param [Mihari::Services::Rule] rule
-      # @param [Hash] **options
+      # @param [Hash, nil] options
+      # @param [Hash] **params
       #
-      def initialize(artifacts:, rule:, **options)
-        super(artifacts: artifacts, rule: rule, **options)
+      def initialize(artifacts:, rule:, options: nil, **params)
+        super(artifacts: artifacts, rule: rule, options: options)
 
-        @url = options[:url] || Mihari.config.thehive_url
-        @api_key = options[:api_key] || Mihari.config.thehive_api_key
-        @api_version = options[:api_version] || Mihari.config.thehive_api_version
+        @url = params[:url] || Mihari.config.thehive_url
+        @api_key = params[:api_key] || Mihari.config.thehive_api_key
+        @api_version = params[:api_version] || Mihari.config.thehive_api_version
       end
 
       # @return [Boolean]
@@ -79,7 +80,7 @@ module Mihari
       end
 
       def client
-        @client ||= Clients::TheHive.new(url, api_key: api_key, api_version: normalized_api_version)
+        @client ||= Clients::TheHive.new(url, api_key: api_key, api_version: normalized_api_version, timeout: timeout)
       end
 
       #

data/lib/mihari/emitters/webhook.rb

@@ -5,28 +5,30 @@ require "erb"
 module Mihari
   module Emitters
     class PayloadTemplate < ERB
-      def self.template
-        %{
-					{
-						"rule": {
-              "id": "<%= @rule.id %>",
-              "title": "<%= @rule.title %>",
-              "description": "<%= @rule.description %>"
-            },
-						"artifacts": [
-							<% @artifacts.each_with_index do |artifact, idx| %>
-								"<%= artifact.data %>"
-								<%= ',' if idx < (@artifacts.length - 1) %>
-							<% end %>
-						],
-						"tags": [
-							<% @rule.tags.each_with_index do |tag, idx| %>
-								"<%= tag %>"
-								<%= ',' if idx < (@rule.tags.length - 1) %>
-							<% end %>
-						]
-					}
-				}
+      class << self
+        def template
+          %{
+            {
+              "rule": {
+                "id": "<%= @rule.id %>",
+                "title": "<%= @rule.title %>",
+                "description": "<%= @rule.description %>"
+              },
+              "artifacts": [
+                <% @artifacts.each_with_index do |artifact, idx| %>
+                  "<%= artifact.data %>"
+                  <%= ',' if idx < (@artifacts.length - 1) %>
+                <% end %>
+              ],
+              "tags": [
+                <% @rule.tags.each_with_index do |tag, idx| %>
+                  "<%= tag %>"
+                  <%= ',' if idx < (@rule.tags.length - 1) %>
+                <% end %>
+              ]
+            }
+          }
+        end
       end
 
       def initialize(artifacts:, rule:, options: {})
@@ -60,13 +62,13 @@ module Mihari
       # @param [Mihari::Services::Rule] rule
       # @param [Hash] **options
       #
-      def initialize(artifacts:, rule:, **options)
-        super(artifacts: artifacts, rule: rule, **options)
+      def initialize(artifacts:, rule:, options: nil, **params)
+        super(artifacts: artifacts, rule: rule, options: options)
 
-        @url = Addressable::URI.parse(options[:url])
-        @headers = options[:headers] || {}
-        @method = options[:method] || "POST"
-        @template = options[:template]
+        @url = Addressable::URI.parse(params[:url])
+        @headers = params[:headers] || {}
+        @method = params[:method] || "POST"
+        @template = params[:template]
       end
 
       def emit
@@ -90,7 +92,7 @@ module Mihari
       private
 
       def http
-        HTTP::Factory.build headers: headers
+        HTTP::Factory.build headers: headers, timeout: timeout
       end
 
       #

data/lib/mihari/enrichers/base.rb

@@ -4,31 +4,68 @@ module Mihari
   module Enrichers
     class Base
       include Mixins::Configurable
+      include Mixins::Retriable
 
-      class << self
-        include Dry::Monads[:result, :try]
+      include Dry::Monads[:result, :try]
 
-        def inherited(child)
-          super
-          Mihari.enrichers << child
-        end
+      # @return [Hash]
+      attr_reader :options
 
-        def query_result(value)
-          Try[StandardError] { query(value) }.to_result
-        end
+      def initialize(options: nil)
+        @options = options || {}
+      end
 
-        #
-        # @param [String] value
-        #
-        def query(value)
-          raise NotImplementedError, "You must implement #{self.class}##{__method__}"
-        end
+      #
+      # @return [Integer]
+      #
+      def retry_interval
+        options[:retry_interval] || Mihari.config.retry_interval
       end
 
+      #
       # @return [Boolean]
-      def valid?
+      #
+      def retry_exponential_backoff
+        options[:retry_exponential_backoff] || Mihari.config.retry_exponential_backoff
+      end
+
+      #
+      # @return [Integer]
+      #
+      def retry_times
+        options[:retry_times] || Mihari.config.retry_times
+      end
+
+      #
+      # @return [Integer, nil]
+      #
+      def timeout
+        options[:timeout]
+      end
+
+      def query_result(value)
+        Try[StandardError] do
+          retry_on_error(
+            times: retry_times,
+            interval: retry_interval,
+            exponential_backoff: retry_exponential_backoff
+          ) { query value }
+        end.to_result
+      end
+
+      #
+      # @param [String] value
+      #
+      def query(value)
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
       end
+
+      class << self
+        def inherited(child)
+          super
+          Mihari.enrichers << child
+        end
+      end
     end
   end
 end

data/lib/mihari/enrichers/google_public_dns.rb

@@ -5,52 +5,43 @@ require "net/https"
 module Mihari
   module Enrichers
     class GooglePublicDNS < Base
-      # @return [Boolean]
-      def valid?
-        true
-      end
-
-      class << self
-        include Dry::Monads[:result]
-
-        #
-        # Query Google Public DNS
-        #
-        # @param [String] name
-        #
-        # @return [Array<Mihari::Structs::Shodan::GooglePublicDNS::Response>]
-        #
-        def query(name)
-          %w[A AAAA CNAME TXT NS].filter_map do |resource_type|
-            query_by_type(name, resource_type)
-          end
+      #
+      # Query Google Public DNS
+      #
+      # @param [String] name
+      #
+      # @return [Array<Mihari::Structs::Shodan::GooglePublicDNS::Response>]
+      #
+      def query(name)
+        %w[A AAAA CNAME TXT NS].filter_map do |resource_type|
+          query_by_type(name, resource_type)
         end
+      end
 
-        #
-        # Query Google Public DNS by resource type
-        #
-        # @param [String] name
-        # @param [String] resource_type
-        #
-        # @return [Mihari::Structs::Shodan::GooglePublicDNS::Response, nil]
-        #
-        def query_by_type(name, resource_type)
-          url = "https://dns.google/resolve"
-          params = { name: name, type: resource_type }
-          res = http.get(url, params: params)
-
-          data = JSON.parse(res.body.to_s)
-
-          Structs::GooglePublicDNS::Response.from_dynamic! data
-        rescue HTTPError
-          nil
-        end
+      #
+      # Query Google Public DNS by resource type
+      #
+      # @param [String] name
+      # @param [String] resource_type
+      #
+      # @return [Mihari::Structs::Shodan::GooglePublicDNS::Response, nil]
+      #
+      def query_by_type(name, resource_type)
+        url = "https://dns.google/resolve"
+        params = { name: name, type: resource_type }
+        res = http.get(url, params: params)
+
+        data = JSON.parse(res.body.to_s)
+
+        Structs::GooglePublicDNS::Response.from_dynamic! data
+      rescue HTTPError
+        nil
+      end
 
-        private
+      private
 
-        def http
-          HTTP::Factory.build
-        end
+      def http
+        HTTP::Factory.build timeout: timeout
       end
     end
   end

data/lib/mihari/enrichers/ipinfo.rb

@@ -5,9 +5,15 @@ require "net/https"
 module Mihari
   module Enrichers
     class IPInfo < Base
-      # @return [Boolean]
-      def valid?
-        Mihari.config.ipinfo_api_key.nil?
+      include Memist::Memoizable
+
+      # @return [String, nil]
+      attr_reader :api_key
+
+      def initialize(options: nil, api_key: nil)
+        @api_key = api_key || Mihari.config.ipinfo_api_key
+
+        super(options: options)
       end
 
       private
@@ -16,37 +22,29 @@ module Mihari
         %w[ipinfo_api_key]
       end
 
-      class << self
-        include Dry::Monads[:result]
-        include Memist::Memoizable
-
-        #
-        # Query IPInfo
-        #
-        # @param [String] ip
-        #
-        # @return [Mihari::Structs::IPInfo::Response, nil]
-        #
-        def query(ip)
-          url = "https://ipinfo.io/#{ip}/json"
-          res = http.get(url)
-          data = JSON.parse(res.body.to_s)
-
-          Structs::IPInfo::Response.from_dynamic! data
-        end
-        memoize :query
-
-        private
-
-        def headers
-          token = Mihari.config.ipinfo_api_key
-          authorization = token.nil? ? nil : "Bearer #{token}"
-          { authorization: authorization }.compact
-        end
-
-        def http
-          HTTP::Factory.build headers: headers
-        end
+      #
+      # Query IPInfo
+      #
+      # @param [String] ip
+      #
+      # @return [Mihari::Structs::IPInfo::Response, nil]
+      #
+      def query(ip)
+        url = "https://ipinfo.io/#{ip}/json"
+        res = http.get(url)
+        data = JSON.parse(res.body.to_s)
+
+        Structs::IPInfo::Response.from_dynamic! data
+      end
+      memoize :query
+
+      def headers
+        authorization = api_key.nil? ? nil : "Bearer #{api_key}"
+        { authorization: authorization }.compact
+      end
+
+      def http
+        HTTP::Factory.build headers: headers, timeout: timeout
       end
     end
   end

data/lib/mihari/enrichers/shodan.rb

@@ -5,36 +5,28 @@ require "net/https"
 module Mihari
   module Enrichers
     class Shodan < Base
-      # @return [Boolean]
-      def valid?
-        true
-      end
-
-      class << self
-        include Dry::Monads[:result]
-        include Memist::Memoizable
+      include Memist::Memoizable
 
-        #
-        # Query Shodan Internet DB
-        #
-        # @param [String] ip
-        #
-        # @return [Mihari::Structs::Shodan::InternetDBResponse, nil]
-        #
-        def query(ip)
-          url = "https://internetdb.shodan.io/#{ip}"
-          res = http.get(url)
-          data = JSON.parse(res.body.to_s)
+      #
+      # Query Shodan Internet DB
+      #
+      # @param [String] ip
+      #
+      # @return [Mihari::Structs::Shodan::InternetDBResponse, nil]
+      #
+      def query(ip)
+        url = "https://internetdb.shodan.io/#{ip}"
+        res = http.get(url)
+        data = JSON.parse(res.body.to_s)
 
-          Structs::Shodan::InternetDBResponse.from_dynamic! data
-        end
-        memoize :query
+        Structs::Shodan::InternetDBResponse.from_dynamic! data
+      end
+      memoize :query
 
-        private
+      private
 
-        def http
-          HTTP::Factory.build
-        end
+      def http
+        HTTP::Factory.build timeout: timeout
       end
     end
   end