mihari 5.2.3 → 5.3.0

data/lib/mihari/analyzers/base.rb

@@ -3,11 +3,45 @@
 module Mihari
   module Analyzers
     class Base
-      extend Dry::Initializer
-
       include Mixins::Configurable
       include Mixins::Retriable
 
+      # @return [String]
+      attr_reader :query
+
+      # @return [Hash]
+      attr_reader :options
+
+      #
+      # @param [String] query
+      # @param [Hash, nil] options
+      #
+      def initialize(query, options: nil)
+        @query = query
+        @options = options || {}
+      end
+
+      #
+      # @return [Integer, nil]
+      #
+      def interval
+        @interval = options[:interval]
+      end
+
+      #
+      # @return [Integer]
+      #
+      def retry_interval
+        @retry_interval ||= options[:retry_interval] || DEFAULT_RETRY_INTERVAL
+      end
+
+      #
+      # @return [Integer]
+      #
+      def retry_times
+        @retry_times ||= options[:retry_times] || DEFAULT_RETRY_TIMES
+      end
+
       # @return [Array<String>, Array<Mihari::Artifact>]
       def artifacts
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
@@ -21,31 +55,48 @@ module Mihari
       # @return [Array<Mihari::Artifact>]
       #
       def normalized_artifacts
-        retry_on_error do
+        retry_on_error(times: retry_times, interval: retry_interval) do
           @normalized_artifacts ||= artifacts.compact.sort.map do |artifact|
             # No need to set data_type manually
             # It is set automatically in #initialize
             artifact = artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact)
-            artifact
-          end.select(&:valid?).uniq(&:data).map do |artifact|
-            # set source
             artifact.source = source
             artifact
-          end
+          end.select(&:valid?).uniq(&:data)
         end
       end
 
-      # @return [String]
-      def source
-        self.class.to_s.split("::").last.to_s
-      end
-
       # @return [String]
       def class_name
         self.class.to_s.split("::").last
       end
 
+      alias_method :source, :class_name
+
       class << self
+        #
+        # Initialize an analyzer by query params
+        #
+        # @param [Hash] params
+        #
+        # @return [Mihari::Analyzers::Base]
+        #
+        def from_query(params)
+          copied = params.deep_dup
+
+          # convert params into arguments for initialization
+          query = copied[:query]
+
+          # delete analyzer and query
+          %i[analyzer query].each do |key|
+            copied.delete key
+          end
+
+          copied[:options] = copied[:options] || nil
+
+          new(query, **copied)
+        end
+
         def inherited(child)
           super
           Mihari.analyzers << child

data/lib/mihari/analyzers/binaryedge.rb

@@ -3,23 +3,18 @@
 module Mihari
   module Analyzers
     class BinaryEdge < Base
-      param :query
-
-      option :interval, default: proc { 0 }
-
       # @return [String, nil]
       attr_reader :api_key
 
-      # @return [String]
-      attr_reader :query
-
-      # @return [Integer]
-      attr_reader :interval
-
-      def initialize(*args, **kwargs)
-        super(*args, **kwargs)
+      #
+      # @param [String] query
+      # @param [Hash, nil] options
+      # @param [String, nil] api_key
+      #
+      def initialize(query, options: nil, api_key: nil)
+        super(query, options: options)
 
-        @api_key = kwargs[:api_key] || Mihari.config.binaryedge_api_key
+        @api_key = api_key || Mihari.config.binaryedge_api_key
       end
 
       def artifacts
@@ -48,7 +43,7 @@ module Mihari
       #
       def search_with_page(page: 1)
         client.search(query, page: page)
-      rescue UnsuccessfulStatusCodeError => e
+      rescue StatusCodeError => e
         raise RetryableError, e if e.message.include?("Request time limit exceeded")
 
         raise e
@@ -69,7 +64,7 @@ module Mihari
           break if total <= page * PAGE_SIZE
 
           # sleep #{interval} seconds to avoid the rate limitation (if it is set)
-          sleep interval
+          sleep(interval) if interval
         end
         responses
       end

data/lib/mihari/analyzers/censys.rb

@@ -3,27 +3,24 @@
 module Mihari
   module Analyzers
     class Censys < Base
-      param :query
-
-      option :interval, default: proc { 0 }
-
       # @return [String, nil]
       attr_reader :id
 
       # @return [String, nil]
       attr_reader :secret
 
-      # @return [Integer]
-      attr_reader :interval
-
-      # @return [String]
-      attr_reader :query
-
-      def initialize(*args, **kwargs)
-        super(*args, **kwargs)
+      #
+      # @param [String] query
+      # @param [hash, nil] options
+      # @param [String, nil] api_key
+      # @param [String, nil] id
+      # @param [String, nil] secret
+      #
+      def initialize(query, options: nil, id: nil, secret: nil)
+        super(query, options: options)
 
-        @id = kwargs[:id] || Mihari.config.censys_id
-        @secret = kwargs[:secret] || Mihari.config.censys_secret
+        @id = id || Mihari.config.censys_id
+        @secret = secret || Mihari.config.censys_secret
       end
 
       #
@@ -45,7 +42,7 @@ module Mihari
           break if cursor.nil? || cursor.empty?
 
           # sleep #{interval} seconds to avoid the rate limitation (if it is set)
-          sleep interval
+          sleep(interval) if interval
         end
 
         artifacts.flatten.uniq(&:data)

data/lib/mihari/analyzers/circl.rb

@@ -5,8 +5,6 @@ module Mihari
     class CIRCL < Base
       include Mixins::Refang
 
-      param :query
-
       # @return [String, nil]
       attr_reader :type
 
@@ -16,17 +14,19 @@ module Mihari
       # @return [String, nil]
       attr_reader :password
 
-      # @return [String]
-      attr_reader :query
-
-      def initialize(*args, **kwargs)
-        super
+      #
+      # @param [String] query
+      # @param [Hash, nil] options
+      # @param [String, nil] username
+      # @param [String, nil] password
+      #
+      def initialize(query, options: nil, username: nil, password: nil)
+        super(refang(query), options: options)
 
-        @query = refang(query)
         @type = TypeChecker.type(query)
 
-        @username = kwargs[:username] || Mihari.config.circl_passive_username
-        @password = kwargs[:password] || Mihari.config.circl_passive_password
+        @username = username || Mihari.config.circl_passive_username
+        @password = password || Mihari.config.circl_passive_password
       end
 
       def artifacts

data/lib/mihari/analyzers/crtsh.rb

@@ -3,15 +3,19 @@
 module Mihari
   module Analyzers
     class Crtsh < Base
-      param :query
-
-      option :exclude_expired, default: proc { true }
-
       # @return [Boolean]
       attr_reader :exclude_expired
 
-      # @return [String]
-      attr_reader :query
+      #
+      # @param [String] query
+      # @param [Hash, nil] options
+      # @param [Bool] exclude_expired
+      #
+      def initialize(query, options: nil, exclude_expired: true)
+        super(query, options: options)
+
+        @exclude_expired = exclude_expired
+      end
 
       def artifacts
         results = search

data/lib/mihari/analyzers/dnstwister.rb

@@ -5,18 +5,16 @@ module Mihari
     class DNSTwister < Base
       include Mixins::Refang
 
-      param :query
-
       # @return [String]
       attr_reader :type
 
-      # @return [String]
-      attr_reader :query
-
-      def initialize(*args, **kwargs)
-        super
+      #
+      # @param [String] query
+      # @param [Hash, nil] options
+      #
+      def initialize(query, options: nil)
+        super(refang(query), options: options)
 
-        @query = refang(query)
         @type = TypeChecker.type(query)
       end
 

data/lib/mihari/analyzers/feed.rb

@@ -6,16 +6,6 @@ require "mihari/feed/parser"
 module Mihari
   module Analyzers
     class Feed < Base
-      param :query
-
-      option :method, default: proc { "GET" }
-      option :headers, default: proc { {} }
-      option :params, default: proc {}
-      option :json, default: proc {}
-      option :data, default: proc {}
-
-      option :selector, default: proc { "" }
-
       # @return [Hash, nil]
       attr_reader :data
 
@@ -37,6 +27,27 @@ module Mihari
       # @return [String]
       attr_reader :query
 
+      #
+      # @param [String] query
+      # @param [Hash, nil] options
+      # @param [String] method
+      # @param [Hash] headers
+      # @param [Hash] params
+      # @param [Hash] json
+      # @param [Hash] data
+      # @param [String] selector
+      #
+      def initialize(query, options: nil, method: "GET", headers: {}, params: {}, json: {}, data: {}, selector: "")
+        super(query, options: options)
+
+        @method = method
+        @headers = headers
+        @params = params
+        @json = json
+        @data = data
+        @selector = selector
+      end
+
       def artifacts
         Mihari::Feed::Parser.new(results).parse selector
       end

data/lib/mihari/analyzers/greynoise.rb

@@ -3,29 +3,28 @@
 module Mihari
   module Analyzers
     class GreyNoise < Base
-      param :query
+      PAGE_SIZE = 10_000
 
       # @return [String, nil]
       attr_reader :api_key
 
-      # @return [String]
-      attr_reader :query
-
-      def initialize(*args, **kwargs)
-        super(*args, **kwargs)
+      #
+      # @param [String] query
+      # @param [Hash, nil] options
+      # @param [String, nil] api_key
+      #
+      def initialize(query, options: nil, api_key: nil)
+        super(query, options: options)
 
-        @api_key = kwargs[:api_key] || Mihari.config.greynoise_api_key
+        @api_key = api_key || Mihari.config.greynoise_api_key
       end
 
       def artifacts
-        res = search
-        res.to_artifacts
+        client.gnql_search(query, size: PAGE_SIZE).to_artifacts
       end
 
       private
 
-      PAGE_SIZE = 10_000
-
       def configuration_keys
         %w[greynoise_api_key]
       end
@@ -33,15 +32,6 @@ module Mihari
       def client
         @client ||= Clients::GreyNoise.new(api_key: api_key)
       end
-
-      #
-      # Search
-      #
-      # @return [Hash]
-      #
-      def search
-        client.gnql_search(query, size: PAGE_SIZE)
-      end
     end
   end
 end

data/lib/mihari/analyzers/onyphe.rb

@@ -5,23 +5,18 @@ require "normalize_country"
 module Mihari
   module Analyzers
     class Onyphe < Base
-      param :query
-
-      option :interval, default: proc { 0 }
-
       # @return [String, nil]
       attr_reader :api_key
 
-      # @return [String]
-      attr_reader :query
-
-      # @return [Integer]
-      attr_reader :interval
-
-      def initialize(*args, **kwargs)
-        super(*args, **kwargs)
+      #
+      # @param [String] query
+      # @param [Hash, nil] options
+      # @param [String, nil] api_key
+      #
+      def initialize(query, options: nil, api_key: nil)
+        super(query, options: options)
 
-        @api_key = kwargs[:api_key] || Mihari.config.onyphe_api_key
+        @api_key = api_key || Mihari.config.onyphe_api_key
       end
 
       def artifacts
@@ -70,7 +65,7 @@ module Mihari
           break if total <= page * PAGE_SIZE
 
           # sleep #{interval} seconds to avoid the rate limitation (if it is set)
-          sleep interval
+          sleep(interval) if interval
         end
         responses
       end

data/lib/mihari/analyzers/otx.rb

@@ -5,24 +5,23 @@ module Mihari
     class OTX < Base
       include Mixins::Refang
 
-      param :query
-
       # @return [String, nil]
       attr_reader :type
 
       # @return [String, nil]
       attr_reader :api_key
 
-      # @return [String]
-      attr_reader :query
-
-      def initialize(*args, **kwargs)
-        super
+      #
+      # @param [String] query
+      # @param [Hash, nil] options
+      # @param [String, nil] api_key
+      #
+      def initialize(query, options: nil, api_key: nil)
+        super(refang(query), options: options)
 
-        @query = refang(query)
         @type = TypeChecker.type(query)
 
-        @api_key = kwargs[:api_key] || Mihari.config.otx_api_key
+        @api_key = api_key || Mihari.config.otx_api_key
       end
 
       def artifacts

data/lib/mihari/analyzers/passivetotal.rb

@@ -5,8 +5,6 @@ module Mihari
     class PassiveTotal < Base
       include Mixins::Refang
 
-      param :query
-
       # @return [String, nil]
       attr_reader :type
 
@@ -16,17 +14,19 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
-      # @return [String]
-      attr_reader :query
-
-      def initialize(*args, **kwargs)
-        super(*args, **kwargs)
+      #
+      # @param [String] query
+      # @param [Hash, nil] options
+      # @param [String, nil] api_key
+      # @param [String, nil] username
+      #
+      def initialize(query, options: nil, api_key: nil, username: nil)
+        super(refang(query), options: options)
 
-        @query = refang(query)
         @type = TypeChecker.type(query)
 
-        @username = kwargs[:username] || Mihari.config.passivetotal_username
-        @api_key = kwargs[:api_key] || Mihari.config.passivetotal_api_key
+        @username = username || Mihari.config.passivetotal_username
+        @api_key = api_key || Mihari.config.passivetotal_api_key
       end
 
       def artifacts

data/lib/mihari/analyzers/pulsedive.rb

@@ -5,28 +5,39 @@ module Mihari
     class Pulsedive < Base
       include Mixins::Refang
 
-      param :query
-
       # @return [String, nil]
       attr_reader :type
 
       # @return [String, nil]
       attr_reader :api_key
 
-      # @return [Integer]
-      attr_reader :query
-
-      def initialize(*args, **kwargs)
-        super
+      #
+      # @param [String] query
+      # @param [Hash, nil] options
+      # @param [String, nil] api_key
+      #
+      def initialize(query, options: nil, api_key: nil)
+        super(refang(query), options: options)
 
-        @query = refang(query)
         @type = TypeChecker.type(query)
 
-        @api_key = kwargs[:api_key] || Mihari.config.pulsedive_api_key
+        @api_key = api_key || Mihari.config.pulsedive_api_key
       end
 
       def artifacts
-        search || []
+        raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
+
+        indicator = client.get_indicator(query)
+        iid = indicator["iid"]
+        properties = client.get_properties(iid)
+        (properties["dns"] || []).filter_map do |property|
+          if %w[A PTR].include?(property["name"])
+            nil
+          else
+            data = property["value"]
+            Artifact.new(data: data, source: source, metadata: property)
+          end
+        end
       end
 
       private
@@ -47,27 +58,6 @@ module Mihari
       def valid_type?
         %w[ip domain].include? type
       end
-
-      #
-      # Search
-      #
-      # @return [Array<Mihari::Artifact>]
-      #
-      def search
-        raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
-
-        indicator = client.get_indicator(query)
-        iid = indicator["iid"]
-        properties = client.get_properties(iid)
-        (properties["dns"] || []).filter_map do |property|
-          if %w[A PTR].include?(property["name"])
-            nil
-          else
-            data = property["value"]
-            Artifact.new(data: data, source: source, metadata: property)
-          end
-        end
-      end
     end
   end
 end

data/lib/mihari/analyzers/rule.rb

@@ -46,7 +46,7 @@ module Mihari
       #
       # @param [Mihari::Structs::Rule] rule
       #
-      def initialize(rule:)
+      def initialize(rule)
         @rule = rule
         @base_time = Time.now.utc
 
@@ -153,15 +153,6 @@ module Mihari
         end
       end
 
-      #
-      # Deep copied queries
-      #
-      # @return [Array<Hash>]
-      #
-      def queries
-        rule.queries.map(&:deep_dup)
-      end
-
       #
       # Get analyzer class
       #
@@ -177,26 +168,13 @@ module Mihari
       end
 
       #
-      # @return [Array<Mihari::Analyzers::Base>] <description>
+      # @return [Array<Mihari::Analyzers::Base>]
       #
       def analyzers
-        @analyzers ||= queries.map do |params|
-          analyzer_name = params[:analyzer]
+        @analyzers ||= rule.queries.map do |query_params|
+          analyzer_name = query_params[:analyzer]
           klass = get_analyzer_class(analyzer_name)
-
-          # set interval in the top level
-          options = params[:options] || {}
-          interval = options[:interval]
-          params[:interval] = interval if interval
-
-          # set rule
-          params[:rule] = rule
-          query = params[:query]
-
-          analyzer = klass.new(query, **params)
-          raise ConfigurationError, "#{analyzer.source} is not configured correctly" unless analyzer.configured?
-
-          analyzer
+          klass.from_query(query_params)
         end
       end
 
@@ -240,8 +218,9 @@ module Mihari
       # Validate configuration of analyzers
       #
       def validate_analyzer_configurations
-        # memoize analyzers & raise ConfigurationError if there is an analyzer which is not configured
-        analyzers
+        analyzers.map do |analyzer|
+          raise ConfigurationError, "#{analyzer.source} is not configured correctly" unless analyzer.configured?
+        end
       end
     end
   end

data/lib/mihari/analyzers/securitytrails.rb

@@ -5,8 +5,6 @@ module Mihari
     class SecurityTrails < Base
       include Mixins::Refang
 
-      param :query
-
       # @return [String, nil]
       attr_reader :type
 
@@ -16,13 +14,17 @@ module Mihari
       # @return [String]
       attr_reader :query
 
-      def initialize(*args, **kwargs)
-        super
+      #
+      # @param [String] query
+      # @param [Hash, nil] options
+      # @param [String, nil] api_key
+      #
+      def initialize(query, options: nil, api_key: nil)
+        super(refang(query), options: options)
 
-        @query = refang(query)
         @type = TypeChecker.type(query)
 
-        @api_key = kwargs[:api_key] || Mihari.config.securitytrails_api_key
+        @api_key = api_key || Mihari.config.securitytrails_api_key
       end
 
       def artifacts