mihari 5.4.9 → 5.5.0

data/lib/mihari/enrichers/whois.rb

@@ -5,122 +5,132 @@ require "whois-parser"
 module Mihari
   module Enrichers
     class Whois < Base
-      # @type [Hash]
-      @memo = {}
+      # @return [Hash]
+      attr_accessor :memo
 
-      # @return [Boolean]
-      def valid?
-        true
+      #
+      # @param [Hash, nil] options
+      #
+      def initialize(options: nil)
+        super(options: options)
+
+        @memo = {}
+      end
+
+      #
+      # Query IAIA Whois API
+      #
+      # @param [String] name
+      #
+      # @return [Mihari::WhoisRecord, nil]
+      #
+      def query(domain)
+        domain = PublicSuffix.domain(domain)
+
+        # check memo
+        return memo[domain].dup if memo.key?(domain)
+
+        record = whois.lookup(domain)
+        parser = record.parser
+        return nil if parser.available?
+
+        whois_record = WhoisRecord.new(
+          domain: domain,
+          created_on: get_created_on(parser),
+          updated_on: get_updated_on(parser),
+          expires_on: get_expires_on(parser),
+          registrar: get_registrar(parser),
+          contacts: get_contacts(parser)
+        )
+
+        # set memo
+        memo[domain] = whois_record
+
+        whois_record
       end
 
-      class << self
-        include Dry::Monads[:result]
-
-        #
-        # Query IAIA Whois API
-        #
-        # @param [String] name
-        #
-        # @return [Mihari::WhoisRecord, nil]
-        #
-        def query(domain)
-          domain = PublicSuffix.domain(domain)
-
-          # check memo
-          if @memo.key?(domain)
-            whois_record = @memo[domain]
-            # return clone of the record
-            return whois_record.dup
+      def reset_cache
+        @memo = {}
+      end
+
+      private
+
+      #
+      # @return [::Whois::Client]
+      #
+      def whois
+        @whois ||= [].tap do |out|
+          out << if timeout.nil?
+            ::Whois::Client.new
+          else
+            ::Whois::Client.new(timeout: timeout)
           end
+        end.last
+      end
+
+      #
+      # Get created_on
+      #
+      # @param [::Whois::Parser:] parser
+      #
+      # @return [Date, nil]
+      #
+      def get_created_on(parser)
+        parser.created_on
+      rescue ::Whois::AttributeNotImplemented
+        nil
+      end
+
+      #
+      # Get updated_on
+      #
+      # @param [::Whois::Parser:] parser
+      #
+      # @return [Date, nil]
+      #
+      def get_updated_on(parser)
+        parser.updated_on
+      rescue ::Whois::AttributeNotImplemented
+        nil
+      end
+
+      #
+      # Get expires_on
+      #
+      # @param [::Whois::Parser:] parser
+      #
+      # @return [Date, nil]
+      #
+      def get_expires_on(parser)
+        parser.expires_on
+      rescue ::Whois::AttributeNotImplemented
+        nil
+      end
+
+      #
+      # Get registrar
+      #
+      # @param [::Whois::Parser:] parser
+      #
+      # @return [Hash, nil]
+      #
+      def get_registrar(parser)
+        parser.registrar&.to_h
+      rescue ::Whois::AttributeNotImplemented
+        nil
+      end
 
-          record = ::Whois.whois(domain)
-          parser = record.parser
-
-          return nil if parser.available?
-
-          whois_record = WhoisRecord.new(
-            domain: domain,
-            created_on: get_created_on(parser),
-            updated_on: get_updated_on(parser),
-            expires_on: get_expires_on(parser),
-            registrar: get_registrar(parser),
-            contacts: get_contacts(parser)
-          )
-          # set memo
-          @memo[domain] = whois_record
-          whois_record
-        end
-
-        def reset_cache
-          @memo = {}
-        end
-
-        private
-
-        #
-        # Get created_on
-        #
-        # @param [::Whois::Parser:] parser
-        #
-        # @return [Date, nil]
-        #
-        def get_created_on(parser)
-          parser.created_on
-        rescue ::Whois::AttributeNotImplemented
-          nil
-        end
-
-        #
-        # Get updated_on
-        #
-        # @param [::Whois::Parser:] parser
-        #
-        # @return [Date, nil]
-        #
-        def get_updated_on(parser)
-          parser.updated_on
-        rescue ::Whois::AttributeNotImplemented
-          nil
-        end
-
-        #
-        # Get expires_on
-        #
-        # @param [::Whois::Parser:] parser
-        #
-        # @return [Date, nil]
-        #
-        def get_expires_on(parser)
-          parser.expires_on
-        rescue ::Whois::AttributeNotImplemented
-          nil
-        end
-
-        #
-        # Get registrar
-        #
-        # @param [::Whois::Parser:] parser
-        #
-        # @return [Hash, nil]
-        #
-        def get_registrar(parser)
-          parser.registrar&.to_h
-        rescue ::Whois::AttributeNotImplemented
-          nil
-        end
-
-        #
-        # Get contacts
-        #
-        # @param [::Whois::Parser:] parser
-        #
-        # @return [Array[Hash], nil]
-        #
-        def get_contacts(parser)
-          parser.contacts.map(&:to_h)
-        rescue ::Whois::AttributeNotImplemented
-          nil
-        end
+      #
+      # Get contacts
+      #
+      # @param [::Whois::Parser:] parser
+      #
+      # @return [Array[Hash], nil]
+      #
+      def get_contacts(parser)
+        parser.contacts.map(&:to_h)
+      rescue ::Whois::AttributeNotImplemented
+        nil
       end
     end
   end

data/lib/mihari/mixins/retriable.rb

@@ -19,15 +19,17 @@ module Mihari
       #
       # @param [Integer] times
       # @param [Integer] interval
+      # @param [Boolean] exponential_backoff
       # @param [Array<StandardError>] on
       #
-      def retry_on_error(times: 3, interval: 5, on: DEFAULT_ON)
+      def retry_on_error(times: 3, interval: 5, exponential_backoff: true, on: DEFAULT_ON)
         try = 0
         begin
           try += 1
           yield
         rescue *on => e
-          sleep interval
+          sleep_seconds = exponential_backoff ? interval * (2**(try - 1)) : interval
+          sleep sleep_seconds
           retry if try < times
           raise e
         end

data/lib/mihari/models/artifact.rb

@@ -73,64 +73,78 @@ module Mihari
     #
     # Enrich(add) whois record
     #
-    def enrich_whois
+    # @param [Mihari::Enrichers::Whois] enricher
+    #
+    def enrich_whois(enricher = Enrichers::Whois.new)
       return unless can_enrich_whois?
 
-      self.whois_record = WhoisRecord.build_by_domain(normalize_as_domain(data))
+      self.whois_record = WhoisRecord.build_by_domain(normalize_as_domain(data), enricher: enricher)
     end
 
     #
     # Enrich(add) DNS records
     #
-    def enrich_dns
+    # @param [Mihari::Enrichers::GooglePublicDNS] enricher
+    #
+    def enrich_dns(enricher = Enrichers::GooglePublicDNS.new)
       return unless can_enrich_dns?
 
-      self.dns_records = DnsRecord.build_by_domain(normalize_as_domain(data))
+      self.dns_records = DnsRecord.build_by_domain(normalize_as_domain(data), enricher: enricher)
     end
 
     #
     # Enrich(add) reverse DNS names
     #
-    def enrich_reverse_dns
+    # @param [Mihari::Enrichers::Shodan] enricher
+    #
+    def enrich_reverse_dns(enricher = Enrichers::Shodan.new)
       return unless can_enrich_revese_dns?
 
-      self.reverse_dns_names = ReverseDnsName.build_by_ip(data)
+      self.reverse_dns_names = ReverseDnsName.build_by_ip(data, enricher: enricher)
     end
 
     #
     # Enrich(add) geolocation
     #
-    def enrich_geolocation
+    # @param [Mihari::Enrichers::IPInfo] enricher
+    #
+    def enrich_geolocation(enricher = Enrichers::IPInfo.new)
       return unless can_enrich_geolocation?
 
-      self.geolocation = Geolocation.build_by_ip(data)
+      self.geolocation = Geolocation.build_by_ip(data, enricher: enricher)
     end
 
     #
     # Enrich AS
     #
-    def enrich_autonomous_system
+    # @param [Mihari::Enrichers::IPInfo] enricher
+    #
+    def enrich_autonomous_system(enricher = Enrichers::IPInfo.new)
       return unless can_enrich_autonomous_system?
 
-      self.autonomous_system = AutonomousSystem.build_by_ip(data)
+      self.autonomous_system = AutonomousSystem.build_by_ip(data, enricher: enricher)
     end
 
     #
     # Enrich ports
     #
-    def enrich_ports
+    # @param [Mihari::Enrichers::Shodan] enricher
+    #
+    def enrich_ports(enricher = Enrichers::Shodan.new)
       return unless can_enrich_ports?
 
-      self.ports = Port.build_by_ip(data)
+      self.ports = Port.build_by_ip(data, enricher: enricher)
     end
 
     #
     # Enrich CPEs
     #
-    def enrich_cpes
+    # @param [Mihari::Enrichers::Shodan] enricher
+    #
+    def enrich_cpes(enricher = Enrichers::Shodan.new)
       return unless can_enrich_cpes?
 
-      self.cpes = CPE.build_by_ip(data)
+      self.cpes = CPE.build_by_ip(data, enricher: enricher)
     end
 
     #
@@ -147,32 +161,32 @@ module Mihari
     end
 
     ENRICH_METHODS_BY_ENRICHER = {
-      whois: [
-        :enrich_whois
+      Enrichers::Whois => %i[
+        enrich_whois
       ],
-      ipinfo: %i[
+      Enrichers::IPInfo => %i[
         enrich_autonomous_system
         enrich_geolocation
       ],
-      shodan: %i[
+      Enrichers::Shodan => %i[
         enrich_ports
         enrich_cpes
         enrich_reverse_dns
       ],
-      google_public_dns: [
-        :enrich_dns
+      Enrichers::GooglePublicDNS => %i[
+        enrich_dns
       ]
     }.freeze
 
     #
     # Enrich by name of enricher
     #
-    # @param [String] enricher
+    # @param [Mihari::Enrichers::Base] enricher
     #
     def enrich_by_enricher(enricher)
-      methods = ENRICH_METHODS_BY_ENRICHER[enricher.downcase.to_sym] || []
+      methods = ENRICH_METHODS_BY_ENRICHER[enricher.class] || []
       methods.each do |method|
-        send(method) if respond_to?(method)
+        send(method, enricher) if respond_to?(method)
       end
     end
 

data/lib/mihari/models/autonomous_system.rb

@@ -11,11 +11,12 @@ module Mihari
       # Build AS
       #
       # @param [String] ip
+      # @param [Mihari::Enrichers::IPInfo] enricher
       #
       # @return [Mihari::AutonomousSystem, nil]
       #
-      def build_by_ip(ip)
-        result = Enrichers::IPInfo.query_result(ip).bind do |res|
+      def build_by_ip(ip, enricher: Enrichers::IPInfo.new)
+        result = enricher.query_result(ip).bind do |res|
           value = res&.asn
           if value.nil?
             Success nil

data/lib/mihari/models/cpe.rb

@@ -11,11 +11,12 @@ module Mihari
       # Build CPEs
       #
       # @param [String] ip
+      # @param [Mihari::Enrichers::Shodan] enricher
       #
       # @return [Array<Mihari::CPE>]
       #
-      def build_by_ip(ip)
-        result = Enrichers::Shodan.query_result(ip).bind do |res|
+      def build_by_ip(ip, enricher: Enrichers::Shodan.new)
+        result = enricher.query_result(ip).bind do |res|
           if res.nil?
             Success []
           else

data/lib/mihari/models/dns.rb

@@ -11,11 +11,12 @@ module Mihari
       # Build DNS records
       #
       # @param [String] domain
+      # @param [Mihari::Enrichers::Shodan] enricher
       #
       # @return [Array<Mihari::DnsRecord>]
       #
-      def build_by_domain(domain)
-        result = Enrichers::GooglePublicDNS.query_result(domain).bind do |responses|
+      def build_by_domain(domain, enricher: Enrichers::GooglePublicDNS.new)
+        result = enricher.query_result(domain).bind do |responses|
           Success(
             responses.map do |res|
               res.answers.map do |answer|

data/lib/mihari/models/geolocation.rb

@@ -13,11 +13,12 @@ module Mihari
       # Build Geolocation
       #
       # @param [String] ip
+      # @param [Mihari::Enrichers::IPinfo] enricher
       #
       # @return [Mihari::Geolocation, nil]
       #
-      def build_by_ip(ip)
-        result = Enrichers::IPInfo.query_result(ip).bind do |res|
+      def build_by_ip(ip, enricher: Enrichers::IPInfo.new)
+        result = enricher.query_result(ip).bind do |res|
           value = res&.country_code
           if value.nil?
             Success nil

data/lib/mihari/models/port.rb

@@ -11,11 +11,12 @@ module Mihari
       # Build ports
       #
       # @param [String] ip
+      # @param [Mihari::Enrichers::Shodan] enricher
       #
       # @return [Array<Mihari::Port>]
       #
-      def build_by_ip(ip)
-        result = Enrichers::Shodan.query_result(ip).bind do |res|
+      def build_by_ip(ip, enricher: Enrichers::Shodan.new)
+        result = enricher.query_result(ip).bind do |res|
           if res.nil?
             Success []
           else

data/lib/mihari/models/reverse_dns.rb

@@ -11,11 +11,12 @@ module Mihari
       # Build reverse DNS names
       #
       # @param [String] ip
+      # @param [Mihari::Enrichers::Shodan] enricher
       #
       # @return [Array<Mihari::ReverseDnsName>]
       #
-      def build_by_ip(ip)
-        result = Enrichers::Shodan.query_result(ip).bind do |res|
+      def build_by_ip(ip, enricher: Enrichers::Shodan.new)
+        result = enricher.query_result(ip).bind do |res|
           if res.nil?
             Success []
           else

data/lib/mihari/models/whois.rb

@@ -12,12 +12,13 @@ module Mihari
       #
       # Build whois record
       #
-      # @param [Stinrg] domain
+      # @param [String] domain
+      # @param [Mihari::Enrichers::Whois] enricher
       #
       # @return [WhoisRecord, nil]
       #
-      def build_by_domain(domain)
-        result = Enrichers::Whois.query_result(domain)
+      def build_by_domain(domain, enricher: Enrichers::Whois.new)
+        result = enricher.query_result(domain)
         result.value_or nil
       end
     end

data/lib/mihari/schemas/analyzer.rb

@@ -3,10 +3,11 @@
 module Mihari
   module Schemas
     AnalyzerOptions = Dry::Schema.Params do
-      optional(:interval).value(:integer)
+      optional(:pagination_interval).value(:integer).default(Mihari.config.pagination_interval)
       optional(:pagination_limit).value(:integer).default(Mihari.config.pagination_limit)
       optional(:retry_times).value(:integer).default(Mihari.config.retry_times)
       optional(:retry_interval).value(:integer).default(Mihari.config.retry_interval)
+      optional(:retry_exponential_backoff).value(:bool).default(Mihari.config.retry_exponential_backoff)
       optional(:ignore_error).value(:bool).default(Mihari.config.ignore_error)
       optional(:timeout).value(:integer)
     end

data/lib/mihari/schemas/emitter.rb

@@ -2,35 +2,49 @@
 
 module Mihari
   module Schemas
-    Database = Dry::Schema.Params do
-      required(:emitter).value(Types::String.enum("database"))
-    end
+    module Emitters
+      EmitterOptions = Dry::Schema.Params do
+        optional(:retry_times).value(:integer).default(Mihari.config.retry_times)
+        optional(:retry_interval).value(:integer).default(Mihari.config.retry_interval)
+        optional(:retry_exponential_backoff).value(:bool).default(Mihari.config.retry_exponential_backoff)
+        optional(:timeout).value(:integer)
+      end
 
-    MISP = Dry::Schema.Params do
-      required(:emitter).value(Types::String.enum("misp"))
-      optional(:url).value(:string)
-      optional(:api_key).value(:string)
-    end
+      Database = Dry::Schema.Params do
+        required(:emitter).value(Types::String.enum("database"))
+        optional(:options).hash(EmitterOptions)
+      end
 
-    TheHive = Dry::Schema.Params do
-      required(:emitter).value(Types::String.enum("the_hive"))
-      optional(:url).value(:string)
-      optional(:api_key).value(:string)
-      optional(:api_version).value(Types::String.enum("v4", "v5")).default("v4")
-    end
+      MISP = Dry::Schema.Params do
+        required(:emitter).value(Types::String.enum("misp"))
+        optional(:url).value(:string)
+        optional(:api_key).value(:string)
+        optional(:options).hash(EmitterOptions)
+      end
 
-    Slack = Dry::Schema.Params do
-      required(:emitter).value(Types::String.enum("slack"))
-      optional(:webhook_url).value(:string)
-      optional(:channel).value(:string)
-    end
+      TheHive = Dry::Schema.Params do
+        required(:emitter).value(Types::String.enum("the_hive"))
+        optional(:url).value(:string)
+        optional(:api_key).value(:string)
+        optional(:api_version).value(Types::String.enum("v4", "v5")).default("v4")
+        optional(:options).hash(EmitterOptions)
+      end
+
+      Slack = Dry::Schema.Params do
+        required(:emitter).value(Types::String.enum("slack"))
+        optional(:webhook_url).value(:string)
+        optional(:channel).value(:string)
+        optional(:options).hash(EmitterOptions)
+      end
 
-    Webhook = Dry::Schema.Params do
-      required(:emitter).value(Types::String.enum("webhook"))
-      required(:url).value(:string)
-      optional(:method).value(Types::HTTPRequestMethods).default("POST")
-      optional(:headers).value(:hash).default({})
-      optional(:template).value(:string)
+      Webhook = Dry::Schema.Params do
+        required(:emitter).value(Types::String.enum("webhook"))
+        required(:url).value(:string)
+        optional(:method).value(Types::HTTPRequestMethods).default("POST")
+        optional(:headers).value(:hash).default({})
+        optional(:template).value(:string)
+        optional(:options).hash(EmitterOptions)
+      end
     end
   end
 end

data/lib/mihari/schemas/enricher.rb

@@ -2,8 +2,34 @@
 
 module Mihari
   module Schemas
-    Enricher = Dry::Schema.Params do
-      required(:enricher).value(Types::EnricherTypes)
+    module Enrichers
+      EnricherOptions = Dry::Schema.Params do
+        optional(:retry_times).value(:integer).default(Mihari.config.retry_times)
+        optional(:retry_interval).value(:integer).default(Mihari.config.retry_interval)
+        optional(:retry_exponential_backoff).value(:bool).default(Mihari.config.retry_exponential_backoff)
+        optional(:timeout).value(:integer)
+      end
+
+      IPInfo = Dry::Schema.Params do
+        required(:enricher).value(Types::String.enum("ipinfo"))
+        optional(:api_key).value(:string)
+        optional(:options).hash(EnricherOptions)
+      end
+
+      Whois = Dry::Schema.Params do
+        required(:enricher).value(Types::String.enum("whois"))
+        optional(:options).hash(EnricherOptions)
+      end
+
+      Shodan = Dry::Schema.Params do
+        required(:enricher).value(Types::String.enum("shodan"))
+        optional(:options).hash(EnricherOptions)
+      end
+
+      GooglePublicDNS = Dry::Schema.Params do
+        required(:enricher).value(Types::String.enum("google_public_dns"))
+        optional(:options).hash(EnricherOptions)
+      end
     end
   end
 end

data/lib/mihari/schemas/rule.rb

@@ -25,9 +25,13 @@ module Mihari
         AnalyzerWithoutAPIKey | AnalyzerWithAPIKey | Censys | CIRCL | PassiveTotal | ZoomEye | Crtsh | Feed | HunterHow
       end
 
-      optional(:emitters).value(:array).each { Database | MISP | TheHive | Slack | Webhook }.default(DEFAULT_EMITTERS)
+      optional(:emitters).value(:array).each do
+        Emitters::Database | Emitters::MISP | Emitters::TheHive | Emitters::Slack | Emitters::Webhook
+      end.default(DEFAULT_EMITTERS)
 
-      optional(:enrichers).value(:array).each(Enricher).default(DEFAULT_ENRICHERS)
+      optional(:enrichers).value(:array).each do
+        Enrichers::Whois | Enrichers::IPInfo | Enrichers::Shodan | Enrichers::GooglePublicDNS
+      end.default(DEFAULT_ENRICHERS)
 
       optional(:data_types).value(array[Types::DataTypes]).default(DEFAULT_DATA_TYPES)
       optional(:falsepositives).value(array[:string]).default([])

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "5.4.9"
+  VERSION = "5.5.0"
 end

data/lib/mihari/web/endpoints/ip_addresses.rb

@@ -15,7 +15,7 @@ module Mihari
         get "/:ip", requirements: { ip: %r{[^/]+} } do
           ip = params[:ip].to_s
 
-          data = Enrichers::IPInfo.query(ip)
+          data = Enrichers::IPInfo.new.query(ip)
           error!({ message: "IP:#{ip} is not found" }, 404) if data.nil?
 
           present data, with: Entities::IPAddress