mihari 5.4.9 → 5.6.0

data/lib/mihari/enrichers/whois.rb

@@ -5,122 +5,132 @@ require "whois-parser"
 module Mihari
   module Enrichers
     class Whois < Base
-      # @type [Hash]
-      @memo = {}
+      # @return [Hash]
+      attr_accessor :memo
 
-      # @return [Boolean]
-      def valid?
-        true
+      #
+      # @param [Hash, nil] options
+      #
+      def initialize(options: nil)
+        super(options: options)
+
+        @memo = {}
+      end
+
+      #
+      # Query IAIA Whois API
+      #
+      # @param [String] name
+      #
+      # @return [Mihari::WhoisRecord, nil]
+      #
+      def query(domain)
+        domain = PublicSuffix.domain(domain)
+
+        # check memo
+        return memo[domain].dup if memo.key?(domain)
+
+        record = whois.lookup(domain)
+        parser = record.parser
+        return nil if parser.available?
+
+        whois_record = WhoisRecord.new(
+          domain: domain,
+          created_on: get_created_on(parser),
+          updated_on: get_updated_on(parser),
+          expires_on: get_expires_on(parser),
+          registrar: get_registrar(parser),
+          contacts: get_contacts(parser)
+        )
+
+        # set memo
+        memo[domain] = whois_record
+
+        whois_record
       end
 
-      class << self
-        include Dry::Monads[:result]
-
-        #
-        # Query IAIA Whois API
-        #
-        # @param [String] name
-        #
-        # @return [Mihari::WhoisRecord, nil]
-        #
-        def query(domain)
-          domain = PublicSuffix.domain(domain)
-
-          # check memo
-          if @memo.key?(domain)
-            whois_record = @memo[domain]
-            # return clone of the record
-            return whois_record.dup
+      def reset_cache
+        @memo = {}
+      end
+
+      private
+
+      #
+      # @return [::Whois::Client]
+      #
+      def whois
+        @whois ||= [].tap do |out|
+          out << if timeout.nil?
+            ::Whois::Client.new
+          else
+            ::Whois::Client.new(timeout: timeout)
           end
+        end.last
+      end
+
+      #
+      # Get created_on
+      #
+      # @param [::Whois::Parser:] parser
+      #
+      # @return [Date, nil]
+      #
+      def get_created_on(parser)
+        parser.created_on
+      rescue ::Whois::AttributeNotImplemented
+        nil
+      end
+
+      #
+      # Get updated_on
+      #
+      # @param [::Whois::Parser:] parser
+      #
+      # @return [Date, nil]
+      #
+      def get_updated_on(parser)
+        parser.updated_on
+      rescue ::Whois::AttributeNotImplemented
+        nil
+      end
+
+      #
+      # Get expires_on
+      #
+      # @param [::Whois::Parser:] parser
+      #
+      # @return [Date, nil]
+      #
+      def get_expires_on(parser)
+        parser.expires_on
+      rescue ::Whois::AttributeNotImplemented
+        nil
+      end
+
+      #
+      # Get registrar
+      #
+      # @param [::Whois::Parser:] parser
+      #
+      # @return [Hash, nil]
+      #
+      def get_registrar(parser)
+        parser.registrar&.to_h
+      rescue ::Whois::AttributeNotImplemented
+        nil
+      end
 
-          record = ::Whois.whois(domain)
-          parser = record.parser
-
-          return nil if parser.available?
-
-          whois_record = WhoisRecord.new(
-            domain: domain,
-            created_on: get_created_on(parser),
-            updated_on: get_updated_on(parser),
-            expires_on: get_expires_on(parser),
-            registrar: get_registrar(parser),
-            contacts: get_contacts(parser)
-          )
-          # set memo
-          @memo[domain] = whois_record
-          whois_record
-        end
-
-        def reset_cache
-          @memo = {}
-        end
-
-        private
-
-        #
-        # Get created_on
-        #
-        # @param [::Whois::Parser:] parser
-        #
-        # @return [Date, nil]
-        #
-        def get_created_on(parser)
-          parser.created_on
-        rescue ::Whois::AttributeNotImplemented
-          nil
-        end
-
-        #
-        # Get updated_on
-        #
-        # @param [::Whois::Parser:] parser
-        #
-        # @return [Date, nil]
-        #
-        def get_updated_on(parser)
-          parser.updated_on
-        rescue ::Whois::AttributeNotImplemented
-          nil
-        end
-
-        #
-        # Get expires_on
-        #
-        # @param [::Whois::Parser:] parser
-        #
-        # @return [Date, nil]
-        #
-        def get_expires_on(parser)
-          parser.expires_on
-        rescue ::Whois::AttributeNotImplemented
-          nil
-        end
-
-        #
-        # Get registrar
-        #
-        # @param [::Whois::Parser:] parser
-        #
-        # @return [Hash, nil]
-        #
-        def get_registrar(parser)
-          parser.registrar&.to_h
-        rescue ::Whois::AttributeNotImplemented
-          nil
-        end
-
-        #
-        # Get contacts
-        #
-        # @param [::Whois::Parser:] parser
-        #
-        # @return [Array[Hash], nil]
-        #
-        def get_contacts(parser)
-          parser.contacts.map(&:to_h)
-        rescue ::Whois::AttributeNotImplemented
-          nil
-        end
+      #
+      # Get contacts
+      #
+      # @param [::Whois::Parser:] parser
+      #
+      # @return [Array[Hash], nil]
+      #
+      def get_contacts(parser)
+        parser.contacts.map(&:to_h)
+      rescue ::Whois::AttributeNotImplemented
+        nil
       end
     end
   end

data/lib/mihari/mixins/retriable.rb

@@ -19,15 +19,17 @@ module Mihari
       #
       # @param [Integer] times
       # @param [Integer] interval
+      # @param [Boolean] exponential_backoff
       # @param [Array<StandardError>] on
       #
-      def retry_on_error(times: 3, interval: 5, on: DEFAULT_ON)
+      def retry_on_error(times: 3, interval: 5, exponential_backoff: true, on: DEFAULT_ON)
         try = 0
         begin
           try += 1
           yield
         rescue *on => e
-          sleep interval
+          sleep_seconds = exponential_backoff ? interval * (2**(try - 1)) : interval
+          sleep sleep_seconds
           retry if try < times
           raise e
         end

data/lib/mihari/models/artifact.rb

@@ -73,64 +73,78 @@ module Mihari
     #
     # Enrich(add) whois record
     #
-    def enrich_whois
+    # @param [Mihari::Enrichers::Whois] enricher
+    #
+    def enrich_whois(enricher = Enrichers::Whois.new)
       return unless can_enrich_whois?
 
-      self.whois_record = WhoisRecord.build_by_domain(normalize_as_domain(data))
+      self.whois_record = WhoisRecord.build_by_domain(normalize_as_domain(data), enricher: enricher)
     end
 
     #
     # Enrich(add) DNS records
     #
-    def enrich_dns
+    # @param [Mihari::Enrichers::GooglePublicDNS] enricher
+    #
+    def enrich_dns(enricher = Enrichers::GooglePublicDNS.new)
       return unless can_enrich_dns?
 
-      self.dns_records = DnsRecord.build_by_domain(normalize_as_domain(data))
+      self.dns_records = DnsRecord.build_by_domain(normalize_as_domain(data), enricher: enricher)
     end
 
     #
     # Enrich(add) reverse DNS names
     #
-    def enrich_reverse_dns
+    # @param [Mihari::Enrichers::Shodan] enricher
+    #
+    def enrich_reverse_dns(enricher = Enrichers::Shodan.new)
       return unless can_enrich_revese_dns?
 
-      self.reverse_dns_names = ReverseDnsName.build_by_ip(data)
+      self.reverse_dns_names = ReverseDnsName.build_by_ip(data, enricher: enricher)
     end
 
     #
     # Enrich(add) geolocation
     #
-    def enrich_geolocation
+    # @param [Mihari::Enrichers::IPInfo] enricher
+    #
+    def enrich_geolocation(enricher = Enrichers::IPInfo.new)
       return unless can_enrich_geolocation?
 
-      self.geolocation = Geolocation.build_by_ip(data)
+      self.geolocation = Geolocation.build_by_ip(data, enricher: enricher)
     end
 
     #
     # Enrich AS
     #
-    def enrich_autonomous_system
+    # @param [Mihari::Enrichers::IPInfo] enricher
+    #
+    def enrich_autonomous_system(enricher = Enrichers::IPInfo.new)
       return unless can_enrich_autonomous_system?
 
-      self.autonomous_system = AutonomousSystem.build_by_ip(data)
+      self.autonomous_system = AutonomousSystem.build_by_ip(data, enricher: enricher)
     end
 
     #
     # Enrich ports
     #
-    def enrich_ports
+    # @param [Mihari::Enrichers::Shodan] enricher
+    #
+    def enrich_ports(enricher = Enrichers::Shodan.new)
       return unless can_enrich_ports?
 
-      self.ports = Port.build_by_ip(data)
+      self.ports = Port.build_by_ip(data, enricher: enricher)
     end
 
     #
     # Enrich CPEs
     #
-    def enrich_cpes
+    # @param [Mihari::Enrichers::Shodan] enricher
+    #
+    def enrich_cpes(enricher = Enrichers::Shodan.new)
       return unless can_enrich_cpes?
 
-      self.cpes = CPE.build_by_ip(data)
+      self.cpes = CPE.build_by_ip(data, enricher: enricher)
     end
 
     #
@@ -147,32 +161,32 @@ module Mihari
     end
 
     ENRICH_METHODS_BY_ENRICHER = {
-      whois: [
-        :enrich_whois
+      Enrichers::Whois => %i[
+        enrich_whois
       ],
-      ipinfo: %i[
+      Enrichers::IPInfo => %i[
         enrich_autonomous_system
         enrich_geolocation
       ],
-      shodan: %i[
+      Enrichers::Shodan => %i[
         enrich_ports
         enrich_cpes
         enrich_reverse_dns
       ],
-      google_public_dns: [
-        :enrich_dns
+      Enrichers::GooglePublicDNS => %i[
+        enrich_dns
       ]
     }.freeze
 
     #
     # Enrich by name of enricher
     #
-    # @param [String] enricher
+    # @param [Mihari::Enrichers::Base] enricher
     #
     def enrich_by_enricher(enricher)
-      methods = ENRICH_METHODS_BY_ENRICHER[enricher.downcase.to_sym] || []
+      methods = ENRICH_METHODS_BY_ENRICHER[enricher.class] || []
       methods.each do |method|
-        send(method) if respond_to?(method)
+        send(method, enricher) if respond_to?(method)
       end
     end
 

data/lib/mihari/models/autonomous_system.rb

@@ -11,11 +11,12 @@ module Mihari
       # Build AS
       #
       # @param [String] ip
+      # @param [Mihari::Enrichers::IPInfo] enricher
       #
       # @return [Mihari::AutonomousSystem, nil]
       #
-      def build_by_ip(ip)
-        result = Enrichers::IPInfo.query_result(ip).bind do |res|
+      def build_by_ip(ip, enricher: Enrichers::IPInfo.new)
+        result = enricher.query_result(ip).bind do |res|
           value = res&.asn
           if value.nil?
             Success nil

data/lib/mihari/models/cpe.rb

@@ -11,11 +11,12 @@ module Mihari
       # Build CPEs
       #
       # @param [String] ip
+      # @param [Mihari::Enrichers::Shodan] enricher
       #
       # @return [Array<Mihari::CPE>]
       #
-      def build_by_ip(ip)
-        result = Enrichers::Shodan.query_result(ip).bind do |res|
+      def build_by_ip(ip, enricher: Enrichers::Shodan.new)
+        result = enricher.query_result(ip).bind do |res|
           if res.nil?
             Success []
           else

data/lib/mihari/models/dns.rb

@@ -11,11 +11,12 @@ module Mihari
       # Build DNS records
       #
       # @param [String] domain
+      # @param [Mihari::Enrichers::Shodan] enricher
       #
       # @return [Array<Mihari::DnsRecord>]
       #
-      def build_by_domain(domain)
-        result = Enrichers::GooglePublicDNS.query_result(domain).bind do |responses|
+      def build_by_domain(domain, enricher: Enrichers::GooglePublicDNS.new)
+        result = enricher.query_result(domain).bind do |responses|
           Success(
             responses.map do |res|
               res.answers.map do |answer|

data/lib/mihari/models/geolocation.rb

@@ -13,11 +13,12 @@ module Mihari
       # Build Geolocation
       #
       # @param [String] ip
+      # @param [Mihari::Enrichers::IPinfo] enricher
       #
       # @return [Mihari::Geolocation, nil]
       #
-      def build_by_ip(ip)
-        result = Enrichers::IPInfo.query_result(ip).bind do |res|
+      def build_by_ip(ip, enricher: Enrichers::IPInfo.new)
+        result = enricher.query_result(ip).bind do |res|
           value = res&.country_code
           if value.nil?
             Success nil

data/lib/mihari/models/port.rb

@@ -11,11 +11,12 @@ module Mihari
       # Build ports
       #
       # @param [String] ip
+      # @param [Mihari::Enrichers::Shodan] enricher
       #
       # @return [Array<Mihari::Port>]
       #
-      def build_by_ip(ip)
-        result = Enrichers::Shodan.query_result(ip).bind do |res|
+      def build_by_ip(ip, enricher: Enrichers::Shodan.new)
+        result = enricher.query_result(ip).bind do |res|
           if res.nil?
             Success []
           else

data/lib/mihari/models/reverse_dns.rb

@@ -11,11 +11,12 @@ module Mihari
       # Build reverse DNS names
       #
       # @param [String] ip
+      # @param [Mihari::Enrichers::Shodan] enricher
       #
       # @return [Array<Mihari::ReverseDnsName>]
       #
-      def build_by_ip(ip)
-        result = Enrichers::Shodan.query_result(ip).bind do |res|
+      def build_by_ip(ip, enricher: Enrichers::Shodan.new)
+        result = enricher.query_result(ip).bind do |res|
           if res.nil?
             Success []
           else

data/lib/mihari/models/whois.rb

@@ -12,12 +12,13 @@ module Mihari
       #
       # Build whois record
       #
-      # @param [Stinrg] domain
+      # @param [String] domain
+      # @param [Mihari::Enrichers::Whois] enricher
       #
       # @return [WhoisRecord, nil]
       #
-      def build_by_domain(domain)
-        result = Enrichers::Whois.query_result(domain)
+      def build_by_domain(domain, enricher: Enrichers::Whois.new)
+        result = enricher.query_result(domain)
         result.value_or nil
       end
     end

data/lib/mihari/schemas/analyzer.rb

@@ -2,36 +2,31 @@
 
 module Mihari
   module Schemas
-    AnalyzerOptions = Dry::Schema.Params do
-      optional(:interval).value(:integer)
-      optional(:pagination_limit).value(:integer).default(Mihari.config.pagination_limit)
-      optional(:retry_times).value(:integer).default(Mihari.config.retry_times)
-      optional(:retry_interval).value(:integer).default(Mihari.config.retry_interval)
-      optional(:ignore_error).value(:bool).default(Mihari.config.ignore_error)
-      optional(:timeout).value(:integer)
-    end
-
-    AnalyzerWithoutAPIKey = Dry::Schema.Params do
-      required(:analyzer).value(Types::String.enum("dnstwister"))
-      required(:query).value(:string)
-      optional(:options).hash(AnalyzerOptions)
-    end
-
-    AnalyzerWithAPIKey = Dry::Schema.Params do
+    AnalyzerAPIKeyPagination = Dry::Schema.Params do
       required(:analyzer).value(
         Types::String.enum(
           "binaryedge",
           "greynoise",
           "onyphe",
+          "shodan",
+          "urlscan",
+          "virustotal_intelligence",
+          "vt_intel"
+        )
+      )
+      required(:query).value(:string)
+      optional(:api_key).value(:string)
+      optional(:options).hash(AnalyzerPaginationOptions)
+    end
+
+    AnalyzerAPIKey = Dry::Schema.Params do
+      required(:analyzer).value(
+        Types::String.enum(
           "otx",
           "pulsedive",
           "securitytrails",
-          "shodan",
           "st",
-          "urlscan",
-          "virustotal_intelligence",
           "virustotal",
-          "vt_intel",
           "vt"
         )
       )
@@ -40,12 +35,18 @@ module Mihari
       optional(:options).hash(AnalyzerOptions)
     end
 
+    DNSTwister = Dry::Schema.Params do
+      required(:analyzer).value(Types::String.enum("dnstwister"))
+      required(:query).value(:string)
+      optional(:options).hash(AnalyzerOptions)
+    end
+
     Censys = Dry::Schema.Params do
       required(:analyzer).value(Types::String.enum("censys"))
       required(:query).value(:string)
       optional(:id).value(:string)
       optional(:secret).value(:string)
-      optional(:options).hash(AnalyzerOptions)
+      optional(:options).hash(AnalyzerPaginationOptions)
     end
 
     CIRCL = Dry::Schema.Params do
@@ -68,7 +69,7 @@ module Mihari
       required(:analyzer).value(Types::String.enum("zoomeye"))
       required(:query).value(:string)
       required(:type).value(Types::String.enum("host", "web"))
-      optional(:options).hash(AnalyzerOptions)
+      optional(:options).hash(AnalyzerPaginationOptions)
     end
 
     Crtsh = Dry::Schema.Params do
@@ -84,7 +85,7 @@ module Mihari
       required(:start_time).value(:date)
       required(:end_time).value(:date)
       optional(:api_key).value(:string)
-      optional(:options).hash(AnalyzerOptions)
+      optional(:options).hash(AnalyzerPaginationOptions)
     end
 
     Feed = Dry::Schema.Params do

data/lib/mihari/schemas/emitter.rb

@@ -2,35 +2,42 @@
 
 module Mihari
   module Schemas
-    Database = Dry::Schema.Params do
-      required(:emitter).value(Types::String.enum("database"))
-    end
+    module Emitters
+      Database = Dry::Schema.Params do
+        required(:emitter).value(Types::String.enum("database"))
+        optional(:options).hash(Options)
+      end
 
-    MISP = Dry::Schema.Params do
-      required(:emitter).value(Types::String.enum("misp"))
-      optional(:url).value(:string)
-      optional(:api_key).value(:string)
-    end
+      MISP = Dry::Schema.Params do
+        required(:emitter).value(Types::String.enum("misp"))
+        optional(:url).value(:string)
+        optional(:api_key).value(:string)
+        optional(:options).hash(Options)
+      end
 
-    TheHive = Dry::Schema.Params do
-      required(:emitter).value(Types::String.enum("the_hive"))
-      optional(:url).value(:string)
-      optional(:api_key).value(:string)
-      optional(:api_version).value(Types::String.enum("v4", "v5")).default("v4")
-    end
+      TheHive = Dry::Schema.Params do
+        required(:emitter).value(Types::String.enum("thehive"))
+        optional(:url).value(:string)
+        optional(:api_key).value(:string)
+        optional(:api_version).value(Types::String.enum("v4", "v5")).default("v4")
+        optional(:options).hash(Options)
+      end
 
-    Slack = Dry::Schema.Params do
-      required(:emitter).value(Types::String.enum("slack"))
-      optional(:webhook_url).value(:string)
-      optional(:channel).value(:string)
-    end
+      Slack = Dry::Schema.Params do
+        required(:emitter).value(Types::String.enum("slack"))
+        optional(:webhook_url).value(:string)
+        optional(:channel).value(:string)
+        optional(:options).hash(Options)
+      end
 
-    Webhook = Dry::Schema.Params do
-      required(:emitter).value(Types::String.enum("webhook"))
-      required(:url).value(:string)
-      optional(:method).value(Types::HTTPRequestMethods).default("POST")
-      optional(:headers).value(:hash).default({})
-      optional(:template).value(:string)
+      Webhook = Dry::Schema.Params do
+        required(:emitter).value(Types::String.enum("webhook"))
+        required(:url).value(:string)
+        optional(:method).value(Types::HTTPRequestMethods).default("POST")
+        optional(:headers).value(:hash).default({})
+        optional(:template).value(:string)
+        optional(:options).hash(Options)
+      end
     end
   end
 end