mihari 7.2.0 → 7.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6b91d99562526b653b793f71e8ef5575f113d26859ca86b91f1980d1c44e898c
-  data.tar.gz: 07c302ce446b0f0986bfc82dd575ebde203f4b25b73a1aee72a2ce37a08b9b87
+  metadata.gz: ed9b24457d01edc4d1643e6c31c654ad7c13bf71fc849b10bb02276abf45852c
+  data.tar.gz: 1be36d2083e0b0209ad16475a8f20eb4d2ee9bfcb37eae43fa76091207526def
 SHA512:
-  metadata.gz: 67d607a09ab2992b6721358b9e96e0c049a355221b2e97358440d70f96ef4933ac86b337bcc12bdc4b2aafccf4e5d6cf8cd68f4b2bbe567523bfba22b7fbd88c
-  data.tar.gz: 6f73de19824d31e21bae4ee7ce456c1b15fa0a875d1f07025c33ac0356ef257ac2eb819ffd8b685bf8914a472da04f691e5ae02a361397da9a83253d5345b108
+  metadata.gz: 4482938e33386e24054cb215f78f065e3190ba32269872aea6a9f543745a2c71777bb609120dcaa3872dba9b6ebd307651f80239342d6ac9427db205f03c80e0
+  data.tar.gz: 5723cbca9f18c519fc4cf91775d751f417a85161add0c0c6d499fd03075c7985c1cb73ef68e24b4913b1d8d3a9652a0e4e1025a32b83c987d3e9e22886923e38

data/Dockerfile

@@ -1,4 +1,4 @@
-FROM ruby:3.2.2-alpine3.19
+FROM ruby:3.3.0-alpine3.19
 
 ARG MIHARI_VERSION=0.0.0
 

data/lib/mihari/emitters/base.rb

@@ -19,6 +19,13 @@ module Mihari
         @rule = rule
       end
 
+      #
+      # @return [Boolean]
+      #
+      def parallel?
+        options[:parallel] || Mihari.config.parallel
+      end
+
       # A target to emit the data
       #
       # @return [String]

data/lib/mihari/enrichers/base.rb

@@ -6,46 +6,88 @@ module Mihari
     # Base class for enrichers
     #
     class Base < Actor
-      prepend MemoWise
-
+      #
+      # @param [Hash, nil] options
+      #
       def initialize(options: nil)
         super(options: options)
       end
 
       #
-      # @param [String] value
+      # Enrich an artifact
+      #
+      # @param [Mihari::Models::Artifact] artifact
       #
-      def call(value)
+      # @return [Mihari::Models::Artifact]
+      #
+      def call(artifact)
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
       end
 
       #
-      # @param [Mihari::Models::Artifact] value
+      # @param [Mihari::Models::Artifact] artifact
       #
       # @return [Dry::Monads::Result::Success<Object>, Dry::Monads::Result::Failure]
       #
-      def result(value)
+      def result(artifact)
+        return unless callable?(artifact)
+
         result = Try[StandardError] do
-          retry_on_error(
-            times: retry_times,
-            interval: retry_interval,
-            exponential_backoff: retry_exponential_backoff
-          ) { call value }
+          retry_on_error(times: retry_times, interval: retry_interval,
+            exponential_backoff: retry_exponential_backoff) do
+            call artifact
+          end
         end.to_result
 
         if result.failure?
-          Mihari.logger.warn("Enricher:#{self.class.key} for #{value.truncate(32)} failed: #{result.failure}")
+          Mihari.logger.warn("Enricher:#{self.class.key} for #{artifact.data.truncate(32)} failed: #{result.failure}")
         end
 
         result
       end
 
+      #
+      # @param [Mihari::Models::Artifact] artifact
+      #
+      # @return [Boolean]
+      #
+      def callable?(artifact)
+        callable_data_type?(artifact) && callable_relationships?(artifact)
+      end
+
       class << self
         def inherited(child)
           super
           Mihari.enrichers << child
         end
       end
+
+      private
+
+      #
+      # @param [Mihari::Models::Artifact] artifact
+      #
+      # @return [Boolean]
+      #
+      def callable_data_type?(artifact)
+        supported_data_types.include? artifact.data_type
+      end
+
+      #
+      # @param [Mihari::Models::Artifact] artifact
+      #
+      # @return [Boolean]
+      #
+      def callable_relationships?(artifact)
+        raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+      end
+
+      #
+      # @return [Array<String>]
+      #
+      def supported_data_types
+        []
+      end
     end
   end
 end

data/lib/mihari/enrichers/google_public_dns.rb

@@ -7,14 +7,22 @@ module Mihari
     #
     class GooglePublicDNS < Base
       #
-      # Query Google Public DNS
+      # @param [Mihari::Models::Artifact] artifact
       #
-      # @param [String] name
+      # @return [Mihari::Models::Artifact]
       #
-      # @return [Mihari::Structs::GooglePublicDNS::Response]
-      #
-      def call(name)
-        client.query_all name
+      def call(artifact)
+        return if artifact.domain.nil?
+
+        res = client.query_all(artifact.domain)
+
+        artifact.tap do |tapped|
+          if tapped.dns_records.empty?
+            tapped.dns_records = res.answers.map do |answer|
+              Models::DnsRecord.new(resource: answer.resource_type, value: answer.data)
+            end
+          end
+        end
       end
 
       class << self
@@ -28,8 +36,21 @@ module Mihari
 
       private
 
+      #
+      # @param [Mihari::Models::Artifact] artifact
+      #
+      # @return [Boolean]
+      #
+      def callable_relationships?(artifact)
+        artifact.dns_records.empty?
+      end
+
+      def supported_data_types
+        %w[url domain]
+      end
+
       def client
-        Clients::GooglePublicDNS.new(timeout: timeout)
+        @client ||= Clients::GooglePublicDNS.new(timeout: timeout)
       end
     end
   end

data/lib/mihari/enrichers/mmdb.rb

@@ -7,18 +7,36 @@ module Mihari
     #
     class MMDB < Base
       #
-      # Query MMDB
+      # @param [Mihari::Models::Artifact] artifact
       #
-      # @param [String] ip
+      def call(artifact)
+        res = client.query(artifact.data)
+
+        artifact.tap do |tapped|
+          tapped.autonomous_system ||= Models::AutonomousSystem.new(number: res.asn) if res.asn
+          if res.country_code
+            tapped.geolocation ||= Models::Geolocation.new(
+              country: NormalizeCountry(res.country_code, to: :short),
+              country_code: res.country_code
+            )
+          end
+        end
+      end
+
+      private
+
+      #
+      # @param [Mihari::Models::Artifact] artifact
       #
-      # @return [Mihari::Structs::MMDB::Response]
+      # @return [Boolean]
       #
-      def call(ip)
-        client.query ip
+      def callable_relationships?(artifact)
+        artifact.geolocation.nil? || artifact.autonomous_system.nil?
       end
-      memo_wise :call
 
-      private
+      def supported_data_types
+        %w[ip]
+      end
 
       def client
         @client ||= Clients::MMDB.new(timeout: timeout)

data/lib/mihari/enrichers/shodan.rb

@@ -9,17 +9,48 @@ module Mihari
       #
       # Query Shodan Internet DB
       #
-      # @param [String] ip
+      # @param [Mihari::Models::Artifact] artifact
       #
       # @return [Mihari::Structs::Shodan::InternetDBResponse, nil]
       #
-      def call(ip)
-        client.query ip
+      def call(artifact)
+        res = client.query(artifact.data)
+
+        artifact.tap do |tapped|
+          tapped.cpes = (res&.cpes || []).map { |cpe| Models::CPE.new(name: cpe) } if tapped.cpes.empty?
+          tapped.ports = (res&.ports || []).map { |port| Models::Port.new(number: port) } if tapped.ports.empty?
+          if tapped.reverse_dns_names.empty?
+            tapped.reverse_dns_names = (res&.hostnames || []).map do |name|
+              Models::ReverseDnsName.new(name: name)
+            end
+          end
+        end
+      end
+
+      #
+      # @param [Mihari::Models::Artifact] artifact
+      #
+      # @return [Boolean]
+      #
+      def callable?(artifact)
+        false unless supported_data_types.include?(artifact.data_type)
       end
-      memo_wise :call
 
       private
 
+      #
+      # @param [Mihari::Models::Artifact] artifact
+      #
+      # @return [Boolean]
+      #
+      def callable_relationships?(artifact)
+        artifact.cpes.empty? || artifact.ports.empty? || artifact.reverse_dns_names.empty?
+      end
+
+      def supported_data_types
+        %w[ip]
+      end
+
       def client
         @client ||= Clients::ShodanInternetDB.new(timeout: timeout)
       end

data/lib/mihari/enrichers/whois.rb

@@ -8,46 +8,54 @@ module Mihari
     # Whois enricher
     #
     class Whois < Base
+      prepend MemoWise
+
+      #
+      # Query IAIA Whois API
       #
-      # @param [Hash, nil] options
+      # @param [Mihari::Models::Artifact] artifact
       #
-      def initialize(options: nil)
-        super(options: options)
+      def call(artifact)
+        return if artifact.domain.nil?
+
+        domain = PublicSuffix.domain(artifact.domain)
+        record = memoized_lookup(domain)
+        return if record.parser.available?
+
+        artifact.whois_record ||= Models::WhoisRecord.new(
+          domain: domain,
+          created_on: get_created_on(record.parser),
+          updated_on: get_updated_on(record.parser),
+          expires_on: get_expires_on(record.parser),
+          registrar: get_registrar(record.parser),
+          contacts: get_contacts(record.parser)
+        )
       end
 
+      private
+
       #
-      # Query IAIA Whois API
-      #
-      # @param [String] domain
+      # @param [Mihari::Models::Artifact] artifact
       #
-      # @return [Mihari::Models::WhoisRecord, nil]
+      # @return [Boolean]
       #
-      def call(domain)
-        memoized_call PublicSuffix.domain(domain)
+      def callable_relationships?(artifact)
+        artifact.whois_record.nil?
       end
 
-      private
+      def supported_data_types
+        %w[url domain]
+      end
 
       #
       # @param [String] domain
       #
       # @return [Mihari::Models::WhoisRecord, nil]
       #
-      def memoized_call(domain)
-        record = whois.lookup(domain)
-        parser = record.parser
-        return nil if parser.available?
-
-        Models::WhoisRecord.new(
-          domain: domain,
-          created_on: get_created_on(parser),
-          updated_on: get_updated_on(parser),
-          expires_on: get_expires_on(parser),
-          registrar: get_registrar(parser),
-          contacts: get_contacts(parser)
-        )
+      def memoized_lookup(domain)
+        whois.lookup domain
       end
-      memo_wise :memoized_call
+      memo_wise :memoized_lookup
 
       #
       # @return [::Whois::Client]

data/lib/mihari/models/alert.rb

@@ -6,6 +6,18 @@ module Mihari
     # Alert model
     #
     class Alert < ActiveRecord::Base
+      # @!attribute [r] id
+      #   @return [Integer, nil]
+
+      # @!attribute [rw] created_at
+      #   @return [DateTime]
+
+      # @!attribute [r] rule
+      #   @return [Mihari::Models::Rule]
+
+      # @!attribute [r] artifacts
+      #   @return [Array<Mihari::Models::Artifact>]
+
       belongs_to :rule
 
       has_many :artifacts, dependent: :destroy

data/lib/mihari/models/artifact.rb

@@ -17,6 +17,90 @@ module Mihari
     # Artifact model
     #
     class Artifact < ActiveRecord::Base
+      # @!attribute [r] id
+      #   @return [Integer, nil]
+
+      # @!attribute [rw] data
+      #   @return [String]
+
+      # @!attribute [rw] data_type
+      #   @return [String]
+
+      # @!attribute [rw] source
+      #   @return [String, nil]
+
+      # @!attribute [rw] query
+      #   @return [String, nil]
+
+      # @!attribute [rw] metadata
+      #   @return [Hash, nil]
+
+      # @!attribute [rw] created_at
+      #   @return [DateTime]
+
+      # @!attribute [r] alert
+      #   @return [Mihari::Models::Alert]
+
+      # @!attribute [r] rule
+      #   @return [Mihari::Models::Rule]
+
+      # @!attribute [rw] autonomous_system
+      #   @return [Mihari::Models::AutonomousSystem, nil]
+
+      # @!attribute [rw] geolocation
+      #   @return [Mihari::Models::Geolocation, nil]
+
+      # @!attribute [rw] whois_record
+      #   @return [Mihari::Models::WhoisRecord, nil]
+
+      # @!attribute [rw] cpes
+      #   @return [Array<Mihari::Models::CPE>]
+
+      # @!attribute [rw] dns_records
+      #   @return [Array<Mihari::Models::DnsRecord>]
+
+      # @!attribute [rw] ports
+      #   @return [Array<Mihari::Models::Port>]
+
+      # @!attribute [rw] reverse_dns_names
+      #   @return [Array<Mihari::Models::ReverseDnsName>]
+
+      # @!attribute [rw] vulnerabilities
+      #   @return [Array<Mihari::Models::Vulnerability>]
+
+      # @!attribute [r] alert
+      #   @return [Mihari::Models::Alert]
+
+      # @!attribute [r] rule
+      #   @return [Mihari::Models::Rule]
+
+      # @!attribute [rw] autonomous_system
+      #   @return [Mihari::Models::AutonomousSystem, nil]
+
+      # @!attribute [rw] geolocation
+      #   @return [Mihari::Models::Geolocation, nil]
+
+      # @!attribute [rw] whois_record
+      #   @return [Mihari::Models::WhoisRecord, nil]
+
+      # @!attribute [rw] cpes
+      #   @return [Array<Mihari::Models::CPE>]
+
+      # @!attribute [rw] dns_records
+      #   @return [Array<Mihari::Models::DnsRecord>]
+
+      # @!attribute [rw] ports
+      #   @return [Array<Mihari::Models::Port>]
+
+      # @!attribute [rw] reverse_dns_names
+      #   @return [Array<Mihari::Models::ReverseDnsName>]
+
+      # @!attribute [rw] vulnerabilities
+      #   @return [Array<Mihari::Models::Vulnerability>]
+
+      # @!attribute [rw] tags
+      #   @return [Array<Mihari::Models::Tag>]
+
       belongs_to :alert
 
       has_one :autonomous_system, dependent: :destroy
@@ -90,173 +174,24 @@ module Mihari
         artifact.created_at < decayed_at
       end
 
-      #
-      # Enrich whois record
-      #
-      # @param [Mihari::Enrichers::Whois] enricher
-      #
-      def enrich_whois(enricher = Enrichers::Whois.new)
-        return unless can_enrich_whois?
-
-        self.whois_record = Services::WhoisRecordBuilder.call(domain, enricher: enricher)
-      end
-
-      #
-      # Enrich DNS records
-      #
-      # @param [Mihari::Enrichers::GooglePublicDNS] enricher
-      #
-      def enrich_dns(enricher = Enrichers::GooglePublicDNS.new)
-        return unless can_enrich_dns?
-
-        self.dns_records = Services::DnsRecordBuilder.call(domain, enricher: enricher)
-      end
-
-      #
-      # Enrich reverse DNS names
-      #
-      # @param [Mihari::Enrichers::Shodan] enricher
-      #
-      def enrich_reverse_dns(enricher = Enrichers::Shodan.new)
-        return unless can_enrich_reverse_dns?
-
-        self.reverse_dns_names = Services::ReverseDnsNameBuilder.call(data, enricher: enricher)
-      end
-
-      #
-      # Enrich geolocation
-      #
-      # @param [Mihari::Enrichers::IPInfo] enricher
-      #
-      def enrich_geolocation(enricher = Enrichers::MMDB.new)
-        return unless can_enrich_geolocation?
-
-        self.geolocation = Services::GeolocationBuilder.call(data, enricher: enricher)
-      end
-
-      #
-      # Enrich AS
-      #
-      # @param [Mihari::Enrichers::IPInfo] enricher
-      #
-      def enrich_autonomous_system(enricher = Enrichers::MMDB.new)
-        return unless can_enrich_autonomous_system?
-
-        self.autonomous_system = Services::AutonomousSystemBuilder.call(data, enricher: enricher)
-      end
-
-      #
-      # Enrich ports
-      #
-      # @param [Mihari::Enrichers::Shodan] enricher
-      #
-      def enrich_ports(enricher = Enrichers::Shodan.new)
-        return unless can_enrich_ports?
-
-        self.ports = Services::PortBuilder.call(data, enricher: enricher)
-      end
-
-      #
-      # Enrich CPEs
-      #
-      # @param [Mihari::Enrichers::Shodan] enricher
-      #
-      def enrich_cpes(enricher = Enrichers::Shodan.new)
-        return unless can_enrich_cpes?
-
-        self.cpes = Services::CPEBuilder.call(data, enricher: enricher)
-      end
-
-      #
-      # Enrich vulnerabilities
-      #
-      # @param [Mihari::Enrichers::Shodan] enricher
-      #
-      def enrich_vulnerabilities(enricher = Enrichers::Shodan.new)
-        return unless can_enrich_vulnerabilities?
-
-        self.vulnerabilities = Services::VulnerabilityBuilder.call(data, enricher: enricher)
+      def enrichable?
+        !callable_enrichers.empty?
       end
 
-      #
-      # Enrich all the enrichable relationships of the artifact
-      #
-      def enrich_all
-        enrich_autonomous_system mmdb
-        enrich_dns
-        enrich_geolocation mmdb
-        enrich_reverse_dns shodan
-        enrich_whois
-        enrich_ports shodan
-        enrich_cpes shodan
-        enrich_vulnerabilities shodan
+      def enrich
+        callable_enrichers.each { |enricher| enricher.result self }
       end
 
-      ENRICH_METHODS_BY_ENRICHER = {
-        Enrichers::Whois => %i[
-          enrich_whois
-        ],
-        Enrichers::MMDB => %i[
-          enrich_autonomous_system
-          enrich_geolocation
-        ],
-        Enrichers::Shodan => %i[
-          enrich_ports
-          enrich_cpes
-          enrich_reverse_dns
-          enrich_vulnerabilities
-        ],
-        Enrichers::GooglePublicDNS => %i[
-          enrich_dns
-        ]
-      }.freeze
-
       #
-      # Enrich by name of enricher
-      #
-      # @param [Mihari::Enrichers::Base] enricher
+      # @return [String, nil]
       #
-      def enrich_by_enricher(enricher)
-        methods = ENRICH_METHODS_BY_ENRICHER[enricher.class] || []
-        methods.each { |method| send(method, enricher) if respond_to?(method) }
-      end
-
-      def can_enrich_whois?
-        %w[domain url].include?(data_type) && whois_record.nil?
-      end
-
-      def can_enrich_dns?
-        %w[domain url].include?(data_type) && dns_records.empty?
-      end
-
-      def can_enrich_reverse_dns?
-        data_type == "ip" && reverse_dns_names.empty?
-      end
-
-      def can_enrich_geolocation?
-        data_type == "ip" && geolocation.nil?
-      end
-
-      def can_enrich_autonomous_system?
-        data_type == "ip" && autonomous_system.nil?
-      end
-
-      def can_enrich_ports?
-        data_type == "ip" && ports.empty?
-      end
-
-      def can_enrich_cpes?
-        data_type == "ip" && cpes.empty?
-      end
-
-      def can_enrich_vulnerabilities?
-        data_type == "ip" && vulnerabilities.empty?
-      end
-
-      def enrichable?
-        enrich_methods = methods.map(&:to_s).select { |method| method.start_with?("can_enrich_") }
-        enrich_methods.map(&:to_sym).any? do |method|
-          send(method) if respond_to?(method)
+      def domain
+        case data_type
+        when "domain"
+          data
+        when "url"
+          host = Addressable::URI.parse(data).host
+          (DataType.type(host) == "ip") ? nil : host
         end
       end
 
@@ -272,6 +207,15 @@ module Mihari
 
       private
 
+      #
+      # @return [Array<Mihari::Enrichers::Base>]
+      #
+      def callable_enrichers
+        @callable_enrichers ||= Mihari.enrichers.map(&:new).select do |enricher|
+          enricher.callable?(self)
+        end
+      end
+
       def set_data_type
         self.data_type = DataType.type(data)
       end
@@ -279,26 +223,6 @@ module Mihari
       def set_rule_id
         @set_rule_id ||= nil
       end
-
-      def mmdb
-        @mmdb ||= Enrichers::MMDB.new
-      end
-
-      def shodan
-        @shodan ||= Enrichers::Shodan.new
-      end
-
-      #
-      # @return [String, nil]
-      #
-      def domain
-        case data_type
-        when "domain"
-          data
-        when "url"
-          Addressable::URI.parse(data).host
-        end
-      end
     end
   end
 end