mihari 7.1.3 → 7.3.0

data/lib/mihari/enrichers/google_public_dns.rb

@@ -7,14 +7,22 @@ module Mihari
     #
     class GooglePublicDNS < Base
       #
-      # Query Google Public DNS
+      # @param [Mihari::Models::Artifact] artifact
       #
-      # @param [String] name
+      # @return [Mihari::Models::Artifact]
       #
-      # @return [Mihari::Structs::GooglePublicDNS::Response]
-      #
-      def call(name)
-        client.query_all name
+      def call(artifact)
+        return if artifact.domain.nil?
+
+        res = client.query_all(artifact.domain)
+
+        artifact.tap do |tapped|
+          if tapped.dns_records.empty?
+            tapped.dns_records = res.answers.map do |answer|
+              Models::DnsRecord.new(resource: answer.resource_type, value: answer.data)
+            end
+          end
+        end
       end
 
       class << self
@@ -28,8 +36,21 @@ module Mihari
 
       private
 
+      #
+      # @param [Mihari::Models::Artifact] artifact
+      #
+      # @return [Boolean]
+      #
+      def callable_relationships?(artifact)
+        artifact.dns_records.empty?
+      end
+
+      def supported_data_types
+        %w[url domain]
+      end
+
       def client
-        Clients::GooglePublicDNS.new(timeout: timeout)
+        @client ||= Clients::GooglePublicDNS.new(timeout: timeout)
       end
     end
   end

data/lib/mihari/enrichers/mmdb.rb

@@ -7,18 +7,36 @@ module Mihari
     #
     class MMDB < Base
       #
-      # Query MMDB
+      # @param [Mihari::Models::Artifact] artifact
       #
-      # @param [String] ip
+      def call(artifact)
+        res = client.query(artifact.data)
+
+        artifact.tap do |tapped|
+          tapped.autonomous_system ||= Models::AutonomousSystem.new(number: res.asn) if res.asn
+          if res.country_code
+            tapped.geolocation ||= Models::Geolocation.new(
+              country: NormalizeCountry(res.country_code, to: :short),
+              country_code: res.country_code
+            )
+          end
+        end
+      end
+
+      private
+
+      #
+      # @param [Mihari::Models::Artifact] artifact
       #
-      # @return [Mihari::Structs::MMDB::Response]
+      # @return [Boolean]
       #
-      def call(ip)
-        client.query ip
+      def callable_relationships?(artifact)
+        artifact.geolocation.nil? || artifact.autonomous_system.nil?
       end
-      memo_wise :call
 
-      private
+      def supported_data_types
+        %w[ip]
+      end
 
       def client
         @client ||= Clients::MMDB.new(timeout: timeout)

data/lib/mihari/enrichers/shodan.rb

@@ -9,17 +9,48 @@ module Mihari
       #
       # Query Shodan Internet DB
       #
-      # @param [String] ip
+      # @param [Mihari::Models::Artifact] artifact
       #
       # @return [Mihari::Structs::Shodan::InternetDBResponse, nil]
       #
-      def call(ip)
-        client.query ip
+      def call(artifact)
+        res = client.query(artifact.data)
+
+        artifact.tap do |tapped|
+          tapped.cpes = (res&.cpes || []).map { |cpe| Models::CPE.new(name: cpe) } if tapped.cpes.empty?
+          tapped.ports = (res&.ports || []).map { |port| Models::Port.new(number: port) } if tapped.ports.empty?
+          if tapped.reverse_dns_names.empty?
+            tapped.reverse_dns_names = (res&.hostnames || []).map do |name|
+              Models::ReverseDnsName.new(name: name)
+            end
+          end
+        end
+      end
+
+      #
+      # @param [Mihari::Models::Artifact] artifact
+      #
+      # @return [Boolean]
+      #
+      def callable?(artifact)
+        false unless supported_data_types.include?(artifact.data_type)
       end
-      memo_wise :call
 
       private
 
+      #
+      # @param [Mihari::Models::Artifact] artifact
+      #
+      # @return [Boolean]
+      #
+      def callable_relationships?(artifact)
+        artifact.cpes.empty? || artifact.ports.empty? || artifact.reverse_dns_names.empty?
+      end
+
+      def supported_data_types
+        %w[ip]
+      end
+
       def client
         @client ||= Clients::ShodanInternetDB.new(timeout: timeout)
       end

data/lib/mihari/enrichers/whois.rb

@@ -8,58 +8,64 @@ module Mihari
     # Whois enricher
     #
     class Whois < Base
+      prepend MemoWise
+
+      #
+      # Query IAIA Whois API
       #
-      # @param [Hash, nil] options
+      # @param [Mihari::Models::Artifact] artifact
       #
-      def initialize(options: nil)
-        super(options: options)
+      def call(artifact)
+        return if artifact.domain.nil?
+
+        domain = PublicSuffix.domain(artifact.domain)
+        record = memoized_lookup(domain)
+        return if record.parser.available?
+
+        artifact.whois_record ||= Models::WhoisRecord.new(
+          domain: domain,
+          created_on: get_created_on(record.parser),
+          updated_on: get_updated_on(record.parser),
+          expires_on: get_expires_on(record.parser),
+          registrar: get_registrar(record.parser),
+          contacts: get_contacts(record.parser)
+        )
       end
 
+      private
+
       #
-      # Query IAIA Whois API
-      #
-      # @param [String] domain
+      # @param [Mihari::Models::Artifact] artifact
       #
-      # @return [Mihari::Models::WhoisRecord, nil]
+      # @return [Boolean]
       #
-      def call(domain)
-        memoized_call PublicSuffix.domain(domain)
+      def callable_relationships?(artifact)
+        artifact.whois_record.nil?
       end
 
-      private
+      def supported_data_types
+        %w[url domain]
+      end
 
       #
       # @param [String] domain
       #
       # @return [Mihari::Models::WhoisRecord, nil]
       #
-      def memoized_call(domain)
-        record = whois.lookup(domain)
-        parser = record.parser
-        return nil if parser.available?
-
-        Models::WhoisRecord.new(
-          domain: domain,
-          created_on: get_created_on(parser),
-          updated_on: get_updated_on(parser),
-          expires_on: get_expires_on(parser),
-          registrar: get_registrar(parser),
-          contacts: get_contacts(parser)
-        )
+      def memoized_lookup(domain)
+        whois.lookup domain
       end
-      memo_wise :memoized_call
+      memo_wise :memoized_lookup
 
       #
       # @return [::Whois::Client]
       #
       def whois
-        @whois ||= [].tap do |out|
-          out << if timeout.nil?
-            ::Whois::Client.new
-          else
-            ::Whois::Client.new(timeout: timeout)
-          end
-        end.last
+        @whois ||= lambda do
+          return ::Whois::Client.new if timeout.nil?
+
+          ::Whois::Client.new(timeout: timeout)
+        end.call
       end
 
       #

data/lib/mihari/entities/artifact.rb

@@ -8,12 +8,12 @@ module Mihari
       expose :data_type, documentation: { type: String, required: true }, as: :dataType
       expose :source, documentation: { type: String, required: true }
       expose :query, documentation: { type: String, required: false }
-      expose :metadata, documentation: { type: Hash }
       expose :created_at, documentation: { type: DateTime, required: true }, as: :createdAt
+      expose :tags, using: Entities::Tag, documentation: { type: Entities::Tag, is_array: true, required: true }
     end
 
     class Artifact < BaseArtifact
-      expose :tags, using: Entities::Tag, documentation: { type: Entities::Tag, is_array: true, required: true }
+      expose :metadata, documentation: { type: Hash }
       expose :autonomous_system, using: Entities::AutonomousSystem,
         documentation: { type: Entities::AutonomousSystem, required: false }, as: :autonomousSystem
       expose :geolocation, using: Entities::Geolocation, documentation: { type: Entities::Geolocation, required: false }
@@ -36,6 +36,10 @@ module Mihari
         as: :ports do |status, _options|
         status.ports.empty? ? nil : status.ports
       end
+      expose :vulnerabilities, using: Vulnerability, documentation: { type: Vulnerability, is_array: true, required: false },
+        as: :vulnerabilities do |status, _options|
+        status.vulnerabilities.empty? ? nil : status.vulnerabilities
+      end
     end
 
     class ArtifactsWithPagination < Pagination

data/lib/mihari/entities/autonomous_system.rb

@@ -3,7 +3,7 @@
 module Mihari
   module Entities
     class AutonomousSystem < Grape::Entity
-      expose :asn, documentation: { type: Integer, required: true }
+      expose :number, documentation: { type: Integer, required: true }
     end
   end
 end

data/lib/mihari/entities/cpe.rb

@@ -3,7 +3,7 @@
 module Mihari
   module Entities
     class CPE < Grape::Entity
-      expose :cpe, documentation: { type: String, required: true }
+      expose :name, documentation: { type: String, required: true }
       expose :created_at, documentation: { type: DateTime, required: true }, as: :createdAt
     end
   end

data/lib/mihari/entities/port.rb

@@ -3,7 +3,7 @@
 module Mihari
   module Entities
     class Port < Grape::Entity
-      expose :port, documentation: { type: Integer, required: true }
+      expose :number, documentation: { type: Integer, required: true }
       expose :created_at, documentation: { type: DateTime, required: true }, as: :createdAt
     end
   end

data/lib/mihari/entities/vulnerability.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Entities
+    class Vulnerability < Grape::Entity
+      expose :name, documentation: { type: String, required: true }
+      expose :created_at, documentation: { type: DateTime, required: true }, as: :createdAt
+    end
+  end
+end

data/lib/mihari/errors.rb

@@ -7,6 +7,8 @@ module Mihari
 
   class ValueError < Error; end
 
+  class UnenrichableError < Error; end
+
   class ConfigurationError < Error
     # @return [Array<String>, nil]
     attr_reader :detail

data/lib/mihari/models/alert.rb

@@ -6,6 +6,18 @@ module Mihari
     # Alert model
     #
     class Alert < ActiveRecord::Base
+      # @!attribute [r] id
+      #   @return [Integer, nil]
+
+      # @!attribute [rw] created_at
+      #   @return [DateTime]
+
+      # @!attribute [r] rule
+      #   @return [Mihari::Models::Rule]
+
+      # @!attribute [r] artifacts
+      #   @return [Array<Mihari::Models::Artifact>]
+
       belongs_to :rule
 
       has_many :artifacts, dependent: :destroy

data/lib/mihari/models/artifact.rb

@@ -17,6 +17,90 @@ module Mihari
     # Artifact model
     #
     class Artifact < ActiveRecord::Base
+      # @!attribute [r] id
+      #   @return [Integer, nil]
+
+      # @!attribute [rw] data
+      #   @return [String]
+
+      # @!attribute [rw] data_type
+      #   @return [String]
+
+      # @!attribute [rw] source
+      #   @return [String, nil]
+
+      # @!attribute [rw] query
+      #   @return [String, nil]
+
+      # @!attribute [rw] metadata
+      #   @return [Hash, nil]
+
+      # @!attribute [rw] created_at
+      #   @return [DateTime]
+
+      # @!attribute [r] alert
+      #   @return [Mihari::Models::Alert]
+
+      # @!attribute [r] rule
+      #   @return [Mihari::Models::Rule]
+
+      # @!attribute [rw] autonomous_system
+      #   @return [Mihari::Models::AutonomousSystem, nil]
+
+      # @!attribute [rw] geolocation
+      #   @return [Mihari::Models::Geolocation, nil]
+
+      # @!attribute [rw] whois_record
+      #   @return [Mihari::Models::WhoisRecord, nil]
+
+      # @!attribute [rw] cpes
+      #   @return [Array<Mihari::Models::CPE>]
+
+      # @!attribute [rw] dns_records
+      #   @return [Array<Mihari::Models::DnsRecord>]
+
+      # @!attribute [rw] ports
+      #   @return [Array<Mihari::Models::Port>]
+
+      # @!attribute [rw] reverse_dns_names
+      #   @return [Array<Mihari::Models::ReverseDnsName>]
+
+      # @!attribute [rw] vulnerabilities
+      #   @return [Array<Mihari::Models::Vulnerability>]
+
+      # @!attribute [r] alert
+      #   @return [Mihari::Models::Alert]
+
+      # @!attribute [r] rule
+      #   @return [Mihari::Models::Rule]
+
+      # @!attribute [rw] autonomous_system
+      #   @return [Mihari::Models::AutonomousSystem, nil]
+
+      # @!attribute [rw] geolocation
+      #   @return [Mihari::Models::Geolocation, nil]
+
+      # @!attribute [rw] whois_record
+      #   @return [Mihari::Models::WhoisRecord, nil]
+
+      # @!attribute [rw] cpes
+      #   @return [Array<Mihari::Models::CPE>]
+
+      # @!attribute [rw] dns_records
+      #   @return [Array<Mihari::Models::DnsRecord>]
+
+      # @!attribute [rw] ports
+      #   @return [Array<Mihari::Models::Port>]
+
+      # @!attribute [rw] reverse_dns_names
+      #   @return [Array<Mihari::Models::ReverseDnsName>]
+
+      # @!attribute [rw] vulnerabilities
+      #   @return [Array<Mihari::Models::Vulnerability>]
+
+      # @!attribute [rw] tags
+      #   @return [Array<Mihari::Models::Tag>]
+
       belongs_to :alert
 
       has_one :autonomous_system, dependent: :destroy
@@ -28,6 +112,8 @@ module Mihari
       has_many :dns_records, dependent: :destroy
       has_many :ports, dependent: :destroy
       has_many :reverse_dns_names, dependent: :destroy
+      has_many :vulnerabilities, dependent: :destroy
+
       has_many :tags, through: :alert
 
       include ActiveModel::Validations
@@ -38,13 +124,14 @@ module Mihari
         attributes :id, :data, :data_type, :source, :query, :created_at, "alert.id", "rule.id", "rule.title",
           "rule.description"
         attributes tag: "tags.name"
-        attributes asn: "autonomous_system.asn"
+        attributes asn: "autonomous_system.number"
         attributes country_code: "geolocation.country_code"
         attributes "dns_record.value": "dns_records.value"
         attributes "dns_record.resource": "dns_records.resource"
         attributes reverse_dns_name: "reverse_dns_names.name"
         attributes cpe: "cpes.name"
-        attributes port: "ports.port"
+        attributes vuln: "vulnerabilities.name"
+        attributes port: "ports.number"
       end
 
       validates_with ArtifactValidator
@@ -54,6 +141,14 @@ module Mihari
       # @return [String, nil]
       attr_accessor :rule_id
 
+      before_destroy do
+        @alert = alert
+      end
+
+      after_destroy do
+        @alert.destroy unless @alert.artifacts.any?
+      end
+
       #
       # Check uniqueness
       #
@@ -79,122 +174,25 @@ module Mihari
         artifact.created_at < decayed_at
       end
 
-      #
-      # Enrich whois record
-      #
-      # @param [Mihari::Enrichers::Whois] enricher
-      #
-      def enrich_whois(enricher = Enrichers::Whois.new)
-        return unless can_enrich_whois?
-
-        self.whois_record = Services::WhoisRecordBuilder.call(domain, enricher: enricher)
-      end
-
-      #
-      # Enrich DNS records
-      #
-      # @param [Mihari::Enrichers::GooglePublicDNS] enricher
-      #
-      def enrich_dns(enricher = Enrichers::GooglePublicDNS.new)
-        return unless can_enrich_dns?
-
-        self.dns_records = Services::DnsRecordBuilder.call(domain, enricher: enricher)
-      end
-
-      #
-      # Enrich reverse DNS names
-      #
-      # @param [Mihari::Enrichers::Shodan] enricher
-      #
-      def enrich_reverse_dns(enricher = Enrichers::Shodan.new)
-        return unless can_enrich_reverse_dns?
-
-        self.reverse_dns_names = Services::ReverseDnsNameBuilder.call(data, enricher: enricher)
-      end
-
-      #
-      # Enrich geolocation
-      #
-      # @param [Mihari::Enrichers::IPInfo] enricher
-      #
-      def enrich_geolocation(enricher = Enrichers::MMDB.new)
-        return unless can_enrich_geolocation?
-
-        self.geolocation = Services::GeolocationBuilder.call(data, enricher: enricher)
-      end
-
-      #
-      # Enrich AS
-      #
-      # @param [Mihari::Enrichers::IPInfo] enricher
-      #
-      def enrich_autonomous_system(enricher = Enrichers::MMDB.new)
-        return unless can_enrich_autonomous_system?
-
-        self.autonomous_system = Services::AutonomousSystemBuilder.call(data, enricher: enricher)
-      end
-
-      #
-      # Enrich ports
-      #
-      # @param [Mihari::Enrichers::Shodan] enricher
-      #
-      def enrich_ports(enricher = Enrichers::Shodan.new)
-        return unless can_enrich_ports?
-
-        self.ports = Services::PortBuilder.call(data, enricher: enricher)
+      def enrichable?
+        !callable_enrichers.empty?
       end
 
-      #
-      # Enrich CPEs
-      #
-      # @param [Mihari::Enrichers::Shodan] enricher
-      #
-      def enrich_cpes(enricher = Enrichers::Shodan.new)
-        return unless can_enrich_cpes?
-
-        self.cpes = Services::CPEBuilder.call(data, enricher: enricher)
+      def enrich
+        callable_enrichers.each { |enricher| enricher.result self }
       end
 
       #
-      # Enrich all the enrichable relationships of the artifact
-      #
-      def enrich_all
-        enrich_autonomous_system mmdb
-        enrich_dns
-        enrich_geolocation mmdb
-        enrich_reverse_dns shodan
-        enrich_whois
-        enrich_ports shodan
-        enrich_cpes shodan
-      end
-
-      ENRICH_METHODS_BY_ENRICHER = {
-        Enrichers::Whois => %i[
-          enrich_whois
-        ],
-        Enrichers::MMDB => %i[
-          enrich_autonomous_system
-          enrich_geolocation
-        ],
-        Enrichers::Shodan => %i[
-          enrich_ports
-          enrich_cpes
-          enrich_reverse_dns
-        ],
-        Enrichers::GooglePublicDNS => %i[
-          enrich_dns
-        ]
-      }.freeze
-
-      #
-      # Enrich by name of enricher
-      #
-      # @param [Mihari::Enrichers::Base] enricher
+      # @return [String, nil]
       #
-      def enrich_by_enricher(enricher)
-        methods = ENRICH_METHODS_BY_ENRICHER[enricher.class] || []
-        methods.each { |method| send(method, enricher) if respond_to?(method) }
+      def domain
+        case data_type
+        when "domain"
+          data
+        when "url"
+          host = Addressable::URI.parse(data).host
+          (DataType.type(host) == "ip") ? nil : host
+        end
       end
 
       class << self
@@ -209,60 +207,21 @@ module Mihari
 
       private
 
-      def set_data_type
-        self.data_type = DataType.type(data)
-      end
-
-      def set_rule_id
-        @set_rule_id ||= nil
-      end
-
-      def mmdb
-        @mmdb ||= Enrichers::MMDB.new
-      end
-
-      def shodan
-        @shodan ||= Enrichers::Shodan.new
-      end
-
       #
-      # @return [String, nil]
+      # @return [Array<Mihari::Enrichers::Base>]
       #
-      def domain
-        case data_type
-        when "domain"
-          data
-        when "url"
-          Addressable::URI.parse(data).host
+      def callable_enrichers
+        @callable_enrichers ||= Mihari.enrichers.map(&:new).select do |enricher|
+          enricher.callable?(self)
         end
       end
 
-      def can_enrich_whois?
-        %w[domain url].include?(data_type) && whois_record.nil?
-      end
-
-      def can_enrich_dns?
-        %w[domain url].include?(data_type) && dns_records.empty?
-      end
-
-      def can_enrich_reverse_dns?
-        data_type == "ip" && reverse_dns_names.empty?
-      end
-
-      def can_enrich_geolocation?
-        data_type == "ip" && geolocation.nil?
-      end
-
-      def can_enrich_autonomous_system?
-        data_type == "ip" && autonomous_system.nil?
-      end
-
-      def can_enrich_ports?
-        data_type == "ip" && ports.empty?
+      def set_data_type
+        self.data_type = DataType.type(data)
       end
 
-      def can_enrich_cpes?
-        data_type == "ip" && cpes.empty?
+      def set_rule_id
+        @set_rule_id ||= nil
       end
     end
   end