mihari 7.2.0 → 7.3.0

data/lib/mihari/models/artifact.rb

@@ -17,6 +17,90 @@ module Mihari
     # Artifact model
     #
     class Artifact < ActiveRecord::Base
+      # @!attribute [r] id
+      #   @return [Integer, nil]
+
+      # @!attribute [rw] data
+      #   @return [String]
+
+      # @!attribute [rw] data_type
+      #   @return [String]
+
+      # @!attribute [rw] source
+      #   @return [String, nil]
+
+      # @!attribute [rw] query
+      #   @return [String, nil]
+
+      # @!attribute [rw] metadata
+      #   @return [Hash, nil]
+
+      # @!attribute [rw] created_at
+      #   @return [DateTime]
+
+      # @!attribute [r] alert
+      #   @return [Mihari::Models::Alert]
+
+      # @!attribute [r] rule
+      #   @return [Mihari::Models::Rule]
+
+      # @!attribute [rw] autonomous_system
+      #   @return [Mihari::Models::AutonomousSystem, nil]
+
+      # @!attribute [rw] geolocation
+      #   @return [Mihari::Models::Geolocation, nil]
+
+      # @!attribute [rw] whois_record
+      #   @return [Mihari::Models::WhoisRecord, nil]
+
+      # @!attribute [rw] cpes
+      #   @return [Array<Mihari::Models::CPE>]
+
+      # @!attribute [rw] dns_records
+      #   @return [Array<Mihari::Models::DnsRecord>]
+
+      # @!attribute [rw] ports
+      #   @return [Array<Mihari::Models::Port>]
+
+      # @!attribute [rw] reverse_dns_names
+      #   @return [Array<Mihari::Models::ReverseDnsName>]
+
+      # @!attribute [rw] vulnerabilities
+      #   @return [Array<Mihari::Models::Vulnerability>]
+
+      # @!attribute [r] alert
+      #   @return [Mihari::Models::Alert]
+
+      # @!attribute [r] rule
+      #   @return [Mihari::Models::Rule]
+
+      # @!attribute [rw] autonomous_system
+      #   @return [Mihari::Models::AutonomousSystem, nil]
+
+      # @!attribute [rw] geolocation
+      #   @return [Mihari::Models::Geolocation, nil]
+
+      # @!attribute [rw] whois_record
+      #   @return [Mihari::Models::WhoisRecord, nil]
+
+      # @!attribute [rw] cpes
+      #   @return [Array<Mihari::Models::CPE>]
+
+      # @!attribute [rw] dns_records
+      #   @return [Array<Mihari::Models::DnsRecord>]
+
+      # @!attribute [rw] ports
+      #   @return [Array<Mihari::Models::Port>]
+
+      # @!attribute [rw] reverse_dns_names
+      #   @return [Array<Mihari::Models::ReverseDnsName>]
+
+      # @!attribute [rw] vulnerabilities
+      #   @return [Array<Mihari::Models::Vulnerability>]
+
+      # @!attribute [rw] tags
+      #   @return [Array<Mihari::Models::Tag>]
+
       belongs_to :alert
 
       has_one :autonomous_system, dependent: :destroy
@@ -90,173 +174,24 @@ module Mihari
         artifact.created_at < decayed_at
       end
 
-      #
-      # Enrich whois record
-      #
-      # @param [Mihari::Enrichers::Whois] enricher
-      #
-      def enrich_whois(enricher = Enrichers::Whois.new)
-        return unless can_enrich_whois?
-
-        self.whois_record = Services::WhoisRecordBuilder.call(domain, enricher: enricher)
-      end
-
-      #
-      # Enrich DNS records
-      #
-      # @param [Mihari::Enrichers::GooglePublicDNS] enricher
-      #
-      def enrich_dns(enricher = Enrichers::GooglePublicDNS.new)
-        return unless can_enrich_dns?
-
-        self.dns_records = Services::DnsRecordBuilder.call(domain, enricher: enricher)
-      end
-
-      #
-      # Enrich reverse DNS names
-      #
-      # @param [Mihari::Enrichers::Shodan] enricher
-      #
-      def enrich_reverse_dns(enricher = Enrichers::Shodan.new)
-        return unless can_enrich_reverse_dns?
-
-        self.reverse_dns_names = Services::ReverseDnsNameBuilder.call(data, enricher: enricher)
-      end
-
-      #
-      # Enrich geolocation
-      #
-      # @param [Mihari::Enrichers::IPInfo] enricher
-      #
-      def enrich_geolocation(enricher = Enrichers::MMDB.new)
-        return unless can_enrich_geolocation?
-
-        self.geolocation = Services::GeolocationBuilder.call(data, enricher: enricher)
-      end
-
-      #
-      # Enrich AS
-      #
-      # @param [Mihari::Enrichers::IPInfo] enricher
-      #
-      def enrich_autonomous_system(enricher = Enrichers::MMDB.new)
-        return unless can_enrich_autonomous_system?
-
-        self.autonomous_system = Services::AutonomousSystemBuilder.call(data, enricher: enricher)
-      end
-
-      #
-      # Enrich ports
-      #
-      # @param [Mihari::Enrichers::Shodan] enricher
-      #
-      def enrich_ports(enricher = Enrichers::Shodan.new)
-        return unless can_enrich_ports?
-
-        self.ports = Services::PortBuilder.call(data, enricher: enricher)
-      end
-
-      #
-      # Enrich CPEs
-      #
-      # @param [Mihari::Enrichers::Shodan] enricher
-      #
-      def enrich_cpes(enricher = Enrichers::Shodan.new)
-        return unless can_enrich_cpes?
-
-        self.cpes = Services::CPEBuilder.call(data, enricher: enricher)
-      end
-
-      #
-      # Enrich vulnerabilities
-      #
-      # @param [Mihari::Enrichers::Shodan] enricher
-      #
-      def enrich_vulnerabilities(enricher = Enrichers::Shodan.new)
-        return unless can_enrich_vulnerabilities?
-
-        self.vulnerabilities = Services::VulnerabilityBuilder.call(data, enricher: enricher)
+      def enrichable?
+        !callable_enrichers.empty?
       end
 
-      #
-      # Enrich all the enrichable relationships of the artifact
-      #
-      def enrich_all
-        enrich_autonomous_system mmdb
-        enrich_dns
-        enrich_geolocation mmdb
-        enrich_reverse_dns shodan
-        enrich_whois
-        enrich_ports shodan
-        enrich_cpes shodan
-        enrich_vulnerabilities shodan
+      def enrich
+        callable_enrichers.each { |enricher| enricher.result self }
       end
 
-      ENRICH_METHODS_BY_ENRICHER = {
-        Enrichers::Whois => %i[
-          enrich_whois
-        ],
-        Enrichers::MMDB => %i[
-          enrich_autonomous_system
-          enrich_geolocation
-        ],
-        Enrichers::Shodan => %i[
-          enrich_ports
-          enrich_cpes
-          enrich_reverse_dns
-          enrich_vulnerabilities
-        ],
-        Enrichers::GooglePublicDNS => %i[
-          enrich_dns
-        ]
-      }.freeze
-
       #
-      # Enrich by name of enricher
-      #
-      # @param [Mihari::Enrichers::Base] enricher
+      # @return [String, nil]
       #
-      def enrich_by_enricher(enricher)
-        methods = ENRICH_METHODS_BY_ENRICHER[enricher.class] || []
-        methods.each { |method| send(method, enricher) if respond_to?(method) }
-      end
-
-      def can_enrich_whois?
-        %w[domain url].include?(data_type) && whois_record.nil?
-      end
-
-      def can_enrich_dns?
-        %w[domain url].include?(data_type) && dns_records.empty?
-      end
-
-      def can_enrich_reverse_dns?
-        data_type == "ip" && reverse_dns_names.empty?
-      end
-
-      def can_enrich_geolocation?
-        data_type == "ip" && geolocation.nil?
-      end
-
-      def can_enrich_autonomous_system?
-        data_type == "ip" && autonomous_system.nil?
-      end
-
-      def can_enrich_ports?
-        data_type == "ip" && ports.empty?
-      end
-
-      def can_enrich_cpes?
-        data_type == "ip" && cpes.empty?
-      end
-
-      def can_enrich_vulnerabilities?
-        data_type == "ip" && vulnerabilities.empty?
-      end
-
-      def enrichable?
-        enrich_methods = methods.map(&:to_s).select { |method| method.start_with?("can_enrich_") }
-        enrich_methods.map(&:to_sym).any? do |method|
-          send(method) if respond_to?(method)
+      def domain
+        case data_type
+        when "domain"
+          data
+        when "url"
+          host = Addressable::URI.parse(data).host
+          (DataType.type(host) == "ip") ? nil : host
         end
       end
 
@@ -272,6 +207,15 @@ module Mihari
 
       private
 
+      #
+      # @return [Array<Mihari::Enrichers::Base>]
+      #
+      def callable_enrichers
+        @callable_enrichers ||= Mihari.enrichers.map(&:new).select do |enricher|
+          enricher.callable?(self)
+        end
+      end
+
       def set_data_type
         self.data_type = DataType.type(data)
       end
@@ -279,26 +223,6 @@ module Mihari
       def set_rule_id
         @set_rule_id ||= nil
       end
-
-      def mmdb
-        @mmdb ||= Enrichers::MMDB.new
-      end
-
-      def shodan
-        @shodan ||= Enrichers::Shodan.new
-      end
-
-      #
-      # @return [String, nil]
-      #
-      def domain
-        case data_type
-        when "domain"
-          data
-        when "url"
-          Addressable::URI.parse(data).host
-        end
-      end
     end
   end
 end

data/lib/mihari/models/rule.rb

@@ -6,6 +6,27 @@ module Mihari
     # Rule model
     #
     class Rule < ActiveRecord::Base
+      # @!attribute [rw] id
+      #   @return [String]
+
+      # @!attribute [rw] title
+      #   @return [String]
+
+      # @!attribute [rw] description
+      #   @return [String]
+
+      # @!attribute [rw] data
+      #   @return [Hash]
+
+      # @!attribute [rw] created_at
+      #   @return [DateTime]
+
+      # @!attribute [rw] updated_at
+      #   @return [DateTime]
+
+      # @!attribute [r] alerts
+      #   @return [Array<Mihari::Models::Alert>]
+
       has_many :alerts, dependent: :destroy
       has_many :taggings, dependent: :destroy
       has_many :tags, through: :taggings

data/lib/mihari/rule.rb

@@ -174,8 +174,10 @@ module Mihari
     # @return [Array<Mihari::Models::Artifact>]
     #
     def enriched_artifacts
-      @enriched_artifacts ||= Parallel.map(unique_artifacts) do |artifact|
-        enrichers.each { |enricher| artifact.enrich_by_enricher enricher }
+      @enriched_artifacts ||= unique_artifacts.map do |artifact|
+        serial_enrichers.each { |enricher| enricher.result(artifact) }
+        Parallel.each(parallel_enrichers) { |enricher| enricher.result(artifact) }
+
         artifact
       end
     end
@@ -188,7 +190,10 @@ module Mihari
     def bulk_emit
       return [] if enriched_artifacts.empty?
 
-      Parallel.map(emitters) { |emitter| emitter.result(enriched_artifacts).value_or nil }.compact
+      [].tap do |out|
+        out << serial_emitters.map { |emitter| emitter.result(enriched_artifacts).value_or(nil) }
+        out << Parallel.map(parallel_emitters) { |emitter| emitter.result(enriched_artifacts).value_or(nil) }
+      end.flatten.compact
     end
 
     #
@@ -289,11 +294,11 @@ module Mihari
     #
     # @return [Boolean]
     #
-    def falsepositive?(value)
-      return true if falsepositives.include?(value)
+    def falsepositive?(artifact)
+      return true if falsepositives.include?(artifact)
 
       regexps = falsepositives.select { |fp| fp.is_a?(Regexp) }
-      regexps.any? { |fp| fp.match?(value) }
+      regexps.any? { |fp| fp.match?(artifact) }
     end
 
     #
@@ -365,6 +370,14 @@ module Mihari
       end
     end
 
+    def parallel_emitters
+      emitters.select(&:parallel?)
+    end
+
+    def serial_emitters
+      emitters.reject(&:parallel?)
+    end
+
     #
     # Get enricher class
     #
@@ -391,6 +404,14 @@ module Mihari
       end
     end
 
+    def parallel_enrichers
+      enrichers.select(&:parallel?)
+    end
+
+    def serial_enrichers
+      enrichers.reject(&:parallel?)
+    end
+
     #
     # Validate the data format
     #

data/lib/mihari/schemas/alert.rb

@@ -3,9 +3,9 @@
 module Mihari
   module Schemas
     Alert = Dry::Schema.Params do
-      required(:rule_id).value(:string)
-      required(:artifacts).value(array[:string])
-      optional(:source).value(:string)
+      required(:rule_id).filled(:string)
+      required(:artifacts).array { filled(:string) }
+      optional(:source).filled(:string)
     end
 
     #

data/lib/mihari/schemas/analyzer.rb

@@ -20,8 +20,8 @@ module Mihari
         key = keys.first
         const_set(key.upcase, Dry::Schema.Params do
           required(:analyzer).value(Types::String.enum(*keys))
-          required(:query).value(:string)
-          optional(:api_key).value(:string)
+          required(:query).filled(:string)
+          optional(:api_key).filled(:string)
           optional(:options).hash(AnalyzerPaginationOptions)
         end)
       end
@@ -36,60 +36,60 @@ module Mihari
         key = keys.first
         const_set(key.upcase, Dry::Schema.Params do
           required(:analyzer).value(Types::String.enum(*keys))
-          required(:query).value(:string)
-          optional(:api_key).value(:string)
+          required(:query).filled(:string)
+          optional(:api_key).filled(:string)
           optional(:options).hash(AnalyzerOptions)
         end)
       end
 
       DNSTwister = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::DNSTwister.keys))
-        required(:query).value(:string)
+        required(:query).filled(:string)
         optional(:options).hash(AnalyzerOptions)
       end
 
       Censys = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::Censys.keys))
-        required(:query).value(:string)
-        optional(:id).value(:string)
-        optional(:secret).value(:string)
+        required(:query).filled(:string)
+        optional(:id).filled(:string)
+        optional(:secret).filled(:string)
         optional(:options).hash(AnalyzerPaginationOptions)
       end
 
       CIRCL = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::CIRCL.keys))
-        required(:query).value(:string)
-        optional(:username).value(:string)
-        optional(:password).value(:string)
+        required(:query).filled(:string)
+        optional(:username).filled(:string)
+        optional(:password).filled(:string)
         optional(:options).hash(AnalyzerOptions)
       end
 
       Fofa = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::Fofa.keys))
-        required(:query).value(:string)
-        optional(:api_key).value(:string)
-        optional(:email).value(:string)
+        required(:query).filled(:string)
+        optional(:api_key).filled(:string)
+        optional(:email).filled(:string)
         optional(:options).hash(AnalyzerPaginationOptions)
       end
 
       PassiveTotal = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::PassiveTotal.keys))
-        required(:query).value(:string)
-        optional(:username).value(:string)
-        optional(:api_key).value(:string)
+        required(:query).filled(:string)
+        optional(:username).filled(:string)
+        optional(:api_key).filled(:string)
         optional(:options).hash(AnalyzerOptions)
       end
 
       ZoomEye = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::ZoomEye.keys))
-        required(:query).value(:string)
+        required(:query).filled(:string)
         required(:type).value(Types::String.enum("host", "web"))
         optional(:options).hash(AnalyzerPaginationOptions)
       end
 
       Crtsh = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::Crtsh.keys))
-        required(:query).value(:string)
+        required(:query).filled(:string)
         optional(:exclude_expired).value(:bool).default(true)
         optional(:match).value(Types::String.enum("=", "ILIKE", "LIKE", "single", "any", "FTS")).default(nil)
         optional(:options).hash(AnalyzerOptions)
@@ -97,22 +97,22 @@ module Mihari
 
       HunterHow = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::HunterHow.keys))
-        required(:query).value(:string)
+        required(:query).filled(:string)
         required(:start_time).value(:date)
         required(:end_time).value(:date)
-        optional(:api_key).value(:string)
+        optional(:api_key).filled(:string)
         optional(:options).hash(AnalyzerPaginationOptions)
       end
 
       Feed = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::Feed.keys))
-        required(:query).value(:string)
-        required(:selector).value(:string)
+        required(:query).filled(:string)
+        required(:selector).filled(:string)
         optional(:method).value(Types::HTTPRequestMethods).default("GET")
-        optional(:headers).value(:hash).default({})
-        optional(:params).value(:hash)
-        optional(:form).value(:hash)
-        optional(:json).value(:hash)
+        optional(:headers).filled(:hash)
+        optional(:params).filled(:hash)
+        optional(:form).filled(:hash)
+        optional(:json).filled(:hash)
         optional(:options).hash(AnalyzerOptions)
       end
     end

data/lib/mihari/schemas/emitter.rb

@@ -15,31 +15,31 @@ module Mihari
 
       MISP = Dry::Schema.Params do
         required(:emitter).value(Types::String.enum(*Mihari::Emitters::MISP.keys))
-        optional(:url).value(:string)
-        optional(:api_key).value(:string)
+        optional(:url).filled(:string)
+        optional(:api_key).filled(:string)
         optional(:options).hash(Options)
       end
 
       TheHive = Dry::Schema.Params do
         required(:emitter).value(Types::String.enum(*Mihari::Emitters::TheHive.keys))
-        optional(:url).value(:string)
-        optional(:api_key).value(:string)
+        optional(:url).filled(:string)
+        optional(:api_key).filled(:string)
         optional(:options).hash(Options)
       end
 
       Slack = Dry::Schema.Params do
         required(:emitter).value(Types::String.enum(*Mihari::Emitters::Slack.keys))
-        optional(:webhook_url).value(:string)
-        optional(:channel).value(:string)
+        optional(:webhook_url).filled(:string)
+        optional(:channel).filled(:string)
         optional(:options).hash(Options)
       end
 
       Webhook = Dry::Schema.Params do
         required(:emitter).value(Types::String.enum(*Mihari::Emitters::Webhook.keys))
-        required(:url).value(:string)
+        required(:url).filled(:string)
         optional(:method).value(Types::HTTPRequestMethods).default("POST")
-        optional(:headers).value(:hash).default({})
-        optional(:template).value(:string)
+        optional(:headers).filled(:hash)
+        optional(:template).filled(:string)
         optional(:options).hash(Options)
       end
     end

data/lib/mihari/schemas/macros.rb

@@ -8,9 +8,9 @@ module Dry
       # (see https://github.com/dry-rb/dry-schema/issues/70)
       #
       class DSL
-        def default(value)
+        def default(artifact)
           schema_dsl.before(:rule_applier) do |result|
-            result.update(name => value) if result.output && !result[name]
+            result.update(name => artifact) if result.output && !result[name]
           end
         end
       end

data/lib/mihari/schemas/options.rb

@@ -7,17 +7,14 @@ module Mihari
       optional(:retry_interval).value(:integer).default(Mihari.config.retry_interval)
       optional(:retry_exponential_backoff).value(:bool).default(Mihari.config.retry_exponential_backoff)
       optional(:timeout).value(:integer)
+      optional(:parallel).value(:bool).default(Mihari.config.parallel)
     end
 
     IgnoreErrorOptions = Dry::Schema.Params do
       optional(:ignore_error).value(:bool).default(Mihari.config.ignore_error)
     end
 
-    ParallelOptions = Dry::Schema.Params do
-      optional(:parallel).value(:bool).default(Mihari.config.parallel)
-    end
-
-    AnalyzerOptions = Options | IgnoreErrorOptions | ParallelOptions
+    AnalyzerOptions = Options | IgnoreErrorOptions
 
     PaginationOptions = Dry::Schema.Params do
       optional(:pagination_interval).value(:integer).default(Mihari.config.pagination_interval)

data/lib/mihari/schemas/rule.rb

@@ -7,27 +7,27 @@ require "mihari/schemas/enricher"
 module Mihari
   module Schemas
     Rule = Dry::Schema.Params do
-      required(:id).value(:string)
-      required(:title).value(:string)
-      required(:description).value(:string)
+      required(:id).filled(:string)
+      required(:title).filled(:string)
+      required(:description).filled(:string)
 
-      optional(:tags).value(array[:string]).default([])
+      optional(:author).filled(:string)
+      optional(:status).filled(:string)
 
-      optional(:author).value(:string)
-      optional(:references).value(array[:string])
-      optional(:related).value(array[:string])
-      optional(:status).value(:string)
+      optional(:tags).array { filled(:string) }.default([])
+      optional(:references).array { filled(:string) }
+      optional(:related).array { filled(:string) }
 
       optional(:created_on).value(:date)
       optional(:updated_on).value(:date)
 
       required(:queries).value(:array).each { Analyzer } # rubocop:disable Lint/Void
-
       optional(:emitters).value(:array).each { Emitter }.default(DEFAULT_EMITTERS) # rubocop:disable Lint/Void
       optional(:enrichers).value(:array).each { Enricher }.default(DEFAULT_ENRICHERS) # rubocop:disable Lint/Void
 
-      optional(:data_types).value(array[Types::DataTypes]).default(Mihari::Types::DataTypes.values)
-      optional(:falsepositives).value(array[:string]).default([])
+      optional(:data_types).filled(array[Types::DataTypes]).default(Mihari::Types::DataTypes.values)
+
+      optional(:falsepositives).array { filled(:string) }.default([])
 
       optional(:artifact_ttl).value(:integer)
     end
@@ -42,7 +42,7 @@ module Mihari
 
       rule(:falsepositives) do
         value.each do |v|
-          key.failure("#{v} is not a valid format.") unless valid_falsepositive?(v)
+          key.failure("#{v} is not a valid format") unless valid_falsepositive?(v)
         end
       end