mihari 7.2.0 → 7.3.1

data/lib/mihari/models/rule.rb

@@ -6,6 +6,27 @@ module Mihari
     # Rule model
     #
     class Rule < ActiveRecord::Base
+      # @!attribute [rw] id
+      #   @return [String]
+
+      # @!attribute [rw] title
+      #   @return [String]
+
+      # @!attribute [rw] description
+      #   @return [String]
+
+      # @!attribute [rw] data
+      #   @return [Hash]
+
+      # @!attribute [rw] created_at
+      #   @return [DateTime]
+
+      # @!attribute [rw] updated_at
+      #   @return [DateTime]
+
+      # @!attribute [r] alerts
+      #   @return [Array<Mihari::Models::Alert>]
+
       has_many :alerts, dependent: :destroy
       has_many :taggings, dependent: :destroy
       has_many :tags, through: :taggings

data/lib/mihari/rule.rb

@@ -33,9 +33,9 @@ module Mihari
     # @return [Boolean]
     #
     def errors?
-      return false if @errors.nil?
+      return false if errors.nil?
 
-      !@errors.empty?
+      !errors.empty?
     end
 
     def [](key)
@@ -163,9 +163,7 @@ module Mihari
     # @return [Array<Mihari::Models::Artifact>]
     #
     def unique_artifacts
-      normalized_artifacts.select do |artifact|
-        artifact.unique?(base_time: base_time, artifact_ttl: artifact_ttl)
-      end
+      normalized_artifacts.select { |artifact| artifact.unique?(base_time: base_time, artifact_ttl: artifact_ttl) }
     end
 
     #
@@ -175,8 +173,10 @@ module Mihari
     #
     def enriched_artifacts
       @enriched_artifacts ||= Parallel.map(unique_artifacts) do |artifact|
-        enrichers.each { |enricher| artifact.enrich_by_enricher enricher }
-        artifact
+        artifact.tap do |tapped|
+          # NOTE: To apply changes correctly, enrichers should be applied to an artifact serially
+          enrichers.each { |enricher| enricher.result(tapped) }
+        end
       end
     end
 
@@ -188,7 +188,10 @@ module Mihari
     def bulk_emit
       return [] if enriched_artifacts.empty?
 
-      Parallel.map(emitters) { |emitter| emitter.result(enriched_artifacts).value_or nil }.compact
+      [].tap do |out|
+        out << serial_emitters.map { |emitter| emitter.result(enriched_artifacts).value_or(nil) }
+        out << Parallel.map(parallel_emitters) { |emitter| emitter.result(enriched_artifacts).value_or(nil) }
+      end.flatten.compact
     end
 
     #
@@ -289,11 +292,11 @@ module Mihari
     #
     # @return [Boolean]
     #
-    def falsepositive?(value)
-      return true if falsepositives.include?(value)
+    def falsepositive?(artifact)
+      return true if falsepositives.include?(artifact)
 
       regexps = falsepositives.select { |fp| fp.is_a?(Regexp) }
-      regexps.any? { |fp| fp.match?(value) }
+      regexps.any? { |fp| fp.match?(artifact) }
     end
 
     #
@@ -332,9 +335,10 @@ module Mihari
 
     # @return [Array<Dry::Monads::Result::Success<Array<Mihari::Models::Artifact>>, Dry::Monads::Result::Failure>]
     def analyzer_results
-      parallel_results = Parallel.map(parallel_analyzers, &:result)
-      serial_results = serial_analyzers.map(&:result)
-      parallel_results + serial_results
+      [].tap do |out|
+        out << Parallel.map(parallel_analyzers, &:result)
+        out << serial_analyzers.map(&:result)
+      end.flatten
     end
 
     #
@@ -365,6 +369,14 @@ module Mihari
       end
     end
 
+    def parallel_emitters
+      emitters.select(&:parallel?)
+    end
+
+    def serial_emitters
+      emitters.reject(&:parallel?)
+    end
+
     #
     # Get enricher class
     #

data/lib/mihari/schemas/alert.rb

@@ -3,9 +3,9 @@
 module Mihari
   module Schemas
     Alert = Dry::Schema.Params do
-      required(:rule_id).value(:string)
-      required(:artifacts).value(array[:string])
-      optional(:source).value(:string)
+      required(:rule_id).filled(:string)
+      required(:artifacts).array { filled(:string) }
+      optional(:source).filled(:string)
     end
 
     #

data/lib/mihari/schemas/analyzer.rb

@@ -20,8 +20,8 @@ module Mihari
         key = keys.first
         const_set(key.upcase, Dry::Schema.Params do
           required(:analyzer).value(Types::String.enum(*keys))
-          required(:query).value(:string)
-          optional(:api_key).value(:string)
+          required(:query).filled(:string)
+          optional(:api_key).filled(:string)
           optional(:options).hash(AnalyzerPaginationOptions)
         end)
       end
@@ -36,60 +36,60 @@ module Mihari
         key = keys.first
         const_set(key.upcase, Dry::Schema.Params do
           required(:analyzer).value(Types::String.enum(*keys))
-          required(:query).value(:string)
-          optional(:api_key).value(:string)
+          required(:query).filled(:string)
+          optional(:api_key).filled(:string)
           optional(:options).hash(AnalyzerOptions)
         end)
       end
 
       DNSTwister = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::DNSTwister.keys))
-        required(:query).value(:string)
+        required(:query).filled(:string)
         optional(:options).hash(AnalyzerOptions)
       end
 
       Censys = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::Censys.keys))
-        required(:query).value(:string)
-        optional(:id).value(:string)
-        optional(:secret).value(:string)
+        required(:query).filled(:string)
+        optional(:id).filled(:string)
+        optional(:secret).filled(:string)
         optional(:options).hash(AnalyzerPaginationOptions)
       end
 
       CIRCL = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::CIRCL.keys))
-        required(:query).value(:string)
-        optional(:username).value(:string)
-        optional(:password).value(:string)
+        required(:query).filled(:string)
+        optional(:username).filled(:string)
+        optional(:password).filled(:string)
         optional(:options).hash(AnalyzerOptions)
       end
 
       Fofa = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::Fofa.keys))
-        required(:query).value(:string)
-        optional(:api_key).value(:string)
-        optional(:email).value(:string)
+        required(:query).filled(:string)
+        optional(:api_key).filled(:string)
+        optional(:email).filled(:string)
         optional(:options).hash(AnalyzerPaginationOptions)
       end
 
       PassiveTotal = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::PassiveTotal.keys))
-        required(:query).value(:string)
-        optional(:username).value(:string)
-        optional(:api_key).value(:string)
+        required(:query).filled(:string)
+        optional(:username).filled(:string)
+        optional(:api_key).filled(:string)
         optional(:options).hash(AnalyzerOptions)
       end
 
       ZoomEye = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::ZoomEye.keys))
-        required(:query).value(:string)
+        required(:query).filled(:string)
         required(:type).value(Types::String.enum("host", "web"))
         optional(:options).hash(AnalyzerPaginationOptions)
       end
 
       Crtsh = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::Crtsh.keys))
-        required(:query).value(:string)
+        required(:query).filled(:string)
         optional(:exclude_expired).value(:bool).default(true)
         optional(:match).value(Types::String.enum("=", "ILIKE", "LIKE", "single", "any", "FTS")).default(nil)
         optional(:options).hash(AnalyzerOptions)
@@ -97,26 +97,26 @@ module Mihari
 
       HunterHow = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::HunterHow.keys))
-        required(:query).value(:string)
+        required(:query).filled(:string)
         required(:start_time).value(:date)
         required(:end_time).value(:date)
-        optional(:api_key).value(:string)
+        optional(:api_key).filled(:string)
         optional(:options).hash(AnalyzerPaginationOptions)
       end
 
       Feed = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::Feed.keys))
-        required(:query).value(:string)
-        required(:selector).value(:string)
+        required(:query).filled(:string)
+        required(:selector).filled(:string)
         optional(:method).value(Types::HTTPRequestMethods).default("GET")
-        optional(:headers).value(:hash).default({})
-        optional(:params).value(:hash)
-        optional(:form).value(:hash)
-        optional(:json).value(:hash)
+        optional(:headers).filled(:hash)
+        optional(:params).filled(:hash)
+        optional(:form).filled(:hash)
+        optional(:json).filled(:hash)
         optional(:options).hash(AnalyzerOptions)
       end
     end
 
-    Analyzer = Schemas::Analyzers.get_or_composition
+    Analyzer = Schemas::Analyzers.compose_by_or
   end
 end

data/lib/mihari/schemas/concerns/orrable.rb

@@ -9,7 +9,7 @@ module Mihari
       module Orrable
         extend ActiveSupport::Concern
 
-        def get_or_composition
+        def compose_by_or
           schemas = constants.map { |sym| const_get sym }
           return schemas.first if schemas.length <= 1
 

data/lib/mihari/schemas/emitter.rb

@@ -10,40 +10,40 @@ module Mihari
 
       Database = Dry::Schema.Params do
         required(:emitter).value(Types::String.enum(*Mihari::Emitters::Database.keys))
-        optional(:options).hash(Options)
+        optional(:options).hash(EmitterOptions)
       end
 
       MISP = Dry::Schema.Params do
         required(:emitter).value(Types::String.enum(*Mihari::Emitters::MISP.keys))
-        optional(:url).value(:string)
-        optional(:api_key).value(:string)
-        optional(:options).hash(Options)
+        optional(:url).filled(:string)
+        optional(:api_key).filled(:string)
+        optional(:options).hash(EmitterOptions)
       end
 
       TheHive = Dry::Schema.Params do
         required(:emitter).value(Types::String.enum(*Mihari::Emitters::TheHive.keys))
-        optional(:url).value(:string)
-        optional(:api_key).value(:string)
-        optional(:options).hash(Options)
+        optional(:url).filled(:string)
+        optional(:api_key).filled(:string)
+        optional(:options).hash(EmitterOptions)
       end
 
       Slack = Dry::Schema.Params do
         required(:emitter).value(Types::String.enum(*Mihari::Emitters::Slack.keys))
-        optional(:webhook_url).value(:string)
-        optional(:channel).value(:string)
-        optional(:options).hash(Options)
+        optional(:webhook_url).filled(:string)
+        optional(:channel).filled(:string)
+        optional(:options).hash(EmitterOptions)
       end
 
       Webhook = Dry::Schema.Params do
         required(:emitter).value(Types::String.enum(*Mihari::Emitters::Webhook.keys))
-        required(:url).value(:string)
+        required(:url).filled(:string)
         optional(:method).value(Types::HTTPRequestMethods).default("POST")
-        optional(:headers).value(:hash).default({})
-        optional(:template).value(:string)
-        optional(:options).hash(Options)
+        optional(:headers).filled(:hash)
+        optional(:template).filled(:string)
+        optional(:options).hash(EmitterOptions)
       end
     end
 
-    Emitter = Schemas::Emitters.get_or_composition
+    Emitter = Schemas::Emitters.compose_by_or
   end
 end

data/lib/mihari/schemas/enricher.rb

@@ -29,6 +29,6 @@ module Mihari
       end
     end
 
-    Enricher = Schemas::Enrichers.get_or_composition
+    Enricher = Schemas::Enrichers.compose_by_or
   end
 end

data/lib/mihari/schemas/macros.rb

@@ -8,9 +8,9 @@ module Dry
       # (see https://github.com/dry-rb/dry-schema/issues/70)
       #
       class DSL
-        def default(value)
+        def default(artifact)
           schema_dsl.before(:rule_applier) do |result|
-            result.update(name => value) if result.output && !result[name]
+            result.update(name => artifact) if result.output && !result[name]
           end
         end
       end

data/lib/mihari/schemas/options.rb

@@ -3,27 +3,29 @@
 module Mihari
   module Schemas
     Options = Dry::Schema.Params do
-      optional(:retry_times).value(:integer).default(Mihari.config.retry_times)
-      optional(:retry_interval).value(:integer).default(Mihari.config.retry_interval)
-      optional(:retry_exponential_backoff).value(:bool).default(Mihari.config.retry_exponential_backoff)
+      optional(:retry_times).value(:integer)
+      optional(:retry_interval).value(:integer)
+      optional(:retry_exponential_backoff).value(:bool)
       optional(:timeout).value(:integer)
     end
 
-    IgnoreErrorOptions = Dry::Schema.Params do
-      optional(:ignore_error).value(:bool).default(Mihari.config.ignore_error)
-    end
-
     ParallelOptions = Dry::Schema.Params do
-      optional(:parallel).value(:bool).default(Mihari.config.parallel)
+      optional(:parallel).value(:bool)
     end
 
-    AnalyzerOptions = Options | IgnoreErrorOptions | ParallelOptions
+    IgnoreErrorOptions = Dry::Schema.Params do
+      optional(:ignore_error).value(:bool)
+    end
 
     PaginationOptions = Dry::Schema.Params do
-      optional(:pagination_interval).value(:integer).default(Mihari.config.pagination_interval)
-      optional(:pagination_limit).value(:integer).default(Mihari.config.pagination_limit)
+      optional(:pagination_interval).value(:integer)
+      optional(:pagination_limit).value(:integer)
     end
 
-    AnalyzerPaginationOptions = AnalyzerOptions | PaginationOptions
+    AnalyzerOptions = Options & IgnoreErrorOptions & ParallelOptions
+
+    AnalyzerPaginationOptions = AnalyzerOptions & PaginationOptions
+
+    EmitterOptions = Options & ParallelOptions
   end
 end

data/lib/mihari/schemas/rule.rb

@@ -7,27 +7,27 @@ require "mihari/schemas/enricher"
 module Mihari
   module Schemas
     Rule = Dry::Schema.Params do
-      required(:id).value(:string)
-      required(:title).value(:string)
-      required(:description).value(:string)
+      required(:id).filled(:string)
+      required(:title).filled(:string)
+      required(:description).filled(:string)
 
-      optional(:tags).value(array[:string]).default([])
+      optional(:author).filled(:string)
+      optional(:status).filled(:string)
 
-      optional(:author).value(:string)
-      optional(:references).value(array[:string])
-      optional(:related).value(array[:string])
-      optional(:status).value(:string)
+      optional(:tags).array { filled(:string) }.default([])
+      optional(:references).array { filled(:string) }
+      optional(:related).array { filled(:string) }
 
       optional(:created_on).value(:date)
       optional(:updated_on).value(:date)
 
-      required(:queries).value(:array).each { Analyzer } # rubocop:disable Lint/Void
+      required(:queries).array { Analyzer }
+      optional(:emitters).array { Emitter }.default(DEFAULT_EMITTERS)
+      optional(:enrichers).array { Enricher }.default(DEFAULT_ENRICHERS)
 
-      optional(:emitters).value(:array).each { Emitter }.default(DEFAULT_EMITTERS) # rubocop:disable Lint/Void
-      optional(:enrichers).value(:array).each { Enricher }.default(DEFAULT_ENRICHERS) # rubocop:disable Lint/Void
+      optional(:data_types).filled(array[Types::DataTypes]).default(Mihari::Types::DataTypes.values)
 
-      optional(:data_types).value(array[Types::DataTypes]).default(Mihari::Types::DataTypes.values)
-      optional(:falsepositives).value(array[:string]).default([])
+      optional(:falsepositives).array { filled(:string) }.default([])
 
       optional(:artifact_ttl).value(:integer)
     end
@@ -42,7 +42,7 @@ module Mihari
 
       rule(:falsepositives) do
         value.each do |v|
-          key.failure("#{v} is not a valid format.") unless valid_falsepositive?(v)
+          key.failure("#{v} is not a valid format") unless valid_falsepositive?(v)
         end
       end
 

data/lib/mihari/services/builders.rb

@@ -20,158 +20,5 @@ module Mihari
         Rule.from_yaml ERB.new(File.read(path_or_id)).result
       end
     end
-
-    #
-    # Autonomous system builder
-    #
-    class AutonomousSystemBuilder < Service
-      #
-      # @param [String] ip
-      # @param [Mihari::Enrichers::MMDB] enricher
-      #
-      # @return [Mihari::Models::AutonomousSystem, nil]
-      #
-      def call(ip, enricher: Enrichers::MMDB.new)
-        enricher.result(ip).fmap do |res|
-          Models::AutonomousSystem.new(number: res.asn) if res.asn
-        end.value_or nil
-      end
-    end
-
-    #
-    # CPE builder
-    #
-    class CPEBuilder < Service
-      #
-      # Build CPEs
-      #
-      # @param [String] ip
-      # @param [Mihari::Enrichers::Shodan] enricher
-      #
-      # @return [Array<Mihari::Models::CPE>]
-      #
-      def call(ip, enricher: Enrichers::Shodan.new)
-        enricher.result(ip).fmap do |res|
-          (res&.cpes || []).map { |cpe| Models::CPE.new(name: cpe) }
-        end.value_or []
-      end
-    end
-
-    #
-    # DNS record builder
-    #
-    class DnsRecordBuilder < Service
-      #
-      # Build DNS records
-      #
-      # @param [String] domain
-      # @param [Mihari::Enrichers::Shodan] enricher
-      #
-      # @return [Array<Mihari::Models::DnsRecord>]
-      #
-      def call(domain, enricher: Enrichers::GooglePublicDNS.new)
-        enricher.result(domain).fmap do |res|
-          res.answers.map { |answer| Models::DnsRecord.new(resource: answer.resource_type, value: answer.data) }
-        end.value_or []
-      end
-    end
-
-    #
-    # Geolocation builder
-    #
-    class GeolocationBuilder < Service
-      #
-      # Build Geolocation
-      #
-      # @param [String] ip
-      # @param [Mihari::Enrichers::MMDB] enricher
-      #
-      # @return [Mihari::Models::Geolocation, nil]
-      #
-      def call(ip, enricher: Enrichers::MMDB.new)
-        enricher.result(ip).fmap do |res|
-          if res.country_code
-            Models::Geolocation.new(
-              country: NormalizeCountry(res.country_code, to: :short),
-              country_code: res.country_code
-            )
-          end
-        end.value_or nil
-      end
-    end
-
-    #
-    # Port builder
-    #
-    class PortBuilder < Service
-      #
-      # Build ports
-      #
-      # @param [String] ip
-      # @param [Mihari::Enrichers::Shodan] enricher
-      #
-      # @return [Array<Mihari::Models::Port>]
-      #
-      def call(ip, enricher: Enrichers::Shodan.new)
-        enricher.result(ip).fmap do |res|
-          (res&.ports || []).map { |port| Models::Port.new(number: port) }
-        end.value_or []
-      end
-    end
-
-    #
-    # Reverse DNS name builder
-    #
-    class ReverseDnsNameBuilder < Service
-      #
-      # Build reverse DNS names
-      #
-      # @param [String] ip
-      # @param [Mihari::Enrichers::Shodan] enricher
-      #
-      # @return [Array<Mihari::Models::ReverseDnsName>]
-      #
-      def call(ip, enricher: Enrichers::Shodan.new)
-        enricher.result(ip).fmap do |res|
-          (res&.hostnames || []).map { |name| Models::ReverseDnsName.new(name: name) }
-        end.value_or []
-      end
-    end
-
-    #
-    # Vulnerability builder
-    #
-    class VulnerabilityBuilder < Service
-      #
-      # Build vulnerabilities
-      #
-      # @param [String] ip
-      # @param [Mihari::Enrichers::Shodan] enricher
-      #
-      # @return [Array<Mihari::Models::Vulnerability>]
-      #
-      def call(ip, enricher: Enrichers::Shodan.new)
-        enricher.result(ip).fmap do |res|
-          (res&.vulns || []).map { |name| Models::Vulnerability.new(name: name) }
-        end.value_or []
-      end
-    end
-
-    #
-    # Whois record builder
-    #
-    class WhoisRecordBuilder < Service
-      #
-      # Build whois record
-      #
-      # @param [String] domain
-      # @param [Mihari::Enrichers::Whois] enricher
-      #
-      # @return [Mihari::Models::WhoisRecord, nil]
-      #
-      def call(domain, enricher: Enrichers::Whois.new)
-        enricher.result(domain).value_or nil
-      end
-    end
   end
 end

data/lib/mihari/services/enrichers.rb

@@ -19,7 +19,7 @@ module Mihari
 
         raise UnenrichableError.new, "#{artifact.id} is already enriched or unenrichable" unless artifact.enrichable?
 
-        artifact.enrich_all
+        artifact.enrich
         artifact.save
       end
     end

data/lib/mihari/services/getters.rb

@@ -51,7 +51,7 @@ module Mihari
       # @return [Mihari::Structs::MMDB::Response]
       #
       def call(ip)
-        Mihari::Enrichers::MMDB.new.call ip
+        Clients::MMDB.new.query ip
       end
     end
   end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "7.2.0"
+  VERSION = "7.3.1"
 end