mihari 7.1.3 → 7.3.0

data/lib/mihari/models/rule.rb

@@ -6,6 +6,27 @@ module Mihari
     # Rule model
     #
     class Rule < ActiveRecord::Base
+      # @!attribute [rw] id
+      #   @return [String]
+
+      # @!attribute [rw] title
+      #   @return [String]
+
+      # @!attribute [rw] description
+      #   @return [String]
+
+      # @!attribute [rw] data
+      #   @return [Hash]
+
+      # @!attribute [rw] created_at
+      #   @return [DateTime]
+
+      # @!attribute [rw] updated_at
+      #   @return [DateTime]
+
+      # @!attribute [r] alerts
+      #   @return [Array<Mihari::Models::Alert>]
+
       has_many :alerts, dependent: :destroy
       has_many :taggings, dependent: :destroy
       has_many :tags, through: :taggings

data/lib/mihari/models/vulnerability.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Models
+    #
+    # Vulnerability model
+    #
+    class Vulnerability < ActiveRecord::Base
+      belongs_to :artifact
+    end
+  end
+end

data/lib/mihari/rule.rb

@@ -2,6 +2,7 @@
 
 module Mihari
   class Rule < Service
+    include Concerns::FalsePositiveNormalizable
     include Concerns::FalsePositiveValidatable
 
     # @return [Hash]
@@ -136,8 +137,7 @@ module Mihari
       analyzer_results.flat_map do |result|
         artifacts = result.value!
         artifacts.map do |artifact|
-          artifact.rule_id = id
-          artifact
+          artifact.tap { |tapped| tapped.rule_id = id }
         end
       end
     end
@@ -174,8 +174,10 @@ module Mihari
     # @return [Array<Mihari::Models::Artifact>]
     #
     def enriched_artifacts
-      @enriched_artifacts ||= Parallel.map(unique_artifacts) do |artifact|
-        enrichers.each { |enricher| artifact.enrich_by_enricher enricher }
+      @enriched_artifacts ||= unique_artifacts.map do |artifact|
+        serial_enrichers.each { |enricher| enricher.result(artifact) }
+        Parallel.each(parallel_enrichers) { |enricher| enricher.result(artifact) }
+
         artifact
       end
     end
@@ -188,9 +190,10 @@ module Mihari
     def bulk_emit
       return [] if enriched_artifacts.empty?
 
-      Parallel.map(emitters) do |emitter|
-        emitter.result(enriched_artifacts).value_or nil
-      end.compact
+      [].tap do |out|
+        out << serial_emitters.map { |emitter| emitter.result(enriched_artifacts).value_or(nil) }
+        out << Parallel.map(parallel_emitters) { |emitter| emitter.result(enriched_artifacts).value_or(nil) }
+      end.flatten.compact
     end
 
     #
@@ -291,11 +294,11 @@ module Mihari
     #
     # @return [Boolean]
     #
-    def falsepositive?(value)
-      return true if falsepositives.include?(value)
+    def falsepositive?(artifact)
+      return true if falsepositives.include?(artifact)
 
       regexps = falsepositives.select { |fp| fp.is_a?(Regexp) }
-      regexps.any? { |fp| fp.match?(value) }
+      regexps.any? { |fp| fp.match?(artifact) }
     end
 
     #
@@ -315,12 +318,12 @@ module Mihari
     # @return [Array<Mihari::Analyzers::Base>]
     #
     def analyzers
-      @analyzers ||= queries.map do |params|
-        analyzer_name = params[:analyzer]
-        klass = get_analyzer_class(analyzer_name)
-        analyzer = klass.from_query(params)
-        analyzer.validate_configuration!
-        analyzer
+      @analyzers ||= queries.deep_dup.map do |params|
+        name = params.delete(:analyzer)
+        klass = get_analyzer_class(name)
+        klass.from_params(params).tap do |analyzer|
+          analyzer.validate_configuration!
+        end
       end
     end
 
@@ -356,19 +359,25 @@ module Mihari
     # @return [Array<Mihari::Emitters::Base>]
     #
     def emitters
-      @emitters ||= data[:emitters].map(&:deep_dup).map do |params|
-        name = params[:emitter]
-        options = params[:options]
-
-        %i[emitter options].each { |key| params.delete key }
+      @emitters ||= data[:emitters].deep_dup.map do |params|
+        name = params.delete(:emitter)
+        options = params.delete(:options)
 
         klass = get_emitter_class(name)
-        emitter = klass.new(rule: self, options: options, **params)
-        emitter.validate_configuration!
-        emitter
+        klass.new(rule: self, options: options, **params).tap do |emitter|
+          emitter.validate_configuration!
+        end
       end
     end
 
+    def parallel_emitters
+      emitters.select(&:parallel?)
+    end
+
+    def serial_emitters
+      emitters.reject(&:parallel?)
+    end
+
     #
     # Get enricher class
     #
@@ -386,17 +395,23 @@ module Mihari
     # @return [Array<Mihari::Enrichers::Base>] enrichers
     #
     def enrichers
-      @enrichers ||= data[:enrichers].map(&:deep_dup).map do |params|
-        name = params[:enricher]
-        options = params[:options]
-
-        %i[enricher options].each { |key| params.delete key }
+      @enrichers ||= data[:enrichers].deep_dup.map do |params|
+        name = params.delete(:enricher)
+        options = params.delete(:options)
 
         klass = get_enricher_class(name)
         klass.new(options: options, **params)
       end
     end
 
+    def parallel_enrichers
+      enrichers.select(&:parallel?)
+    end
+
+    def serial_enrichers
+      enrichers.reject(&:parallel?)
+    end
+
     #
     # Validate the data format
     #

data/lib/mihari/schemas/alert.rb

@@ -3,9 +3,9 @@
 module Mihari
   module Schemas
     Alert = Dry::Schema.Params do
-      required(:rule_id).value(:string)
-      required(:artifacts).value(array[:string])
-      optional(:source).value(:string)
+      required(:rule_id).filled(:string)
+      required(:artifacts).array { filled(:string) }
+      optional(:source).filled(:string)
     end
 
     #

data/lib/mihari/schemas/analyzer.rb

@@ -20,8 +20,8 @@ module Mihari
         key = keys.first
         const_set(key.upcase, Dry::Schema.Params do
           required(:analyzer).value(Types::String.enum(*keys))
-          required(:query).value(:string)
-          optional(:api_key).value(:string)
+          required(:query).filled(:string)
+          optional(:api_key).filled(:string)
           optional(:options).hash(AnalyzerPaginationOptions)
         end)
       end
@@ -36,60 +36,60 @@ module Mihari
         key = keys.first
         const_set(key.upcase, Dry::Schema.Params do
           required(:analyzer).value(Types::String.enum(*keys))
-          required(:query).value(:string)
-          optional(:api_key).value(:string)
+          required(:query).filled(:string)
+          optional(:api_key).filled(:string)
           optional(:options).hash(AnalyzerOptions)
         end)
       end
 
       DNSTwister = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::DNSTwister.keys))
-        required(:query).value(:string)
+        required(:query).filled(:string)
         optional(:options).hash(AnalyzerOptions)
       end
 
       Censys = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::Censys.keys))
-        required(:query).value(:string)
-        optional(:id).value(:string)
-        optional(:secret).value(:string)
+        required(:query).filled(:string)
+        optional(:id).filled(:string)
+        optional(:secret).filled(:string)
         optional(:options).hash(AnalyzerPaginationOptions)
       end
 
       CIRCL = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::CIRCL.keys))
-        required(:query).value(:string)
-        optional(:username).value(:string)
-        optional(:password).value(:string)
+        required(:query).filled(:string)
+        optional(:username).filled(:string)
+        optional(:password).filled(:string)
         optional(:options).hash(AnalyzerOptions)
       end
 
       Fofa = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::Fofa.keys))
-        required(:query).value(:string)
-        optional(:api_key).value(:string)
-        optional(:email).value(:string)
+        required(:query).filled(:string)
+        optional(:api_key).filled(:string)
+        optional(:email).filled(:string)
         optional(:options).hash(AnalyzerPaginationOptions)
       end
 
       PassiveTotal = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::PassiveTotal.keys))
-        required(:query).value(:string)
-        optional(:username).value(:string)
-        optional(:api_key).value(:string)
+        required(:query).filled(:string)
+        optional(:username).filled(:string)
+        optional(:api_key).filled(:string)
         optional(:options).hash(AnalyzerOptions)
       end
 
       ZoomEye = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::ZoomEye.keys))
-        required(:query).value(:string)
+        required(:query).filled(:string)
         required(:type).value(Types::String.enum("host", "web"))
         optional(:options).hash(AnalyzerPaginationOptions)
       end
 
       Crtsh = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::Crtsh.keys))
-        required(:query).value(:string)
+        required(:query).filled(:string)
         optional(:exclude_expired).value(:bool).default(true)
         optional(:match).value(Types::String.enum("=", "ILIKE", "LIKE", "single", "any", "FTS")).default(nil)
         optional(:options).hash(AnalyzerOptions)
@@ -97,22 +97,22 @@ module Mihari
 
       HunterHow = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::HunterHow.keys))
-        required(:query).value(:string)
+        required(:query).filled(:string)
         required(:start_time).value(:date)
         required(:end_time).value(:date)
-        optional(:api_key).value(:string)
+        optional(:api_key).filled(:string)
         optional(:options).hash(AnalyzerPaginationOptions)
       end
 
       Feed = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::Feed.keys))
-        required(:query).value(:string)
-        required(:selector).value(:string)
+        required(:query).filled(:string)
+        required(:selector).filled(:string)
         optional(:method).value(Types::HTTPRequestMethods).default("GET")
-        optional(:headers).value(:hash).default({})
-        optional(:params).value(:hash)
-        optional(:form).value(:hash)
-        optional(:json).value(:hash)
+        optional(:headers).filled(:hash)
+        optional(:params).filled(:hash)
+        optional(:form).filled(:hash)
+        optional(:json).filled(:hash)
         optional(:options).hash(AnalyzerOptions)
       end
     end

data/lib/mihari/schemas/emitter.rb

@@ -15,31 +15,31 @@ module Mihari
 
       MISP = Dry::Schema.Params do
         required(:emitter).value(Types::String.enum(*Mihari::Emitters::MISP.keys))
-        optional(:url).value(:string)
-        optional(:api_key).value(:string)
+        optional(:url).filled(:string)
+        optional(:api_key).filled(:string)
         optional(:options).hash(Options)
       end
 
       TheHive = Dry::Schema.Params do
         required(:emitter).value(Types::String.enum(*Mihari::Emitters::TheHive.keys))
-        optional(:url).value(:string)
-        optional(:api_key).value(:string)
+        optional(:url).filled(:string)
+        optional(:api_key).filled(:string)
         optional(:options).hash(Options)
       end
 
       Slack = Dry::Schema.Params do
         required(:emitter).value(Types::String.enum(*Mihari::Emitters::Slack.keys))
-        optional(:webhook_url).value(:string)
-        optional(:channel).value(:string)
+        optional(:webhook_url).filled(:string)
+        optional(:channel).filled(:string)
         optional(:options).hash(Options)
       end
 
       Webhook = Dry::Schema.Params do
         required(:emitter).value(Types::String.enum(*Mihari::Emitters::Webhook.keys))
-        required(:url).value(:string)
+        required(:url).filled(:string)
         optional(:method).value(Types::HTTPRequestMethods).default("POST")
-        optional(:headers).value(:hash).default({})
-        optional(:template).value(:string)
+        optional(:headers).filled(:hash)
+        optional(:template).filled(:string)
         optional(:options).hash(Options)
       end
     end

data/lib/mihari/schemas/macros.rb

@@ -8,9 +8,9 @@ module Dry
       # (see https://github.com/dry-rb/dry-schema/issues/70)
       #
       class DSL
-        def default(value)
+        def default(artifact)
           schema_dsl.before(:rule_applier) do |result|
-            result.update(name => value) if result.output && !result[name]
+            result.update(name => artifact) if result.output && !result[name]
           end
         end
       end

data/lib/mihari/schemas/options.rb

@@ -7,17 +7,14 @@ module Mihari
       optional(:retry_interval).value(:integer).default(Mihari.config.retry_interval)
       optional(:retry_exponential_backoff).value(:bool).default(Mihari.config.retry_exponential_backoff)
       optional(:timeout).value(:integer)
+      optional(:parallel).value(:bool).default(Mihari.config.parallel)
     end
 
     IgnoreErrorOptions = Dry::Schema.Params do
       optional(:ignore_error).value(:bool).default(Mihari.config.ignore_error)
     end
 
-    ParallelOptions = Dry::Schema.Params do
-      optional(:parallel).value(:bool).default(Mihari.config.parallel)
-    end
-
-    AnalyzerOptions = Options | IgnoreErrorOptions | ParallelOptions
+    AnalyzerOptions = Options | IgnoreErrorOptions
 
     PaginationOptions = Dry::Schema.Params do
       optional(:pagination_interval).value(:integer).default(Mihari.config.pagination_interval)

data/lib/mihari/schemas/rule.rb

@@ -7,27 +7,27 @@ require "mihari/schemas/enricher"
 module Mihari
   module Schemas
     Rule = Dry::Schema.Params do
-      required(:id).value(:string)
-      required(:title).value(:string)
-      required(:description).value(:string)
+      required(:id).filled(:string)
+      required(:title).filled(:string)
+      required(:description).filled(:string)
 
-      optional(:tags).value(array[:string]).default([])
+      optional(:author).filled(:string)
+      optional(:status).filled(:string)
 
-      optional(:author).value(:string)
-      optional(:references).value(array[:string])
-      optional(:related).value(array[:string])
-      optional(:status).value(:string)
+      optional(:tags).array { filled(:string) }.default([])
+      optional(:references).array { filled(:string) }
+      optional(:related).array { filled(:string) }
 
       optional(:created_on).value(:date)
       optional(:updated_on).value(:date)
 
       required(:queries).value(:array).each { Analyzer } # rubocop:disable Lint/Void
-
       optional(:emitters).value(:array).each { Emitter }.default(DEFAULT_EMITTERS) # rubocop:disable Lint/Void
       optional(:enrichers).value(:array).each { Enricher }.default(DEFAULT_ENRICHERS) # rubocop:disable Lint/Void
 
-      optional(:data_types).value(array[Types::DataTypes]).default(Mihari::Types::DataTypes.values)
-      optional(:falsepositives).value(array[:string]).default([])
+      optional(:data_types).filled(array[Types::DataTypes]).default(Mihari::Types::DataTypes.values)
+
+      optional(:falsepositives).array { filled(:string) }.default([])
 
       optional(:artifact_ttl).value(:integer)
     end
@@ -42,7 +42,14 @@ module Mihari
 
       rule(:falsepositives) do
         value.each do |v|
-          key.failure("#{v} is not a valid format.") unless valid_falsepositive?(v)
+          key.failure("#{v} is not a valid format") unless valid_falsepositive?(v)
+        end
+      end
+
+      rule(:emitters) do
+        emitters = value.filter_map { |v| v[:emitter] }
+        unless emitters.include?(Mihari::Emitters::Database.key)
+          key.failure("Emitter:#{Mihari::Emitters::Database.key} should be included in emitters")
         end
       end
     end

data/lib/mihari/services/builders.rb

@@ -20,139 +20,5 @@ module Mihari
         Rule.from_yaml ERB.new(File.read(path_or_id)).result
       end
     end
-
-    #
-    # Autonomous system builder
-    #
-    class AutonomousSystemBuilder < Service
-      #
-      # @param [String] ip
-      # @param [Mihari::Enrichers::MMDB] enricher
-      #
-      # @return [Mihari::Models::AutonomousSystem, nil]
-      #
-      def call(ip, enricher: Enrichers::MMDB.new)
-        enricher.result(ip).fmap do |res|
-          Models::AutonomousSystem.new(asn: res.asn) if res.asn
-        end.value_or nil
-      end
-    end
-
-    #
-    # CPE builder
-    #
-    class CPEBuilder < Service
-      #
-      # Build CPEs
-      #
-      # @param [String] ip
-      # @param [Mihari::Enrichers::Shodan] enricher
-      #
-      # @return [Array<Mihari::Models::CPE>]
-      #
-      def call(ip, enricher: Enrichers::Shodan.new)
-        enricher.result(ip).fmap do |res|
-          (res&.cpes || []).map { |cpe| Models::CPE.new(cpe: cpe) }
-        end.value_or []
-      end
-    end
-
-    #
-    # DNS record builder
-    #
-    class DnsRecordBuilder < Service
-      #
-      # Build DNS records
-      #
-      # @param [String] domain
-      # @param [Mihari::Enrichers::Shodan] enricher
-      #
-      # @return [Array<Mihari::Models::DnsRecord>]
-      #
-      def call(domain, enricher: Enrichers::GooglePublicDNS.new)
-        enricher.result(domain).fmap do |res|
-          res.answers.map { |answer| Models::DnsRecord.new(resource: answer.resource_type, value: answer.data) }
-        end.value_or []
-      end
-    end
-
-    #
-    # Geolocation builder
-    #
-    class GeolocationBuilder < Service
-      #
-      # Build Geolocation
-      #
-      # @param [String] ip
-      # @param [Mihari::Enrichers::MMDB] enricher
-      #
-      # @return [Mihari::Models::Geolocation, nil]
-      #
-      def call(ip, enricher: Enrichers::MMDB.new)
-        enricher.result(ip).fmap do |res|
-          if res.country_code
-            Models::Geolocation.new(
-              country: NormalizeCountry(res.country_code, to: :short),
-              country_code: res.country_code
-            )
-          end
-        end.value_or nil
-      end
-    end
-
-    #
-    # Port builder
-    #
-    class PortBuilder < Service
-      #
-      # Build ports
-      #
-      # @param [String] ip
-      # @param [Mihari::Enrichers::Shodan] enricher
-      #
-      # @return [Array<Mihari::Models::Port>]
-      #
-      def call(ip, enricher: Enrichers::Shodan.new)
-        enricher.result(ip).fmap do |res|
-          (res&.ports || []).map { |port| Models::Port.new(port: port) }
-        end.value_or []
-      end
-    end
-
-    #
-    # Reverse DNS name builder
-    #
-    class ReverseDnsNameBuilder < Service
-      #
-      # Build reverse DNS names
-      #
-      # @param [String] ip
-      # @param [Mihari::Enrichers::Shodan] enricher
-      #
-      # @return [Array<Mihari::Models::ReverseDnsName>]
-      #
-      def call(ip, enricher: Enrichers::Shodan.new)
-        enricher.result(ip).fmap do |res|
-          (res&.hostnames || []).map { |name| Models::ReverseDnsName.new(name: name) }
-        end.value_or []
-      end
-    end
-
-    #
-    # Whois record builder
-    #
-    class WhoisRecordBuilder < Service
-      #
-      # Build whois record
-      #
-      # @param [String] domain
-      # @param [Mihari::Enrichers::Whois] enricher
-      #
-      # @return [Mihari::Models::WhoisRecord, nil]
-      #
-      def call(domain, enricher: Enrichers::Whois.new)
-        enricher.result(domain).value_or nil
-      end
-    end
   end
 end

data/lib/mihari/services/enrichers.rb

@@ -17,7 +17,9 @@ module Mihari
           :ports
         ).find(id)
 
-        artifact.enrich_all
+        raise UnenrichableError.new, "#{artifact.id} is already enriched or unenrichable" unless artifact.enrichable?
+
+        artifact.enrich
         artifact.save
       end
     end

data/lib/mihari/services/feed.rb

@@ -93,12 +93,9 @@ module Mihari
       #
       # @param [Object] read_data
       def call(input_enumerator, selector)
-        parsed = proc do
-          $SAFE = 1
-          input_enumerator.instance_eval(selector)
-        end.call
+        parsed = input_enumerator.instance_eval(selector)
 
-        raise TypeError unless parsed.all?(String)
+        raise TypeError unless parsed.is_a?(Array) || parsed.all?(String)
 
         parsed
       end

data/lib/mihari/services/getters.rb

@@ -51,7 +51,7 @@ module Mihari
       # @return [Mihari::Structs::MMDB::Response]
       #
       def call(ip)
-        Mihari::Enrichers::MMDB.new.call ip
+        Clients::MMDB.new.query ip
       end
     end
   end

data/lib/mihari/services/proxies.rb

@@ -79,10 +79,10 @@ module Mihari
       # @return [Mihari::Rule]
       #
       def rule
-        @rule ||= [].tap do |out|
+        @rule ||= lambda do
           data = Mihari::Models::Rule.find(rule_id).data
-          out << Rule.new(**data)
-        end.first
+          Rule.new(**data)
+        end.call
       end
     end
   end

data/lib/mihari/structs/censys.rb

@@ -14,7 +14,7 @@ module Mihari
         # @return [Mihari::AutonomousSystem]
         #
         def as
-          Mihari::Models::AutonomousSystem.new(asn: normalize_asn(asn))
+          Mihari::Models::AutonomousSystem.new(number: normalize_asn(asn))
         end
 
         class << self
@@ -76,7 +76,7 @@ module Mihari
         # @return [Mihari::Port]
         #
         def _port
-          Models::Port.new(port: port)
+          Models::Port.new(number: port)
         end
 
         class << self

data/lib/mihari/structs/greynoise.rb

@@ -22,7 +22,7 @@ module Mihari
         # @return [Mihari::AutonomousSystem]
         #
         def as
-          Mihari::Models::AutonomousSystem.new(asn: normalize_asn(asn))
+          Mihari::Models::AutonomousSystem.new(number: normalize_asn(asn))
         end
 
         #

data/lib/mihari/structs/onyphe.rb

@@ -51,7 +51,7 @@ module Mihari
         # @return [Mihari::AutonomousSystem]
         #
         def as
-          Mihari::Models::AutonomousSystem.new(asn: normalize_asn(asn))
+          Mihari::Models::AutonomousSystem.new(number: normalize_asn(asn))
         end
 
         class << self