mihari 5.6.1 → 5.7.0

data/lib/mihari/rule.rb

@@ -0,0 +1,357 @@
+# frozen_string_literal: true
+
+module Mihari
+  class Rule < Service
+    include Mixins::FalsePositive
+
+    # @return [Hash]
+    attr_reader :data
+
+    # @return [Array, nil]
+    attr_reader :errors
+
+    # @return [Time]
+    attr_reader :base_time
+
+    #
+    # Initialize
+    #
+    # @param [Hash] data
+    #
+    def initialize(**data)
+      super()
+
+      @data = data.deep_symbolize_keys
+      @errors = nil
+      @base_time = Time.now.utc
+
+      validate!
+    end
+
+    #
+    # @return [Boolean]
+    #
+    def errors?
+      return false if @errors.nil?
+
+      !@errors.empty?
+    end
+
+    def [](key)
+      data key.to_sym
+    end
+
+    #
+    # @return [String]
+    #
+    def id
+      data[:id]
+    end
+
+    #
+    # @return [String]
+    #
+    def title
+      data[:title]
+    end
+
+    #
+    # @return [String]
+    #
+    def description
+      data[:description]
+    end
+
+    #
+    # @return [String]
+    #
+    def yaml
+      data.deep_stringify_keys.to_yaml
+    end
+
+    #
+    # @return [Array<Hash>]
+    #
+    def queries
+      data[:queries]
+    end
+
+    #
+    # @return [Array<String>]
+    #
+    def data_types
+      data[:data_types]
+    end
+
+    #
+    # @return [Array<String>]
+    #
+    def tags
+      data[:tags]
+    end
+
+    #
+    # @return [Array<String, RegExp>]
+    #
+    def falsepositives
+      @falsepositives ||= data[:falsepositives].map { |fp| normalize_falsepositive fp }
+    end
+
+    #
+    # @return [Integer, nil]
+    #
+    def artifact_lifetime
+      data[:artifact_lifetime] || data[:artifact_ttl]
+    end
+
+    #
+    # Returns a list of artifacts matched with queries/analyzers (with the rule ID)
+    #
+    # @return [Array<Mihari::Models::Artifact>]
+    #
+    def artifacts
+      analyzers.flat_map do |analyzer|
+        # @type [Dry::Monads::Result::Success<Array<Mihari::Models::Artifact>>, Dry::Monads::Result::Failure]
+        result = analyzer.result
+
+        if result.failure?
+          raise result.failure unless analyzer.ignore_error?
+        else
+          artifacts = result.value!
+          artifacts.map do |artifact|
+            artifact.rule_id = id
+            artifact
+          end
+        end
+      end.compact
+    end
+
+    #
+    # Normalize artifacts
+    # - Reject invalid artifacts (for just in case)
+    # - Select artifacts with allowed data types
+    # - Reject artifacts with false positive values
+    # - Set rule ID
+    #
+    # @return [Array<Mihari::Models::Artifact>]
+    #
+    def normalized_artifacts
+      valid_artifacts = artifacts.uniq(&:data).select(&:valid?)
+      date_type_allowed_artifacts = valid_artifacts.select { |artifact| data_types.include? artifact.data_type }
+      date_type_allowed_artifacts.reject { |artifact| falsepositive? artifact.data }
+    end
+
+    #
+    # Uniquify artifacts (assure rule level uniqueness)
+    #
+    # @return [Array<Mihari::Models::Artifact>]
+    #
+    def unique_artifacts
+      normalized_artifacts.select do |artifact|
+        artifact.unique?(base_time: base_time, artifact_lifetime: artifact_lifetime)
+      end
+    end
+
+    #
+    # Enriched artifacts
+    #
+    # @return [Array<Mihari::Models::Artifact>]
+    #
+    def enriched_artifacts
+      @enriched_artifacts ||= Parallel.map(unique_artifacts) do |artifact|
+        enrichers.each { |enricher| artifact.enrich_by_enricher enricher }
+        artifact
+      end
+    end
+
+    #
+    # Bulk emit
+    #
+    # @return [Array<Mihari::Models::Alert>]
+    #
+    def bulk_emit
+      return [] if enriched_artifacts.empty?
+
+      # NOTE: separate parallel execution and logging
+      #       because the logger does not work along with Parallel
+      results = Parallel.map(emitters) { |emitter| emitter.result enriched_artifacts }
+      results.zip(emitters).map do |result_and_emitter|
+        result, emitter = result_and_emitter
+        Mihari.logger.info "Emission by #{emitter.class} is failed: #{result.failure}" if result.failure?
+        Mihari.logger.info "Emission by #{emitter.class} is succeeded" if result.success?
+        result.value_or nil
+      end.compact
+    end
+
+    #
+    # Set artifacts & run emitters in parallel
+    #
+    # @return [Mihari::Models::Alert, nil]
+    #
+    def call
+      # Validate analyzers & emitters before using them
+      analyzers
+      emitters
+
+      alert_or_something = bulk_emit
+      # Return Mihari::Models::Alert created by the database emitter
+      alert_or_something.find { |res| res.is_a?(Mihari::Models::Alert) }
+    end
+
+    #
+    # @return [Mihari::Models::Rule]
+    #
+    def model
+      rule = Mihari::Models::Rule.find(id)
+
+      rule.title = title
+      rule.description = description
+      rule.data = data
+
+      rule
+    rescue ActiveRecord::RecordNotFound
+      Mihari::Models::Rule.new(
+        id: id,
+        title: title,
+        description: description,
+        data: data
+      )
+    end
+
+    class << self
+      #
+      # Load rule from YAML string
+      #
+      # @param [String] yaml
+      #
+      # @return [Mihari::Rule]
+      #
+      def from_yaml(yaml)
+        data = YAML.safe_load(ERB.new(yaml).result, permitted_classes: [Date, Symbol])
+        new(**data)
+      end
+
+      #
+      # @param [Mihari::Models::Rule] model
+      #
+      # @return [Mihari::Rule]
+      #
+      def from_model(model)
+        new(**model.data)
+      end
+    end
+
+    private
+
+    #
+    # Check whether a value is a falsepositive value or not
+    #
+    # @return [Boolean]
+    #
+    def falsepositive?(value)
+      return true if falsepositives.include?(value)
+
+      regexps = falsepositives.select { |fp| fp.is_a?(Regexp) }
+      regexps.any? { |fp| fp.match?(value) }
+    end
+
+    #
+    # Get analyzer class
+    #
+    # @param [String] key
+    #
+    # @return [Class<Mihari::Analyzers::Base>] analyzer class
+    #
+    def get_analyzer_class(key)
+      raise ArgumentError, "#{key} is not supported" unless Mihari.analyzer_to_class.key?(key)
+
+      Mihari.analyzer_to_class[key]
+    end
+
+    #
+    # @return [Array<Mihari::Analyzers::Base>]
+    #
+    def analyzers
+      @analyzers ||= queries.map do |query_params|
+        analyzer_name = query_params[:analyzer]
+        klass = get_analyzer_class(analyzer_name)
+        klass.from_query(query_params)
+      end.map do |analyzer|
+        analyzer.validate_configuration!
+        analyzer
+      end
+    end
+
+    #
+    # Get emitter class
+    #
+    # @param [String] key
+    #
+    # @return [Class<Mihari::Emitters::Base>] emitter class
+    #
+    def get_emitter_class(key)
+      raise ArgumentError, "#{key} is not supported" unless Mihari.emitter_to_class.key?(key)
+
+      Mihari.emitter_to_class[key]
+    end
+
+    #
+    # @return [Array<Mihari::Emitters::Base>]
+    #
+    def emitters
+      @emitters ||= data[:emitters].map(&:deep_dup).map do |params|
+        name = params[:emitter]
+        options = params[:options]
+
+        %i[emitter options].each { |key| params.delete key }
+
+        klass = get_emitter_class(name)
+        klass.new(rule: self, options: options, **params)
+      end.map do |emitter|
+        emitter.validate_configuration!
+        emitter
+      end
+    end
+
+    #
+    # Get enricher class
+    #
+    # @param [String] key
+    #
+    # @return [Class<Mihari::Enrichers::Base>] enricher class
+    #
+    def get_enricher_class(key)
+      raise ArgumentError, "#{key} is not supported" unless Mihari.enricher_to_class.key?(key)
+
+      Mihari.enricher_to_class[key]
+    end
+
+    #
+    # @return [Array<Mihari::Enrichers::Base>] enrichers
+    #
+    def enrichers
+      @enrichers ||= data[:enrichers].map(&:deep_dup).map do |params|
+        name = params[:enricher]
+        options = params[:options]
+
+        %i[enricher options].each { |key| params.delete key }
+
+        klass = get_enricher_class(name)
+        klass.new(options: options, **params)
+      end
+    end
+
+    #
+    # Validate the data format
+    #
+    def validate!
+      contract = Schemas::RuleContract.new
+      result = contract.call(data)
+
+      @data = result.to_h
+      @errors = result.errors
+
+      raise ValidationError.new("Validation failed", errors) if errors?
+    end
+  end
+end

data/lib/mihari/schemas/alert.rb

@@ -7,6 +7,9 @@ module Mihari
       required(:artifacts).value(array[:string])
     end
 
+    #
+    # Alert schema contract
+    #
     class AlertContract < Dry::Validation::Contract
       params(Alert)
     end

data/lib/mihari/schemas/analyzer.rb

@@ -2,102 +2,120 @@
 
 module Mihari
   module Schemas
-    AnalyzerAPIKeyPagination = Dry::Schema.Params do
-      required(:analyzer).value(
-        Types::String.enum(
-          "binaryedge",
-          "greynoise",
-          "onyphe",
-          "shodan",
-          "urlscan",
-          "virustotal_intelligence",
-          "vt_intel"
-        )
-      )
-      required(:query).value(:string)
-      optional(:api_key).value(:string)
-      optional(:options).hash(AnalyzerPaginationOptions)
-    end
+    #
+    # Analyzer schemas
+    #
+    module Analyzers
+      extend Schemas::Mixins
 
-    AnalyzerAPIKey = Dry::Schema.Params do
-      required(:analyzer).value(
-        Types::String.enum(
-          "otx",
-          "pulsedive",
-          "securitytrails",
-          "st",
-          "virustotal",
-          "vt"
-        )
-      )
-      required(:query).value(:string)
-      optional(:api_key).value(:string)
-      optional(:options).hash(AnalyzerOptions)
-    end
+      # Analyzer with API key and pagination
+      [
+        Mihari::Analyzers::BinaryEdge.class_keys,
+        Mihari::Analyzers::GreyNoise.class_keys,
+        Mihari::Analyzers::Onyphe.class_keys,
+        Mihari::Analyzers::Shodan.class_keys,
+        Mihari::Analyzers::Urlscan.class_keys,
+        Mihari::Analyzers::VirusTotalIntelligence.class_keys
+      ].each do |keys|
+        key = keys.first
+        const_set(key.upcase, Dry::Schema.Params do
+          required(:analyzer).value(Types::String.enum(*keys))
+          required(:query).value(:string)
+          optional(:api_key).value(:string)
+          optional(:options).hash(AnalyzerPaginationOptions)
+        end)
+      end
 
-    DNSTwister = Dry::Schema.Params do
-      required(:analyzer).value(Types::String.enum("dnstwister"))
-      required(:query).value(:string)
-      optional(:options).hash(AnalyzerOptions)
-    end
+      # Analyzer with API key
+      [
+        Mihari::Analyzers::OTX.class_keys,
+        Mihari::Analyzers::Pulsedive.class_keys,
+        Mihari::Analyzers::VirusTotal.class_keys,
+        Mihari::Analyzers::SecurityTrails.class_keys
+      ].each do |keys|
+        key = keys.first
+        const_set(key.upcase, Dry::Schema.Params do
+          required(:analyzer).value(Types::String.enum(*keys))
+          required(:query).value(:string)
+          optional(:api_key).value(:string)
+          optional(:options).hash(AnalyzerOptions)
+        end)
+      end
 
-    Censys = Dry::Schema.Params do
-      required(:analyzer).value(Types::String.enum("censys"))
-      required(:query).value(:string)
-      optional(:id).value(:string)
-      optional(:secret).value(:string)
-      optional(:options).hash(AnalyzerPaginationOptions)
-    end
+      DNSTwister = Dry::Schema.Params do
+        required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::DNSTwister.class_keys))
+        required(:query).value(:string)
+        optional(:options).hash(AnalyzerOptions)
+      end
 
-    CIRCL = Dry::Schema.Params do
-      required(:analyzer).value(Types::String.enum("circl"))
-      required(:query).value(:string)
-      optional(:username).value(:string)
-      optional(:password).value(:string)
-      optional(:options).hash(AnalyzerOptions)
-    end
+      Censys = Dry::Schema.Params do
+        required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::Censys.class_keys))
+        required(:query).value(:string)
+        optional(:id).value(:string)
+        optional(:secret).value(:string)
+        optional(:options).hash(AnalyzerPaginationOptions)
+      end
 
-    PassiveTotal = Dry::Schema.Params do
-      required(:analyzer).value(Types::String.enum("passivetotal", "pt"))
-      required(:query).value(:string)
-      optional(:username).value(:string)
-      optional(:api_key).value(:string)
-      optional(:options).hash(AnalyzerOptions)
-    end
+      CIRCL = Dry::Schema.Params do
+        required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::CIRCL.class_keys))
+        required(:query).value(:string)
+        optional(:username).value(:string)
+        optional(:password).value(:string)
+        optional(:options).hash(AnalyzerOptions)
+      end
 
-    ZoomEye = Dry::Schema.Params do
-      required(:analyzer).value(Types::String.enum("zoomeye"))
-      required(:query).value(:string)
-      required(:type).value(Types::String.enum("host", "web"))
-      optional(:options).hash(AnalyzerPaginationOptions)
-    end
+      Fofa = Dry::Schema.Params do
+        required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::Fofa.class_keys))
+        required(:query).value(:string)
+        optional(:api_key).value(:string)
+        optional(:email).value(:string)
+        optional(:options).hash(AnalyzerPaginationOptions)
+      end
 
-    Crtsh = Dry::Schema.Params do
-      required(:analyzer).value(Types::String.enum("crtsh"))
-      required(:query).value(:string)
-      optional(:exclude_expired).value(:bool).default(true)
-      optional(:options).hash(AnalyzerOptions)
-    end
+      PassiveTotal = Dry::Schema.Params do
+        required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::PassiveTotal.class_keys))
+        required(:query).value(:string)
+        optional(:username).value(:string)
+        optional(:api_key).value(:string)
+        optional(:options).hash(AnalyzerOptions)
+      end
 
-    HunterHow = Dry::Schema.Params do
-      required(:analyzer).value(Types::String.enum("hunterhow"))
-      required(:query).value(:string)
-      required(:start_time).value(:date)
-      required(:end_time).value(:date)
-      optional(:api_key).value(:string)
-      optional(:options).hash(AnalyzerPaginationOptions)
-    end
+      ZoomEye = Dry::Schema.Params do
+        required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::ZoomEye.class_keys))
+        required(:query).value(:string)
+        required(:type).value(Types::String.enum("host", "web"))
+        optional(:options).hash(AnalyzerPaginationOptions)
+      end
 
-    Feed = Dry::Schema.Params do
-      required(:analyzer).value(Types::String.enum("feed"))
-      required(:query).value(:string)
-      required(:selector).value(:string)
-      optional(:method).value(Types::HTTPRequestMethods).default("GET")
-      optional(:headers).value(:hash).default({})
-      optional(:params).value(:hash)
-      optional(:data).value(:hash)
-      optional(:json).value(:hash)
-      optional(:options).hash(AnalyzerOptions)
+      Crtsh = Dry::Schema.Params do
+        required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::Crtsh.class_keys))
+        required(:query).value(:string)
+        optional(:exclude_expired).value(:bool).default(true)
+        optional(:options).hash(AnalyzerOptions)
+      end
+
+      HunterHow = Dry::Schema.Params do
+        required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::HunterHow.class_keys))
+        required(:query).value(:string)
+        required(:start_time).value(:date)
+        required(:end_time).value(:date)
+        optional(:api_key).value(:string)
+        optional(:options).hash(AnalyzerPaginationOptions)
+      end
+
+      Feed = Dry::Schema.Params do
+        required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::Feed.class_keys))
+        required(:query).value(:string)
+        required(:selector).value(:string)
+        optional(:method).value(Types::HTTPRequestMethods).default("GET")
+        optional(:headers).value(:hash).default({})
+        optional(:params).value(:hash)
+        optional(:data).value(:hash)
+        optional(:json).value(:hash)
+        optional(:options).hash(AnalyzerOptions)
+      end
     end
+
+    Analyzer = Schemas::Analyzers.get_or_composition
   end
 end

data/lib/mihari/schemas/emitter.rb

@@ -2,21 +2,26 @@
 
 module Mihari
   module Schemas
+    #
+    # Emitter schemas
+    #
     module Emitters
+      extend Schemas::Mixins
+
       Database = Dry::Schema.Params do
-        required(:emitter).value(Types::String.enum("database"))
+        required(:emitter).value(Types::String.enum(*Mihari::Emitters::Database.class_keys))
         optional(:options).hash(Options)
       end
 
       MISP = Dry::Schema.Params do
-        required(:emitter).value(Types::String.enum("misp"))
+        required(:emitter).value(Types::String.enum(*Mihari::Emitters::MISP.class_keys))
         optional(:url).value(:string)
         optional(:api_key).value(:string)
         optional(:options).hash(Options)
       end
 
       TheHive = Dry::Schema.Params do
-        required(:emitter).value(Types::String.enum("thehive"))
+        required(:emitter).value(Types::String.enum(*Mihari::Emitters::TheHive.class_keys))
         optional(:url).value(:string)
         optional(:api_key).value(:string)
         optional(:api_version).value(Types::String.enum("v4", "v5")).default("v4")
@@ -24,14 +29,14 @@ module Mihari
       end
 
       Slack = Dry::Schema.Params do
-        required(:emitter).value(Types::String.enum("slack"))
+        required(:emitter).value(Types::String.enum(*Mihari::Emitters::Slack.class_keys))
         optional(:webhook_url).value(:string)
         optional(:channel).value(:string)
         optional(:options).hash(Options)
       end
 
       Webhook = Dry::Schema.Params do
-        required(:emitter).value(Types::String.enum("webhook"))
+        required(:emitter).value(Types::String.enum(*Mihari::Emitters::Webhook.class_keys))
         required(:url).value(:string)
         optional(:method).value(Types::HTTPRequestMethods).default("POST")
         optional(:headers).value(:hash).default({})
@@ -39,5 +44,7 @@ module Mihari
         optional(:options).hash(Options)
       end
     end
+
+    Emitter = Schemas::Emitters.get_or_composition
   end
 end

data/lib/mihari/schemas/enricher.rb

@@ -2,27 +2,34 @@
 
 module Mihari
   module Schemas
+    #
+    # Enricher schemas
+    #
     module Enrichers
+      extend Schemas::Mixins
+
       IPInfo = Dry::Schema.Params do
-        required(:enricher).value(Types::String.enum("ipinfo"))
+        required(:enricher).value(Types::String.enum(*Mihari::Enrichers::IPInfo.class_keys))
         optional(:api_key).value(:string)
         optional(:options).hash(Options)
       end
 
       Whois = Dry::Schema.Params do
-        required(:enricher).value(Types::String.enum("whois"))
+        required(:enricher).value(Types::String.enum(*Mihari::Enrichers::Whois.class_keys))
         optional(:options).hash(Options)
       end
 
       Shodan = Dry::Schema.Params do
-        required(:enricher).value(Types::String.enum("shodan"))
+        required(:enricher).value(Types::String.enum(*Mihari::Enrichers::Shodan.class_keys))
         optional(:options).hash(Options)
       end
 
       GooglePublicDNS = Dry::Schema.Params do
-        required(:enricher).value(Types::String.enum("google_public_dns"))
+        required(:enricher).value(Types::String.enum(*Mihari::Enrichers::GooglePublicDNS.class_keys))
         optional(:options).hash(Options)
       end
     end
+
+    Enricher = Schemas::Enrichers.get_or_composition
   end
 end

data/lib/mihari/schemas/macros.rb

@@ -3,6 +3,10 @@
 module Dry
   module Schema
     module Macros
+      #
+      # Macros DSL for options with default values
+      # (see https://github.com/dry-rb/dry-schema/issues/70)
+      #
       class DSL
         def default(value)
           schema_dsl.before(:rule_applier) do |result|