mihari 5.6.1 → 5.6.2

data/lib/mihari/models/reverse_dns.rb

@@ -1,29 +1,31 @@
 # frozen_string_literal: true
 
 module Mihari
-  class ReverseDnsName < ActiveRecord::Base
-    belongs_to :artifact
+  module Models
+    class ReverseDnsName < ActiveRecord::Base
+      belongs_to :artifact
 
-    class << self
-      include Dry::Monads[:result]
+      class << self
+        include Dry::Monads[:result]
 
-      #
-      # Build reverse DNS names
-      #
-      # @param [String] ip
-      # @param [Mihari::Enrichers::Shodan] enricher
-      #
-      # @return [Array<Mihari::ReverseDnsName>]
-      #
-      def build_by_ip(ip, enricher: Enrichers::Shodan.new)
-        result = enricher.query_result(ip).bind do |res|
-          if res.nil?
-            Success []
-          else
-            Success(res.hostnames.map { |name| new(name: name) })
+        #
+        # Build reverse DNS names
+        #
+        # @param [String] ip
+        # @param [Mihari::Enrichers::Shodan] enricher
+        #
+        # @return [Array<Mihari::Models::ReverseDnsName>]
+        #
+        def build_by_ip(ip, enricher: Enrichers::Shodan.new)
+          result = enricher.query_result(ip).bind do |res|
+            if res.nil?
+              Success []
+            else
+              Success(res.hostnames.map { |name| new(name: name) })
+            end
           end
+          result.value_or []
         end
-        result.value_or []
       end
     end
   end

data/lib/mihari/models/rule.rb

@@ -3,77 +3,79 @@
 require "yaml"
 
 module Mihari
-  class Rule < ActiveRecord::Base
-    has_many :alerts, dependent: :destroy
+  module Models
+    class Rule < ActiveRecord::Base
+      has_many :alerts, dependent: :destroy
 
-    def symbolized_data
-      @symbolized_data ||= data.deep_symbolize_keys
-    end
-
-    def yaml
-      data.to_yaml
-    end
-
-    def tags
-      (data["tags"] || []).map { |tag| { name: tag } }
-    end
-
-    class << self
-      #
-      # Search rules
-      #
-      # @param [Structs::Filters::Rule::SearchFilterWithPagination] filter
-      #
-      # @return [Array<Rule>]
-      #
-      def search(filter)
-        limit = filter.limit.to_i
-        raise ArgumentError, "limit should be bigger than zero" unless limit.positive?
-
-        page = filter.page.to_i
-        raise ArgumentError, "page should be bigger than zero" unless page.positive?
-
-        offset = (page - 1) * limit
-
-        relation = build_relation(filter.without_pagination)
-
-        # TODO: improve queires
-        rule_ids = relation.limit(limit).offset(offset).order(created_at: :desc).pluck(:id).uniq
-        where(id: [rule_ids]).order(created_at: :desc)
+      def symbolized_data
+        @symbolized_data ||= data.deep_symbolize_keys
       end
 
-      #
-      # Count alerts
-      #
-      # @param [Structs::Filters::Rule::SearchFilterWithPagination] filter
-      #
-      # @return [Integer]
-      #
-      def count(filter)
-        relation = build_relation(filter)
-        relation.distinct("rules.id").count
+      def yaml
+        data.to_yaml
       end
 
-      private
-
-      #
-      # @param [Structs::Filters::Rule::SearchFilter] filter
-      #
-      # @return [Mihari::Rule]
-      #
-      def build_relation(filter)
-        relation = self
-        relation = relation.includes(alerts: :tags)
-
-        relation = relation.where(alerts: { tags: { name: filter.tag_name } }) if filter.tag_name
-
-        relation = relation.where("rules.title LIKE ?", "%#{filter.title}%") if filter.title
-        relation = relation.where("rules.description LIKE ?", "%#{filter.description}%") if filter.description
-
-        relation = relation.where("rules.created_at >= ?", filter.from_at) if filter.from_at
-        relation = relation.where("rules.created_at <= ?", filter.to_at) if filter.to_at
+      def tags
+        (data["tags"] || []).map { |tag| { name: tag } }
+      end
 
-        relation
+      class << self
+        #
+        # Search rules
+        #
+        # @param [Structs::Filters::Rule::SearchFilterWithPagination] filter
+        #
+        # @return [Array<Rule>]
+        #
+        def search(filter)
+          limit = filter.limit.to_i
+          raise ArgumentError, "limit should be bigger than zero" unless limit.positive?
+
+          page = filter.page.to_i
+          raise ArgumentError, "page should be bigger than zero" unless page.positive?
+
+          offset = (page - 1) * limit
+
+          relation = build_relation(filter.without_pagination)
+
+          # TODO: improve queires
+          rule_ids = relation.limit(limit).offset(offset).order(created_at: :desc).pluck(:id).uniq
+          where(id: [rule_ids]).order(created_at: :desc)
+        end
+
+        #
+        # Count alerts
+        #
+        # @param [Structs::Filters::Rule::SearchFilterWithPagination] filter
+        #
+        # @return [Integer]
+        #
+        def count(filter)
+          relation = build_relation(filter)
+          relation.distinct("rules.id").count
+        end
+
+        private
+
+        #
+        # @param [Structs::Filters::Rule::SearchFilter] filter
+        #
+        # @return [Mihari::Models::Rule]
+        #
+        def build_relation(filter)
+          relation = self
+          relation = relation.includes(alerts: :tags)
+
+          relation = relation.where(alerts: { tags: { name: filter.tag_name } }) if filter.tag_name
+
+          relation = relation.where("rules.title LIKE ?", "%#{filter.title}%") if filter.title
+          relation = relation.where("rules.description LIKE ?", "%#{filter.description}%") if filter.description
+
+          relation = relation.where("rules.created_at >= ?", filter.from_at) if filter.from_at
+          relation = relation.where("rules.created_at <= ?", filter.to_at) if filter.to_at
+
+          relation
+        end
       end
     end
   end

data/lib/mihari/models/tag.rb

@@ -1,8 +1,10 @@
 # frozen_string_literal: true
 
 module Mihari
-  class Tag < ActiveRecord::Base
-    has_many :taggings, dependent: :destroy
-    has_many :tags, through: :taggings
+  module Models
+    class Tag < ActiveRecord::Base
+      has_many :taggings, dependent: :destroy
+      has_many :tags, through: :taggings
+    end
   end
 end

data/lib/mihari/models/tagging.rb

@@ -1,8 +1,10 @@
 # frozen_string_literal: true
 
 module Mihari
-  class Tagging < ActiveRecord::Base
-    belongs_to :alert
-    belongs_to :tag
+  module Models
+    class Tagging < ActiveRecord::Base
+      belongs_to :alert
+      belongs_to :tag
+    end
   end
 end

data/lib/mihari/models/whois.rb

@@ -1,25 +1,27 @@
 # frozen_string_literal: true
 
 module Mihari
-  class WhoisRecord < ActiveRecord::Base
-    belongs_to :artifact
+  module Models
+    class WhoisRecord < ActiveRecord::Base
+      belongs_to :artifact
 
-    @memo = {}
+      @memo = {}
 
-    class << self
-      include Dry::Monads[:result]
+      class << self
+        include Dry::Monads[:result]
 
-      #
-      # Build whois record
-      #
-      # @param [String] domain
-      # @param [Mihari::Enrichers::Whois] enricher
-      #
-      # @return [WhoisRecord, nil]
-      #
-      def build_by_domain(domain, enricher: Enrichers::Whois.new)
-        result = enricher.query_result(domain)
-        result.value_or nil
+        #
+        # Build whois record
+        #
+        # @param [String] domain
+        # @param [Mihari::Enrichers::Whois] enricher
+        #
+        # @return [WhoisRecord, nil]
+        #
+        def build_by_domain(domain, enricher: Enrichers::Whois.new)
+          result = enricher.query_result(domain)
+          result.value_or nil
+        end
       end
     end
   end

data/lib/mihari/rule.rb

@@ -0,0 +1,352 @@
+# frozen_string_literal: true
+
+module Mihari
+  class Rule
+    include Mixins::FalsePositive
+
+    # @return [Hash]
+    attr_reader :data
+
+    # @return [Array, nil]
+    attr_reader :errors
+
+    # @return [Time]
+    attr_reader :base_time
+
+    #
+    # Initialize
+    #
+    # @param [Hash] data
+    #
+    def initialize(**data)
+      @data = data.deep_symbolize_keys
+      @errors = nil
+      @base_time = Time.now.utc
+
+      validate!
+    end
+
+    #
+    # @return [Boolean]
+    #
+    def errors?
+      return false if @errors.nil?
+
+      !@errors.empty?
+    end
+
+    def [](key)
+      data key.to_sym
+    end
+
+    #
+    # @return [String]
+    #
+    def id
+      data[:id]
+    end
+
+    #
+    # @return [String]
+    #
+    def title
+      data[:title]
+    end
+
+    #
+    # @return [String]
+    #
+    def description
+      data[:description]
+    end
+
+    #
+    # @return [String]
+    #
+    def yaml
+      data.deep_stringify_keys.to_yaml
+    end
+
+    #
+    # @return [Array<Hash>]
+    #
+    def queries
+      data[:queries]
+    end
+
+    #
+    # @return [Array<String>]
+    #
+    def data_types
+      data[:data_types]
+    end
+
+    #
+    # @return [Array<String>]
+    #
+    def tags
+      data[:tags]
+    end
+
+    #
+    # @return [Array<String, RegExp>]
+    #
+    def falsepositives
+      @falsepositives ||= data[:falsepositives].map { |fp| normalize_falsepositive fp }
+    end
+
+    #
+    # @return [Integer, nil]
+    #
+    def artifact_lifetime
+      data[:artifact_lifetime] || data[:artifact_ttl]
+    end
+
+    #
+    # Returns a list of artifacts matched with queries/analyzers (with the rule ID)
+    #
+    # @return [Array<Mihari::Models::Artifact>]
+    #
+    def artifacts
+      analyzers.flat_map do |analyzer|
+        result = analyzer.result
+
+        raise result.failure if result.failure? && !analyzer.ignore_error?
+
+        artifacts = result.value!
+        artifacts.map do |artifact|
+          artifact.rule_id = id
+          artifact
+        end
+      end
+    end
+
+    #
+    # Normalize artifacts
+    # - Reject invalid artifacts (for just in case)
+    # - Select artifacts with allowed data types
+    # - Reject artifacts with false positive values
+    # - Set rule ID
+    #
+    # @return [Array<Mihari::Models::Artifact>]
+    #
+    def normalized_artifacts
+      valid_artifacts = artifacts.uniq(&:data).select(&:valid?)
+      date_type_allowed_artifacts = valid_artifacts.select { |artifact| data_types.include? artifact.data_type }
+      date_type_allowed_artifacts.reject { |artifact| falsepositive? artifact.data }
+    end
+
+    #
+    # Uniquify artifacts (assure rule level uniqueness)
+    #
+    # @return [Array<Mihari::Models::Artifact>]
+    #
+    def unique_artifacts
+      normalized_artifacts.select do |artifact|
+        artifact.unique?(base_time: base_time, artifact_lifetime: artifact_lifetime)
+      end
+    end
+
+    #
+    # Enriched artifacts
+    #
+    # @return [Array<Mihari::Models::Artifact>]
+    #
+    def enriched_artifacts
+      @enriched_artifacts ||= Parallel.map(unique_artifacts) do |artifact|
+        enrichers.each { |enricher| artifact.enrich_by_enricher enricher }
+        artifact
+      end
+    end
+
+    #
+    # Bulk emit
+    #
+    # @return [Array<Mihari::Models::Alert>]
+    #
+    def bulk_emit
+      return [] if enriched_artifacts.empty?
+
+      # NOTE: separate parallel execution and logging
+      #       because the logger does not work along with Parallel
+      results = Parallel.map(emitters) { |emitter| emitter.emit_result(enriched_artifacts) }
+      results.zip(emitters).map do |result_and_emitter|
+        result, emitter = result_and_emitter
+        Mihari.logger.info "Emission by #{emitter.class} is failed: #{result.failure}" if result.failure?
+        Mihari.logger.info "Emission by #{emitter.class} is succeeded" if result.success?
+        result.value_or nil
+      end.compact
+    end
+
+    #
+    # Set artifacts & run emitters in parallel
+    #
+    # @return [Mihari::Models::Alert, nil]
+    #
+    def run
+      # Validate analyzers & emitters before using them
+      analyzers
+      emitters
+
+      alert_or_something = bulk_emit
+      # Return Mihari::Models::Alert created by the database emitter
+      alert_or_something.find { |res| res.is_a?(Mihari::Models::Alert) }
+    end
+
+    #
+    # @return [Mihari::Models::Rule]
+    #
+    def model
+      rule = Mihari::Models::Rule.find(id)
+
+      rule.title = title
+      rule.description = description
+      rule.data = data
+
+      rule
+    rescue ActiveRecord::RecordNotFound
+      Mihari::Models::Rule.new(
+        id: id,
+        title: title,
+        description: description,
+        data: data
+      )
+    end
+
+    class << self
+      #
+      # Load rule from YAML string
+      #
+      # @param [String] yaml
+      #
+      # @return [Mihari::Services::Rule]
+      #
+      def from_yaml(yaml)
+        data = YAML.safe_load(ERB.new(yaml).result, permitted_classes: [Date, Symbol])
+        new(**data)
+      end
+
+      #
+      # @param [Mihari::Models::Rule] model
+      #
+      # @return [Mihari::Services::Rule]
+      #
+      def from_model(model)
+        new(**model.data)
+      end
+    end
+
+    private
+
+    #
+    # Check whether a value is a falsepositive value or not
+    #
+    # @return [Boolean]
+    #
+    def falsepositive?(value)
+      return true if falsepositives.include?(value)
+
+      regexps = falsepositives.select { |fp| fp.is_a?(Regexp) }
+      regexps.any? { |fp| fp.match?(value) }
+    end
+
+    #
+    # Get analyzer class
+    #
+    # @param [String] key
+    #
+    # @return [Class<Mihari::Analyzers::Base>] analyzer class
+    #
+    def get_analyzer_class(key)
+      raise ArgumentError, "#{key} is not supported" unless Mihari.analyzer_to_class.key?(key)
+
+      Mihari.analyzer_to_class[key]
+    end
+
+    #
+    # @return [Array<Mihari::Analyzers::Base>]
+    #
+    def analyzers
+      @analyzers ||= queries.map do |query_params|
+        analyzer_name = query_params[:analyzer]
+        klass = get_analyzer_class(analyzer_name)
+        klass.from_query(query_params)
+      end.map do |analyzer|
+        analyzer.validate_configuration!
+        analyzer
+      end
+    end
+
+    #
+    # Get emitter class
+    #
+    # @param [String] key
+    #
+    # @return [Class<Mihari::Emitters::Base>] emitter class
+    #
+    def get_emitter_class(key)
+      raise ArgumentError, "#{key} is not supported" unless Mihari.emitter_to_class.key?(key)
+
+      Mihari.emitter_to_class[key]
+    end
+
+    #
+    # @return [Array<Mihari::Emitters::Base>]
+    #
+    def emitters
+      @emitters ||= data[:emitters].map(&:deep_dup).map do |params|
+        name = params[:emitter]
+        options = params[:options]
+
+        %i[emitter options].each { |key| params.delete key }
+
+        klass = get_emitter_class(name)
+        klass.new(rule: self, options: options, **params)
+      end.map do |emitter|
+        emitter.validate_configuration!
+        emitter
+      end
+    end
+
+    #
+    # Get enricher class
+    #
+    # @param [String] key
+    #
+    # @return [Class<Mihari::Enrichers::Base>] enricher class
+    #
+    def get_enricher_class(key)
+      raise ArgumentError, "#{key} is not supported" unless Mihari.enricher_to_class.key?(key)
+
+      Mihari.enricher_to_class[key]
+    end
+
+    #
+    # @return [Array<Mihari::Enrichers::Base>] enrichers
+    #
+    def enrichers
+      @enrichers ||= data[:enrichers].map(&:deep_dup).map do |params|
+        name = params[:enricher]
+        options = params[:options]
+
+        %i[enricher options].each { |key| params.delete key }
+
+        klass = get_enricher_class(name)
+        klass.new(options: options, **params)
+      end
+    end
+
+    #
+    # Validate the data format
+    #
+    def validate!
+      contract = Schemas::RuleContract.new
+      result = contract.call(data)
+
+      @data = result.to_h
+      @errors = result.errors
+
+      raise ValidationError.new("Validation failed", errors) if errors?
+    end
+  end
+end