mihari 4.12.0 → 5.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 16412e44a6aa5eb9fb7022aa531b950249df76f074136e71c6621e5f2d3c7d44
-  data.tar.gz: 403f780911934e891ef072c87a3a6ae48d31f1854c8806d4c9ece884faf9ec62
+  metadata.gz: b7944fcbb2ef6b1ff7fccbe5c8158bd21a186b05e8fae70a6700256dce10adbb
+  data.tar.gz: 36605153506952b323be6e3a7646fd4446f280943fc6c587d7885e8eec413c33
 SHA512:
-  metadata.gz: edde8ae3fb93a7e3719788c6727782f7f9a9e2ae53eaa9d804342e545e0ca5e9ec6bb96d633f9c26a33f98e4addaf8693b6da37de8930f8770847a531665baa0
-  data.tar.gz: afda1bef59be058cbb5972d771bf7615820cc4588cae6cc53c2f2d39333e02edce9a3165dee30caf0996195ab09565392316d116040ce5cc93dbdac28b04c239
+  metadata.gz: ba53c1fb987ffd933017ccc64a3a72adc032de81a377e34684c4927304d295d85531ad11e796abd3c17489411fcf15eedd494d7ff7a8b7d0c53fdeb511eb5a8d
+  data.tar.gz: 6dff687f32dfef7f0cc19b76cba77a6233ebbefa9b90f522f37092468370eb5abb0e7f185d86f74077944cb0c76609b01390e7878ca494a82419ac44e59d93e5

data/Steepfile

@@ -1,5 +1,4 @@
 target :lib do
-  signature "sig"
   check "lib"
 
   repo_path "vendor/rbs/gem_rbs_collection/gems"

data/lib/mihari/analyzers/base.rb

@@ -5,18 +5,19 @@ module Mihari
     class Base
       extend Dry::Initializer
 
+      option :rule, default: proc {}
+
       include Mixins::AutonomousSystem
       include Mixins::Configurable
-      include Mixins::Database
       include Mixins::Retriable
 
-      attr_accessor :ignore_old_artifacts, :ignore_threshold
+      # @return [Mihari::Structs::Rule, nil]
+      attr_reader :rule
 
       def initialize(*args, **kwargs)
-        super
+        super(*args, **kwargs)
 
-        @ignore_old_artifacts = false
-        @ignore_threshold = 0
+        @base_time = Time.now.utc
       end
 
       # @return [Array<String>, Array<Mihari::Artifact>]
@@ -24,26 +25,11 @@ module Mihari
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
       end
 
-      # @return [String]
-      def title
-        self.class.to_s.split("::").last.to_s
-      end
-
-      # @return [String]
-      def description
-        raise NotImplementedError, "You must implement #{self.class}##{__method__}"
-      end
-
       # @return [String]
       def source
         self.class.to_s.split("::").last.to_s
       end
 
-      # @return [Array<String>]
-      def tags
-        []
-      end
-
       #
       # Set artifacts & run emitters in parallel
       #
@@ -55,16 +41,14 @@ module Mihari
           raise ConfigurationError, "#{class_name} is not configured correctly"
         end
 
-        with_db_connection do
-          set_enriched_artifacts
-
-          responses = Parallel.map(valid_emitters) do |emitter|
-            run_emitter emitter
-          end
+        set_enriched_artifacts
 
-          # returns Mihari::Alert created by the database emitter
-          responses.find { |res| res.is_a?(Mihari::Alert) }
+        responses = Parallel.map(valid_emitters) do |emitter|
+          run_emitter emitter
         end
+
+        # returns Mihari::Alert created by the database emitter
+        responses.find { |res| res.is_a?(Mihari::Alert) }
       end
 
       #
@@ -77,13 +61,7 @@ module Mihari
       def run_emitter(emitter)
         return if enriched_artifacts.empty?
 
-        alert_or_something = emitter.run(
-          title: title,
-          description: description,
-          artifacts: enriched_artifacts,
-          source: source,
-          tags: tags
-        )
+        alert_or_something = emitter.run(artifacts: enriched_artifacts, rule: rule)
 
         Mihari.logger.info "Emission by #{emitter.class} is succedded"
 
@@ -112,7 +90,10 @@ module Mihari
           # No need to set data_type manually
           # It is set automatically in #initialize
           artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact, source: source)
-        end.select(&:valid?).uniq(&:data)
+        end.select(&:valid?).uniq(&:data).map do |artifact|
+          artifact.rule_id = rule&.id
+          artifact
+        end
       end
 
       private
@@ -124,7 +105,7 @@ module Mihari
       #
       def unique_artifacts
         @unique_artifacts ||= normalized_artifacts.select do |artifact|
-          artifact.unique?(ignore_old_artifacts: ignore_old_artifacts, ignore_threshold: ignore_threshold)
+          artifact.unique?(base_time: @base_time, artifact_lifetime: rule&.artifact_lifetime)
         end
       end
 

data/lib/mihari/analyzers/rule.rb

@@ -29,38 +29,21 @@ module Mihari
 
     EMITTER_TO_CLASS = {
       "database" => Emitters::Database,
-      "http" => Emitters::HTTP,
       "misp" => Emitters::MISP,
       "slack" => Emitters::Slack,
       "the_hive" => Emitters::TheHive,
       "webhook" => Emitters::Webhook
     }.freeze
 
-    class Rule < Base
-      include Mixins::DisallowedDataValue
-
-      option :title
-      option :description
-      option :queries
-
-      option :id, default: proc { "" }
-      option :tags, default: proc { [] }
-      option :allowed_data_types, default: proc { ALLOWED_DATA_TYPES }
-      option :disallowed_data_values, default: proc { [] }
+    # @return [Mihari::Structs::Rule]
+    attr_reader :rule
 
-      option :emitters, optional: true
-      option :enrichers, optional: true
-
-      attr_reader :source
+    class Rule < Base
+      include Mixins::FalsePositive
 
       def initialize(**kwargs)
         super(**kwargs)
 
-        @source = id
-
-        @emitters = emitters || DEFAULT_EMITTERS
-        @enrichers = enrichers || DEFAULT_ENRICHERS
-
         validate_analyzer_configurations
       end
 
@@ -72,7 +55,7 @@ module Mihari
       def artifacts
         artifacts = []
 
-        queries.each do |original_params|
+        rule.queries.each do |original_params|
           parmas = original_params.deep_dup
 
           analyzer_name = parmas[:analyzer]
@@ -83,8 +66,12 @@ module Mihari
           # set interval in the top level
           options = parmas[:options] || {}
           interval = options[:interval]
+
           parmas[:interval] = interval if interval
 
+          # set rule
+          parmas[:rule] = rule
+
           analyzer = klass.new(query, **parmas)
 
           # Use #normalized_artifacts method to get atrifacts as Array<Mihari::Artifact>
@@ -106,9 +93,9 @@ module Mihari
       #
       def normalized_artifacts
         @normalized_artifacts ||= artifacts.uniq(&:data).select(&:valid?).select do |artifact|
-          allowed_data_types.include? artifact.data_type
+          rule.data_types.include? artifact.data_type
         end.reject do |artifact|
-          disallowed_data_value? artifact.data
+          falsepositive? artifact.data
         end
       end
 
@@ -119,7 +106,7 @@ module Mihari
       #
       def enriched_artifacts
         @enriched_artifacts ||= Parallel.map(unique_artifacts) do |artifact|
-          enrichers.each do |enricher|
+          rule.enrichers.each do |enricher|
             artifact.enrich_by_enricher(enricher[:enricher])
           end
 
@@ -132,22 +119,22 @@ module Mihari
       #
       # @return [Array<Regexp, String>]
       #
-      def normalized_disallowed_data_values
-        @normalized_disallowed_data_values ||= disallowed_data_values.map { |v| normalize_disallowed_data_value v }
+      def normalized_falsepositives
+        @normalized_falsepositives ||= rule.falsepositives.map { |v| normalize_falsepositive v }
       end
 
       #
-      # Check whether a value is a disallowed data value or not
+      # Check whether a value is a falsepositive value or not
       #
       # @return [Boolean]
       #
-      def disallowed_data_value?(value)
-        return true if normalized_disallowed_data_values.include?(value)
+      def falsepositive?(value)
+        return true if normalized_falsepositives.include?(value)
 
-        normalized_disallowed_data_values.select do |disallowed_data_value|
-          disallowed_data_value.is_a?(Regexp)
-        end.any? do |disallowed_data_value|
-          disallowed_data_value.match?(value)
+        normalized_falsepositives.select do |falsepositive|
+          falsepositive.is_a?(Regexp)
+        end.any? do |falseposistive|
+          falseposistive.match?(value)
         end
       end
 
@@ -168,7 +155,7 @@ module Mihari
       end
 
       def valid_emitters
-        @valid_emitters ||= emitters.filter_map do |original_params|
+        @valid_emitters ||= rule.emitters.filter_map do |original_params|
           params = original_params.deep_dup
 
           name = params[:emitter]
@@ -199,7 +186,7 @@ module Mihari
       # Validate configuration of analyzers
       #
       def validate_analyzer_configurations
-        queries.each do |params|
+        rule.queries.each do |params|
           analyzer_name = params[:analyzer]
           klass = get_analyzer_class(analyzer_name)
 

data/lib/mihari/cli/main.rb

@@ -3,28 +3,23 @@
 require "thor"
 
 # Commands
-require "mihari/commands/search"
+require "mihari/commands/initializer"
+require "mihari/commands/searcher"
+require "mihari/commands/validator"
 require "mihari/commands/version"
 require "mihari/commands/web"
 
 # CLIs
 require "mihari/cli/base"
 
-require "mihari/cli/init"
-require "mihari/cli/validator"
-
 module Mihari
   module CLI
     class Main < Base
-      include Mihari::Commands::Search
+      include Mihari::Commands::Searcher
       include Mihari::Commands::Version
       include Mihari::Commands::Web
-
-      desc "init", "Sub commands to initialize a rule"
-      subcommand "init", Initialization
-
-      desc "validate", "Sub commands to validate format of a rule"
-      subcommand "validate", Validator
+      include Mihari::Commands::Validator
+      include Mihari::Commands::Initializer
     end
   end
 end

data/lib/mihari/commands/initializer.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+module Mihari
+  module Commands
+    module Initializer
+      def self.included(thor)
+        thor.class_eval do
+          desc "init", "Initialize a new rule"
+          method_option :path, type: :string, default: "./rule.yml"
+          def init
+            path = options["path"]
+
+            warning = "#{path} exists. Do you want to overwrite it? (y/n)"
+            return if Pathname(path).exist? && !(yes? warning)
+
+            initialize_rule path
+
+            Mihari.logger.info "A new rule is initialized as #{path}."
+          end
+
+          no_commands do
+            #
+            # @return [Mihari::Structs::Rule]
+            #
+            def rule_template
+              Structs::Rule.from_path File.expand_path("../templates/rule.yml.erb", __dir__)
+            end
+
+            #
+            # Create a new rule
+            #
+            # @param [String] path
+            # @param [Dry::Files] files
+            #
+            # @return [nil]
+            #
+            def initialize_rule(path, files = Dry::Files.new)
+              files.write(path, rule_template.yaml)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/commands/searcher.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Commands
+    module Searcher
+      include Mixins::Database
+      include Mixins::ErrorNotification
+
+      def self.included(thor)
+        thor.class_eval do
+          desc "search [PATH]", "Search by a rule"
+          method_option :force_overwrite, type: :boolean, aliases: "-f", desc: "Force an overwrite the rule"
+          def search(path_or_id)
+            with_db_connection do
+              rule = Structs::Rule.from_path_or_id path_or_id
+
+              # validate
+              begin
+                rule.validate!
+              rescue RuleValidationError
+                return
+              end
+
+              force_overwrite = options["force_overwrite"] || false
+
+              begin
+                rule_model = Mihari::Rule.find(rule.id)
+                has_change = rule_model.data != rule.data.deep_stringify_keys
+                has_change_and_not_force_overwrite = has_change & !force_overwrite
+
+                if has_change_and_not_force_overwrite && !yes?("This operation will overwrite the rule in the database (Rule ID: #{rule.id}). Are you sure you want to update the rule? (y/n)")
+                  return
+                end
+
+                # update the rule
+                rule.model.save
+              rescue ActiveRecord::RecordNotFound
+                # create a new rule
+                rule.model.save
+              end
+
+              with_error_notification do
+                alert = rule.analyzer.run
+                if alert
+                  data = Mihari::Entities::Alert.represent(alert)
+                  puts JSON.pretty_generate(data.as_json)
+                else
+                  Mihari.logger.info "There is no new alert created in the database"
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/commands/validator.rb

@@ -5,7 +5,7 @@ module Mihari
     module Validator
       def self.included(thor)
         thor.class_eval do
-          desc "rule [PATH]", "Validate rule file format"
+          desc "validate [PATH]", "Validate a rule file"
           #
           # Validate format of a rule
           #
@@ -13,7 +13,7 @@ module Mihari
           #
           # @return [nil]
           #
-          def rule(path)
+          def validate(path)
             rule = Structs::Rule.from_path_or_id(path)
 
             begin

data/lib/mihari/constants.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
 module Mihari
-  ALLOWED_DATA_TYPES = ["hash", "ip", "domain", "url", "mail"].freeze
+  DEFAULT_DATA_TYPES = %w[hash ip domain url mail].freeze
 
-  DEFAULT_EMITTERS = ["database", "misp", "slack", "the_hive", "webhook"].map { |name| { emitter: name } }.freeze
+  DEFAULT_EMITTERS = %w[database misp slack the_hive].map { |name| { emitter: name } }.freeze
 
-  DEFAULT_ENRICHERS = ["whois", "ipinfo", "shodan", "google_public_dns"].map { |name| { enricher: name } }.freeze
+  DEFAULT_ENRICHERS = %w[whois ipinfo shodan google_public_dns].map { |name| { enricher: name } }.freeze
 end

data/lib/mihari/database.rb

@@ -17,53 +17,44 @@ def development_env?
   env == "development"
 end
 
-class InitialSchema < ActiveRecord::Migration[7.0]
+class V5Schema < ActiveRecord::Migration[7.0]
   def change
-    create_table :tags, if_not_exists: true do |t|
-      t.string :name, null: false
+    create_table :rules, id: :string, if_not_exists: true do |t|
+      t.string :title, null: false
+      t.string :description, null: false
+      t.json :data, null: false
+      t.timestamps
     end
 
     create_table :alerts, if_not_exists: true do |t|
-      t.string :title, null: false
-      t.string :description, null: true
-      t.string :source, null: false
       t.timestamps
+
+      t.belongs_to :rule, foreign_key: true, type: :string, null: false
     end
 
     create_table :artifacts, if_not_exists: true do |t|
       t.string :data, null: false
       t.string :data_type, null: false
-      t.belongs_to :alert, foreign_key: true
+      t.string :source
+      t.json :metadata
       t.timestamps
-    end
 
-    create_table :taggings, if_not_exists: true do |t|
-      t.integer :tag_id
-      t.integer :alert_id
+      t.belongs_to :alert, foreign_key: true, null: false
     end
 
-    add_index :taggings, :tag_id, if_not_exists: true
-    add_index :taggings, [:tag_id, :alert_id], unique: true, if_not_exists: true
-  end
-end
-
-class AddeSourceToArtifactSchema < ActiveRecord::Migration[7.0]
-  def change
-    add_column :artifacts, :source, :string, if_not_exists: true
-  end
-end
-
-class EnrichmentsSchema < ActiveRecord::Migration[7.0]
-  def change
     create_table :autonomous_systems, if_not_exists: true do |t|
       t.integer :asn, null: false
-      t.belongs_to :artifact, foreign_key: true
+      t.datetime :created_at
+
+      t.belongs_to :artifact, foreign_key: true, null: false
     end
 
     create_table :geolocations, if_not_exists: true do |t|
       t.string :country, null: false
       t.string :country_code, null: false
-      t.belongs_to :artifact, foreign_key: true
+      t.datetime :created_at
+
+      t.belongs_to :artifact, foreign_key: true, null: false
     end
 
     create_table :whois_records, if_not_exists: true do |t|
@@ -73,73 +64,59 @@ class EnrichmentsSchema < ActiveRecord::Migration[7.0]
       t.date :expires_on
       t.json :registrar
       t.json :contacts
-      t.belongs_to :artifact, foreign_key: true
+      t.datetime :created_at
+
+      t.belongs_to :artifact, foreign_key: true, null: false
     end
 
     create_table :dns_records, if_not_exists: true do |t|
       t.string :resource, null: false
       t.string :value, null: false
-      t.belongs_to :artifact, foreign_key: true
+      t.datetime :created_at
+
+      t.belongs_to :artifact, foreign_key: true, null: false
     end
 
     create_table :reverse_dns_names, if_not_exists: true do |t|
       t.string :name, null: false
-      t.belongs_to :artifact, foreign_key: true
-    end
-  end
-end
+      t.datetime :created_at
 
-class EnrichmentCreatedAtSchema < ActiveRecord::Migration[7.0]
-  def change
-    # Add created_at column because now it is able to enrich an atrifact after the creation
-    add_column :autonomous_systems, :created_at, :datetime, if_not_exists: true
-    add_column :geolocations, :created_at, :datetime, if_not_exists: true
-    add_column :whois_records, :created_at, :datetime, if_not_exists: true
-    add_column :dns_records, :created_at, :datetime, if_not_exists: true
-    add_column :reverse_dns_names, :created_at, :datetime, if_not_exists: true
-  end
-end
-
-class RuleSchema < ActiveRecord::Migration[7.0]
-  def change
-    create_table :rules, id: :string, if_not_exists: true do |t|
-      t.string :title, null: false
-      t.string :description, null: false
-      t.json :data, null: false
-      t.timestamps
+      t.belongs_to :artifact, foreign_key: true, null: false
     end
-  end
-end
-
-class AddeMetadataToArtifactSchema < ActiveRecord::Migration[7.0]
-  def change
-    add_column :artifacts, :metadata, :json, if_not_exists: true
-  end
-end
 
-class AddYAMLToRulesSchema < ActiveRecord::Migration[7.0]
-  def change
-    add_column :rules, :yaml, :text, if_not_exists: true
-  end
-end
-
-class EnrichmentsV45Schema < ActiveRecord::Migration[7.0]
-  def change
     create_table :cpes, if_not_exists: true do |t|
       t.string :cpe, null: false
-      t.belongs_to :artifact, foreign_key: true
+      t.datetime :created_at
+
+      t.belongs_to :artifact, foreign_key: true, null: false
     end
 
     create_table :ports, if_not_exists: true do |t|
       t.integer :port, null: false
-      t.belongs_to :artifact, foreign_key: true
+      t.datetime :created_at
+
+      t.belongs_to :artifact, foreign_key: true, null: false
+    end
+
+    create_table :tags, if_not_exists: true do |t|
+      t.string :name, null: false
+      t.datetime :created_at
     end
+
+    create_table :taggings, if_not_exists: true do |t|
+      t.integer :tag_id
+      t.integer :alert_id
+      t.datetime :created_at
+    end
+
+    add_index :taggings, :tag_id, if_not_exists: true
+    add_index :taggings, %i[tag_id alert_id], unique: true, if_not_exists: true
   end
 end
 
 def adapter
-  return "postgresql" if Mihari.config.database.start_with?("postgresql://", "postgres://")
-  return "mysql2" if Mihari.config.database.start_with?("mysql2://")
+  return "postgresql" if %w[postgresql postgres].include?(Mihari.config.database_url.scheme)
+  return "mysql2" if Mihari.config.database_url.scheme == "mysql2"
 
   "sqlite3"
 end
@@ -157,19 +134,7 @@ module Mihari
       def migrate(direction)
         ActiveRecord::Migration.verbose = false
 
-        [
-          InitialSchema,
-          AddeSourceToArtifactSchema,
-          EnrichmentsSchema,
-          EnrichmentCreatedAtSchema,
-          # v4.0
-          RuleSchema,
-          AddeMetadataToArtifactSchema,
-          # v4.4
-          AddYAMLToRulesSchema,
-          # v4.5
-          EnrichmentsV45Schema
-        ].each { |schema| schema.migrate direction }
+        [V5Schema].each { |schema| schema.migrate direction }
       end
       memoize :migrate unless test_env?
 
@@ -181,19 +146,19 @@ module Mihari
 
         case adapter
         when "postgresql", "mysql2"
-          ActiveRecord::Base.establish_connection(Mihari.config.database)
+          ActiveRecord::Base.establish_connection(Mihari.config.database_url.to_s)
         else
           ActiveRecord::Base.establish_connection(
             adapter: adapter,
-            database: Mihari.config.database
+            database: Mihari.config.database_url.path[1..]
           )
         end
 
         ActiveRecord::Base.logger = Logger.new($stdout) if development_env?
 
         migrate :up
-      rescue StandardError
-        # Do nothing
+      rescue StandardError => e
+        Mihari.logger.error e
       end
 
       #