mihari 4.12.0 → 5.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 16412e44a6aa5eb9fb7022aa531b950249df76f074136e71c6621e5f2d3c7d44
-  data.tar.gz: 403f780911934e891ef072c87a3a6ae48d31f1854c8806d4c9ece884faf9ec62
+  metadata.gz: 41784cbe3811a8f5f2a6a4de663ceaa03e2635c329c455b523e5fc55fda0c9e7
+  data.tar.gz: '052987fcb8805a8f71a0716d5e0970b0c4c7451059dded4b5149679ccda6c999'
 SHA512:
-  metadata.gz: edde8ae3fb93a7e3719788c6727782f7f9a9e2ae53eaa9d804342e545e0ca5e9ec6bb96d633f9c26a33f98e4addaf8693b6da37de8930f8770847a531665baa0
-  data.tar.gz: afda1bef59be058cbb5972d771bf7615820cc4588cae6cc53c2f2d39333e02edce9a3165dee30caf0996195ab09565392316d116040ce5cc93dbdac28b04c239
+  metadata.gz: f16447e83adb4630baf9587eecea0dbb6f580de23894d70fa2c1729483faa6dc1c86fe5de2bd782100f8cba89e2a50e7ce2f0ab675de822fe5fb767fb2c6f45f
+  data.tar.gz: af70fcc510ed4fd6a434481a8372c56d20a54b605ee0b49425a1315531d2261db6df9588f1961ba34796728ff40f0ce188ad58e6c542f4c3644a8c8251b37eb5

data/Steepfile

@@ -1,5 +1,4 @@
 target :lib do
-  signature "sig"
   check "lib"
 
   repo_path "vendor/rbs/gem_rbs_collection/gems"

data/lib/mihari/analyzers/base.rb

@@ -5,18 +5,20 @@ module Mihari
     class Base
       extend Dry::Initializer
 
+      option :rule, default: proc {}
+
       include Mixins::AutonomousSystem
       include Mixins::Configurable
       include Mixins::Database
       include Mixins::Retriable
 
-      attr_accessor :ignore_old_artifacts, :ignore_threshold
+      # @return [Mihari::Structs::Rule, nil]
+      attr_reader :rule
 
       def initialize(*args, **kwargs)
-        super
+        super(*args, **kwargs)
 
-        @ignore_old_artifacts = false
-        @ignore_threshold = 0
+        @base_time = Time.now.utc
       end
 
       # @return [Array<String>, Array<Mihari::Artifact>]
@@ -24,26 +26,11 @@ module Mihari
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
       end
 
-      # @return [String]
-      def title
-        self.class.to_s.split("::").last.to_s
-      end
-
-      # @return [String]
-      def description
-        raise NotImplementedError, "You must implement #{self.class}##{__method__}"
-      end
-
       # @return [String]
       def source
         self.class.to_s.split("::").last.to_s
       end
 
-      # @return [Array<String>]
-      def tags
-        []
-      end
-
       #
       # Set artifacts & run emitters in parallel
       #
@@ -77,13 +64,7 @@ module Mihari
       def run_emitter(emitter)
         return if enriched_artifacts.empty?
 
-        alert_or_something = emitter.run(
-          title: title,
-          description: description,
-          artifacts: enriched_artifacts,
-          source: source,
-          tags: tags
-        )
+        alert_or_something = emitter.run(artifacts: enriched_artifacts, rule: rule)
 
         Mihari.logger.info "Emission by #{emitter.class} is succedded"
 
@@ -112,7 +93,10 @@ module Mihari
           # No need to set data_type manually
           # It is set automatically in #initialize
           artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact, source: source)
-        end.select(&:valid?).uniq(&:data)
+        end.select(&:valid?).uniq(&:data).map do |artifact|
+          artifact.rule_id = rule&.id
+          artifact
+        end
       end
 
       private
@@ -124,7 +108,7 @@ module Mihari
       #
       def unique_artifacts
         @unique_artifacts ||= normalized_artifacts.select do |artifact|
-          artifact.unique?(ignore_old_artifacts: ignore_old_artifacts, ignore_threshold: ignore_threshold)
+          artifact.unique?(base_time: @base_time, artifact_lifetime: rule&.artifact_lifetime)
         end
       end
 

data/lib/mihari/analyzers/rule.rb

@@ -29,38 +29,21 @@ module Mihari
 
     EMITTER_TO_CLASS = {
       "database" => Emitters::Database,
-      "http" => Emitters::HTTP,
       "misp" => Emitters::MISP,
       "slack" => Emitters::Slack,
       "the_hive" => Emitters::TheHive,
       "webhook" => Emitters::Webhook
     }.freeze
 
-    class Rule < Base
-      include Mixins::DisallowedDataValue
-
-      option :title
-      option :description
-      option :queries
-
-      option :id, default: proc { "" }
-      option :tags, default: proc { [] }
-      option :allowed_data_types, default: proc { ALLOWED_DATA_TYPES }
-      option :disallowed_data_values, default: proc { [] }
+    # @return [Mihari::Structs::Rule]
+    attr_reader :rule
 
-      option :emitters, optional: true
-      option :enrichers, optional: true
-
-      attr_reader :source
+    class Rule < Base
+      include Mixins::FalsePositive
 
       def initialize(**kwargs)
         super(**kwargs)
 
-        @source = id
-
-        @emitters = emitters || DEFAULT_EMITTERS
-        @enrichers = enrichers || DEFAULT_ENRICHERS
-
         validate_analyzer_configurations
       end
 
@@ -72,7 +55,7 @@ module Mihari
       def artifacts
         artifacts = []
 
-        queries.each do |original_params|
+        rule.queries.each do |original_params|
           parmas = original_params.deep_dup
 
           analyzer_name = parmas[:analyzer]
@@ -83,8 +66,12 @@ module Mihari
           # set interval in the top level
           options = parmas[:options] || {}
           interval = options[:interval]
+
           parmas[:interval] = interval if interval
 
+          # set rule
+          parmas[:rule] = rule
+
           analyzer = klass.new(query, **parmas)
 
           # Use #normalized_artifacts method to get atrifacts as Array<Mihari::Artifact>
@@ -106,9 +93,9 @@ module Mihari
       #
       def normalized_artifacts
         @normalized_artifacts ||= artifacts.uniq(&:data).select(&:valid?).select do |artifact|
-          allowed_data_types.include? artifact.data_type
+          rule.data_types.include? artifact.data_type
         end.reject do |artifact|
-          disallowed_data_value? artifact.data
+          falsepositive? artifact.data
         end
       end
 
@@ -119,7 +106,7 @@ module Mihari
       #
       def enriched_artifacts
         @enriched_artifacts ||= Parallel.map(unique_artifacts) do |artifact|
-          enrichers.each do |enricher|
+          rule.enrichers.each do |enricher|
             artifact.enrich_by_enricher(enricher[:enricher])
           end
 
@@ -132,22 +119,22 @@ module Mihari
       #
       # @return [Array<Regexp, String>]
       #
-      def normalized_disallowed_data_values
-        @normalized_disallowed_data_values ||= disallowed_data_values.map { |v| normalize_disallowed_data_value v }
+      def normalized_falsepositives
+        @normalized_falsepositives ||= rule.falsepositives.map { |v| normalize_falsepositive v }
       end
 
       #
-      # Check whether a value is a disallowed data value or not
+      # Check whether a value is a falsepositive value or not
       #
       # @return [Boolean]
       #
-      def disallowed_data_value?(value)
-        return true if normalized_disallowed_data_values.include?(value)
+      def falsepositive?(value)
+        return true if normalized_falsepositives.include?(value)
 
-        normalized_disallowed_data_values.select do |disallowed_data_value|
-          disallowed_data_value.is_a?(Regexp)
-        end.any? do |disallowed_data_value|
-          disallowed_data_value.match?(value)
+        normalized_falsepositives.select do |falsepositive|
+          falsepositive.is_a?(Regexp)
+        end.any? do |falseposistive|
+          falseposistive.match?(value)
         end
       end
 
@@ -168,7 +155,7 @@ module Mihari
       end
 
       def valid_emitters
-        @valid_emitters ||= emitters.filter_map do |original_params|
+        @valid_emitters ||= rule.emitters.filter_map do |original_params|
           params = original_params.deep_dup
 
           name = params[:emitter]
@@ -199,7 +186,7 @@ module Mihari
       # Validate configuration of analyzers
       #
       def validate_analyzer_configurations
-        queries.each do |params|
+        rule.queries.each do |params|
           analyzer_name = params[:analyzer]
           klass = get_analyzer_class(analyzer_name)
 

data/lib/mihari/cli/main.rb

@@ -3,28 +3,23 @@
 require "thor"
 
 # Commands
-require "mihari/commands/search"
+require "mihari/commands/initializer"
+require "mihari/commands/searcher"
+require "mihari/commands/validator"
 require "mihari/commands/version"
 require "mihari/commands/web"
 
 # CLIs
 require "mihari/cli/base"
 
-require "mihari/cli/init"
-require "mihari/cli/validator"
-
 module Mihari
   module CLI
     class Main < Base
-      include Mihari::Commands::Search
+      include Mihari::Commands::Searcher
       include Mihari::Commands::Version
       include Mihari::Commands::Web
-
-      desc "init", "Sub commands to initialize a rule"
-      subcommand "init", Initialization
-
-      desc "validate", "Sub commands to validate format of a rule"
-      subcommand "validate", Validator
+      include Mihari::Commands::Validator
+      include Mihari::Commands::Initializer
     end
   end
 end

data/lib/mihari/commands/initializer.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+module Mihari
+  module Commands
+    module Initializer
+      def self.included(thor)
+        thor.class_eval do
+          desc "init", "Initialize a new rule"
+          method_option :path, type: :string, default: "./rule.yml"
+          def init
+            path = options["path"]
+
+            warning = "#{path} exists. Do you want to overwrite it? (y/n)"
+            return if Pathname(path).exist? && !(yes? warning)
+
+            initialize_rule path
+
+            Mihari.logger.info "A new rule is initialized as #{path}."
+          end
+
+          no_commands do
+            #
+            # @return [Mihari::Structs::Rule]
+            #
+            def rule_template
+              Structs::Rule.from_path File.expand_path("../templates/rule.yml.erb", __dir__)
+            end
+
+            #
+            # Create a new rule
+            #
+            # @param [String] path
+            # @param [Dry::Files] files
+            #
+            # @return [nil]
+            #
+            def initialize_rule(path, files = Dry::Files.new)
+              files.write(path, rule_template.yaml)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/commands/{search.rb → searcher.rb}

@@ -2,15 +2,15 @@
 
 module Mihari
   module Commands
-    module Search
+    module Searcher
       include Mixins::Database
       include Mixins::ErrorNotification
 
       def self.included(thor)
         thor.class_eval do
-          desc "search [RULE]", "Search by a rule"
+          desc "search [PATH]", "Search by a rule"
           method_option :yes, type: :boolean, aliases: "-y", desc: "yes to overwrite the rule in the database"
-          def search_by_rule(path_or_id)
+          def search(path_or_id)
             rule = Structs::Rule.from_path_or_id path_or_id
 
             # validate
@@ -21,38 +21,27 @@ module Mihari
             end
 
             # check update
-            id = rule.id
             yes = options["yes"] || false
             unless yes
               with_db_connection do
-                rule_ = Mihari::Rule.find(id)
-                next if rule.yaml == rule_.yaml
-                unless yes?("This operation will overwrite the rule in the database (Rule ID: #{id}). Are you sure you want to update the rule? (yes/no)")
+                next if Mihari::Rule.find(rule.id).data == rule.data.deep_stringify_keys
+                unless yes?("This operation will overwrite the rule in the database (Rule ID: #{rule.id}). Are you sure you want to update the rule? (y/n)")
                   return
                 end
               rescue ActiveRecord::RecordNotFound
                 next
               end
             end
-
-            analyzer = rule.to_analyzer
+            # update rule model
+            rule.model.save
 
             with_error_notification do
-              alert = analyzer.run
-
+              alert = rule.analyzer.run
               if alert
                 data = Mihari::Entities::Alert.represent(alert)
                 puts JSON.pretty_generate(data.as_json)
               else
-                Mihari.logger.info "No new alert created in the database"
-              end
-
-              # record a rule
-              with_db_connection do
-                model = rule.to_model
-                model.save
-              rescue ActiveRecord::RecordNotUnique
-                nil
+                Mihari.logger.info "There is no new alert created in the database"
               end
             end
           end

data/lib/mihari/commands/validator.rb

@@ -5,7 +5,7 @@ module Mihari
     module Validator
       def self.included(thor)
         thor.class_eval do
-          desc "rule [PATH]", "Validate rule file format"
+          desc "validate [PATH]", "Validate a rule file"
           #
           # Validate format of a rule
           #
@@ -13,7 +13,7 @@ module Mihari
           #
           # @return [nil]
           #
-          def rule(path)
+          def validate(path)
             rule = Structs::Rule.from_path_or_id(path)
 
             begin

data/lib/mihari/constants.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
 module Mihari
-  ALLOWED_DATA_TYPES = ["hash", "ip", "domain", "url", "mail"].freeze
+  DEFAULT_DATA_TYPES = %w[hash ip domain url mail].freeze
 
-  DEFAULT_EMITTERS = ["database", "misp", "slack", "the_hive", "webhook"].map { |name| { emitter: name } }.freeze
+  DEFAULT_EMITTERS = %w[database misp slack the_hive].map { |name| { emitter: name } }.freeze
 
-  DEFAULT_ENRICHERS = ["whois", "ipinfo", "shodan", "google_public_dns"].map { |name| { enricher: name } }.freeze
+  DEFAULT_ENRICHERS = %w[whois ipinfo shodan google_public_dns].map { |name| { enricher: name } }.freeze
 end

data/lib/mihari/database.rb

@@ -17,53 +17,44 @@ def development_env?
   env == "development"
 end
 
-class InitialSchema < ActiveRecord::Migration[7.0]
+class V5Schema < ActiveRecord::Migration[7.0]
   def change
-    create_table :tags, if_not_exists: true do |t|
-      t.string :name, null: false
+    create_table :rules, id: :string, if_not_exists: true do |t|
+      t.string :title, null: false
+      t.string :description, null: false
+      t.json :data, null: false
+      t.timestamps
     end
 
     create_table :alerts, if_not_exists: true do |t|
-      t.string :title, null: false
-      t.string :description, null: true
-      t.string :source, null: false
       t.timestamps
+
+      t.belongs_to :rule, foreign_key: true, type: :string, null: false
     end
 
     create_table :artifacts, if_not_exists: true do |t|
       t.string :data, null: false
       t.string :data_type, null: false
-      t.belongs_to :alert, foreign_key: true
+      t.string :source
+      t.json :metadata
       t.timestamps
-    end
 
-    create_table :taggings, if_not_exists: true do |t|
-      t.integer :tag_id
-      t.integer :alert_id
+      t.belongs_to :alert, foreign_key: true, null: false
     end
 
-    add_index :taggings, :tag_id, if_not_exists: true
-    add_index :taggings, [:tag_id, :alert_id], unique: true, if_not_exists: true
-  end
-end
-
-class AddeSourceToArtifactSchema < ActiveRecord::Migration[7.0]
-  def change
-    add_column :artifacts, :source, :string, if_not_exists: true
-  end
-end
-
-class EnrichmentsSchema < ActiveRecord::Migration[7.0]
-  def change
     create_table :autonomous_systems, if_not_exists: true do |t|
       t.integer :asn, null: false
-      t.belongs_to :artifact, foreign_key: true
+      t.datetime :created_at
+
+      t.belongs_to :artifact, foreign_key: true, null: false
     end
 
     create_table :geolocations, if_not_exists: true do |t|
       t.string :country, null: false
       t.string :country_code, null: false
-      t.belongs_to :artifact, foreign_key: true
+      t.datetime :created_at
+
+      t.belongs_to :artifact, foreign_key: true, null: false
     end
 
     create_table :whois_records, if_not_exists: true do |t|
@@ -73,73 +64,59 @@ class EnrichmentsSchema < ActiveRecord::Migration[7.0]
       t.date :expires_on
       t.json :registrar
       t.json :contacts
-      t.belongs_to :artifact, foreign_key: true
+      t.datetime :created_at
+
+      t.belongs_to :artifact, foreign_key: true, null: false
     end
 
     create_table :dns_records, if_not_exists: true do |t|
       t.string :resource, null: false
       t.string :value, null: false
-      t.belongs_to :artifact, foreign_key: true
+      t.datetime :created_at
+
+      t.belongs_to :artifact, foreign_key: true, null: false
     end
 
     create_table :reverse_dns_names, if_not_exists: true do |t|
       t.string :name, null: false
-      t.belongs_to :artifact, foreign_key: true
-    end
-  end
-end
+      t.datetime :created_at
 
-class EnrichmentCreatedAtSchema < ActiveRecord::Migration[7.0]
-  def change
-    # Add created_at column because now it is able to enrich an atrifact after the creation
-    add_column :autonomous_systems, :created_at, :datetime, if_not_exists: true
-    add_column :geolocations, :created_at, :datetime, if_not_exists: true
-    add_column :whois_records, :created_at, :datetime, if_not_exists: true
-    add_column :dns_records, :created_at, :datetime, if_not_exists: true
-    add_column :reverse_dns_names, :created_at, :datetime, if_not_exists: true
-  end
-end
-
-class RuleSchema < ActiveRecord::Migration[7.0]
-  def change
-    create_table :rules, id: :string, if_not_exists: true do |t|
-      t.string :title, null: false
-      t.string :description, null: false
-      t.json :data, null: false
-      t.timestamps
+      t.belongs_to :artifact, foreign_key: true, null: false
     end
-  end
-end
-
-class AddeMetadataToArtifactSchema < ActiveRecord::Migration[7.0]
-  def change
-    add_column :artifacts, :metadata, :json, if_not_exists: true
-  end
-end
 
-class AddYAMLToRulesSchema < ActiveRecord::Migration[7.0]
-  def change
-    add_column :rules, :yaml, :text, if_not_exists: true
-  end
-end
-
-class EnrichmentsV45Schema < ActiveRecord::Migration[7.0]
-  def change
     create_table :cpes, if_not_exists: true do |t|
       t.string :cpe, null: false
-      t.belongs_to :artifact, foreign_key: true
+      t.datetime :created_at
+
+      t.belongs_to :artifact, foreign_key: true, null: false
     end
 
     create_table :ports, if_not_exists: true do |t|
       t.integer :port, null: false
-      t.belongs_to :artifact, foreign_key: true
+      t.datetime :created_at
+
+      t.belongs_to :artifact, foreign_key: true, null: false
+    end
+
+    create_table :tags, if_not_exists: true do |t|
+      t.string :name, null: false
+      t.datetime :created_at
     end
+
+    create_table :taggings, if_not_exists: true do |t|
+      t.integer :tag_id
+      t.integer :alert_id
+      t.datetime :created_at
+    end
+
+    add_index :taggings, :tag_id, if_not_exists: true
+    add_index :taggings, %i[tag_id alert_id], unique: true, if_not_exists: true
   end
 end
 
 def adapter
-  return "postgresql" if Mihari.config.database.start_with?("postgresql://", "postgres://")
-  return "mysql2" if Mihari.config.database.start_with?("mysql2://")
+  return "postgresql" if %w[postgresql postgres].include?(Mihari.config.database_url.scheme)
+  return "mysql2" if Mihari.config.database_url.scheme == "mysql2"
 
   "sqlite3"
 end
@@ -157,19 +134,7 @@ module Mihari
       def migrate(direction)
         ActiveRecord::Migration.verbose = false
 
-        [
-          InitialSchema,
-          AddeSourceToArtifactSchema,
-          EnrichmentsSchema,
-          EnrichmentCreatedAtSchema,
-          # v4.0
-          RuleSchema,
-          AddeMetadataToArtifactSchema,
-          # v4.4
-          AddYAMLToRulesSchema,
-          # v4.5
-          EnrichmentsV45Schema
-        ].each { |schema| schema.migrate direction }
+        [V5Schema].each { |schema| schema.migrate direction }
       end
       memoize :migrate unless test_env?
 
@@ -181,19 +146,19 @@ module Mihari
 
         case adapter
         when "postgresql", "mysql2"
-          ActiveRecord::Base.establish_connection(Mihari.config.database)
+          ActiveRecord::Base.establish_connection(Mihari.config.database_url.to_s)
         else
           ActiveRecord::Base.establish_connection(
             adapter: adapter,
-            database: Mihari.config.database
+            database: Mihari.config.database_url.path[1..]
           )
         end
 
         ActiveRecord::Base.logger = Logger.new($stdout) if development_env?
 
         migrate :up
-      rescue StandardError
-        # Do nothing
+      rescue StandardError => e
+        Mihari.logger.error e
       end
 
       #

data/lib/mihari/emitters/database.rb

@@ -7,23 +7,32 @@ module Mihari
         true
       end
 
-      def emit(title:, description:, artifacts:, source:, tags: [])
+      #
+      # Create an alert
+      #
+      # @param [Arra<Mihari::Artifact>] artifacts
+      # @param [Mihari::Structs::Rule] rule
+      #
+      # @return [Mihari::Alert]
+      #
+      def emit(artifacts:, rule:)
         return if artifacts.empty?
 
-        tags = tags.filter_map { |name| Tag.find_or_create_by(name: name) }.uniq
+        tags = rule.tags.filter_map { |name| Tag.find_or_create_by(name: name) }.uniq
         taggings = tags.map { |tag| Tagging.new(tag_id: tag.id) }
 
         alert = Alert.new(
-          title: title,
-          description: description,
           artifacts: artifacts,
-          source: source,
-          taggings: taggings
+          taggings: taggings,
+          rule_id: rule.id
         )
-
         alert.save
         alert
       end
+
+      def configuration_keys
+        %w[database_url]
+      end
     end
   end
 end