mihari 3.12.0 → 4.0.0

data/lib/mihari/cli/mixins/utils.rb

@@ -27,19 +27,6 @@ module Mihari
           %w[title description artifacts].all? { |key| json.key? key }
         end
 
-        #
-        # Load configuration and establish DB connection
-        #
-        # @return [Hash]
-        #
-        def load_configuration
-          config = options["config"]
-          return unless config
-
-          Mihari.load_config_from_yaml config
-          Database.connect
-        end
-
         #
         # Run analyzer
         #
@@ -50,14 +37,12 @@ module Mihari
         # @return [nil]
         #
         def run_analyzer(analyzer_class, query:, options:)
-          load_configuration
-
           # options = Thor::CoreExt::HashWithIndifferentAccess
           # ref. https://www.rubydoc.info/github/wycats/thor/Thor/CoreExt/HashWithIndifferentAccess
           # so need to covert it to a plain hash
           hash_options = options.to_hash
 
-          hash_options = symbolize_hash(hash_options)
+          hash_options = hash_options.symbolize_keys
           hash_options = normalize_options(hash_options)
 
           analyzer = analyzer_class.new(query, **hash_options)
@@ -76,8 +61,7 @@ module Mihari
         # @return [Hash]
         #
         def normalize_options(options)
-          # Delete :config because it is not intended to use for running an analyzer
-          [:config, :ignore_old_artifacts, :ignore_threshold].each do |ignore_key|
+          [:ignore_old_artifacts, :ignore_threshold].each do |ignore_key|
             options.delete(ignore_key)
           end
           options

data/lib/mihari/commands/init.rb

@@ -1,30 +1,12 @@
 # frozen_string_literal: true
 
-require "colorize"
-
 module Mihari
   module Commands
     module Initialization
-      include Mixins::Configuration
       include Mixins::Rule
 
       def self.included(thor)
         thor.class_eval do
-          desc "config", "Create a config file"
-          method_option :filename, type: :string, default: "mihari.yml"
-          def config
-            filename = options["filename"]
-
-            warning = "#{filename} exists. Do you want to overwrite it? (y/n)"
-            if File.exist?(filename) && !(yes? warning)
-              return
-            end
-
-            initialize_config_yaml filename
-
-            puts "The config file is initialized as #{filename}.".colorize(:blue)
-          end
-
           desc "rule", "Create a rule file"
           method_option :filename, type: :string, default: "rule.yml"
           def rule

data/lib/mihari/commands/search.rb

@@ -3,22 +3,22 @@
 module Mihari
   module Commands
     module Search
+      include Mixins::Database
       include Mixins::Rule
 
       def self.included(thor)
         thor.class_eval do
           desc "search [RULE]", "Search by a rule"
-          method_option :config, type: :string, desc: "Path to the config file"
-          def search_by_rule(rule)
-            # load configuration
-            load_configuration
-
-            # convert str(YAML) to hash or str(path/YAML file) to hash
-            rule = load_rule(rule)
-
-            # validate rule schema
-            rule = validate_rule(rule)
+          def search_by_rule(path_or_id)
+            rule = load_rule(path_or_id)
+            # validate
+            begin
+              validate_rule! rule
+            rescue RuleValidationError => e
+              raise e
+            end
 
+            # build and run the analyzer
             analyzer = build_rule_analyzer(
               title: rule[:title],
               description: rule[:description],
@@ -26,8 +26,7 @@ module Mihari
               tags: rule[:tags],
               allowed_data_types: rule[:allowed_data_types],
               disallowed_data_values: rule[:disallowed_data_values],
-              source: rule[:source],
-              id: rule[:id]
+              id: rule.id
             )
 
             ignore_old_artifacts = rule[:ignore_old_artifacts]
@@ -35,6 +34,14 @@ module Mihari
 
             with_error_handling do
               run_rule_analyzer analyzer, ignore_old_artifacts: ignore_old_artifacts, ignore_threshold: ignore_threshold
+
+              # record a rule
+              with_db_connection do
+                model = rule.to_model
+                model.save
+              rescue ActiveRecord::RecordNotUnique
+                nil
+              end
             end
           end
         end
@@ -51,11 +58,10 @@ module Mihari
       # @param [Array<String>, nil] tags
       # @param [Array<String>, nil] allowed_data_types
       # @param [Array<String>, nil] disallowed_data_values
-      # @param [String, nil] source
       #
       # @return [Mihari::Analyzers::Rule]
       #
-      def build_rule_analyzer(title:, description:, queries:, tags: nil, allowed_data_types: nil, disallowed_data_values: nil, source: nil, id: nil)
+      def build_rule_analyzer(title:, description:, queries:, tags: nil, allowed_data_types: nil, disallowed_data_values: nil, id: nil)
         tags = [] if tags.nil?
         allowed_data_types = ALLOWED_DATA_TYPES if allowed_data_types.nil?
         disallowed_data_values = [] if disallowed_data_values.nil?
@@ -67,7 +73,6 @@ module Mihari
           queries: queries,
           allowed_data_types: allowed_data_types,
           disallowed_data_values: disallowed_data_values,
-          source: source,
           id: id
         )
       end

data/lib/mihari/commands/validator.rb

@@ -4,32 +4,20 @@ module Mihari
   module Commands
     module Validator
       include Mixins::Rule
-      include Mixins::Configuration
 
       def self.included(thor)
         thor.class_eval do
           desc "rule [PATH]", "Validate format of a rule file"
           def rule(path)
-            # convert str(YAML) to hash or str(path/YAML file) to hash
             rule = load_rule(path)
 
-            # validate rule schema
-            validate_rule rule
-
-            puts "Valid format. The input is parsed as the following:"
-            puts rule.to_yaml
-          end
-
-          desc "config [PATH]", "Validate format of a config file"
-          def config(path)
-            # convert str(YAML) to hash or str(path/YAML file) to hash
-            config = load_config(path)
-
-            # validate config schema
-            validate_config config
-
-            puts "Valid format. The input is parsed as the following:"
-            puts config.to_yaml
+            begin
+              validate_rule! rule
+              puts "Valid format. The input is parsed as the following:"
+              puts rule.data.to_yaml
+            rescue RuleValidationError
+              nil
+            end
           end
         end
       end

data/lib/mihari/commands/web.rb

@@ -10,15 +10,12 @@ module Mihari
           method_option :host, type: :string, default: "localhost", desc: "Port to listen on"
           method_option :threads, type: :string, default: "0:16", desc: "min:max threads to use"
           method_option :verbose, type: :boolean, default: true, desc: "Report each request"
-          method_option :config, type: :string, desc: "Path to the config file"
           def web
             port = options["port"]
             host = options["host"]
             threads = options["threads"]
             verbose = options["verbose"]
 
-            load_configuration
-
             # set rack env as production
             ENV["RACK_ENV"] ||= "production"
 

data/lib/mihari/database.rb

@@ -1,6 +1,16 @@
 # frozen_string_literal: true
 
-require "active_record"
+def env
+  ENV["APP_ENV"] || ENV["RACK_ENV"]
+end
+
+def test_env?
+  env == "test"
+end
+
+def development_env?
+  env == "development"
+end
 
 class InitialSchema < ActiveRecord::Migration[7.0]
   def change
@@ -85,6 +95,23 @@ class EnrichmentCreatedAtSchema < ActiveRecord::Migration[7.0]
   end
 end
 
+class RuleSchema < ActiveRecord::Migration[7.0]
+  def change
+    create_table :rules, id: :string, if_not_exists: true do |t|
+      t.string :title, null: false
+      t.string :description, null: false
+      t.json :data, null: false
+      t.timestamps
+    end
+  end
+end
+
+class AddeMetadataToArtifactSchema < ActiveRecord::Migration[7.0]
+  def change
+    add_column :artifacts, :metadata, :json, if_not_exists: true
+  end
+end
+
 def adapter
   return "postgresql" if Mihari.config.database.start_with?("postgresql://", "postgres://")
   return "mysql2" if Mihari.config.database.start_with?("mysql2://")
@@ -95,7 +122,34 @@ end
 module Mihari
   class Database
     class << self
+      include Memist::Memoizable
+
+      #
+      # DB migraration
+      #
+      # @param [Symbol] direction
+      #
+      def migrate(direction)
+        ActiveRecord::Migration.verbose = false
+
+        [
+          InitialSchema,
+          AddeSourceToArtifactSchema,
+          EnrichmentsSchema,
+          EnrichmentCreatedAtSchema,
+          # v4
+          RuleSchema,
+          AddeMetadataToArtifactSchema
+        ].each { |schema| schema.migrate direction }
+      end
+      memoize :migrate unless test_env?
+
+      #
+      # Establish DB connection
+      #
       def connect
+        return if ActiveRecord::Base.connected?
+
         case adapter
         when "postgresql", "mysql2"
           ActiveRecord::Base.establish_connection(Mihari.config.database)
@@ -106,32 +160,30 @@ module Mihari
           )
         end
 
-        ActiveRecord::Base.logger = Logger.new($stdout) if ENV["RACK_ENV"] == "development"
-        ActiveRecord::Migration.verbose = false
+        ActiveRecord::Base.logger = Logger.new($stdout) if development_env?
 
-        InitialSchema.migrate(:up)
-        AddeSourceToArtifactSchema.migrate(:up)
-        EnrichmentsSchema.migrate(:up)
-        EnrichmentCreatedAtSchema.migrate(:up)
+        migrate :up
       rescue StandardError
         # Do nothing
       end
 
+      #
+      # Close DB connection(s)
+      #
       def close
-        ActiveRecord::Base.clear_active_connections!
-        ActiveRecord::Base.connection.close
+        return unless ActiveRecord::Base.connected?
+
+        ActiveRecord::Base.clear_all_connections!
       end
 
+      #
+      # Destory DB
+      #
       def destroy!
         return unless ActiveRecord::Base.connected?
 
-        InitialSchema.migrate(:down)
-        AddeSourceToArtifactSchema.migrate(:down)
-        EnrichmentsSchema.migrate(:down)
-        EnrichmentCreatedAtSchema.migrate(:down)
+        migrate :down
       end
     end
   end
 end
-
-Mihari::Database.connect

data/lib/mihari/emitters/misp.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "misp"
-require "net/ping"
 
 module Mihari
   module Emitters

data/lib/mihari/emitters/slack.rb

@@ -1,13 +1,12 @@
 # frozen_string_literal: true
 
-require "slack-notifier"
 require "digest/sha2"
-require "dry-initializer"
+require "slack-notifier"
 
 module Mihari
   module Emitters
     class Attachment
-      include Mem
+      include Memist::Memoizable
 
       extend Dry::Initializer
 
@@ -63,7 +62,7 @@ module Mihari
         when "domain"
           "https://urlscan.io/domain/#{data}"
         when "url"
-          uri = URI(data)
+          uri = Addressable::URI.parse(data)
           "https://urlscan.io/domain/#{uri.hostname}"
         end
       end

data/lib/mihari/emitters/stdout.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "json"
-
 module Mihari
   module Emitters
     class StandardOutput < Base

data/lib/mihari/emitters/the_hive.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "hachi"
-require "net/ping"
 
 module Mihari
   module Emitters

data/lib/mihari/emitters/webhook.rb

@@ -1,9 +1,5 @@
 # frozen_string_literal: true
 
-require "json"
-require "net/http"
-require "uri"
-
 module Mihari
   module Emitters
     class Webhook < Base
@@ -15,7 +11,7 @@ module Mihari
       def emit(title:, description:, artifacts:, source:, tags:)
         return if artifacts.empty?
 
-        uri = URI(Mihari.config.webhook_url)
+        uri = Addressable::URI.parse(Mihari.config.webhook_url)
         data = {
           title: title,
           description: description,

data/lib/mihari/enrichers/ipinfo.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 require "http"
-require "json"
-require "memist"
 
 module Mihari
   module Enrichers

data/lib/mihari/errors.rb

@@ -12,4 +12,6 @@ module Mihari
   class HttpError < Error; end
 
   class FeedParseError < Error; end
+
+  class RuleValidationError < Error; end
 end

data/lib/mihari/feed/reader.rb

@@ -1,17 +1,14 @@
 # frozen_string_literal: true
 
 require "csv"
-require "json"
-require "net/https"
-require "uri"
 
 module Mihari
   module Feed
     class Reader
       attr_reader :uri, :http_request_headers, :http_request_method, :http_request_payload_type, :http_request_payload
 
-      def initialize(url, http_request_headers: {}, http_request_method: "GET", http_request_payload_type: nil, http_request_payload: {})
-        @uri = URI(url)
+      def initialize(uri, http_request_headers: {}, http_request_method: "GET", http_request_payload_type: nil, http_request_payload: {})
+        @uri = Addressable::URI.parse(uri)
         @http_request_headers = http_request_headers
         @http_request_method = http_request_method
         @http_request_payload_type = http_request_payload_type
@@ -19,13 +16,15 @@ module Mihari
       end
 
       def read
+        return read_file(uri.path) if uri.scheme == "file"
+
         return get if http_request_method == "GET"
 
         post
       end
 
       def get
-        uri.query = URI.encode_www_form(http_request_payload)
+        uri.query = Addressable::URI.form_encode(http_request_payload)
         get = Net::HTTP::Get.new(uri)
 
         request(get)
@@ -99,13 +98,28 @@ module Mihari
           body = response.body
 
           content_type = response["Content-Type"].to_s
-          data = if content_type.include?("application/json")
+          if content_type.include?("application/json")
             convert_as_json(body)
           else
             convert_as_csv(body)
           end
+        end
+      end
+
+      #
+      # Read & convert a file
+      #
+      # @param [String] path
+      #
+      # @return [Array<Hash>]
+      #
+      def read_file(path)
+        text = File.read(path)
 
-          data
+        if path.end_with?(".json")
+          convert_as_json text
+        else
+          convert_as_csv text
         end
       end
     end

data/lib/mihari/mixins/database.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Mixins
+    module Database
+      def with_db_connection
+        Mihari::Database.connect
+        yield
+      ensure
+        Mihari::Database.close
+      end
+    end
+  end
+end

data/lib/mihari/mixins/disallowed_data_value.rb

@@ -1,11 +1,9 @@
 # frozen_string_literal: true
 
-require "mem"
-
 module Mihari
   module Mixins
     module DisallowedDataValue
-      include Mem
+      include Memist::Memoizable
 
       #
       # Normalize a value as a disallowed data value
@@ -21,7 +19,6 @@ module Mihari
         value_without_slashes = value[1..-2]
         Regexp.compile value_without_slashes.to_s
       end
-
       memoize :normalize_disallowed_data_value
 
       #

data/lib/mihari/mixins/rule.rb

@@ -1,53 +1,56 @@
 # frozen_string_literal: true
 
 require "date"
-require "pathname"
-require "yaml"
 require "erb"
+require "pathname"
 
 module Mihari
   module Mixins
     module Rule
+      include Mixins::Database
+
       #
       # Load rule into hash
       #
-      # @param [String] path Path to YAML file or YAML string
+      # @param [String] path_or_id Path to YAML file or YAML string or ID of a rule in the database
       #
-      # @return [Hash]
+      # @return [Mihari::Structs::Rule::Rule]
       #
-      def load_rule(path)
+      def load_rule(path_or_id)
+        data = nil
+
+        data = load_data_from_file(path_or_id) if File.exist?(path_or_id)
+        data = load_data_from_db(path_or_id) if data.nil?
+
+        Structs::Rule::Rule.new(data)
+      end
+
+      def load_data_from_file(path)
         return YAML.safe_load(File.read(path), permitted_classes: [Date], symbolize_names: true) if Pathname(path).exist?
 
         YAML.safe_load(path, symbolize_names: true)
       end
 
+      def load_data_from_db(id)
+        with_db_connection do
+          rule = Mihari::Rule.find(id)
+          rule.data
+        rescue ActiveRecord::RecordNotFound
+          raise ArgumentError, "ID:#{id} is not found in the database"
+        end
+      end
+
       #
-      # Validate rule schema and return a normalized rule
-      #
-      # @param [Hash] rule
+      # Validate a rule
       #
-      # @return [Hash]
+      # @param [Mihari::Structs::Rule::Rule] rule
       #
-      def validate_rule(rule)
-        error_message = "Failed to parse the input as a rule!"
-
-        contract = Schemas::RuleContract.new
-        begin
-          result = contract.call(rule)
-          unless result.errors.empty?
-            messages = result.errors.messages.map do |message|
-              path = message.path.map(&:to_s).join
-              "#{path} #{message.text}"
-            end
-            puts error_message.colorize(:red)
-            raise ArgumentError, messages.join("\n")
-          end
-        rescue NoMethodError
-          puts error_message.colorize(:red)
-          raise ArgumentError, "Invalid rule schema"
-        end
-
-        result.to_h
+      def validate_rule!(rule)
+        rule.validate!
+      rescue RuleValidationError => e
+        error_message = "Failed to parse the input as a rule:\n#{e.message}"
+        puts error_message.colorize(:red)
+        raise e
       end
 
       #
@@ -62,8 +65,8 @@ module Mihari
         data = template.result
 
         # validate the template of rule for just in case
-        rule = YAML.safe_load(data, permitted_classes: [Date], symbolize_names: true)
-        validate_rule rule
+        hashed_data = YAML.safe_load(data, permitted_classes: [Date], symbolize_names: true)
+        Structs::Rule::Rule.new(hashed_data)
 
         data
       end

data/lib/mihari/models/alert.rb

@@ -1,20 +1,20 @@
 # frozen_string_literal: true
 
-require "active_record"
-
 module Mihari
   class Alert < ActiveRecord::Base
     has_many :taggings, dependent: :destroy
     has_many :artifacts, dependent: :destroy
     has_many :tags, through: :taggings
 
+    belongs_to :rule
+
     class << self
       #
       # Search alerts
       #
       # @param [Structs::Alert::SearchFilterWithPagination] filter
       #
-      # @return [Array<Hash>]
+      # @return [Array<Alert>]
       #
       def search(filter)
         limit = filter.limit.to_i

data/lib/mihari/models/artifact.rb

@@ -1,10 +1,5 @@
 # frozen_string_literal: true
 
-require "active_record"
-require "active_support/core_ext/integer/time"
-require "active_support/core_ext/numeric/time"
-require "addressable/uri"
-
 class ArtifactValidator < ActiveModel::Validator
   def validate(record)
     return if record.data_type

data/lib/mihari/models/autonomous_system.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "active_record"
-
 module Mihari
   class AutonomousSystem < ActiveRecord::Base
     belongs_to :artifact

data/lib/mihari/models/dns.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require "active_record"
-require "resolv"
-
 module Mihari
   class DnsRecord < ActiveRecord::Base
     belongs_to :artifact