mihari 6.3.0 → 7.0.0

data/lib/mihari/commands/rule.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "pathname"
-
 module Mihari
   module Commands
     #
@@ -9,9 +7,10 @@ module Mihari
     #
     module Rule
       class << self
+        # rubocop:disable Metrics/AbcSize
         def included(thor)
           thor.class_eval do
-            include Dry::Monads[:try, :result]
+            include Concerns::DatabaseConnectable
 
             desc "validate [PATH]", "Validate a rule file"
             #
@@ -20,8 +19,7 @@ module Mihari
             # @param [String] path
             #
             def validate(path)
-              res = Dry::Monads::Try[ValidationError] { Mihari::Rule.from_yaml File.read(path) }
-              rule = res.value!
+              rule = Dry::Monads::Try[ValidationError] { Mihari::Rule.from_yaml File.read(path) }.value!
               puts rule.data.to_yaml
             end
 
@@ -31,38 +29,53 @@ module Mihari
             #
             # @param [String] path
             #
-            #
             def init(path = "./rule.yml")
-              warning = "#{path} exists. Do you want to overwrite it? (y/n)"
+              warning = "Do you want to overwrite it? (y/n)"
               return if Pathname(path).exist? && !(yes? warning)
 
-              initialize_rule path
+              Services::RuleInitializer.call(path)
 
               puts "A new rule file has been initialized: #{path}."
             end
 
-            no_commands do
-              #
-              # Create a new rule
-              #
-              # @param [String] path
-              # @param [Dry::Files] files
-              #
-              # @return [nil]
-              #
-              def initialize_rule(path, files = Dry::Files.new)
-                rule = Mihari::Rule.new(
-                  id: SecureRandom.uuid,
-                  title: "Title goes here",
-                  description: "Description goes here",
-                  created_on: Date.today,
-                  queries: []
-                )
-                files.write(path, rule.yaml)
-              end
+            desc "list [QUERY]", "List/search rules"
+            around :with_db_connection
+            method_option :page, type: :numeric, default: 1
+            method_option :limit, type: :numeric, default: 10
+            #
+            # @param [String] q
+            #
+            def list(q = "")
+              filter = Structs::Filters::Search.new(q: q, page: options["page"], limit: options["limit"])
+              value = Services::RuleSearcher.result(filter).value!
+              data = Entities::RulesWithPagination.represent(
+                results: value.results,
+                total: value.total,
+                current_page: value.filter[:page].to_i,
+                page_size: value.filter[:limit].to_i
+              )
+              puts JSON.pretty_generate(data.as_json)
+            end
+
+            desc "get [ID]", "Get a rule"
+            around :with_db_connection
+            def get(id)
+              value = Services::RuleGetter.result(id).value!
+              data = Entities::Rule.represent(value)
+              puts JSON.pretty_generate(data.as_json)
+            end
+
+            desc "delete [ID]", "Delete a rule"
+            around :with_db_connection
+            #
+            # @param [String] id
+            #
+            def delete(id)
+              Services::RuleDestroyer.result(id).value!
             end
           end
         end
+        # rubocop:enable Metrics/AbcSize
       end
     end
   end

data/lib/mihari/commands/search.rb

@@ -9,32 +9,31 @@ module Mihari
       class << self
         def included(thor)
           thor.class_eval do
-            include Dry::Monads[:try, :result]
-            include Mixins
+            include Concerns::DatabaseConnectable
 
-            desc "search [PATH_OR_ID]", "Search by a rule (Outputs null if there is no new finding)"
+            desc "search [PATH_OR_ID]", "Search by a rule"
             around :with_db_connection
-            method_option :force_overwrite, type: :boolean, aliases: "-f", desc: "Force overwriting a rule"
+            method_option :force_overwrite, type: :boolean, default: false, aliases: "-f",
+              desc: "Force overwriting a rule"
             #
             # Search by a rule
             #
             # @param [String] path_or_id
             #
             def search(path_or_id)
-              result = Dry::Monads::Try[StandardError] do
+              force_overwrite = options["force_overwrite"] || false
+              message = "Are you sure you want to overwrite this rule? (y/n)"
+
+              # @type [Mihari::Models::Alert]
+              alert = Dry::Monads::Try[StandardError] do
                 # @type [Mihari::Rule]
                 rule = Services::RuleBuilder.call(path_or_id)
 
-                force_overwrite = options["force_overwrite"] || false
-                message = "There is a diff in the rule. Are you sure you want to overwrite the rule? (y/n)"
                 exit 0 if rule.diff? && !force_overwrite && !yes?(message)
 
                 rule.update_or_create
                 rule.call
-              end.to_result
-
-              # @type [Mihari::Models::Alert]
-              alert = result.value!
+              end.value!
               data = Entities::Alert.represent(alert)
               puts JSON.pretty_generate(data.as_json)
             end

data/lib/mihari/commands/sidekiq.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Commands
+    #
+    # Sidekiq sub-commands
+    #
+    module Sidekiq
+      class << self
+        def included(thor)
+          thor.class_eval do
+            desc "sidekiq", "Start Sidekiq"
+            method_option :env, type: :string, default: "production", desc: "Environment"
+            method_option :concurrency, type: :numeric, default: 5, desc: "Sidekiq concurrency", aliases: "-c"
+            def sidekiq
+              require "sidekiq/cli"
+
+              ENV["APP_ENV"] ||= options["env"]
+              concurrency = options["concurrency"].to_s
+
+              cli = ::Sidekiq::CLI.instance
+              cli.parse ["-r", File.expand_path(File.join(__dir__, "..", "sidekiq", "application.rb")), "-c",
+                concurrency]
+              cli.run
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/commands/tag.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Commands
+    #
+    # Tag sub-commands
+    #
+    module Tag
+      class << self
+        def included(thor)
+          thor.class_eval do
+            include Concerns::DatabaseConnectable
+
+            desc "list", "List/search tags"
+            around :with_db_connection
+            method_option :page, type: :numeric, default: 1
+            method_option :limit, type: :numeric, default: 10
+            #
+            # @param [String] q
+            #
+            def list(q = "")
+              filter = Structs::Filters::Search.new(q: q, page: options["page"], limit: options["limit"])
+              value = Services::TagSearcher.result(filter).value!
+              data = Entities::TagsWithPagination.represent(
+                results: value.results,
+                total: value.total,
+                current_page: value.filter[:page].to_i,
+                page_size: value.filter[:limit].to_i
+              )
+              puts JSON.pretty_generate(data.as_json)
+            end
+
+            desc "delete [ID]", "Delete a tag"
+            around :with_db_connection
+            #
+            # @param [Integer] id
+            #
+            def delete(id)
+              Services::TagDestroyer.result(id).value!
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/commands/web.rb

@@ -9,20 +9,19 @@ module Mihari
       class << self
         def included(thor)
           thor.class_eval do
-            desc "web", "Launch the web app"
+            desc "web", "Start the web app"
             method_option :port, type: :numeric, default: 9292, desc: "Hostname to listen on"
             method_option :host, type: :string, default: "localhost", desc: "Port to listen on"
             method_option :threads, type: :string, default: "0:5", desc: "min:max threads to use"
             method_option :verbose, type: :boolean, default: true, desc: "Report each request"
             method_option :worker_timeout, type: :numeric, default: 60, desc: "Worker timeout value (in seconds)"
-            method_option :hide_config_values, type: :boolean, default: true,
-              desc: "Whether to hide config values or not"
             method_option :open, type: :boolean, default: true, desc: "Whether to open the app in browser or not"
-            method_option :rack_env, type: :string, default: "production", desc: "Rack environment"
+            method_option :env, type: :string, default: "production", desc: "Environment"
             def web
-              Mihari.config.hide_config_values = options["hide_config_values"]
-              # set rack env as production
-              ENV["RACK_ENV"] ||= options["rack_env"]
+              require "mihari/web/application"
+
+              ENV["APP_ENV"] ||= options["env"]
+
               Mihari::Web::App.run!(
                 port: options["port"],
                 host: options["host"],

data/lib/mihari/{mixins/autonomous_system.rb → concerns/autonomous_system_normalizable.rb}

@@ -1,11 +1,13 @@
 # frozen_string_literal: true
 
 module Mihari
-  module Mixins
+  module Concerns
     #
-    # Autonomous System mixin
+    # Autonomous System concern
     #
-    module AutonomousSystem
+    module AutonomousSystemNormalizable
+      extend ActiveSupport::Concern
+
       #
       # Normalize ASN value
       #

data/lib/mihari/concerns/configurable.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Concerns
+    #
+    # Configurable concern
+    #
+    module Configurable
+      extend ActiveSupport::Concern
+
+      #
+      # Check whether there are configuration key-values or not
+      #
+      # @return [Boolean]
+      #
+      def configuration_keys?
+        return true if self.class.configuration_keys.empty?
+
+        self.class.configuration_keys.all? { |key| Mihari.config.send(key) }
+      end
+
+      #
+      # Check whether it is configured or not
+      #
+      # @return [Boolean]
+      #
+      def configured?
+        configuration_keys? || api_key?
+      end
+
+      class_methods do
+        #
+        # Configuration items
+        #
+        # @return [Array<Hash>] Configuration values as a list of hash. Returns nil if there is any keys.
+        #
+        def configuration_items
+          return nil if configuration_keys.empty?
+
+          configuration_keys.map do |key|
+            value = Mihari.config.send(key)
+            value = "REDACTED" if value && Mihari.config.hide_config_values
+            { key: key.upcase, value: value }
+          end
+        end
+
+        #
+        # Configuration keys
+        #
+        # @return [Array<String>] A list of configuration keys
+        #
+        def configuration_keys
+          []
+        end
+      end
+
+      private
+
+      #
+      # Check whether API key is set or not
+      #
+      # @return [Boolean]
+      #
+      def api_key?
+        value = method(:api_key).call
+        !value.nil?
+      rescue NameError
+        true
+      end
+    end
+  end
+end

data/lib/mihari/concerns/database_connectable.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Concerns
+    #
+    # Database connectable concern
+    #
+    module DatabaseConnectable
+      extend ActiveSupport::Concern
+
+      def with_db_connection(&block)
+        Database.with_db_connection(&block)
+      end
+    end
+  end
+end

data/lib/mihari/{mixins/unwrap_error.rb → concerns/error_unwrappable.rb}

@@ -1,11 +1,13 @@
 # frozen_string_literal: true
 
 module Mihari
-  module Mixins
+  module Concerns
     #
-    # Unwrap error mixins
+    # Error unwrappable concern
     #
-    module UnwrapError
+    module ErrorUnwrappable
+      extend ActiveSupport::Concern
+
       def unwrap_error(err)
         return err unless err.is_a?(Dry::Monads::UnwrapError)
 

data/lib/mihari/{mixins/falsepositive.rb → concerns/falsepositive_validatable.rb}

@@ -1,11 +1,13 @@
 # frozen_string_literal: true
 
 module Mihari
-  module Mixins
+  module Concerns
     #
-    # False positive mixins
+    # False positive validatable concern
     #
-    module FalsePositive
+    module FalsePositiveValidatable
+      extend ActiveSupport::Concern
+
       prepend MemoWise
 
       #

data/lib/mihari/{mixins/refang.rb → concerns/refangable.rb}

@@ -1,11 +1,13 @@
 # frozen_string_literal: true
 
 module Mihari
-  module Mixins
+  module Concerns
     #
-    # Refang mixin
+    # Refangable concern
     #
-    module Refang
+    module Refangable
+      extend ActiveSupport::Concern
+
       #
       # Refang defanged indicator
       #

data/lib/mihari/{mixins → concerns}/retriable.rb

@@ -1,11 +1,13 @@
 # frozen_string_literal: true
 
 module Mihari
-  module Mixins
+  module Concerns
     #
-    # Retriable mixin
+    # Retriable concern
     #
     module Retriable
+      extend ActiveSupport::Concern
+
       DEFAULT_ON = [
         Errno::ECONNRESET,
         Errno::ECONNABORTED,

data/lib/mihari/config.rb

@@ -14,12 +14,11 @@ module Mihari
       censys_secret: nil,
       circl_passive_password: nil,
       circl_passive_username: nil,
-      database_url: URI("sqlite3:///mihari.db"),
+      database_url: URI("sqlite3:mihari.db"),
       fofa_api_key: nil,
       fofa_email: nil,
       greynoise_api_key: nil,
       hunterhow_api_key: nil,
-      ipinfo_api_key: nil,
       misp_api_key: nil,
       misp_url: nil,
       onyphe_api_key: nil,
@@ -32,11 +31,12 @@ module Mihari
       slack_channel: nil,
       slack_webhook_url: nil,
       thehive_api_key: nil,
-      thehive_api_version: nil,
       thehive_url: nil,
       urlscan_api_key: nil,
       virustotal_api_key: nil,
       zoomeye_api_key: nil,
+      # sidekiq
+      sidekiq_redis_url: nil,
       # others
       hide_config_values: true,
       ignore_error: false,
@@ -63,7 +63,7 @@ module Mihari
     #   @return [String, nil]
 
     # @!attribute [r] database_url
-    #   @return [URI, nil]
+    #   @return [URI]
 
     # @!attribute [r] fofa_api_key
     #   @return [String, nil]
@@ -77,9 +77,6 @@ module Mihari
     # @!attribute [r] hunterhow_api_key
     #   @return [String, nil]
 
-    # @!attribute [r] ipinfo_api_key
-    #   @return [String, nil]
-
     # @!attribute [r] misp_url
     #   @return [String, nil]
 
@@ -119,9 +116,6 @@ module Mihari
     # @!attribute [r] thehive_api_key
     #   @return [String, nil]
 
-    # @!attribute [r] thehive_api_version
-    #   @return [String, nil]
-
     # @!attribute [r] urlscan_api_key
     #   @return [String, nil]
 
@@ -158,11 +152,18 @@ module Mihari
     # @!attribute [r] ignore_error
     #   @return [Boolean]
 
-    # @!attribute [rw] hide_config_values
+    # @!attribute [r] hide_config_values
     #   @return [Boolean]
 
+    # @!attribute [r] sidekiq_redis_url
+    #   @return [URI, nil]
+
     def database_url=(val)
-      super URI(val.to_s)
+      super(URI(val.to_s))
+    end
+
+    def sidekiq_redis_url=(val)
+      super(val.nil? ? val : URI(val.to_s))
     end
 
     #

data/lib/mihari/database.rb

@@ -3,18 +3,10 @@
 # Make possible to use upper case acronyms in class names
 ActiveSupport::Inflector.inflections(:en) { |inflect| inflect.acronym "CPE" }
 
-def env
-  ENV["APP_ENV"] || ENV["RACK_ENV"]
-end
-
-def development_env?
-  env == "development"
-end
-
 #
-# Mihari v5 DB schema
+# Mihari v7 DB schema
 #
-class V5Schema < ActiveRecord::Migration[7.1]
+class V7Schema < ActiveRecord::Migration[7.1]
   def change
     create_table :rules, id: :string, if_not_exists: true do |t|
       t.string :title, null: false
@@ -24,7 +16,7 @@ class V5Schema < ActiveRecord::Migration[7.1]
     end
 
     create_table :alerts, if_not_exists: true do |t|
-      t.timestamps
+      t.datetime :created_at
 
       t.belongs_to :rule, foreign_key: true, type: :string, null: false
     end
@@ -33,8 +25,9 @@ class V5Schema < ActiveRecord::Migration[7.1]
       t.string :data, null: false
       t.string :data_type, null: false
       t.string :source
+      t.string :query
       t.json :metadata
-      t.timestamps
+      t.datetime :created_at
 
       t.belongs_to :alert, foreign_key: true, null: false
     end
@@ -102,33 +95,20 @@ class V5Schema < ActiveRecord::Migration[7.1]
 
     create_table :taggings, if_not_exists: true do |t|
       t.integer :tag_id
-      t.integer :alert_id
+      t.string :rule_id
       t.datetime :created_at
     end
 
     add_index :taggings, :tag_id, if_not_exists: true
-    add_index :taggings, %i[tag_id alert_id], unique: true, if_not_exists: true
-  end
-end
-
-class V61Schema < ActiveRecord::Migration[7.1]
-  def change
-    add_column :artifacts, :query, :string
+    add_index :taggings, %i[tag_id rule_id], unique: true, if_not_exists: true
   end
 end
 
-def adapter
-  return "postgresql" if %w[postgresql postgres].include?(Mihari.config.database_url.scheme)
-  return "mysql2" if Mihari.config.database_url.scheme == "mysql2"
-
-  "sqlite3"
-end
-
 #
 # @return [Array<ActiveRecord::Migration>] schemas
 #
 def schemas
-  [V5Schema, V61Schema]
+  [V7Schema]
 end
 
 module Mihari
@@ -150,37 +130,45 @@ module Mihari
       # Establish DB connection
       #
       def connect
-        return if ActiveRecord::Base.connected?
-
-        case adapter
-        when "postgresql", "mysql2"
-          ActiveRecord::Base.establish_connection Mihari.config.database_url.to_s
-        else
-          ActiveRecord::Base.establish_connection(
-            adapter: adapter,
-            database: Mihari.config.database_url.path[1..]
-          )
-        end
-        ActiveRecord::Base.logger = Logger.new($stdout) if development_env?
+        return if connected?
+
+        ActiveRecord::Base.establish_connection Mihari.config.database_url.to_s
+        ActiveRecord::Base.logger = Logger.new($stdout) if Mihari.development?
+      end
+
+      #
+      # @return [Boolean]
+      #
+      def connected?
+        ActiveRecord::Base.connected?
       end
 
       #
       # Close DB connection(s)
       #
       def close
-        return unless ActiveRecord::Base.connected?
+        return unless connected?
 
         ActiveRecord::Base.connection_handler.clear_active_connections!
       end
 
       def with_db_connection
-        Mihari::Database.connect
+        Mihari::Database.connect unless connected?
         yield
       rescue ActiveRecord::StatementInvalid
         Mihari.logger.error("The DB migration is not yet complete. Please run 'mihari db migrate'.")
       ensure
         Mihari::Database.close
       end
+
+      private
+
+      def adapter
+        return "postgresql" if %w[postgresql postgres].include?(Mihari.config.database_url.scheme)
+        return "mysql2" if Mihari.config.database_url.scheme == "mysql2"
+
+        "sqlite3"
+      end
     end
   end
 end

data/lib/mihari/emitters/database.rb

@@ -16,16 +16,15 @@ module Mihari
       def call(artifacts)
         return if artifacts.empty?
 
-        tags = rule.tags.filter_map { |name| Models::Tag.find_or_create_by(name: name) }.uniq
-        taggings = tags.map { |tag| Models::Tagging.new(tag_id: tag.id) }
-
-        alert = Models::Alert.new(artifacts: artifacts, taggings: taggings, rule_id: rule.id)
+        alert = Models::Alert.new(artifacts: artifacts, rule_id: rule.id)
         alert.save
         alert
       end
 
-      def configuration_keys
-        %w[database_url]
+      class << self
+        def configuration_keys
+          %w[database_url]
+        end
       end
     end
   end