mihari 5.4.0 → 5.4.2

data/lib/mihari/analyzers/binaryedge.rb

@@ -30,6 +30,10 @@ module Mihari
         end.flatten
       end
 
+      def configuration_keys
+        %w[binaryedge_api_key]
+      end
+
       private
 
       PAGE_SIZE = 20
@@ -69,10 +73,6 @@ module Mihari
         responses
       end
 
-      def configuration_keys
-        %w[binaryedge_api_key]
-      end
-
       #
       #
       # @return [Mihari::Clients::BinaryEdge]

data/lib/mihari/analyzers/censys.rb

@@ -55,8 +55,6 @@ module Mihari
         configuration_keys? || (id? && secret?)
       end
 
-      private
-
       #
       # @return [Array<String>]
       #
@@ -64,6 +62,8 @@ module Mihari
         %w[censys_id censys_secret]
       end
 
+      private
+
       #
       # @return [Mihari::Clients::Censys]
       #

data/lib/mihari/analyzers/circl.rb

@@ -44,12 +44,12 @@ module Mihari
         configuration_keys? || (username? && password?)
       end
 
-      private
-
       def configuration_keys
         %w[circl_passive_password circl_passive_username]
       end
 
+      private
+
       def client
         @client ||= Clients::CIRCL.new(username: username, password: password)
       end

data/lib/mihari/analyzers/greynoise.rb

@@ -23,12 +23,12 @@ module Mihari
         client.gnql_search(query, size: PAGE_SIZE).to_artifacts
       end
 
-      private
-
       def configuration_keys
         %w[greynoise_api_key]
       end
 
+      private
+
       def client
         @client ||= Clients::GreyNoise.new(api_key: api_key)
       end

data/lib/mihari/analyzers/hunterhow.rb

@@ -3,9 +3,6 @@
 module Mihari
   module Analyzers
     class HunterHow < Base
-      # @return [Integer]
-      PAGE_SIZE = 100
-
       # @return [String, nil]
       attr_reader :api_key
 
@@ -54,12 +51,15 @@ module Mihari
         artifacts.flatten
       end
 
-      private
-
       def configuration_keys
         %w[hunterhow_api_key]
       end
 
+      private
+
+      # @return [Integer]
+      PAGE_SIZE = 100
+
       def client
         @client ||= Clients::HunterHow.new(api_key: api_key)
       end

data/lib/mihari/analyzers/onyphe.rb

@@ -26,14 +26,14 @@ module Mihari
         responses.map(&:to_artifacts).flatten
       end
 
-      private
-
-      PAGE_SIZE = 10
-
       def configuration_keys
         %w[onyphe_api_key]
       end
 
+      private
+
+      PAGE_SIZE = 10
+
       def client
         @client ||= Clients::Onyphe.new(api_key: api_key)
       end

data/lib/mihari/analyzers/otx.rb

@@ -35,12 +35,12 @@ module Mihari
         end
       end
 
-      private
-
       def configuration_keys
         %w[otx_api_key]
       end
 
+      private
+
       def client
         @client ||= Mihari::Clients::OTX.new(api_key: api_key)
       end

data/lib/mihari/analyzers/passivetotal.rb

@@ -46,12 +46,12 @@ module Mihari
         configuration_keys? || (username? && api_key?)
       end
 
-      private
-
       def configuration_keys
         %w[passivetotal_username passivetotal_api_key]
       end
 
+      private
+
       def client
         @client ||= Clients::PassiveTotal.new(username: username, api_key: api_key)
       end

data/lib/mihari/analyzers/pulsedive.rb

@@ -40,12 +40,12 @@ module Mihari
         end
       end
 
-      private
-
       def configuration_keys
         %w[pulsedive_api_key]
       end
 
+      private
+
       def client
         @client ||= Clients::PulseDive.new(api_key: api_key)
       end

data/lib/mihari/analyzers/rule.rb

@@ -55,12 +55,15 @@ module Mihari
       end
 
       #
-      # Returns a list of artifacts matched with queries/analyzers
+      # Returns a list of artifacts matched with queries/analyzers (with the rule ID)
       #
       # @return [Array<Mihari::Artifact>]
       #
       def artifacts
-        analyzers.flat_map(&:normalized_artifacts)
+        analyzers.flat_map(&:normalized_artifacts).map do |artifact|
+          artifact.rule_id = rule.id
+          artifact
+        end
       end
 
       #
@@ -73,14 +76,9 @@ module Mihari
       # @return [Array<Mihari::Artifact>]
       #
       def normalized_artifacts
-        @normalized_artifacts ||= artifacts.uniq(&:data).select(&:valid?).select do |artifact|
-          rule.data_types.include? artifact.data_type
-        end.reject do |artifact|
-          falsepositive? artifact.data
-        end.map do |artifact|
-          artifact.rule_id = rule.id
-          artifact
-        end
+        valid_artifacts = artifacts.uniq(&:data).select(&:valid?)
+        date_type_allowed_artifacts = valid_artifacts.select { |artifact| rule.data_types.include? artifact.data_type }
+        date_type_allowed_artifacts.reject { |artifact| falsepositive? artifact.data }
       end
 
       #
@@ -89,7 +87,7 @@ module Mihari
       # @return [Array<Mihari::Artifact>]
       #
       def unique_artifacts
-        @unique_artifacts ||= normalized_artifacts.select do |artifact|
+        normalized_artifacts.select do |artifact|
           artifact.unique?(base_time: base_time, artifact_lifetime: rule.artifact_lifetime)
         end
       end
@@ -217,7 +215,10 @@ module Mihari
       #
       def validate_analyzer_configurations
         analyzers.map do |analyzer|
-          raise ConfigurationError, "#{analyzer.source} is not configured correctly" unless analyzer.configured?
+          next if analyzer.configured?
+
+          message = "#{analyzer.source} is not configured correctly. #{analyzer.configuration_keys.join(", ")} is/are missing."
+          raise ConfigurationError, message
         end
       end
     end

data/lib/mihari/analyzers/securitytrails.rb

@@ -40,12 +40,12 @@ module Mihari
         end
       end
 
-      private
-
       def configuration_keys
         %w[securitytrails_api_key]
       end
 
+      private
+
       def client
         @client ||= Clients::SecurityTrails.new(api_key: api_key)
       end

data/lib/mihari/analyzers/shodan.rb

@@ -24,14 +24,14 @@ module Mihari
         results.map(&:to_artifacts).flatten.uniq(&:data)
       end
 
-      private
-
-      PAGE_SIZE = 100
-
       def configuration_keys
         %w[shodan_api_key]
       end
 
+      private
+
+      PAGE_SIZE = 100
+
       def client
         @client ||= Clients::Shodan.new(api_key: api_key)
       end

data/lib/mihari/analyzers/urlscan.rb

@@ -39,12 +39,12 @@ module Mihari
         end
       end
 
-      private
-
       def configuration_keys
         %w[urlscan_api_key]
       end
 
+      private
+
       def client
         @client ||= Clients::UrlScan.new(api_key: api_key)
       end

data/lib/mihari/analyzers/virustotal.rb

@@ -35,12 +35,12 @@ module Mihari
         end
       end
 
-      private
-
       def configuration_keys
         %w[virustotal_api_key]
       end
 
+      private
+
       def client
         @client = Clients::VirusTotal.new(api_key: api_key)
       end

data/lib/mihari/analyzers/virustotal_intelligence.rb

@@ -21,12 +21,12 @@ module Mihari
         search_with_cursor.map(&:to_artifacts).flatten
       end
 
-      private
-
       def configuration_keys
         %w[virustotal_api_key]
       end
 
+      private
+
       #
       # VT API
       #

data/lib/mihari/analyzers/zoomeye.rb

@@ -33,6 +33,10 @@ module Mihari
         end
       end
 
+      def configuration_keys
+        %w[zoomeye_api_key]
+      end
+
       private
 
       PAGE_SIZE = 10
@@ -46,10 +50,6 @@ module Mihari
         %w[host web].include? type
       end
 
-      def configuration_keys
-        %w[zoomeye_api_key]
-      end
-
       def client
         @client ||= Clients::ZoomEye.new(api_key: api_key)
       end

data/lib/mihari/cli/alert.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require "mihari/commands/alert"
+
+module Mihari
+  module CLI
+    class Alert < Base
+      include Mihari::Commands::Alert
+    end
+  end
+end

data/lib/mihari/cli/main.rb

@@ -3,14 +3,16 @@
 require "thor"
 
 # Commands
+require "mihari/commands/alert"
+require "mihari/commands/database"
 require "mihari/commands/search"
 require "mihari/commands/version"
 require "mihari/commands/web"
-require "mihari/commands/database"
 
 # CLIs
 require "mihari/cli/base"
 
+require "mihari/cli/alert"
 require "mihari/cli/database"
 require "mihari/cli/rule"
 
@@ -26,6 +28,9 @@ module Mihari
 
       desc "rule", "Sub commands for rule"
       subcommand "rule", Rule
+
+      desc "alert", "Sub commands for alert"
+      subcommand "alert", Alert
     end
   end
 end

data/lib/mihari/commands/alert.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Commands
+    module Alert
+      class << self
+        def included(thor)
+          thor.class_eval do
+            desc "add [PATH]", "Add an alert"
+            #
+            # @param [String] path
+            #
+            def add(path)
+              Mihari::Database.with_db_connection do
+                proxy = Mihari::Services::AlertProxy.from_path(path)
+                proxy.validate!
+
+                runner = Mihari::Services::AlertRunner.new(proxy)
+
+                begin
+                  alert = runner.run
+                rescue ActiveRecord::RecordNotFound => e
+                  # if there is a ActiveRecord::RecordNotFound, output that error without the stack trace
+                  Mihari.logger.error e.to_s
+                  return
+                end
+
+                if alert.nil?
+                  Mihari.logger.info "There is no new artifact found"
+                  return
+                end
+
+                data = Mihari::Entities::Alert.represent(alert)
+                puts JSON.pretty_generate(data.as_json)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/commands/rule.rb

@@ -15,7 +15,7 @@ module Mihari
             # @param [String] path
             #
             def validate(path)
-              rule = Services::Rule.from_path_or_id(path)
+              rule = Services::RuleProxy.from_path_or_id(path)
 
               begin
                 rule.validate!
@@ -47,7 +47,7 @@ module Mihari
               # @return [Mihari::Services::Rule]
               #
               def rule_template
-                Services::Rule.from_path File.expand_path("../templates/rule.yml.erb", __dir__)
+                Services::RuleProxy.from_path File.expand_path("../templates/rule.yml.erb", __dir__)
               end
 
               #

data/lib/mihari/commands/search.rb

@@ -4,60 +4,6 @@ module Mihari
   module Commands
     module Search
       class << self
-        class RuleWrapper
-          include Mixins::ErrorNotification
-
-          # @return [Nihari::Structs::Rule]
-          attr_reader :rule
-
-          # @return [Boolean]
-          attr_reader :force_overwrite
-
-          def initialize(rule, force_overwrite:)
-            @rule = rule
-            @force_overwrite = force_overwrite
-          end
-
-          def force_overwrite?
-            force_overwrite
-          end
-
-          #
-          # @return [Boolean]
-          #
-          def diff?
-            model = Mihari::Rule.find(rule.id)
-            model.data != rule.data.deep_stringify_keys
-          rescue ActiveRecord::RecordNotFound
-            false
-          end
-
-          def update_or_create
-            rule.to_model.save
-          end
-
-          def run
-            begin
-              analyzer = rule.to_analyzer
-            rescue ConfigurationError => e
-              # if there is a configuration error, output that error without the stack trace
-              Mihari.logger.error e.to_s
-              return
-            end
-
-            with_error_notification do
-              alert = analyzer.run
-              if alert.nil?
-                Mihari.logger.info "There is no new artifact found"
-                return
-              end
-
-              data = Mihari::Entities::Alert.represent(alert)
-              puts JSON.pretty_generate(data.as_json)
-            end
-          end
-        end
-
         def included(thor)
           thor.class_eval do
             desc "search [PATH]", "Search by a rule"
@@ -69,7 +15,7 @@ module Mihari
             #
             def search(path_or_id)
               Mihari::Database.with_db_connection do
-                rule = Services::Rule.from_path_or_id path_or_id
+                rule = Services::RuleProxy.from_path_or_id path_or_id
 
                 begin
                   rule.validate!
@@ -78,15 +24,30 @@ module Mihari
                 end
 
                 force_overwrite = options["force_overwrite"] || false
-                wrapper = RuleWrapper.new(rule, force_overwrite: force_overwrite)
+                runner = Services::RuleRunner.new(rule, force_overwrite: force_overwrite)
 
-                if wrapper.diff? && !force_overwrite
+                if runner.diff? && !force_overwrite
                   message = "There is diff in the rule (#{rule.id}). Are you sure you want to overwrite the rule? (y/n)"
                   return unless yes?(message)
                 end
 
-                wrapper.update_or_create
-                wrapper.run
+                runner.update_or_create
+
+                begin
+                  alert = runner.run
+                rescue ConfigurationError => e
+                  # if there is a configuration error, output that error without the stack trace
+                  Mihari.logger.error e.to_s
+                  return
+                end
+
+                if alert.nil?
+                  Mihari.logger.info "There is no new artifact found"
+                  return
+                end
+
+                data = Mihari::Entities::Alert.represent(alert)
+                puts JSON.pretty_generate(data.as_json)
               end
             end
           end

data/lib/mihari/config.rb

@@ -82,7 +82,7 @@ module Mihari
     attr_reader :sentry_dsn
 
     # @return [Boolean]
-    attr_reader :hide_config_values
+    attr_accessor :hide_config_values
 
     # @return [Integer]
     attr_reader :retry_interval
@@ -146,7 +146,7 @@ module Mihari
       @retry_times = ENV.fetch("RETRY_TIMES", 3).to_i
       @retry_interval = ENV.fetch("RETRY_INTERVAL", 5).to_i
 
-      @pagination_limit = ENV.fetch("PAGINATION_LIMIT", 1000).to_i
+      @pagination_limit = ENV.fetch("PAGINATION_LIMIT", 100).to_i
     end
   end
 end

data/lib/mihari/emitters/base.rb

@@ -14,7 +14,7 @@ module Mihari
 
       #
       # @param [Array<Mihari::Artifact>] artifacts
-      # @param [Mihari::Services::Rule] rule
+      # @param [Mihari::Services::RuleProxy] rule
       # @param [Hash] **_options
       #
       def initialize(artifacts:, rule:, **_options)

data/lib/mihari/emitters/database.rb

@@ -10,10 +10,10 @@ module Mihari
       #
       # Create an alert
       #
-      # @return [Mihari::Alert]
+      # @return [Mihari::Alert, nil]
       #
       def emit
-        return if artifacts.empty?
+        return nil if artifacts.empty?
 
         tags = rule.tags.filter_map { |name| Tag.find_or_create_by(name: name) }.uniq
         taggings = tags.map { |tag| Tagging.new(tag_id: tag.id) }

data/lib/mihari/errors.rb

@@ -15,17 +15,38 @@ module Mihari
 
   class RuleValidationError < Error; end
 
+  class AlertValidationError < Error; end
+
   class YAMLSyntaxError < Error; end
 
   class ConfigurationError < Error; end
 
+  # errors for HTTP interactions
   class HTTPError < Error; end
 
-  class StatusCodeError < HTTPError; end
-
   class NetworkError < HTTPError; end
 
   class TimeoutError < HTTPError; end
 
   class SSLError < HTTPError; end
+
+  class StatusCodeError < HTTPError
+    # @return [Integer]
+    attr_reader :status_code
+
+    # @return [String, nil]
+    attr_reader :body
+
+    #
+    # @param [String] msg
+    # @param [Integer] status_code
+    # @param [String, nil] body
+    #
+    def initialize(msg, status_code, body)
+      super(msg)
+
+      @status_code = status_code
+      @body = body
+    end
+  end
 end

data/lib/mihari/http.rb

@@ -94,7 +94,13 @@ module Mihari
       Net::HTTP.start(url.host, url.port, https_options) do |http|
         res = http.request(req)
 
-        raise StatusCodeError, "Unsuccessful response code returned: #{res.code}" unless res.is_a?(Net::HTTPSuccess)
+        unless res.is_a?(Net::HTTPSuccess)
+          raise StatusCodeError.new(
+            "Unsuccessful response code returned: #{res.code}",
+            res.code.to_i,
+            res.body
+          )
+        end
 
         res
       end

data/lib/mihari/schemas/alert.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Schemas
+    Alert = Dry::Schema.Params do
+      required(:rule_id).value(:string)
+      required(:artifacts).value(array[:string])
+    end
+
+    class AlertContract < Dry::Validation::Contract
+      params(Alert)
+    end
+  end
+end