mihari 5.2.1 → 5.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5ce7e22d3d54cc3e66fec62b494ce5f9def9107129513cd51803c71b8ae48b18
-  data.tar.gz: fd85f33a666a77a8b43d12dc3234ab2656cd843c8f7e23025c1a5f076947fbcf
+  metadata.gz: 77de9f96ff14a64b2ab7a492fff36ed1515df7def7b18b5d81bac6785a5b75c0
+  data.tar.gz: 9e99189740e4e3b6ee6af97cc09fbd2af1f5395c047df925f528721f2c68595d
 SHA512:
-  metadata.gz: a17809e3c6f52e7d37f6df2fc92b0ad5a1d134556e6ef13cb64b3802074a44f2ecd0c2475ab632e5da0f57e7c68a44fca15f722f31124fbb9e8af512f46aa2ac
-  data.tar.gz: d777cac77ceeb2a98c8f37a8e001d9240403b47a1e2326a6f1bc42c3cf48e7813dd69ab2260ce5a29a34b7b9b3e0ef54fa541785f8b5dbc82b787381e549525e
+  metadata.gz: 59171eef265ace4aab9295b9e3866233138194afc0ee691022d5c6ec9f4ca6b53400a01109f9a745cdf54d70cd8c74a1fd39ff319624bf3f3ff04d2ae409ca26
+  data.tar.gz: a886b191dcab85164e2f7e47d2b9445121254a8b94248ce1f6e54fff31b8e0a13afc6a8dc5013f04cfaac2340957344ba03b3441d09b0018e46bb7a564af4b03

data/.rubocop.yml

@@ -8,3 +8,5 @@ Metrics/BlockLength:
     - "*.gemspec"
 Metrics/ClassLength:
   Enabled: false
+Metrics/MethodLength:
+  Max: 20

data/lib/mihari/analyzers/base.rb

@@ -5,32 +5,34 @@ module Mihari
     class Base
       extend Dry::Initializer
 
-      option :rule, default: proc {}
-
       include Mixins::Configurable
       include Mixins::Retriable
 
-      # @return [Mihari::Structs::Rule, nil]
-      attr_reader :rule
-
-      def initialize(*args, **kwargs)
-        super(*args, **kwargs)
-
-        @base_time = Time.now.utc
+      # @return [Array<String>, Array<Mihari::Artifact>]
+      def artifacts
+        raise NotImplementedError, "You must implement #{self.class}##{__method__}"
       end
 
       #
-      # Load/overwrite rule
+      # Normalize artifacts
+      # - Convert data (string) into an artifact
+      # - Reject an invalid artifact
       #
-      # @param [String] path_or_id
+      # @return [Array<Mihari::Artifact>]
       #
-      def load_rule(path_or_id)
-        @rule = Structs::Rule.from_path_or_id path_or_id
-      end
-
-      # @return [Array<String>, Array<Mihari::Artifact>]
-      def artifacts
-        raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+      def normalized_artifacts
+        retry_on_error do
+          @normalized_artifacts ||= artifacts.compact.sort.map do |artifact|
+            # No need to set data_type manually
+            # It is set automatically in #initialize
+            artifact = artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact)
+            artifact
+          end.select(&:valid?).uniq(&:data).map do |artifact|
+            # set source
+            artifact.source = source
+            artifact
+          end
+        end
       end
 
       # @return [String]
@@ -43,109 +45,12 @@ module Mihari
         self.class.to_s.split("::").last
       end
 
-      #
-      # Set artifacts & run emitters in parallel
-      #
-      # @return [Mihari::Alert, nil]
-      #
-      def run
-        raise ConfigurationError, "#{class_name} is not configured correctly" unless configured?
-
-        alert_or_something = bulk_emit
-        # returns Mihari::Alert created by the database emitter
-        alert_or_something.find { |res| res.is_a?(Mihari::Alert) }
-      end
-
-      #
-      # Bulk emit
-      #
-      # @return [Array<Mihari::Alert>]
-      #
-      def bulk_emit
-        Parallel.map(valid_emitters) { |emitter| emit emitter }.compact
-      end
-
-      #
-      # Emit an alert
-      #
-      # @param [Mihari::Emitters::Base] emitter
-      #
-      # @return [Mihari::Alert, nil]
-      #
-      def emit(emitter)
-        return if enriched_artifacts.empty?
-
-        alert_or_something = emitter.run(artifacts: enriched_artifacts, rule: rule)
-
-        Mihari.logger.info "Emission by #{emitter.class} is succedded"
-
-        alert_or_something
-      rescue StandardError => e
-        Mihari.logger.info "Emission by #{emitter.class} is failed: #{e}"
-      end
-
       class << self
         def inherited(child)
           super
           Mihari.analyzers << child
         end
       end
-
-      #
-      # Normalize artifacts
-      # - Convert data (string) into an artifact
-      # - Set rule ID
-      # - Reject an invalid artifact
-      # - Uniquefy artifacts by data
-      #
-      # @return [Array<Mihari::Artifact>]
-      #
-      def normalized_artifacts
-        @normalized_artifacts ||= artifacts.compact.sort.map do |artifact|
-          # No need to set data_type manually
-          # It is set automatically in #initialize
-          artifact = artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact, source: source)
-          artifact.rule_id = rule&.id
-          artifact
-        end.select(&:valid?).uniq(&:data)
-      end
-
-      private
-
-      #
-      # Uniquefy artifacts (assure rule level uniqueness)
-      #
-      # @return [Array<Mihari::Artifact>]
-      #
-      def unique_artifacts
-        @unique_artifacts ||= normalized_artifacts.select do |artifact|
-          artifact.unique?(base_time: @base_time, artifact_lifetime: rule&.artifact_lifetime)
-        end
-      end
-
-      #
-      # Enriched artifacts
-      #
-      # @return [Array<Mihari::Artifact>]
-      #
-      def enriched_artifacts
-        @enriched_artifacts ||= Parallel.map(unique_artifacts) do |artifact|
-          artifact.enrich_all
-          artifact
-        end
-      end
-
-      #
-      # Select valid emitters
-      #
-      # @return [Array<Mihari::Emitters::Base>]
-      #
-      def valid_emitters
-        @valid_emitters ||= Mihari.emitters.filter_map do |klass|
-          emitter = klass.new
-          emitter.valid? ? emitter : nil
-        end.compact
-      end
     end
   end
 end

data/lib/mihari/analyzers/censys.rb

@@ -26,15 +26,18 @@ module Mihari
         @secret = kwargs[:secret] || Mihari.config.censys_secret
       end
 
+      #
+      # @return [Array<Mihari::Artifact>]
+      #
       def artifacts
         artifacts = []
 
         cursor = nil
         loop do
           response = client.search(query, cursor: cursor)
-          artifacts << response.result.to_artifacts(source)
+          artifacts << response.result.to_artifacts
           cursor = response.result.links.next
-          break if cursor == ""
+          break if cursor.nil?
 
           # sleep #{interval} seconds to avoid the rate limitation (if it is set)
           sleep interval
@@ -43,24 +46,39 @@ module Mihari
         artifacts.flatten.uniq(&:data)
       end
 
+      #
+      # @return [Boolean]
+      #
       def configured?
         configuration_keys.all? { |key| Mihari.config.send(key) } || (id? && secret?)
       end
 
       private
 
+      #
+      # @return [Array<String>]
+      #
       def configuration_keys
         %w[censys_id censys_secret]
       end
 
+      #
+      # @return [Mihari::Clients::Censys]
+      #
       def client
         @client ||= Clients::Censys.new(id: id, secret: secret)
       end
 
+      #
+      # @return [Boolean]
+      #
       def id?
         !id.nil?
       end
 
+      #
+      # @return [Boolean]
+      #
       def secret?
         !secret.nil?
       end

data/lib/mihari/analyzers/onyphe.rb

@@ -28,7 +28,7 @@ module Mihari
         responses = search
         return [] unless responses
 
-        responses.map { |response| response.to_artifacts(source) }.flatten
+        responses.map(&:to_artifacts).flatten
       end
 
       private

data/lib/mihari/analyzers/rule.rb

@@ -34,14 +34,21 @@ module Mihari
       "webhook" => Emitters::Webhook
     }.freeze
 
-    # @return [Mihari::Structs::Rule]
-    attr_reader :rule
-
-    class Rule < Base
+    class Rule
       include Mixins::FalsePositive
 
-      def initialize(**kwargs)
-        super(**kwargs)
+      # @return [Mihari::Structs::Rule]
+      attr_reader :rule
+
+      # @return [Time]
+      attr_reader :base_time
+
+      #
+      # @param [Mihari::Structs::Rule] rule
+      #
+      def initialize(rule:)
+        @rule = rule
+        @base_time = Time.now.utc
 
         validate_analyzer_configurations
       end
@@ -52,41 +59,15 @@ module Mihari
       # @return [Array<Mihari::Artifact>]
       #
       def artifacts
-        artifacts = []
-
-        rule.queries.each do |original_params|
-          parmas = original_params.deep_dup
-
-          analyzer_name = parmas[:analyzer]
-          klass = get_analyzer_class(analyzer_name)
-
-          query = parmas[:query]
-
-          # set interval in the top level
-          options = parmas[:options] || {}
-          interval = options[:interval]
-
-          parmas[:interval] = interval if interval
-
-          # set rule
-          parmas[:rule] = rule
-
-          analyzer = klass.new(query, **parmas)
-
-          # Use #normalized_artifacts method to get atrifacts as Array<Mihari::Artifact>
-          # So Mihari::Artifact object has "source" attribute (e.g. "Shodan")
-          artifacts << analyzer.normalized_artifacts
-        end
-
-        artifacts.flatten
+        rule.queries.map { |params| run_query(params.deep_dup) }.flatten
       end
 
       #
       # Normalize artifacts
-      # - Uniquefy artifacts by #uniq(&:data)
-      # - Reject an invalid artifact (for just in case)
+      # - Reject invalid artifacts (for just in case)
       # - Select artifacts with allowed data types
-      # - Reject artifacts with disallowed data values
+      # - Reject artifacts with false positive values
+      # - Set rule ID
       #
       # @return [Array<Mihari::Artifact>]
       #
@@ -95,6 +76,20 @@ module Mihari
           rule.data_types.include? artifact.data_type
         end.reject do |artifact|
           falsepositive? artifact.data
+        end.map do |artifact|
+          artifact.rule_id = rule.id
+          artifact
+        end
+      end
+
+      #
+      # Uniquify artifacts (assure rule level uniqueness)
+      #
+      # @return [Array<Mihari::Artifact>]
+      #
+      def unique_artifacts
+        @unique_artifacts ||= normalized_artifacts.select do |artifact|
+          artifact.unique?(base_time: base_time, artifact_lifetime: rule.artifact_lifetime)
         end
       end
 
@@ -105,39 +100,93 @@ module Mihari
       #
       def enriched_artifacts
         @enriched_artifacts ||= Parallel.map(unique_artifacts) do |artifact|
-          rule.enrichers.each do |enricher|
-            artifact.enrich_by_enricher(enricher[:enricher])
-          end
-
+          rule.enrichers.each { |enricher| artifact.enrich_by_enricher enricher[:enricher] }
           artifact
         end
       end
 
       #
-      # Normalized disallowed data values
+      # Bulk emit
+      #
+      # @return [Array<Mihari::Alert>]
+      #
+      def bulk_emit
+        Parallel.map(valid_emitters) { |emitter| emit emitter }.compact
+      end
+
+      #
+      # Emit an alert
+      #
+      # @param [Mihari::Emitters::Base] emitter
       #
-      # @return [Array<Regexp, String>]
+      # @return [Mihari::Alert, nil]
       #
-      def normalized_falsepositives
-        @normalized_falsepositives ||= rule.falsepositives.map { |v| normalize_falsepositive v }
+      def emit(emitter)
+        return if enriched_artifacts.empty?
+
+        alert_or_something = emitter.run(artifacts: enriched_artifacts, rule: rule)
+
+        Mihari.logger.info "Emission by #{emitter.class} is succeeded"
+
+        alert_or_something
+      rescue StandardError => e
+        Mihari.logger.info "Emission by #{emitter.class} is failed: #{e}"
       end
 
+      #
+      # Set artifacts & run emitters in parallel
+      #
+      # @return [Mihari::Alert, nil]
+      #
+      def run
+        # memoize enriched artifacts
+        enriched_artifacts
+
+        alert_or_something = bulk_emit
+        # returns Mihari::Alert created by the database emitter
+        alert_or_something.find { |res| res.is_a?(Mihari::Alert) }
+      end
+
+      private
+
       #
       # Check whether a value is a falsepositive value or not
       #
       # @return [Boolean]
       #
       def falsepositive?(value)
-        return true if normalized_falsepositives.include?(value)
+        return true if rule.falsepositives.include?(value)
 
-        normalized_falsepositives.select do |falsepositive|
+        rule.falsepositives.select do |falsepositive|
           falsepositive.is_a?(Regexp)
         end.any? do |falseposistive|
           falseposistive.match?(value)
         end
       end
 
-      private
+      #
+      # @param [Hash] params
+      #
+      # @return [Array<Mihari::Artifact>]
+      #
+      def run_query(params)
+        analyzer_name = params[:analyzer]
+        klass = get_analyzer_class(analyzer_name)
+
+        # set interval in the top level
+        options = params[:options] || {}
+        interval = options[:interval]
+        params[:interval] = interval if interval
+
+        # set rule
+        params[:rule] = rule
+        query = params[:query]
+        analyzer = klass.new(query, **params)
+
+        # Use #normalized_artifacts method to get artifacts as Array<Mihari::Artifact>
+        # So Mihari::Artifact object has "source" attribute (e.g. "Shodan")
+        analyzer.normalized_artifacts
+      end
 
       #
       # Get emitter class
@@ -153,18 +202,26 @@ module Mihari
         raise ArgumentError, "#{emitter_name} is not supported"
       end
 
-      def valid_emitters
-        @valid_emitters ||= rule.emitters.filter_map do |original_params|
-          params = original_params.deep_dup
+      #
+      # @param [Hash] params
+      #
+      # @return [Mihari::Emitter:Base]
+      #
+      def validate_emitter(params)
+        name = params[:emitter]
+        params.delete(:emitter)
 
-          name = params[:emitter]
-          params.delete(:emitter)
+        klass = get_emitter_class(name)
+        emitter = klass.new(**params)
 
-          klass = get_emitter_class(name)
-          emitter = klass.new(**params)
+        emitter.valid? ? emitter : nil
+      end
 
-          emitter.valid? ? emitter : nil
-        end
+      #
+      # @return [Array<Mihari::Emitter::Base>]
+      #
+      def valid_emitters
+        @valid_emitters ||= rule.emitters.filter_map { |params| validate_emitter(params.deep_dup) }
       end
 
       #
@@ -187,13 +244,12 @@ module Mihari
       def validate_analyzer_configurations
         rule.queries.each do |params|
           analyzer_name = params[:analyzer]
+
           klass = get_analyzer_class(analyzer_name)
+          klass_name = klass.to_s.split("::").last
 
           instance = klass.new("dummy")
-          unless instance.configured?
-            klass_name = klass.to_s.split("::").last
-            raise ConfigurationError, "#{klass_name} is not configured correctly"
-          end
+          raise ConfigurationError, "#{klass_name} is not configured correctly" unless instance.configured?
         end
       end
     end

data/lib/mihari/analyzers/shodan.rb

@@ -26,7 +26,7 @@ module Mihari
         results = search
         return [] if results.empty?
 
-        results.map { |result| result.to_artifacts(source) }.flatten.uniq(&:data)
+        results.map(&:to_artifacts).flatten.uniq(&:data)
       end
 
       private

data/lib/mihari/analyzers/urlscan.rb

@@ -36,15 +36,12 @@ module Mihari
 
       def artifacts
         responses = search
-        results = responses.map(&:results).flatten
-
-        allowed_data_types.map do |type|
-          results.filter_map do |result|
-            page = result.page
-            data = page.send(type.to_sym)
-            data.nil? ? nil : Artifact.new(data: data, source: source, metadata: result)
-          end
-        end.flatten
+        # @type [Array<Mihari::Artifact>]
+        artifacts = responses.map { |res| res.to_artifacts }.flatten
+
+        artifacts.select do |artifact|
+          allowed_data_types.include? artifact.data_type
+        end
       end
 
       private

data/lib/mihari/analyzers/virustotal_intelligence.rb

@@ -26,11 +26,7 @@ module Mihari
 
       def artifacts
         responses = search_with_cursor
-        responses.map do |response|
-          response.data.map do |datum|
-            Artifact.new(data: datum.value, source: source, metadata: datum.metadata)
-          end
-        end.flatten
+        responses.map(&:to_artifacts).flatten
       end
 
       private

data/lib/mihari/cli/main.rb

@@ -3,7 +3,7 @@
 require "thor"
 
 # Commands
-require "mihari/commands/searcher"
+require "mihari/commands/search"
 require "mihari/commands/version"
 require "mihari/commands/web"
 require "mihari/commands/database"
@@ -17,7 +17,7 @@ require "mihari/cli/rule"
 module Mihari
   module CLI
     class Main < Base
-      include Mihari::Commands::Searcher
+      include Mihari::Commands::Search
       include Mihari::Commands::Version
       include Mihari::Commands::Web
 

data/lib/mihari/commands/search.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Commands
+    module Search
+      include Mixins::ErrorNotification
+
+      def self.included(thor)
+        thor.class_eval do
+          desc "search [PATH]", "Search by a rule"
+          method_option :force_overwrite, type: :boolean, aliases: "-f", desc: "Force an overwrite the rule"
+          #
+          # Search by a rule
+          #
+          # @param [String] path_or_id
+          #
+          def search(path_or_id)
+            Mihari::Database.with_db_connection do
+              rule = Structs::Rule.from_path_or_id path_or_id
+
+              begin
+                rule.validate!
+              rescue RuleValidationError
+                return
+              end
+
+              update_rule rule
+              run_rule rule
+            end
+          end
+        end
+      end
+
+      # @param [Mihari::Structs::Rule] rule
+      #
+      def update_rule(rule)
+        force_overwrite = options["force_overwrite"] || false
+        begin
+          rule_model = Mihari::Rule.find(rule.id)
+          has_change = rule_model.data != rule.data.deep_stringify_keys
+          has_change_and_not_force_overwrite = has_change & !force_overwrite
+
+          confirm_message = "This operation will overwrite the rule in the database (Rule ID: #{rule.id}). Are you sure you want to update the rule? (y/n)"
+          return if has_change_and_not_force_overwrite && !yes?(confirm_message)
+          # update the rule
+          rule.model.save
+        rescue ActiveRecord::RecordNotFound
+          # create a new rule
+          rule.model.save
+        end
+      end
+
+      #
+      # @param [Mihari::Structs::Rule] rule
+      #
+      def run_rule(rule)
+        with_error_notification do
+          alert = rule.analyzer.run
+          if alert
+            data = Mihari::Entities::Alert.represent(alert)
+            puts JSON.pretty_generate(data.as_json)
+          else
+            Mihari.logger.info "There is no new alert created in the database"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/mixins/error_notification.rb

@@ -6,8 +6,6 @@ module Mihari
       #
       # Send an exception notification if there is any error in a block
       #
-      # @return [Nil]
-      #
       def with_error_notification
         yield
       rescue StandardError => e

data/lib/mihari/models/artifact.rb

@@ -60,7 +60,7 @@ module Mihari
       ).order(created_at: :desc).first
       return true if artifact.nil?
 
-      # check whetehr the artifact is decayed or not
+      # check whether the artifact is decayed or not
       return false if artifact_lifetime.nil?
 
       # use the current UTC time if base_time is not given (for testing)

data/lib/mihari/schemas/rule.rb

@@ -25,30 +25,15 @@ module Mihari
         AnalyzerWithoutAPIKey | AnalyzerWithAPIKey | Censys | CIRCL | PassiveTotal | ZoomEye | Urlscan | Crtsh | Feed
       end
 
-      optional(:emitters).value(:array).each { Database | MISP | TheHive | Slack | Webhook }
+      optional(:emitters).value(:array).each { Database | MISP | TheHive | Slack | Webhook }.default(DEFAULT_EMITTERS)
 
-      optional(:enrichers).value(:array).each(Enricher)
+      optional(:enrichers).value(:array).each(Enricher).default(DEFAULT_ENRICHERS)
 
       optional(:data_types).value(array[Types::DataTypes]).default(DEFAULT_DATA_TYPES)
       optional(:falsepositives).value(array[:string]).default([])
 
       optional(:artifact_lifetime).value(:integer)
       optional(:artifact_ttl).value(:integer)
-
-      before(:key_coercer) do |result|
-        # it looks like that dry-schema v1.9.1 has an issue with setting an array of schemas as a default value
-        # e.g. optional(:emitters).value(:array).each { Emitter | HTTP }.default(DEFAULT_EMITTERS) does not work well
-        # so let's do a dirty hack...
-        h = result.to_h
-
-        emitters = h[:emitters]
-        h[:emitters] = emitters || DEFAULT_EMITTERS
-
-        enrichers = h[:enrichers]
-        h[:enrichers] = enrichers || DEFAULT_ENRICHERS
-
-        h
-      end
     end
 
     class RuleContract < Dry::Validation::Contract