mihari 5.2.1 → 5.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5ce7e22d3d54cc3e66fec62b494ce5f9def9107129513cd51803c71b8ae48b18
-  data.tar.gz: fd85f33a666a77a8b43d12dc3234ab2656cd843c8f7e23025c1a5f076947fbcf
+  metadata.gz: 2a66fb2d71bcae401062277921a8ade6a3e9e9d961b193a80deacd3a8a934d4c
+  data.tar.gz: b4dcccfa58019f819241f8679b5d1ae846002f0a95307cd95856f2b6d04a6dd1
 SHA512:
-  metadata.gz: a17809e3c6f52e7d37f6df2fc92b0ad5a1d134556e6ef13cb64b3802074a44f2ecd0c2475ab632e5da0f57e7c68a44fca15f722f31124fbb9e8af512f46aa2ac
-  data.tar.gz: d777cac77ceeb2a98c8f37a8e001d9240403b47a1e2326a6f1bc42c3cf48e7813dd69ab2260ce5a29a34b7b9b3e0ef54fa541785f8b5dbc82b787381e549525e
+  metadata.gz: e215400c8dce2b864bc26a951ed8ea35757441e7d25bdba6c66632d6716991bad202170f9984d09b7a709f6c9aceb731e5b40396efe621ffc55810a10db45db2
+  data.tar.gz: 745522a9cefaed75e5b266a429dea7afc0efe34f26f8591af18e3bc27a0746659ed8d64f6aa64f6edc0f9195acbf26e4a7ee078c052bab760f985da779c4e6e4

data/.rubocop.yml

@@ -8,3 +8,5 @@ Metrics/BlockLength:
     - "*.gemspec"
 Metrics/ClassLength:
   Enabled: false
+Metrics/MethodLength:
+  Max: 20

data/lib/mihari/analyzers/base.rb

@@ -5,32 +5,34 @@ module Mihari
     class Base
       extend Dry::Initializer
 
-      option :rule, default: proc {}
-
       include Mixins::Configurable
       include Mixins::Retriable
 
-      # @return [Mihari::Structs::Rule, nil]
-      attr_reader :rule
-
-      def initialize(*args, **kwargs)
-        super(*args, **kwargs)
-
-        @base_time = Time.now.utc
+      # @return [Array<String>, Array<Mihari::Artifact>]
+      def artifacts
+        raise NotImplementedError, "You must implement #{self.class}##{__method__}"
       end
 
       #
-      # Load/overwrite rule
+      # Normalize artifacts
+      # - Convert data (string) into an artifact
+      # - Reject an invalid artifact
       #
-      # @param [String] path_or_id
+      # @return [Array<Mihari::Artifact>]
       #
-      def load_rule(path_or_id)
-        @rule = Structs::Rule.from_path_or_id path_or_id
-      end
-
-      # @return [Array<String>, Array<Mihari::Artifact>]
-      def artifacts
-        raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+      def normalized_artifacts
+        retry_on_error do
+          @normalized_artifacts ||= artifacts.compact.sort.map do |artifact|
+            # No need to set data_type manually
+            # It is set automatically in #initialize
+            artifact = artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact)
+            artifact
+          end.select(&:valid?).uniq(&:data).map do |artifact|
+            # set source
+            artifact.source = source
+            artifact
+          end
+        end
       end
 
       # @return [String]
@@ -43,109 +45,12 @@ module Mihari
         self.class.to_s.split("::").last
       end
 
-      #
-      # Set artifacts & run emitters in parallel
-      #
-      # @return [Mihari::Alert, nil]
-      #
-      def run
-        raise ConfigurationError, "#{class_name} is not configured correctly" unless configured?
-
-        alert_or_something = bulk_emit
-        # returns Mihari::Alert created by the database emitter
-        alert_or_something.find { |res| res.is_a?(Mihari::Alert) }
-      end
-
-      #
-      # Bulk emit
-      #
-      # @return [Array<Mihari::Alert>]
-      #
-      def bulk_emit
-        Parallel.map(valid_emitters) { |emitter| emit emitter }.compact
-      end
-
-      #
-      # Emit an alert
-      #
-      # @param [Mihari::Emitters::Base] emitter
-      #
-      # @return [Mihari::Alert, nil]
-      #
-      def emit(emitter)
-        return if enriched_artifacts.empty?
-
-        alert_or_something = emitter.run(artifacts: enriched_artifacts, rule: rule)
-
-        Mihari.logger.info "Emission by #{emitter.class} is succedded"
-
-        alert_or_something
-      rescue StandardError => e
-        Mihari.logger.info "Emission by #{emitter.class} is failed: #{e}"
-      end
-
       class << self
         def inherited(child)
           super
           Mihari.analyzers << child
         end
       end
-
-      #
-      # Normalize artifacts
-      # - Convert data (string) into an artifact
-      # - Set rule ID
-      # - Reject an invalid artifact
-      # - Uniquefy artifacts by data
-      #
-      # @return [Array<Mihari::Artifact>]
-      #
-      def normalized_artifacts
-        @normalized_artifacts ||= artifacts.compact.sort.map do |artifact|
-          # No need to set data_type manually
-          # It is set automatically in #initialize
-          artifact = artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact, source: source)
-          artifact.rule_id = rule&.id
-          artifact
-        end.select(&:valid?).uniq(&:data)
-      end
-
-      private
-
-      #
-      # Uniquefy artifacts (assure rule level uniqueness)
-      #
-      # @return [Array<Mihari::Artifact>]
-      #
-      def unique_artifacts
-        @unique_artifacts ||= normalized_artifacts.select do |artifact|
-          artifact.unique?(base_time: @base_time, artifact_lifetime: rule&.artifact_lifetime)
-        end
-      end
-
-      #
-      # Enriched artifacts
-      #
-      # @return [Array<Mihari::Artifact>]
-      #
-      def enriched_artifacts
-        @enriched_artifacts ||= Parallel.map(unique_artifacts) do |artifact|
-          artifact.enrich_all
-          artifact
-        end
-      end
-
-      #
-      # Select valid emitters
-      #
-      # @return [Array<Mihari::Emitters::Base>]
-      #
-      def valid_emitters
-        @valid_emitters ||= Mihari.emitters.filter_map do |klass|
-          emitter = klass.new
-          emitter.valid? ? emitter : nil
-        end.compact
-      end
     end
   end
 end

data/lib/mihari/analyzers/binaryedge.rb

@@ -42,7 +42,6 @@ module Mihari
       #
       # Search with pagination
       #
-      # @param [String] query
       # @param [Integer] page
       #
       # @return [Hash]

data/lib/mihari/analyzers/censys.rb

@@ -26,15 +26,23 @@ module Mihari
         @secret = kwargs[:secret] || Mihari.config.censys_secret
       end
 
+      #
+      # @return [Array<Mihari::Artifact>]
+      #
       def artifacts
         artifacts = []
 
         cursor = nil
         loop do
           response = client.search(query, cursor: cursor)
-          artifacts << response.result.to_artifacts(source)
+          artifacts << response.result.to_artifacts
           cursor = response.result.links.next
-          break if cursor == ""
+          # NOTE: Censys's search API is unstable recently
+          # it may returns empty links or empty string cursors
+          # - Empty links: "links": {}
+          # - Empty cursors: "links": { "next": "", "prev": "" }
+          # So it needs to check both cases
+          break if cursor.nil? || cursor.empty?
 
           # sleep #{interval} seconds to avoid the rate limitation (if it is set)
           sleep interval
@@ -43,24 +51,39 @@ module Mihari
         artifacts.flatten.uniq(&:data)
       end
 
+      #
+      # @return [Boolean]
+      #
       def configured?
-        configuration_keys.all? { |key| Mihari.config.send(key) } || (id? && secret?)
+        configuration_keys? || (id? && secret?)
       end
 
       private
 
+      #
+      # @return [Array<String>]
+      #
       def configuration_keys
         %w[censys_id censys_secret]
       end
 
+      #
+      # @return [Mihari::Clients::Censys]
+      #
       def client
         @client ||= Clients::Censys.new(id: id, secret: secret)
       end
 
+      #
+      # @return [Boolean]
+      #
       def id?
         !id.nil?
       end
 
+      #
+      # @return [Boolean]
+      #
       def secret?
         !secret.nil?
       end

data/lib/mihari/analyzers/circl.rb

@@ -41,7 +41,7 @@ module Mihari
       end
 
       def configured?
-        configuration_keys.all? { |key| Mihari.config.send(key) } || (username? && password?)
+        configuration_keys? || (username? && password?)
       end
 
       private

data/lib/mihari/analyzers/onyphe.rb

@@ -28,7 +28,7 @@ module Mihari
         responses = search
         return [] unless responses
 
-        responses.map { |response| response.to_artifacts(source) }.flatten
+        responses.map(&:to_artifacts).flatten
       end
 
       private

data/lib/mihari/analyzers/passivetotal.rb

@@ -43,7 +43,7 @@ module Mihari
       end
 
       def configured?
-        configuration_keys.all? { |key| Mihari.config.send(key) } || (username? && api_key?)
+        configuration_keys? || (username? && api_key?)
       end
 
       private

data/lib/mihari/analyzers/rule.rb

@@ -34,59 +34,40 @@ module Mihari
       "webhook" => Emitters::Webhook
     }.freeze
 
-    # @return [Mihari::Structs::Rule]
-    attr_reader :rule
-
-    class Rule < Base
+    class Rule
       include Mixins::FalsePositive
 
-      def initialize(**kwargs)
-        super(**kwargs)
+      # @return [Mihari::Structs::Rule]
+      attr_reader :rule
+
+      # @return [Time]
+      attr_reader :base_time
+
+      #
+      # @param [Mihari::Structs::Rule] rule
+      #
+      def initialize(rule:)
+        @rule = rule
+        @base_time = Time.now.utc
 
         validate_analyzer_configurations
       end
 
       #
-      # Returns a list of artifacts matched with queries
+      # Returns a list of artifacts matched with queries/analyzers
       #
       # @return [Array<Mihari::Artifact>]
       #
       def artifacts
-        artifacts = []
-
-        rule.queries.each do |original_params|
-          parmas = original_params.deep_dup
-
-          analyzer_name = parmas[:analyzer]
-          klass = get_analyzer_class(analyzer_name)
-
-          query = parmas[:query]
-
-          # set interval in the top level
-          options = parmas[:options] || {}
-          interval = options[:interval]
-
-          parmas[:interval] = interval if interval
-
-          # set rule
-          parmas[:rule] = rule
-
-          analyzer = klass.new(query, **parmas)
-
-          # Use #normalized_artifacts method to get atrifacts as Array<Mihari::Artifact>
-          # So Mihari::Artifact object has "source" attribute (e.g. "Shodan")
-          artifacts << analyzer.normalized_artifacts
-        end
-
-        artifacts.flatten
+        analyzers.flat_map(&:normalized_artifacts)
       end
 
       #
       # Normalize artifacts
-      # - Uniquefy artifacts by #uniq(&:data)
-      # - Reject an invalid artifact (for just in case)
+      # - Reject invalid artifacts (for just in case)
       # - Select artifacts with allowed data types
-      # - Reject artifacts with disallowed data values
+      # - Reject artifacts with false positive values
+      # - Set rule ID
       #
       # @return [Array<Mihari::Artifact>]
       #
@@ -95,6 +76,20 @@ module Mihari
           rule.data_types.include? artifact.data_type
         end.reject do |artifact|
           falsepositive? artifact.data
+        end.map do |artifact|
+          artifact.rule_id = rule.id
+          artifact
+        end
+      end
+
+      #
+      # Uniquify artifacts (assure rule level uniqueness)
+      #
+      # @return [Array<Mihari::Artifact>]
+      #
+      def unique_artifacts
+        @unique_artifacts ||= normalized_artifacts.select do |artifact|
+          artifact.unique?(base_time: base_time, artifact_lifetime: rule.artifact_lifetime)
         end
       end
 
@@ -105,39 +100,105 @@ module Mihari
       #
       def enriched_artifacts
         @enriched_artifacts ||= Parallel.map(unique_artifacts) do |artifact|
-          rule.enrichers.each do |enricher|
-            artifact.enrich_by_enricher(enricher[:enricher])
-          end
-
+          rule.enrichers.each { |enricher| artifact.enrich_by_enricher enricher[:enricher] }
           artifact
         end
       end
 
       #
-      # Normalized disallowed data values
+      # Bulk emit
       #
-      # @return [Array<Regexp, String>]
+      # @return [Array<Mihari::Alert>]
       #
-      def normalized_falsepositives
-        @normalized_falsepositives ||= rule.falsepositives.map { |v| normalize_falsepositive v }
+      def bulk_emit
+        return [] if enriched_artifacts.empty?
+
+        Parallel.map(valid_emitters) do |emitter|
+          emission = emitter.emit
+          Mihari.logger.info "Emission by #{emitter.class} is succeeded"
+          emission
+        rescue StandardError => e
+          Mihari.logger.info "Emission by #{emitter.class} is failed: #{e}"
+        end.compact
       end
 
+      #
+      # Set artifacts & run emitters in parallel
+      #
+      # @return [Mihari::Alert, nil]
+      #
+      def run
+        # memoize enriched artifacts
+        enriched_artifacts
+
+        alert_or_something = bulk_emit
+        # returns Mihari::Alert created by the database emitter
+        alert_or_something.find { |res| res.is_a?(Mihari::Alert) }
+      end
+
+      private
+
       #
       # Check whether a value is a falsepositive value or not
       #
       # @return [Boolean]
       #
       def falsepositive?(value)
-        return true if normalized_falsepositives.include?(value)
+        return true if rule.falsepositives.include?(value)
 
-        normalized_falsepositives.select do |falsepositive|
+        rule.falsepositives.select do |falsepositive|
           falsepositive.is_a?(Regexp)
         end.any? do |falseposistive|
           falseposistive.match?(value)
         end
       end
 
-      private
+      #
+      # Deep copied queries
+      #
+      # @return [Array<Hash>]
+      #
+      def queries
+        rule.queries.map(&:deep_dup)
+      end
+
+      #
+      # Get analyzer class
+      #
+      # @param [String] analyzer_name
+      #
+      # @return [Class<Mihari::Analyzers::Base>] analyzer class
+      #
+      def get_analyzer_class(analyzer_name)
+        analyzer = ANALYZER_TO_CLASS[analyzer_name]
+        return analyzer if analyzer
+
+        raise ArgumentError, "#{analyzer_name} is not supported"
+      end
+
+      #
+      # @return [Array<Mihari::Analyzers::Base>] <description>
+      #
+      def analyzers
+        @analyzers ||= queries.map do |params|
+          analyzer_name = params[:analyzer]
+          klass = get_analyzer_class(analyzer_name)
+
+          # set interval in the top level
+          options = params[:options] || {}
+          interval = options[:interval]
+          params[:interval] = interval if interval
+
+          # set rule
+          params[:rule] = rule
+          query = params[:query]
+
+          analyzer = klass.new(query, **params)
+          raise ConfigurationError, "#{analyzer.source} is not configured correctly" unless analyzer.configured?
+
+          analyzer
+        end
+      end
 
       #
       # Get emitter class
@@ -153,48 +214,34 @@ module Mihari
         raise ArgumentError, "#{emitter_name} is not supported"
       end
 
-      def valid_emitters
-        @valid_emitters ||= rule.emitters.filter_map do |original_params|
-          params = original_params.deep_dup
-
+      #
+      # Deep copied emitters
+      #
+      # @return [Array<Mihari::Emitters::Base>]
+      #
+      def emitters
+        rule.emitters.map(&:deep_dup).map do |params|
           name = params[:emitter]
           params.delete(:emitter)
 
           klass = get_emitter_class(name)
-          emitter = klass.new(**params)
-
-          emitter.valid? ? emitter : nil
+          klass.new(artifacts: enriched_artifacts, rule: rule, **params)
         end
       end
 
       #
-      # Get analyzer class
-      #
-      # @param [String] analyzer_name
-      #
-      # @return [Class<Mihari::Analyzers::Base>] analyzer class
+      # @return [Array<Mihari::Emitters::Base>]
       #
-      def get_analyzer_class(analyzer_name)
-        analyzer = ANALYZER_TO_CLASS[analyzer_name]
-        return analyzer if analyzer
-
-        raise ArgumentError, "#{analyzer_name} is not supported"
+      def valid_emitters
+        @valid_emitters ||= emitters.select(&:valid?)
       end
 
       #
       # Validate configuration of analyzers
       #
       def validate_analyzer_configurations
-        rule.queries.each do |params|
-          analyzer_name = params[:analyzer]
-          klass = get_analyzer_class(analyzer_name)
-
-          instance = klass.new("dummy")
-          unless instance.configured?
-            klass_name = klass.to_s.split("::").last
-            raise ConfigurationError, "#{klass_name} is not configured correctly"
-          end
-        end
+        # memoize analyzers & raise ConfigurationError if there is an analyzer which is not configured
+        analyzers
       end
     end
   end

data/lib/mihari/analyzers/shodan.rb

@@ -26,7 +26,7 @@ module Mihari
         results = search
         return [] if results.empty?
 
-        results.map { |result| result.to_artifacts(source) }.flatten.uniq(&:data)
+        results.map(&:to_artifacts).flatten.uniq(&:data)
       end
 
       private

data/lib/mihari/analyzers/urlscan.rb

@@ -36,15 +36,12 @@ module Mihari
 
       def artifacts
         responses = search
-        results = responses.map(&:results).flatten
-
-        allowed_data_types.map do |type|
-          results.filter_map do |result|
-            page = result.page
-            data = page.send(type.to_sym)
-            data.nil? ? nil : Artifact.new(data: data, source: source, metadata: result)
-          end
-        end.flatten
+        # @type [Array<Mihari::Artifact>]
+        artifacts = responses.map { |res| res.to_artifacts }.flatten
+
+        artifacts.select do |artifact|
+          allowed_data_types.include? artifact.data_type
+        end
       end
 
       private

data/lib/mihari/analyzers/virustotal_intelligence.rb

@@ -25,12 +25,7 @@ module Mihari
       end
 
       def artifacts
-        responses = search_with_cursor
-        responses.map do |response|
-          response.data.map do |datum|
-            Artifact.new(data: datum.value, source: source, metadata: datum.metadata)
-          end
-        end.flatten
+        search_with_cursor.map(&:to_artifacts).flatten
       end
 
       private

data/lib/mihari/cli/main.rb

@@ -3,7 +3,7 @@
 require "thor"
 
 # Commands
-require "mihari/commands/searcher"
+require "mihari/commands/search"
 require "mihari/commands/version"
 require "mihari/commands/web"
 require "mihari/commands/database"
@@ -17,7 +17,7 @@ require "mihari/cli/rule"
 module Mihari
   module CLI
     class Main < Base
-      include Mihari::Commands::Searcher
+      include Mihari::Commands::Search
       include Mihari::Commands::Version
       include Mihari::Commands::Web
 

data/lib/mihari/clients/base.rb

@@ -31,7 +31,7 @@ module Mihari
 
       #
       # @param [String] path
-      # @param [Hashk, nil] params
+      # @param [Hash, nil] params
       #
       # @return [String] <description>
       #

data/lib/mihari/commands/database.rb

@@ -3,18 +3,19 @@
 module Mihari
   module Commands
     module Database
-      def self.included(thor)
-        thor.class_eval do
-          desc "migrate", "Migrate DB schemas"
-          method_option :verbose, type: :boolean, default: true
-          #
-          # @param [String] direction
-          #
-          def migrate(direction = "up")
-            verbose = options["verbose"]
-            ActiveRecord::Migration.verbose = verbose
+      class << self
+        def included(thor)
+          thor.class_eval do
+            desc "migrate", "Migrate DB schemas"
+            method_option :verbose, type: :boolean, default: true
+            #
+            # @param [String] direction
+            #
+            def migrate(direction = "up")
+              ActiveRecord::Migration.verbose = options["verbose"]
 
-            Mihari::Database.with_db_connection { Mihari::Database.migrate(direction.to_sym) }
+              Mihari::Database.with_db_connection { Mihari::Database.migrate direction.to_sym }
+            end
           end
         end
       end