mihari 5.2.2 → 5.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 77de9f96ff14a64b2ab7a492fff36ed1515df7def7b18b5d81bac6785a5b75c0
-  data.tar.gz: 9e99189740e4e3b6ee6af97cc09fbd2af1f5395c047df925f528721f2c68595d
+  metadata.gz: 2a66fb2d71bcae401062277921a8ade6a3e9e9d961b193a80deacd3a8a934d4c
+  data.tar.gz: b4dcccfa58019f819241f8679b5d1ae846002f0a95307cd95856f2b6d04a6dd1
 SHA512:
-  metadata.gz: 59171eef265ace4aab9295b9e3866233138194afc0ee691022d5c6ec9f4ca6b53400a01109f9a745cdf54d70cd8c74a1fd39ff319624bf3f3ff04d2ae409ca26
-  data.tar.gz: a886b191dcab85164e2f7e47d2b9445121254a8b94248ce1f6e54fff31b8e0a13afc6a8dc5013f04cfaac2340957344ba03b3441d09b0018e46bb7a564af4b03
+  metadata.gz: e215400c8dce2b864bc26a951ed8ea35757441e7d25bdba6c66632d6716991bad202170f9984d09b7a709f6c9aceb731e5b40396efe621ffc55810a10db45db2
+  data.tar.gz: 745522a9cefaed75e5b266a429dea7afc0efe34f26f8591af18e3bc27a0746659ed8d64f6aa64f6edc0f9195acbf26e4a7ee078c052bab760f985da779c4e6e4

data/lib/mihari/analyzers/binaryedge.rb

@@ -42,7 +42,6 @@ module Mihari
       #
       # Search with pagination
       #
-      # @param [String] query
       # @param [Integer] page
       #
       # @return [Hash]

data/lib/mihari/analyzers/censys.rb

@@ -37,7 +37,12 @@ module Mihari
           response = client.search(query, cursor: cursor)
           artifacts << response.result.to_artifacts
           cursor = response.result.links.next
-          break if cursor.nil?
+          # NOTE: Censys's search API is unstable recently
+          # it may returns empty links or empty string cursors
+          # - Empty links: "links": {}
+          # - Empty cursors: "links": { "next": "", "prev": "" }
+          # So it needs to check both cases
+          break if cursor.nil? || cursor.empty?
 
           # sleep #{interval} seconds to avoid the rate limitation (if it is set)
           sleep interval
@@ -50,7 +55,7 @@ module Mihari
       # @return [Boolean]
       #
       def configured?
-        configuration_keys.all? { |key| Mihari.config.send(key) } || (id? && secret?)
+        configuration_keys? || (id? && secret?)
       end
 
       private

data/lib/mihari/analyzers/circl.rb

@@ -41,7 +41,7 @@ module Mihari
       end
 
       def configured?
-        configuration_keys.all? { |key| Mihari.config.send(key) } || (username? && password?)
+        configuration_keys? || (username? && password?)
       end
 
       private

data/lib/mihari/analyzers/passivetotal.rb

@@ -43,7 +43,7 @@ module Mihari
       end
 
       def configured?
-        configuration_keys.all? { |key| Mihari.config.send(key) } || (username? && api_key?)
+        configuration_keys? || (username? && api_key?)
       end
 
       private

data/lib/mihari/analyzers/rule.rb

@@ -54,12 +54,12 @@ module Mihari
       end
 
       #
-      # Returns a list of artifacts matched with queries
+      # Returns a list of artifacts matched with queries/analyzers
       #
       # @return [Array<Mihari::Artifact>]
       #
       def artifacts
-        rule.queries.map { |params| run_query(params.deep_dup) }.flatten
+        analyzers.flat_map(&:normalized_artifacts)
       end
 
       #
@@ -111,26 +111,15 @@ module Mihari
       # @return [Array<Mihari::Alert>]
       #
       def bulk_emit
-        Parallel.map(valid_emitters) { |emitter| emit emitter }.compact
-      end
-
-      #
-      # Emit an alert
-      #
-      # @param [Mihari::Emitters::Base] emitter
-      #
-      # @return [Mihari::Alert, nil]
-      #
-      def emit(emitter)
-        return if enriched_artifacts.empty?
-
-        alert_or_something = emitter.run(artifacts: enriched_artifacts, rule: rule)
-
-        Mihari.logger.info "Emission by #{emitter.class} is succeeded"
-
-        alert_or_something
-      rescue StandardError => e
-        Mihari.logger.info "Emission by #{emitter.class} is failed: #{e}"
+        return [] if enriched_artifacts.empty?
+
+        Parallel.map(valid_emitters) do |emitter|
+          emission = emitter.emit
+          Mihari.logger.info "Emission by #{emitter.class} is succeeded"
+          emission
+        rescue StandardError => e
+          Mihari.logger.info "Emission by #{emitter.class} is failed: #{e}"
+        end.compact
       end
 
       #
@@ -165,27 +154,50 @@ module Mihari
       end
 
       #
-      # @param [Hash] params
+      # Deep copied queries
       #
-      # @return [Array<Mihari::Artifact>]
+      # @return [Array<Hash>]
+      #
+      def queries
+        rule.queries.map(&:deep_dup)
+      end
+
+      #
+      # Get analyzer class
+      #
+      # @param [String] analyzer_name
+      #
+      # @return [Class<Mihari::Analyzers::Base>] analyzer class
+      #
+      def get_analyzer_class(analyzer_name)
+        analyzer = ANALYZER_TO_CLASS[analyzer_name]
+        return analyzer if analyzer
+
+        raise ArgumentError, "#{analyzer_name} is not supported"
+      end
+
+      #
+      # @return [Array<Mihari::Analyzers::Base>] <description>
       #
-      def run_query(params)
-        analyzer_name = params[:analyzer]
-        klass = get_analyzer_class(analyzer_name)
+      def analyzers
+        @analyzers ||= queries.map do |params|
+          analyzer_name = params[:analyzer]
+          klass = get_analyzer_class(analyzer_name)
 
-        # set interval in the top level
-        options = params[:options] || {}
-        interval = options[:interval]
-        params[:interval] = interval if interval
+          # set interval in the top level
+          options = params[:options] || {}
+          interval = options[:interval]
+          params[:interval] = interval if interval
 
-        # set rule
-        params[:rule] = rule
-        query = params[:query]
-        analyzer = klass.new(query, **params)
+          # set rule
+          params[:rule] = rule
+          query = params[:query]
 
-        # Use #normalized_artifacts method to get artifacts as Array<Mihari::Artifact>
-        # So Mihari::Artifact object has "source" attribute (e.g. "Shodan")
-        analyzer.normalized_artifacts
+          analyzer = klass.new(query, **params)
+          raise ConfigurationError, "#{analyzer.source} is not configured correctly" unless analyzer.configured?
+
+          analyzer
+        end
       end
 
       #
@@ -203,54 +215,33 @@ module Mihari
       end
 
       #
-      # @param [Hash] params
+      # Deep copied emitters
       #
-      # @return [Mihari::Emitter:Base]
+      # @return [Array<Mihari::Emitters::Base>]
       #
-      def validate_emitter(params)
-        name = params[:emitter]
-        params.delete(:emitter)
-
-        klass = get_emitter_class(name)
-        emitter = klass.new(**params)
+      def emitters
+        rule.emitters.map(&:deep_dup).map do |params|
+          name = params[:emitter]
+          params.delete(:emitter)
 
-        emitter.valid? ? emitter : nil
+          klass = get_emitter_class(name)
+          klass.new(artifacts: enriched_artifacts, rule: rule, **params)
+        end
       end
 
       #
-      # @return [Array<Mihari::Emitter::Base>]
+      # @return [Array<Mihari::Emitters::Base>]
       #
       def valid_emitters
-        @valid_emitters ||= rule.emitters.filter_map { |params| validate_emitter(params.deep_dup) }
-      end
-
-      #
-      # Get analyzer class
-      #
-      # @param [String] analyzer_name
-      #
-      # @return [Class<Mihari::Analyzers::Base>] analyzer class
-      #
-      def get_analyzer_class(analyzer_name)
-        analyzer = ANALYZER_TO_CLASS[analyzer_name]
-        return analyzer if analyzer
-
-        raise ArgumentError, "#{analyzer_name} is not supported"
+        @valid_emitters ||= emitters.select(&:valid?)
       end
 
       #
       # Validate configuration of analyzers
       #
       def validate_analyzer_configurations
-        rule.queries.each do |params|
-          analyzer_name = params[:analyzer]
-
-          klass = get_analyzer_class(analyzer_name)
-          klass_name = klass.to_s.split("::").last
-
-          instance = klass.new("dummy")
-          raise ConfigurationError, "#{klass_name} is not configured correctly" unless instance.configured?
-        end
+        # memoize analyzers & raise ConfigurationError if there is an analyzer which is not configured
+        analyzers
       end
     end
   end

data/lib/mihari/analyzers/virustotal_intelligence.rb

@@ -25,8 +25,7 @@ module Mihari
       end
 
       def artifacts
-        responses = search_with_cursor
-        responses.map(&:to_artifacts).flatten
+        search_with_cursor.map(&:to_artifacts).flatten
       end
 
       private

data/lib/mihari/clients/base.rb

@@ -31,7 +31,7 @@ module Mihari
 
       #
       # @param [String] path
-      # @param [Hashk, nil] params
+      # @param [Hash, nil] params
       #
       # @return [String] <description>
       #

data/lib/mihari/commands/database.rb

@@ -3,18 +3,19 @@
 module Mihari
   module Commands
     module Database
-      def self.included(thor)
-        thor.class_eval do
-          desc "migrate", "Migrate DB schemas"
-          method_option :verbose, type: :boolean, default: true
-          #
-          # @param [String] direction
-          #
-          def migrate(direction = "up")
-            verbose = options["verbose"]
-            ActiveRecord::Migration.verbose = verbose
+      class << self
+        def included(thor)
+          thor.class_eval do
+            desc "migrate", "Migrate DB schemas"
+            method_option :verbose, type: :boolean, default: true
+            #
+            # @param [String] direction
+            #
+            def migrate(direction = "up")
+              ActiveRecord::Migration.verbose = options["verbose"]
 
-            Mihari::Database.with_db_connection { Mihari::Database.migrate(direction.to_sym) }
+              Mihari::Database.with_db_connection { Mihari::Database.migrate direction.to_sym }
+            end
           end
         end
       end

data/lib/mihari/commands/rule.rb

@@ -5,60 +5,62 @@ require "pathname"
 module Mihari
   module Commands
     module Rule
-      def self.included(thor)
-        thor.class_eval do
-          desc "validate [PATH]", "Validate a rule file"
-          #
-          # Validate format of a rule
-          #
-          # @param [String] path
-          #
-          def validate(path)
-            rule = Structs::Rule.from_path_or_id(path)
-
-            begin
-              rule.validate!
-              Mihari.logger.info "Valid format. The input is parsed as the following:"
-              Mihari.logger.info rule.data.to_yaml
-            rescue RuleValidationError
-              nil
-            end
-          end
-
-          desc "init [PATH]", "Initialize a new rule file"
-          #
-          # Initialize a new rule file
-          #
-          # @param [String] path
-          #
-          #
-          def init(path = "./rule.yml")
-            warning = "#{path} exists. Do you want to overwrite it? (y/n)"
-            return if Pathname(path).exist? && !(yes? warning)
-
-            initialize_rule path
-
-            Mihari.logger.info "A new rule is initialized: #{path}."
-          end
-
-          no_commands do
+      class << self
+        def included(thor)
+          thor.class_eval do
+            desc "validate [PATH]", "Validate a rule file"
             #
-            # @return [Mihari::Structs::Rule]
+            # Validate format of a rule
             #
-            def rule_template
-              Structs::Rule.from_path File.expand_path("../templates/rule.yml.erb", __dir__)
+            # @param [String] path
+            #
+            def validate(path)
+              rule = Structs::Rule.from_path_or_id(path)
+
+              begin
+                rule.validate!
+                Mihari.logger.info "Valid format. The input is parsed as the following:"
+                Mihari.logger.info rule.data.to_yaml
+              rescue RuleValidationError
+                nil
+              end
             end
 
+            desc "init [PATH]", "Initialize a new rule file"
             #
-            # Create a new rule
+            # Initialize a new rule file
             #
             # @param [String] path
-            # @param [Dry::Files] files
             #
-            # @return [nil]
             #
-            def initialize_rule(path, files = Dry::Files.new)
-              files.write(path, rule_template.yaml)
+            def init(path = "./rule.yml")
+              warning = "#{path} exists. Do you want to overwrite it? (y/n)"
+              return if Pathname(path).exist? && !(yes? warning)
+
+              initialize_rule path
+
+              Mihari.logger.info "A new rule file has been initialized: #{path}."
+            end
+
+            no_commands do
+              #
+              # @return [Mihari::Structs::Rule]
+              #
+              def rule_template
+                Structs::Rule.from_path File.expand_path("../templates/rule.yml.erb", __dir__)
+              end
+
+              #
+              # Create a new rule
+              #
+              # @param [String] path
+              # @param [Dry::Files] files
+              #
+              # @return [nil]
+              #
+              def initialize_rule(path, files = Dry::Files.new)
+                files.write(path, rule_template.yaml)
+              end
             end
           end
         end

data/lib/mihari/commands/search.rb

@@ -3,64 +3,83 @@
 module Mihari
   module Commands
     module Search
-      include Mixins::ErrorNotification
+      class << self
+        class RuleWrapper
+          include Mixins::ErrorNotification
+
+          # @return [Nihari::Structs::Rule]
+          attr_reader :rule
+
+          # @return [Boolean]
+          attr_reader :force_overwrite
+
+          def initialize(rule:, force_overwrite:)
+            @rule = rule
+            @force_overwrite = force_overwrite
+          end
+
+          def force_overwrite?
+            force_overwrite
+          end
 
-      def self.included(thor)
-        thor.class_eval do
-          desc "search [PATH]", "Search by a rule"
-          method_option :force_overwrite, type: :boolean, aliases: "-f", desc: "Force an overwrite the rule"
-          #
-          # Search by a rule
           #
-          # @param [String] path_or_id
+          # @return [Boolean]
           #
-          def search(path_or_id)
-            Mihari::Database.with_db_connection do
-              rule = Structs::Rule.from_path_or_id path_or_id
+          def diff?
+            model = Mihari::Rule.find(rule.id)
+            model.data != rule.data.deep_stringify_keys
+          rescue ActiveRecord::RecordNotFound
+            false
+          end
 
-              begin
-                rule.validate!
-              rescue RuleValidationError
-                return
-              end
+          def update_or_create
+            rule.model.save
+          end
 
-              update_rule rule
-              run_rule rule
+          def run
+            with_error_notification do
+              alert = rule.analyzer.run
+              if alert
+                data = Mihari::Entities::Alert.represent(alert)
+                puts JSON.pretty_generate(data.as_json)
+              else
+                Mihari.logger.info "There is no new artifact found"
+              end
             end
           end
         end
-      end
 
-      # @param [Mihari::Structs::Rule] rule
-      #
-      def update_rule(rule)
-        force_overwrite = options["force_overwrite"] || false
-        begin
-          rule_model = Mihari::Rule.find(rule.id)
-          has_change = rule_model.data != rule.data.deep_stringify_keys
-          has_change_and_not_force_overwrite = has_change & !force_overwrite
+        def included(thor)
+          thor.class_eval do
+            desc "search [PATH]", "Search by a rule"
+            method_option :force_overwrite, type: :boolean, aliases: "-f", desc: "Force an overwrite the rule"
+            #
+            # Search by a rule
+            #
+            # @param [String] path_or_id
+            #
+            def search(path_or_id)
+              Mihari::Database.with_db_connection do
+                rule = Structs::Rule.from_path_or_id path_or_id
 
-          confirm_message = "This operation will overwrite the rule in the database (Rule ID: #{rule.id}). Are you sure you want to update the rule? (y/n)"
-          return if has_change_and_not_force_overwrite && !yes?(confirm_message)
-          # update the rule
-          rule.model.save
-        rescue ActiveRecord::RecordNotFound
-          # create a new rule
-          rule.model.save
-        end
-      end
+                begin
+                  rule.validate!
+                rescue RuleValidationError
+                  return
+                end
 
-      #
-      # @param [Mihari::Structs::Rule] rule
-      #
-      def run_rule(rule)
-        with_error_notification do
-          alert = rule.analyzer.run
-          if alert
-            data = Mihari::Entities::Alert.represent(alert)
-            puts JSON.pretty_generate(data.as_json)
-          else
-            Mihari.logger.info "There is no new alert created in the database"
+                force_overwrite = options["force_overwrite"] || false
+                wrapper = RuleWrapper.new(rule: rule, force_overwrite: force_overwrite)
+
+                if wrapper.diff? && !force_overwrite
+                  message = "There is diff in the rule (#{rule.id}). Are you sure you want to overwrite the rule? (y/n)"
+                  return unless yes?(message)
+                end
+
+                wrapper.update_or_create
+                wrapper.run
+              end
+            end
           end
         end
       end

data/lib/mihari/commands/version.rb

@@ -3,13 +3,15 @@
 module Mihari
   module Commands
     module Version
-      def self.included(thor)
-        thor.class_eval do
-          map %w[--version -v] => :__print_version
+      class << self
+        def included(thor)
+          thor.class_eval do
+            map %w[--version -v] => :__print_version
 
-          desc "--version, -v", "Print the version"
-          def __print_version
-            puts Mihari::VERSION
+            desc "--version, -v", "Print the version"
+            def __print_version
+              puts Mihari::VERSION
+            end
           end
         end
       end

data/lib/mihari/commands/web.rb

@@ -3,29 +3,32 @@
 module Mihari
   module Commands
     module Web
-      def self.included(thor)
-        thor.class_eval do
-          desc "web", "Launch the web app"
-          method_option :port, type: :numeric, default: 9292, desc: "Hostname to listen on"
-          method_option :host, type: :string, default: "localhost", desc: "Port to listen on"
-          method_option :threads, type: :string, default: "0:5", desc: "min:max threads to use"
-          method_option :verbose, type: :boolean, default: true, desc: "Report each request"
-          method_option :worker_timeout, type: :numeric, default: 60, desc: "Worker timeout value (in seconds)"
-          method_option :hide_config_values, type: :boolean, default: false,
-            desc: "Whether to hide config values or not"
-          method_option :open, type: :boolean, default: true, desc: "Whether to open the app in browser or not"
-          def web
-            Mihari.config.hide_config_values = options["hide_config_values"]
-            # set rack env as production
-            ENV["RACK_ENV"] ||= "production"
-            Mihari::App.run!(
-              port: options["port"],
-              host: options["host"],
-              threads: options["threads"],
-              verbose: options["verbose"],
-              worker_timeout: options["worker_timeout"],
-              open: options["open"]
-            )
+      class << self
+        def included(thor)
+          thor.class_eval do
+            desc "web", "Launch the web app"
+            method_option :port, type: :numeric, default: 9292, desc: "Hostname to listen on"
+            method_option :host, type: :string, default: "localhost", desc: "Port to listen on"
+            method_option :threads, type: :string, default: "0:5", desc: "min:max threads to use"
+            method_option :verbose, type: :boolean, default: true, desc: "Report each request"
+            method_option :worker_timeout, type: :numeric, default: 60, desc: "Worker timeout value (in seconds)"
+            method_option :hide_config_values, type: :boolean, default: false,
+              desc: "Whether to hide config values or not"
+            method_option :open, type: :boolean, default: true, desc: "Whether to open the app in browser or not"
+            method_option :rack_env, type: :string, default: "production", desc: "Rack environment"
+            def web
+              Mihari.config.hide_config_values = options["hide_config_values"]
+              # set rack env as production
+              ENV["RACK_ENV"] ||= options["rack_env"]
+              Mihari::App.run!(
+                port: options["port"],
+                host: options["host"],
+                threads: options["threads"],
+                verbose: options["verbose"],
+                worker_timeout: options["worker_timeout"],
+                open: options["open"]
+              )
+            end
           end
         end
       end

data/lib/mihari/emitters/base.rb

@@ -6,7 +6,20 @@ module Mihari
       include Mixins::Configurable
       include Mixins::Retriable
 
-      def initialize(*)
+      # @return [Array<Mihari::Artifact>]
+      attr_reader :artifacts
+
+      # @return [Mihari::Structs::Rule]
+      attr_reader :rule
+
+      #
+      # @param [Array<Mihari::Artifact>] artifacts
+      # @param [Mihari::Structs::Rule] rule
+      # @param [Hash] **_options
+      #
+      def initialize(artifacts:, rule:, **_options)
+        @artifacts = artifacts
+        @rule = rule
       end
 
       class << self