mihari 5.4.3 → 5.4.5

data/frontend/package.json

@@ -17,15 +17,15 @@
     "@fortawesome/fontawesome-svg-core": "^6.4.2",
     "@fortawesome/free-solid-svg-icons": "^6.4.2",
     "@fortawesome/vue-fontawesome": "^3.0.3",
-    "@vueuse/core": "^10.3.0",
-    "@vueuse/router": "^10.3.0",
-    "ace-builds": "^1.24.1",
-    "axios": "^1.4.0",
+    "@vueuse/core": "^10.4.1",
+    "@vueuse/router": "^10.4.1",
+    "ace-builds": "^1.28.0",
+    "axios": "^1.5.1",
     "bulma": "^0.9.4",
     "bulma-helpers": "^0.4.3",
-    "dayjs": "^1.11.9",
+    "dayjs": "^1.11.10",
     "font-awesome-animation": "^1.1.1",
-    "js-sha256": "^0.9.0",
+    "js-sha256": "^0.10.1",
     "truncate": "^3.0.0",
     "ts-dedent": "^2.2.0",
     "url-parse": "^1.5.10",
@@ -33,24 +33,24 @@
     "vue": "^3.3.4",
     "vue-concurrency": "4.0.1",
     "vue-json-pretty": "^2.2.4",
-    "vue-router": "^4.2.4",
+    "vue-router": "^4.2.5",
     "vue3-ace-editor": "^2.2.3"
   },
   "devDependencies": {
-    "@redocly/cli": "1.0.2",
-    "@rushstack/eslint-patch": "^1.3.3",
-    "@tsconfig/node20": "^20.1.1",
-    "@types/jsdom": "^21.1.1",
-    "@types/node": "^20.5.1",
-    "@types/url-parse": "^1.4.8",
-    "@typescript-eslint/eslint-plugin": "^6.4.0",
-    "@typescript-eslint/parser": "^6.4.0",
-    "@vitejs/plugin-vue": "^4.3.1",
+    "@redocly/cli": "1.2.0",
+    "@rushstack/eslint-patch": "^1.5.0",
+    "@tsconfig/node20": "^20.1.2",
+    "@types/jsdom": "^21.1.3",
+    "@types/node": "^20.7.2",
+    "@types/url-parse": "^1.4.9",
+    "@typescript-eslint/eslint-plugin": "^6.7.3",
+    "@typescript-eslint/parser": "^6.7.3",
+    "@vitejs/plugin-vue": "^4.3.4",
     "@vue/eslint-config-prettier": "^8.0.0",
-    "@vue/eslint-config-typescript": "^11.0.3",
+    "@vue/eslint-config-typescript": "^12.0.0",
     "@vue/test-utils": "2.4.1",
     "@vue/tsconfig": "^0.4.0",
-    "eslint": "^8.47.0",
+    "eslint": "^8.50.0",
     "eslint-config-prettier": "^9.0.0",
     "eslint-plugin-prettier": "^5.0.0",
     "eslint-plugin-simple-import-sort": "^10.0.0",
@@ -58,10 +58,10 @@
     "husky": "^8.0.3",
     "jsdom": "^22.1.0",
     "npm-run-all": "^4.1.5",
-    "prettier": "^3.0.2",
-    "typescript": "~5.1.6",
+    "prettier": "^3.0.3",
+    "typescript": "~5.2.2",
     "vite": "^4.4.9",
-    "vitest": "^0.34.2",
-    "vue-tsc": "^1.8.8"
+    "vitest": "^0.34.6",
+    "vue-tsc": "^1.8.15"
   }
 }

data/lib/mihari/analyzers/base.rb

@@ -3,6 +3,8 @@
 module Mihari
   module Analyzers
     class Base
+      include Dry::Monads[:result, :try]
+
       include Mixins::Configurable
       include Mixins::Retriable
 
@@ -25,28 +27,38 @@ module Mihari
       # @return [Integer, nil]
       #
       def interval
-        @interval ||= options[:interval]
+        options[:interval]
       end
 
       #
       # @return [Integer]
       #
       def retry_interval
-        @retry_interval ||= options[:retry_interval] || Mihari.config.retry_interval
+        options[:retry_interval] || Mihari.config.retry_interval
       end
 
       #
       # @return [Integer]
       #
       def retry_times
-        @retry_times ||= options[:retry_times] || Mihari.config.retry_times
+        options[:retry_times] || Mihari.config.retry_times
       end
 
       #
       # @return [Integer]
       #
       def pagination_limit
-        @pagination_limit ||= options[:pagination_limit] || Mihari.config.pagination_limit
+        options[:pagination_limit] || Mihari.config.pagination_limit
+      end
+
+      #
+      # @return [Boolean]
+      #
+      def ignore_error?
+        ignore_error = options[:ignore_error]
+        return ignore_error unless ignore_error.nil?
+
+        Mihari.config.ignore_error
       end
 
       # @return [Array<String>, Array<Mihari::Artifact>]
@@ -63,7 +75,7 @@ module Mihari
       #
       def normalized_artifacts
         retry_on_error(times: retry_times, interval: retry_interval) do
-          @normalized_artifacts ||= artifacts.compact.sort.map do |artifact|
+          artifacts.compact.sort.map do |artifact|
             # No need to set data_type manually
             # It is set automatically in #initialize
             artifact = artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact)
@@ -73,6 +85,13 @@ module Mihari
         end
       end
 
+      #
+      # @return [Dry::Monads::Result::Success<Array<Mihari::Artifact>>, Dry::Monads::Result::Failure]
+      #
+      def result
+        Try[StandardError] { normalized_artifacts }.to_result
+      end
+
       # @return [String]
       def class_name
         self.class.to_s.split("::").last
@@ -80,8 +99,6 @@ module Mihari
 
       alias_method :source, :class_name
 
-      private
-
       class << self
         #
         # Initialize an analyzer by query params
@@ -97,9 +114,7 @@ module Mihari
           query = copied[:query]
 
           # delete analyzer and query
-          %i[analyzer query].each do |key|
-            copied.delete key
-          end
+          %i[analyzer query].each { |key| copied.delete key }
 
           copied[:options] = copied[:options] || nil
 

data/lib/mihari/analyzers/binaryedge.rb

@@ -18,13 +18,7 @@ module Mihari
       end
 
       def artifacts
-        client.search_with_pagination(query, pagination_limit: pagination_limit).map do |res|
-          events = res["events"] || []
-          events.filter_map do |event|
-            data = event.dig("target", "ip")
-            data.nil? ? nil : Artifact.new(data: data, source: source, metadata: event)
-          end
-        end
+        client.search_with_pagination(query, pagination_limit: pagination_limit).map(&:artifacts).flatten
       end
 
       def configuration_keys

data/lib/mihari/analyzers/circl.rb

@@ -36,7 +36,7 @@ module Mihari
         when "hash"
           client.passive_ssl_search query
         else
-          raise InvalidInputError, "#{@query}(type: #{@type || "unknown"}) is not supported."
+          raise ValueError, "#{@query}(type: #{@type || "unknown"}) is not supported."
         end
       end
 

data/lib/mihari/analyzers/dnstwister.rb

@@ -19,7 +19,7 @@ module Mihari
       end
 
       def artifacts
-        raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
+        raise ValueError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
 
         domains = client.fuzz(query)
         Parallel.map(domains) do |domain|

data/lib/mihari/analyzers/otx.rb

@@ -31,7 +31,7 @@ module Mihari
         when "ip"
           client.ip_search(query)
         else
-          raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
+          raise ValueError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
         end
       end
 

data/lib/mihari/analyzers/passivetotal.rb

@@ -38,7 +38,7 @@ module Mihari
         when "hash"
           client.ssl_search query
         else
-          raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
+          raise ValueError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
         end
       end
 

data/lib/mihari/analyzers/pulsedive.rb

@@ -25,7 +25,7 @@ module Mihari
       end
 
       def artifacts
-        raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
+        raise ValueError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
 
         indicator = client.get_indicator(query)
         iid = indicator["iid"]

data/lib/mihari/analyzers/rule.rb

@@ -60,9 +60,16 @@ module Mihari
       # @return [Array<Mihari::Artifact>]
       #
       def artifacts
-        analyzers.flat_map(&:normalized_artifacts).map do |artifact|
-          artifact.rule_id = rule.id
-          artifact
+        analyzers.flat_map do |analyzer|
+          result = analyzer.result
+
+          raise result.failure if result.failure? && !analyzer.ignore_error?
+
+          artifacts = result.value!
+          artifacts.map do |artifact|
+            artifact.rule_id = rule.id
+            artifact
+          end
         end
       end
 
@@ -113,11 +120,12 @@ module Mihari
         return [] if enriched_artifacts.empty?
 
         Parallel.map(valid_emitters) do |emitter|
-          emission = emitter.emit
-          Mihari.logger.info "Emission by #{emitter.class} is succeeded"
-          emission
-        rescue StandardError => e
-          Mihari.logger.info "Emission by #{emitter.class} is failed: #{e}"
+          result = emitter.result
+
+          Mihari.logger.info "Emission by #{emitter.class} is failed: #{result.failure}" if result.failure?
+          Mihari.logger.info "Emission by #{emitter.class} is succeeded" if result.success?
+
+          result.value_or nil
         end.compact
       end
 
@@ -127,9 +135,6 @@ module Mihari
       # @return [Mihari::Alert, nil]
       #
       def run
-        # memoize enriched artifacts
-        enriched_artifacts
-
         alert_or_something = bulk_emit
         # returns Mihari::Alert created by the database emitter
         alert_or_something.find { |res| res.is_a?(Mihari::Alert) }
@@ -167,7 +172,7 @@ module Mihari
       # @return [Array<Mihari::Analyzers::Base>]
       #
       def analyzers
-        @analyzers ||= rule.queries.map do |query_params|
+        rule.queries.map do |query_params|
           analyzer_name = query_params[:analyzer]
           klass = get_analyzer_class(analyzer_name)
           klass.from_query(query_params)
@@ -207,7 +212,7 @@ module Mihari
       # @return [Array<Mihari::Emitters::Base>]
       #
       def valid_emitters
-        @valid_emitters ||= emitters.select(&:valid?)
+        emitters.select(&:valid?)
       end
 
       #

data/lib/mihari/analyzers/securitytrails.rb

@@ -36,7 +36,7 @@ module Mihari
         when "mail"
           client.mail_search query
         else
-          raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
+          raise ValueError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
         end
       end
 

data/lib/mihari/analyzers/urlscan.rb

@@ -25,7 +25,7 @@ module Mihari
 
         return if valid_allowed_data_types?
 
-        raise InvalidInputError, "allowed_data_types should be any of url, domain and ip."
+        raise ValueError, "allowed_data_types should be any of url, domain and ip."
       end
 
       def artifacts

data/lib/mihari/analyzers/virustotal.rb

@@ -31,7 +31,7 @@ module Mihari
         when "ip"
           ip_search
         else
-          raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
+          raise ValueError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
         end
       end
 

data/lib/mihari/analyzers/zoomeye.rb

@@ -33,7 +33,7 @@ module Mihari
             convert(res)
           end.flatten
         else
-          raise InvalidInputError, "#{type} type is not supported." unless valid_type?
+          raise ValueError, "#{type} type is not supported." unless valid_type?
         end
       end
 

data/lib/mihari/clients/binaryedge.rb

@@ -22,7 +22,7 @@ module Mihari
       # @param [Integer] page Default 1, Maximum: 500
       # @param [Integer, nil] only_ips If selected, only output IP addresses, ports and protocols.
       #
-      # @return [Hash]
+      # @return [Structs::BinaryEdge::Response]
       #
       def search(query, page: 1, only_ips: nil)
         params = {
@@ -32,7 +32,7 @@ module Mihari
         }.compact
 
         res = get("/query/search", params: params)
-        JSON.parse(res.body.to_s)
+        Structs::BinaryEdge::Response.from_dynamic! JSON.parse(res.body.to_s)
       end
 
       #
@@ -40,19 +40,16 @@ module Mihari
       # @param [Integer, nil] only_ips
       # @param [Integer] pagination_limit
       #
-      # @return [Enumerable<Hash>]
+      # @return [Enumerable<Structs::BinaryEdge::Response.>]
       #
       def search_with_pagination(query, only_ips: nil, pagination_limit: Mihari.config.pagination_limit)
         Enumerator.new do |y|
           (1..pagination_limit).each do |page|
             res = search(query, page: page, only_ips: only_ips)
 
-            page_size = res["pagesize"].to_i
-            events = res["events"] || []
-
             y.yield res
 
-            break if events.length < page_size
+            break if res.events.length < res.pagesize
 
             sleep_interval
           end

data/lib/mihari/clients/crtsh.rb

@@ -28,9 +28,7 @@ module Mihari
 
         parsed.map do |result|
           values = result["name_value"].to_s.lines.map(&:chomp)
-          values.map do |value|
-            Artifact.new(data: value, metadata: result)
-          end
+          values.map { |value| Artifact.new(data: value, metadata: result) }
         end.flatten
       end
     end

data/lib/mihari/clients/publsedive.rb

@@ -49,7 +49,7 @@ module Mihari
         params["key"] = api_key
 
         res = get(path, params: params)
-        JSON.parse(res.body.to_s)
+        JSON.parse res.body.to_s
       end
     end
   end

data/lib/mihari/clients/shodan.rb

@@ -37,7 +37,7 @@ module Mihari
           key: api_key
         }
         res = get("/shodan/host/search", params: params)
-        Structs::Shodan::Result.from_dynamic! JSON.parse(res.body.to_s)
+        Structs::Shodan::Response.from_dynamic! JSON.parse(res.body.to_s)
       end
 
       #
@@ -45,7 +45,7 @@ module Mihari
       # @param [Boolean] minify
       # @param [Integer] pagination_limit
       #
-      # @return [Enumerable<Structs::Shodan::Result>]
+      # @return [Enumerable<Structs::Shodan::Response>]
       #
       def search_with_pagination(query, minify: true, pagination_limit: Mihari.config.pagination_limit)
         Enumerator.new do |y|

data/lib/mihari/commands/alert.rb

@@ -6,32 +6,61 @@ module Mihari
       class << self
         def included(thor)
           thor.class_eval do
+            include Dry::Monads[:result, :try]
+
             desc "add [PATH]", "Add an alert"
             #
             # @param [String] path
             #
             def add(path)
               Mihari::Database.with_db_connection do
-                proxy = Mihari::Services::AlertProxy.from_path(path)
-                proxy.validate!
+                builder = Mihari::Services::AlertBuilder.new(path)
+
+                run_proxy_l = ->(proxy) { run_proxy proxy }
+                check_nil_l = ->(alert_or_nil) { check_nil alert_or_nil }
 
-                runner = Mihari::Services::AlertRunner.new(proxy)
+                result = builder.result.bind(run_proxy_l).bind(check_nil_l)
 
-                begin
-                  alert = runner.run
-                rescue ActiveRecord::RecordNotFound => e
-                  # if there is a ActiveRecord::RecordNotFound, output that error without the stack trace
-                  Mihari.logger.error e.to_s
+                if result.success?
+                  alert = result.value!
+                  data = Mihari::Entities::Alert.represent(alert)
+                  puts JSON.pretty_generate(data.as_json)
                   return
                 end
 
-                if alert.nil?
-                  Mihari.logger.info "There is no new artifact found"
-                  return
+                failure = result.failure
+                case failure
+                when ValidationError
+                  Mihari.logger.error "Failed to parse the input as an alert:"
+                  Mihari.logger.error JSON.pretty_generate(failure.errors.to_h)
+                when StandardError
+                  raise failure
+                else
+                  Mihari.logger.info failure
                 end
+              end
+            end
+
+            no_commands do
+              #
+              # @param [Mihari::Services::AlertProxy] proxy
+              #
+              def run_proxy(proxy)
+                Dry::Monads::Try[StandardError] do
+                  runner = Mihari::Services::AlertRunner.new(proxy)
+                  runner.run
+                end.to_result
+              end
 
-                data = Mihari::Entities::Alert.represent(alert)
-                puts JSON.pretty_generate(data.as_json)
+              #
+              # @param [Mihari::Alert, nil] alert_or_nil
+              #
+              def check_nil(alert_or_nil)
+                if alert_or_nil.nil?
+                  Failure "There is no new artifact found"
+                else
+                  Success alert_or_nil
+                end
               end
             end
           end

data/lib/mihari/commands/rule.rb

@@ -8,6 +8,8 @@ module Mihari
       class << self
         def included(thor)
           thor.class_eval do
+            include Dry::Monads[:result, :try]
+
             desc "validate [PATH]", "Validate a rule file"
             #
             # Validate format of a rule
@@ -15,15 +17,17 @@ module Mihari
             # @param [String] path
             #
             def validate(path)
-              rule = Services::RuleProxy.from_path_or_id(path)
-
-              begin
-                rule.validate!
+              res = Dry::Monads::Try[ValidationError] do
+                Services::RuleProxy.from_yaml(File.read(path))
+              end.fmap do |rule|
                 Mihari.logger.info "Valid format. The input is parsed as the following:"
                 Mihari.logger.info rule.data.to_yaml
-              rescue RuleValidationError
-                nil
               end
+
+              return unless res.error?
+
+              Mihari.logger.error "Failed to parse the input as a rule:"
+              Mihari.logger.error JSON.pretty_generate(res.exception.errors.to_h)
             end
 
             desc "init [PATH]", "Initialize a new rule file"
@@ -47,7 +51,7 @@ module Mihari
               # @return [Mihari::Services::Rule]
               #
               def rule_template
-                Services::RuleProxy.from_path File.expand_path("../templates/rule.yml.erb", __dir__)
+                Services::RuleProxy.from_yaml File.read(File.expand_path("../templates/rule.yml.erb", __dir__))
               end
 
               #

data/lib/mihari/commands/search.rb

@@ -6,7 +6,9 @@ module Mihari
       class << self
         def included(thor)
           thor.class_eval do
-            desc "search [PATH]", "Search by a rule"
+            include Dry::Monads[:result, :try]
+
+            desc "search [PATH_OR_ID]", "Search by a rule"
             method_option :force_overwrite, type: :boolean, aliases: "-f", desc: "Force an overwrite the rule"
             #
             # Search by a rule
@@ -15,39 +17,69 @@ module Mihari
             #
             def search(path_or_id)
               Mihari::Database.with_db_connection do
-                rule = Services::RuleProxy.from_path_or_id path_or_id
+                builder = Services::RuleBuilder.new(path_or_id)
+
+                check_diff_l = ->(rule) { check_diff rule }
+                update_and_run_l = ->(runner) { update_and_run runner }
+                check_nil_l = ->(alert_or_nil) { check_nil alert_or_nil }
 
-                begin
-                  rule.validate!
-                rescue RuleValidationError
+                result = builder.result.bind(check_diff_l).bind(update_and_run_l).bind(check_nil_l)
+
+                if result.success?
+                  alert = result.value!
+                  data = Mihari::Entities::Alert.represent(alert)
+                  puts JSON.pretty_generate(data.as_json)
                   return
                 end
 
+                failure = result.failure
+                case failure
+                when ValidationError
+                  Mihari.logger.error "Failed to parse the input as a rule:"
+                  Mihari.logger.error JSON.pretty_generate(failure.errors.to_h)
+                when StandardError
+                  raise failure
+                else
+                  Mihari.logger.info failure
+                end
+              end
+            end
+
+            no_commands do
+              #
+              # @param [Mihari::Services::RuleProxy] rule
+              #
+              def check_diff(rule)
                 force_overwrite = options["force_overwrite"] || false
                 runner = Services::RuleRunner.new(rule, force_overwrite: force_overwrite)
+                message = "There is a diff in the rule (#{rule.id}). Are you sure you want to overwrite the rule? (y/n)"
 
-                if runner.diff? && !force_overwrite
-                  message = "There is diff in the rule (#{rule.id}). Are you sure you want to overwrite the rule? (y/n)"
-                  return unless yes?(message)
+                if runner.diff? && !force_overwrite && !yes?(message)
+                  return Failure("Stop overwriting the rule (#{rule.id})")
                 end
 
-                runner.update_or_create
-
-                begin
-                  alert = runner.run
-                rescue ConfigurationError => e
-                  # if there is a configuration error, output that error without the stack trace
-                  Mihari.logger.error e.to_s
-                  return
-                end
+                Success runner
+              end
 
-                if alert.nil?
-                  Mihari.logger.info "There is no new artifact found"
-                  return
+              #
+              # @param [Mihari::Alert, nil] alert_or_nil
+              #
+              def check_nil(alert_or_nil)
+                if alert_or_nil.nil?
+                  Failure "There is no new artifact found"
+                else
+                  Success alert_or_nil
                 end
+              end
 
-                data = Mihari::Entities::Alert.represent(alert)
-                puts JSON.pretty_generate(data.as_json)
+              #
+              # @param [Mihari::Services::RuleRunner] runner
+              #
+              def update_and_run(runner)
+                Dry::Monads::Try[StandardError] do
+                  runner.update_or_create
+                  runner.run
+                end.to_result
               end
             end
           end

data/lib/mihari/config.rb

@@ -93,6 +93,9 @@ module Mihari
     # @return [Integer]
     attr_reader :pagination_limit
 
+    # @return [Boolean]
+    attr_reader :ignore_error
+
     def initialize
       @binaryedge_api_key = ENV.fetch("BINARYEDGE_API_KEY", nil)
 
@@ -147,6 +150,8 @@ module Mihari
       @retry_interval = ENV.fetch("RETRY_INTERVAL", 5).to_i
 
       @pagination_limit = ENV.fetch("PAGINATION_LIMIT", 100).to_i
+
+      @ignore_error = ENV.fetch("IGNORE_ERROR", false)
     end
   end
 end