mihari 7.0.3 → 7.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a5bc182d0f1c5e1ecfb08cd72d083c30a5775ea2890795dc812357da493ffb63
-  data.tar.gz: 87fe1706c56301184b79f02d1d8be0b2deeb5591b8cfd76c3bea6d1a12005311
+  metadata.gz: aca048c797a73db1c57f0a5ad5f29066fbfba69451c71df1a26c3ea57853a1e3
+  data.tar.gz: 5ce2d298f325cc9d0ae50a98a6c18a64ec11cc06386504535c8ca647c7c68ad0
 SHA512:
-  metadata.gz: 543e6850571451e862e95db1697482c39e796783728ddcb48dca6008b0a29e10bbe75dfabb4512dcf0b9c4ff6d31168138bc6285c2ff7131a82a2e61d5353ab1
-  data.tar.gz: fc4e7fe111efa45d68a7b106cc61b5312f5c5be62fdc662e4daf2d3e466054deaf66eac08217f35680fbab0688eaad7642dc1a17df4bcf385b7b61c38eb31906
+  metadata.gz: ac1d6ab49351a2c9c60e703a5fe5fe0f94c84c4294271f297d529a781d3a87a37b515d090a78676bcaf3da225f03617f8900966d108e31557e86319a2407b00b
+  data.tar.gz: f5aad0f8648783f4d3c3cc42a087f646358e5e93586cb394a2644e8f6c32a296b3668ea7758348c0dd55d4868527b4aa6a3d774ee113d5a4a14c39ae3ecd2857

data/lib/mihari/analyzers/base.rb

@@ -92,7 +92,15 @@ module Mihari
 
         return res.recover { [] } if ignore_error?
 
-        res.to_result
+        result = res.to_result
+        return result if result.success?
+
+        # Wrap failure with AnalyzerError to explicitly name a failed analyzer
+        Failure AnalyzerError.new(
+          result.failure.message,
+          self.class.class_key,
+          cause: result.failure
+        )
       end
 
       class << self

data/lib/mihari/cli/application.rb

@@ -41,17 +41,17 @@ module Mihari
         def safe_execute
           yield
         rescue StandardError => e
-          err = unwrap_error(e)
+          error = unwrap_error(e)
 
-          raise err if options["debug"]
+          raise error if options["debug"]
 
-          case err
-          when ValidationError
-            warn JSON.pretty_generate(err.errors.to_h)
-          when StandardError
-            Sentry.capture_exception(err) if Sentry.initialized?
-            warn err
-          end
+          data = Entities::ErrorMessage.represent(
+            message: error.message,
+            detail: error.respond_to?(:detail) ? error.detail : nil
+          )
+          warn JSON.pretty_generate(data.as_json)
+
+          Sentry.capture_exception(error) if Sentry.initialized? && !error.is_a?(ValidationError)
 
           exit 1
         end

data/lib/mihari/clients/fofa.rb

@@ -52,7 +52,10 @@ module Mihari
       def search(query, page:, size: PAGE_SIZE)
         qbase64 = Base64.urlsafe_encode64(query)
         params = { qbase64: qbase64, size: size, page: page, email: email, key: api_key }.compact
-        Structs::Fofa::Response.from_dynamic! get_json("/api/v1/search/all", params: params)
+        res = Structs::Fofa::Response.from_dynamic!(get_json("/api/v1/search/all", params: params))
+        raise ResponseError, res.errmsg if res.error
+
+        res
       end
 
       #

data/lib/mihari/commands/rule.rb

@@ -34,8 +34,6 @@ module Mihari
               return if Pathname(path).exist? && !(yes? warning)
 
               Services::RuleInitializer.call(path)
-
-              puts "A new rule file has been initialized: #{path}."
             end
 
             desc "list [QUERY]", "List/search rules"

data/lib/mihari/concerns/error_unwrappable.rb

@@ -17,9 +17,10 @@ module Mihari
         receiver = err.receiver
         case receiver
         when Dry::Monads::Try::Error
-          receiver.exception
+          # Error may be wrapped like Matryoshka
+          unwrap_error receiver.exception
         when Dry::Monads::Failure
-          receiver.failure
+          unwrap_error receiver.failure
         else
           err
         end

data/lib/mihari/concerns/retriable.rb

@@ -8,35 +8,48 @@ module Mihari
     module Retriable
       extend ActiveSupport::Concern
 
-      DEFAULT_ON = [
-        Errno::ECONNRESET,
-        Errno::ECONNABORTED,
-        Errno::EPIPE,
+      RETRIABLE_ERRORS = [
         OpenSSL::SSL::SSLError,
         Timeout::Error,
-        RetryableError,
-        NetworkError,
-        TimeoutError,
-        StatusCodeError
+        ::HTTP::ConnectionError,
+        ::HTTP::ResponseError,
+        ::HTTP::TimeoutError
       ].freeze
 
+      DEFAULT_CONDITION = lambda do |error|
+        return true if RETRIABLE_ERRORS.any? { |klass| error.is_a? klass }
+
+        case error
+        when StatusCodeError
+          error.status_code != 404
+        else
+          false
+        end
+      end
+
       #
       # Retry on error
       #
       # @param [Integer] times
       # @param [Integer] interval
       # @param [Boolean] exponential_backoff
-      # @param [Array<StandardError>] on
+      # @param [Proc] condition
       #
-      def retry_on_error(times: 3, interval: 5, exponential_backoff: true, on: DEFAULT_ON)
+      # @param [Object] on
+      def retry_on_error(times: 3, interval: 5, exponential_backoff: true, condition: DEFAULT_CONDITION)
         try = 0
         begin
           try += 1
           yield
-        rescue *on => e
+        rescue StandardError => e
+          # Raise error if it's not a retriable error
+          raise e unless condition.call(e)
+
           sleep_seconds = exponential_backoff ? interval * (2**(try - 1)) : interval
           sleep sleep_seconds
           retry if try < times
+
+          # Raise error if retry times exceed a given times
           raise e
         end
       end

data/lib/mihari/errors.rb

@@ -1,27 +1,45 @@
 # frozen_string_literal: true
 
+require "http"
+
 module Mihari
   class Error < StandardError; end
 
   class ValueError < Error; end
 
-  class RetryableError < Error; end
-
   class ConfigurationError < Error; end
 
-  class IntegrityError < Error; end
+  class ResponseError < Error; end
 
-  # errors for HTTP interactions
-  class HTTPError < Error; end
+  class AnalyzerError < Error
+    # @return [StandardException, nil]
+    attr_reader :cause
 
-  class NetworkError < HTTPError; end
+    #
+    # @param [String] msg
+    # @param [String] analyzer
+    # @param [StandardException, nil] cause
+    #
+    def initialize(msg, analyzer, cause: nil)
+      super("#{msg} (from #{analyzer})")
+
+      @cause = cause
+      set_backtrace(cause.backtrace) if cause
+    end
 
-  class TimeoutError < HTTPError; end
+    def detail
+      return nil unless cause.respond_to?(:detail)
+
+      cause.detail
+    end
+  end
+
+  class IntegrityError < Error; end
 
   #
   # HTTP status code error
   #
-  class StatusCodeError < HTTPError
+  class StatusCodeError < ::HTTP::Error
     # @return [Integer]
     attr_reader :status_code
 
@@ -39,6 +57,10 @@ module Mihari
       @status_code = status_code
       @body = body
     end
+
+    def detail
+      { status_code: status_code, body: body }
+    end
   end
 
   #
@@ -56,5 +78,9 @@ module Mihari
 
       @errors = errors
     end
+
+    def detail
+      errors.to_h
+    end
   end
 end

data/lib/mihari/http.rb

@@ -18,11 +18,6 @@ module Mihari
         )
       end
 
-      def on_error(_request, error)
-        raise TimeoutError, error if error.is_a?(::HTTP::TimeoutError)
-        raise NetworkError, error if error.is_a?(::HTTP::Error)
-      end
-
       ::HTTP::Options.register_feature(:better_error, self)
     end
 

data/lib/mihari/rule.rb

@@ -83,6 +83,20 @@ module Mihari
       data[:data_types]
     end
 
+    #
+    # @return [Date, nil]
+    #
+    def created_on
+      data[:created_on]
+    end
+
+    #
+    # @return [Date, nil]
+    #
+    def updated_on
+      data[:updated_on]
+    end
+
     #
     # @return [Array<Mihari::Models::Tag>]
     #
@@ -231,7 +245,7 @@ module Mihari
     #
     def diff?
       model = Mihari::Models::Rule.find(id)
-      model.data != data.deep_stringify_keys
+      model.data != diff_comparable_data
     rescue ActiveRecord::RecordNotFound
       false
     end
@@ -272,6 +286,18 @@ module Mihari
 
     private
 
+    #
+    # @return [Hash]
+    #
+    def diff_comparable_data
+      # data is serialized as JSON so dates (created_on & updated_on) are stringified in there
+      # thus dates & (hash) keys have to be stringified when comparing
+      data.deep_dup.tap do |data|
+        data[:created_on] = created_on.to_s unless created_on.nil?
+        data[:updated_on] = updated_on.to_s unless updated_on.nil?
+      end.deep_stringify_keys
+    end
+
     #
     # Check whether a value is a falsepositive value or not
     #

data/lib/mihari/structs/fofa.rb

@@ -8,6 +8,10 @@ module Mihari
         #   @return [Boolean]
         attribute :error, Types::Bool
 
+        # @!attribute [r] errmsg
+        #   @return [String, nil]
+        attribute? :errmsg, Types::String.optional
+
         # @!attribute [r] size
         #   @return [Integer, nil]
         attribute? :size, Types::Int.optional
@@ -35,6 +39,7 @@ module Mihari
           def from_dynamic!(d)
             new(
               error: d.fetch("error"),
+              errmsg: d["errmsg"],
               size: d["size"],
               page: d["page"],
               mode: d["mode"],

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "7.0.3"
+  VERSION = "7.0.5"
 end

data/lib/mihari/web/endpoints/ip_addresses.rb

@@ -8,13 +8,16 @@ module Mihari
       #
       class IPAddresses < Grape::API
         namespace :ip_addresses do
-          desc "Get an IP address", {
+          desc "Get IP address data", {
             success: Entities::IPAddress,
-            failure: [{ code: 404, model: Entities::ErrorMessage }],
-            summary: "Get an IP address"
+            failure: [
+              { code: 404, model: Entities::ErrorMessage },
+              { code: 422, model: Entities::ErrorMessage }
+            ],
+            summary: "Get IP address data"
           }
           params do
-            requires :ip, type: String, regexp: /\A[0-9.]+\z/
+            requires :ip, type: String
           end
           get "/:ip", requirements: { ip: %r{[^/]+} } do
             ip = params[:ip].to_s
@@ -34,7 +37,8 @@ module Mihari
             failure = result.failure
             case failure
             when Mihari::StatusCodeError
-              error!({ message: "ID:#{id} not found" }, 404) if failure.status_code == 404
+              error!({ message: "IP:#{ip} not found" }, failure.status_code) if failure.status_code == 404
+              error!({ message: "IP format invalid" }, failure.status_code) if failure.status_code == 422
             end
             raise failure
           end

data/lib/mihari/web/endpoints/rules.rb

@@ -17,7 +17,10 @@ module Mihari
           def call(yaml, overwrite: true)
             rule = Rule.from_yaml(yaml)
 
-            raise IntegrityError, "ID:#{rule.id} is already registered" if rule.exists? && !overwrite
+            # To invoke ActiveRecord::RecordNotFound
+            Models::Rule.find(rule.id) if overwrite
+
+            raise IntegrityError, "ID:#{rule.id} already registered" if rule.exists? && !overwrite
 
             rule.update_or_create
             rule
@@ -141,13 +144,11 @@ module Mihari
             summary: "Update a rule"
           }
           params do
-            requires :id, type: String, documentation: { param_type: "body" }
             requires :yaml, type: String, documentation: { param_type: "body" }
           end
           put "/" do
             status 201
 
-            id = params[:id].to_s
             yaml = params[:yaml].to_s
 
             result = RuleCreateUpdater.result(yaml, overwrite: true)
@@ -156,11 +157,11 @@ module Mihari
             failure = result.failure
             case failure
             when ActiveRecord::RecordNotFound
-              error!({ message: "ID:#{id} not found" }, 404)
+              error!({ message: "Rule not found" }, 404)
             when Psych::SyntaxError
               error!({ message: failure.message }, 422)
             when ValidationError
-              error!({ message: "Rule format is invalid", detail: failure.errors.to_h }, 422)
+              error!({ message: "Rule format invalid", detail: failure.errors.to_h }, 422)
             end
             raise failure
           end

data/lib/mihari/web/middleware/capture_exceptions.rb

@@ -9,8 +9,6 @@ module Mihari
       class CaptureExceptions < Sentry::Rack::CaptureExceptions
         include Concerns::ErrorUnwrappable
 
-        private
-
         def capture_exception(exception, env)
           unwrapped = unwrap_error(exception)
           Mihari.logger.error unwrapped