mihari 7.0.4 → 7.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a5859aa404d85ccc45008979195ace2c4104cc86336b65afacbeece226c7def5
-  data.tar.gz: 75f6687159420d8f0d7b5f2f13fbab7f523881ad55a66574a7a9ec612cf720ae
+  metadata.gz: 6930da0e95068ca8e30d1f226be5692e85375f796b7246cafdd2bec566d00ff7
+  data.tar.gz: 2bf34b1231bffcd88d402ffda335960ad929b1de91cd51b257c9c24f7b2fc16f
 SHA512:
-  metadata.gz: 9db820d33af592d9b2953a01e745ce05c3a297a6cbefb409e147fd90ab5d02070722abbba18b3f282bf9abcc1cc85f50d2579ab2774ea26074d74b6aea8d6073
-  data.tar.gz: 8218aaf062e8ecad9cf01f58188de497ffba135ce42269e7ff87b82d49cf96c0a46c0aa273e752da1f97b288ffead8569fa00c8d6452701273e9694bcf4dba74
+  metadata.gz: fd515cdbde67d10c3fcae14c8e45a7929a34931701be176c9f9a294d3db310f522a3e186aefe548d6f119c5c8552723114da4ee6f790dbbe0391138dca9cc00e
+  data.tar.gz: 5f4a3e1049ad55af018cf9ed2186dda49bd18517d37bc84cad6e6f60b10589834ac3bd1f09333e7f9773fab43442400c097c1063f1c304d5be5eb8991af7ec77

data/lib/mihari/analyzers/base.rb

@@ -92,7 +92,15 @@ module Mihari
 
         return res.recover { [] } if ignore_error?
 
-        res.to_result
+        result = res.to_result
+        return result if result.success?
+
+        # Wrap failure with AnalyzerError to explicitly name a failed analyzer
+        Failure AnalyzerError.new(
+          result.failure.message,
+          self.class.class_key,
+          cause: result.failure
+        )
       end
 
       class << self

data/lib/mihari/cli/application.rb

@@ -41,17 +41,20 @@ module Mihari
         def safe_execute
           yield
         rescue StandardError => e
-          err = unwrap_error(e)
+          error = unwrap_error(e)
 
-          raise err if options["debug"]
+          # Raise error if it's a Thor::Error to follow Thor's manner
+          raise error if error.is_a?(Thor::Error)
+          # Raise error if debug is set as true
+          raise error if options["debug"]
 
-          case err
-          when ValidationError
-            warn JSON.pretty_generate(err.errors.to_h)
-          when StandardError
-            Sentry.capture_exception(err) if Sentry.initialized?
-            warn err
-          end
+          data = Entities::ErrorMessage.represent(
+            message: error.message,
+            detail: error.respond_to?(:detail) ? error.detail : nil
+          )
+          warn JSON.pretty_generate(data.as_json)
+
+          Sentry.capture_exception(error) if Sentry.initialized? && !error.is_a?(ValidationError)
 
           exit 1
         end

data/lib/mihari/clients/fofa.rb

@@ -52,7 +52,10 @@ module Mihari
       def search(query, page:, size: PAGE_SIZE)
         qbase64 = Base64.urlsafe_encode64(query)
         params = { qbase64: qbase64, size: size, page: page, email: email, key: api_key }.compact
-        Structs::Fofa::Response.from_dynamic! get_json("/api/v1/search/all", params: params)
+        res = Structs::Fofa::Response.from_dynamic!(get_json("/api/v1/search/all", params: params))
+        raise ResponseError, res.errmsg if res.error
+
+        res
       end
 
       #

data/lib/mihari/commands/alert.rb

@@ -12,6 +12,20 @@ module Mihari
           thor.class_eval do
             include Concerns::DatabaseConnectable
 
+            no_commands do
+              #
+              # @param [String] q
+              # @param [Integer] page
+              # @param [Integer] limit
+              #
+              # @return [Mihari::Services::ResultValue]
+              #
+              def _search(q, page: 1, limit: 10)
+                filter = Structs::Filters::Search.new(q: q, page: page, limit: limit)
+                Services::AlertSearcher.result(filter).value!
+              end
+            end
+
             desc "create [PATH]", "Create an alert"
             around :with_db_connection
             #
@@ -40,8 +54,7 @@ module Mihari
             # @param [String] q
             #
             def list(q = "")
-              filter = Structs::Filters::Search.new(q: q, page: options["page"], limit: options["limit"])
-              value = Services::AlertSearcher.result(filter).value!
+              value = _search(q, page: options["page"], limit: options["limit"])
               data = Entities::AlertsWithPagination.represent(
                 results: value.results,
                 total: value.total,
@@ -51,6 +64,28 @@ module Mihari
               puts JSON.pretty_generate(data.as_json)
             end
 
+            desc "list-transform QUERY", "List/search alerts with transformation"
+            around :with_db_connection
+            method_option :template, type: :string, required: true, aliases: "-t",
+              description: "Jbuilder template itself or a path to a template file"
+            method_option :page, type: :numeric, default: 1
+            method_option :limit, type: :numeric, default: 10
+            #
+            # @param [String] q
+            #
+            def list_transform(q = "")
+              value = _search(q, page: options["page"], limit: options["limit"])
+              puts Services::JbuilderRenderer.call(
+                options["template"],
+                {
+                  results: value.results,
+                  total: value.total,
+                  current_page: value.filter[:page].to_i,
+                  page_size: value.filter[:limit].to_i
+                }
+              )
+            end
+
             desc "get [ID]", "Get an alert"
             around :with_db_connection
             #

data/lib/mihari/commands/artifact.rb

@@ -11,6 +11,20 @@ module Mihari
           thor.class_eval do
             include Concerns::DatabaseConnectable
 
+            no_commands do
+              #
+              # @param [String] q
+              # @param [Integer] page
+              # @param [Integer] limit
+              #
+              # @return [Mihari::Services::ResultValue]
+              #
+              def _search(q, page: 1, limit: 10)
+                filter = Structs::Filters::Search.new(q: q, page: page, limit: limit)
+                Services::ArtifactSearcher.result(filter).value!
+              end
+            end
+
             desc "list [QUERY]", "List/search artifacts"
             around :with_db_connection
             method_option :page, type: :numeric, default: 1
@@ -19,8 +33,7 @@ module Mihari
             # @param [String] q
             #
             def list(q = "")
-              filter = Structs::Filters::Search.new(q: q, page: options["page"], limit: options["limit"])
-              value = Services::ArtifactSearcher.result(filter).value!
+              value = _search(q, page: options["page"], limit: options["limit"])
               data = Entities::ArtifactsWithPagination.represent(
                 results: value.results,
                 total: value.total,
@@ -30,6 +43,28 @@ module Mihari
               puts JSON.pretty_generate(data.as_json)
             end
 
+            desc "list-transform QUERY", "List/search artifacts with transformation"
+            around :with_db_connection
+            method_option :template, type: :string, required: true, aliases: "-t",
+              description: "Jbuilder template itself or a path to a template file"
+            method_option :page, type: :numeric, default: 1
+            method_option :limit, type: :numeric, default: 10
+            #
+            # @param [String] q
+            #
+            def list_transform(q = "")
+              value = _search(q, page: options["page"], limit: options["limit"])
+              puts Services::JbuilderRenderer.call(
+                options["template"],
+                {
+                  results: value.results,
+                  total: value.total,
+                  current_page: value.filter[:page].to_i,
+                  page_size: value.filter[:limit].to_i
+                }
+              )
+            end
+
             desc "get [ID]", "Get an artifact"
             around :with_db_connection
             #

data/lib/mihari/commands/config.rb

@@ -9,7 +9,7 @@ module Mihari
       class << self
         def included(thor)
           thor.class_eval do
-            desc "list", "List config"
+            desc "list", "List configs"
             def list
               configs = Services::ConfigSearcher.call
               data = configs.map { |config| Entities::Config.represent(config) }

data/lib/mihari/commands/rule.rb

@@ -12,6 +12,20 @@ module Mihari
           thor.class_eval do
             include Concerns::DatabaseConnectable
 
+            no_commands do
+              #
+              # @param [String] q
+              # @param [Integer] page
+              # @param [Integer] limit
+              #
+              # @return [Mihari::Services::ResultValue]
+              #
+              def _search(q, page: 1, limit: 10)
+                filter = Structs::Filters::Search.new(q: q, page: page, limit: limit)
+                Services::RuleSearcher.result(filter).value!
+              end
+            end
+
             desc "validate [PATH]", "Validate a rule file"
             #
             # Validate format of a rule
@@ -44,8 +58,7 @@ module Mihari
             # @param [String] q
             #
             def list(q = "")
-              filter = Structs::Filters::Search.new(q: q, page: options["page"], limit: options["limit"])
-              value = Services::RuleSearcher.result(filter).value!
+              value = _search(q, page: options["page"], limit: options["limit"])
               data = Entities::RulesWithPagination.represent(
                 results: value.results,
                 total: value.total,
@@ -55,6 +68,28 @@ module Mihari
               puts JSON.pretty_generate(data.as_json)
             end
 
+            desc "list-transform QUERY", "List/search rules with transformation"
+            around :with_db_connection
+            method_option :template, type: :string, required: true, aliases: "-t",
+              description: "Jbuilder template itself or a path to a template file"
+            method_option :page, type: :numeric, default: 1
+            method_option :limit, type: :numeric, default: 10
+            #
+            # @param [String] q
+            #
+            def list_transform(q = "")
+              value = _search(q, page: options["page"], limit: options["limit"])
+              puts Services::JbuilderRenderer.call(
+                options["template"],
+                {
+                  results: value.results,
+                  total: value.total,
+                  current_page: value.filter[:page].to_i,
+                  page_size: value.filter[:limit].to_i
+                }
+              )
+            end
+
             desc "get [ID]", "Get a rule"
             around :with_db_connection
             def get(id)

data/lib/mihari/commands/tag.rb

@@ -11,6 +11,20 @@ module Mihari
           thor.class_eval do
             include Concerns::DatabaseConnectable
 
+            no_commands do
+              #
+              # @param [String] q
+              # @param [Integer] page
+              # @param [Integer] limit
+              #
+              # @return [Mihari::Services::ResultValue]
+              #
+              def _search(q, page: 1, limit: 10)
+                filter = Structs::Filters::Search.new(q: q, page: page, limit: limit)
+                Services::TagSearcher.result(filter).value!
+              end
+            end
+
             desc "list", "List/search tags"
             around :with_db_connection
             method_option :page, type: :numeric, default: 1
@@ -19,8 +33,7 @@ module Mihari
             # @param [String] q
             #
             def list(q = "")
-              filter = Structs::Filters::Search.new(q: q, page: options["page"], limit: options["limit"])
-              value = Services::TagSearcher.result(filter).value!
+              value = _search(q, page: options["page"], limit: options["limit"])
               data = Entities::TagsWithPagination.represent(
                 results: value.results,
                 total: value.total,
@@ -30,6 +43,28 @@ module Mihari
               puts JSON.pretty_generate(data.as_json)
             end
 
+            desc "list-transform QUERY", "List/search tags with transformation"
+            around :with_db_connection
+            method_option :template, type: :string, required: true, aliases: "-t",
+              description: "Jbuilder template itself or a path to a template file"
+            method_option :page, type: :numeric, default: 1
+            method_option :limit, type: :numeric, default: 10
+            #
+            # @param [String] q
+            #
+            def list_transform(q = "")
+              value = _search(q, page: options["page"], limit: options["limit"])
+              puts Services::JbuilderRenderer.call(
+                options["template"],
+                {
+                  results: value.results,
+                  total: value.total,
+                  current_page: value.filter[:page].to_i,
+                  page_size: value.filter[:limit].to_i
+                }
+              )
+            end
+
             desc "delete [ID]", "Delete a tag"
             around :with_db_connection
             #

data/lib/mihari/concerns/error_unwrappable.rb

@@ -17,9 +17,10 @@ module Mihari
         receiver = err.receiver
         case receiver
         when Dry::Monads::Try::Error
-          receiver.exception
+          # Error may be wrapped like Matryoshka
+          unwrap_error receiver.exception
         when Dry::Monads::Failure
-          receiver.failure
+          unwrap_error receiver.failure
         else
           err
         end

data/lib/mihari/concerns/retriable.rb

@@ -8,35 +8,48 @@ module Mihari
     module Retriable
       extend ActiveSupport::Concern
 
-      DEFAULT_ON = [
-        Errno::ECONNRESET,
-        Errno::ECONNABORTED,
-        Errno::EPIPE,
+      RETRIABLE_ERRORS = [
         OpenSSL::SSL::SSLError,
         Timeout::Error,
-        RetryableError,
-        NetworkError,
-        TimeoutError,
-        StatusCodeError
+        ::HTTP::ConnectionError,
+        ::HTTP::ResponseError,
+        ::HTTP::TimeoutError
       ].freeze
 
+      DEFAULT_CONDITION = lambda do |error|
+        return true if RETRIABLE_ERRORS.any? { |klass| error.is_a? klass }
+
+        case error
+        when StatusError
+          error.status_code != 404
+        else
+          false
+        end
+      end
+
       #
       # Retry on error
       #
       # @param [Integer] times
       # @param [Integer] interval
       # @param [Boolean] exponential_backoff
-      # @param [Array<StandardError>] on
+      # @param [Proc] condition
       #
-      def retry_on_error(times: 3, interval: 5, exponential_backoff: true, on: DEFAULT_ON)
+      # @param [Object] on
+      def retry_on_error(times: 3, interval: 5, exponential_backoff: true, condition: DEFAULT_CONDITION)
         try = 0
         begin
           try += 1
           yield
-        rescue *on => e
+        rescue StandardError => e
+          # Raise error if it's not a retriable error
+          raise e unless condition.call(e)
+
           sleep_seconds = exponential_backoff ? interval * (2**(try - 1)) : interval
           sleep sleep_seconds
           retry if try < times
+
+          # Raise error if retry times exceed a given times
           raise e
         end
       end

data/lib/mihari/data_type.rb

@@ -28,7 +28,7 @@ module Mihari
     def ip?
       Try[IPAddr::InvalidAddressError] do
         IPAddr.new(data).to_s == data
-      end.to_result.value_or(false)
+      end.recover { false }.value!
     end
 
     # @return [Boolean]
@@ -36,7 +36,7 @@ module Mihari
       Try[Addressable::URI::InvalidURIError] do
         uri = Addressable::URI.parse("http://#{data}")
         uri.host == data && PublicSuffix.valid?(uri.host)
-      end.to_result.value_or(false)
+      end.recover { false }.value!
     end
 
     # @return [Boolean]
@@ -44,7 +44,7 @@ module Mihari
       Try[Addressable::URI::InvalidURIError] do
         uri = Addressable::URI.parse(data)
         uri.scheme && uri.host && uri.path && PublicSuffix.valid?(uri.host)
-      end.to_result.value_or(false)
+      end.recover { false }.value!
     end
 
     # @return [Boolean]

data/lib/mihari/emitters/webhook.rb

@@ -1,49 +1,7 @@
 # frozen_string_literal: true
 
-require "erb"
-
 module Mihari
   module Emitters
-    class ERBTemplate < ERB
-      class << self
-        def template
-          %{
-            {
-              "rule": {
-                "id": "<%= @rule.id %>",
-                "title": "<%= @rule.title %>",
-                "description": "<%= @rule.description %>"
-              },
-              "artifacts": [
-                <% @artifacts.each_with_index do |artifact, idx| %>
-                  "<%= artifact.data %>"
-                  <%= ',' if idx < (@artifacts.length - 1) %>
-                <% end %>
-              ],
-              "tags": [
-                <% @rule.tags.each_with_index do |tag, idx| %>
-                  "<%= tag.name %>"
-                  <%= ',' if idx < (@rule.tags.length - 1) %>
-                <% end %>
-              ]
-            }
-          }
-        end
-      end
-
-      def initialize(artifacts:, rule:, options: {})
-        @artifacts = artifacts
-        @rule = rule
-
-        @template = options.fetch(:template, self.class.template)
-        super(@template)
-      end
-
-      def result
-        super(binding)
-      end
-    end
-
     class Webhook < Base
       # @return [Addressable::URI, nil]
       attr_reader :url
@@ -54,12 +12,24 @@ module Mihari
       # @return [String]
       attr_reader :method
 
-      # @return [String, nil]
+      # @return [String]
       attr_reader :template
 
       # @return [Array<Mihari::Models::Artifact>]
       attr_accessor :artifacts
 
+      DEFAULT_TEMPLATE = %{
+          json.rule do
+          json.id rule.id
+          json.title rule.title
+          json.description rule.description
+        end
+
+        json.artifacts artifacts.map(&:data)
+
+        json.tags rule.tags.map(&:name)
+      }
+
       #
       # @param [Mihari::Rule] rule
       # @param [Hash, nil] options
@@ -71,7 +41,7 @@ module Mihari
         @url = Addressable::URI.parse(params[:url])
         @headers = params[:headers] || {}
         @method = params[:method] || "POST"
-        @template = params[:template]
+        @template = params[:template] || DEFAULT_TEMPLATE
 
         @artifacts = []
       end
@@ -114,15 +84,7 @@ module Mihari
       # @return [String]
       #
       def render
-        options = {}
-        options[:template] = File.read(template) unless template.nil?
-
-        erb_template = ERBTemplate.new(
-          artifacts: artifacts,
-          rule: rule,
-          options: options
-        )
-        erb_template.result
+        Services::JbuilderRenderer.call(template, { rule: rule, artifacts: artifacts })
       end
 
       #

data/lib/mihari/errors.rb

@@ -1,27 +1,45 @@
 # frozen_string_literal: true
 
+require "http"
+
 module Mihari
   class Error < StandardError; end
 
   class ValueError < Error; end
 
-  class RetryableError < Error; end
-
   class ConfigurationError < Error; end
 
-  class IntegrityError < Error; end
+  class ResponseError < Error; end
 
-  # errors for HTTP interactions
-  class HTTPError < Error; end
+  class AnalyzerError < Error
+    # @return [StandardException, nil]
+    attr_reader :cause
 
-  class NetworkError < HTTPError; end
+    #
+    # @param [String] msg
+    # @param [String] analyzer
+    # @param [StandardException, nil] cause
+    #
+    def initialize(msg, analyzer, cause: nil)
+      super("#{msg} (from #{analyzer})")
+
+      @cause = cause
+      set_backtrace(cause.backtrace) if cause
+    end
 
-  class TimeoutError < HTTPError; end
+    def detail
+      return nil unless cause.respond_to?(:detail)
+
+      cause.detail
+    end
+  end
+
+  class IntegrityError < Error; end
 
   #
-  # HTTP status code error
+  # HTTP status error
   #
-  class StatusCodeError < HTTPError
+  class StatusError < ::HTTP::Error
     # @return [Integer]
     attr_reader :status_code
 
@@ -39,6 +57,10 @@ module Mihari
       @status_code = status_code
       @body = body
     end
+
+    def detail
+      { status_code: status_code, body: body }
+    end
   end
 
   #
@@ -56,5 +78,9 @@ module Mihari
 
       @errors = errors
     end
+
+    def detail
+      errors.to_h
+    end
   end
 end

data/lib/mihari/http.rb

@@ -11,18 +11,13 @@ module Mihari
       def wrap_response(response)
         return response if response.status.success?
 
-        raise StatusCodeError.new(
+        raise StatusError.new(
           "Unsuccessful response code returned: #{response.code}",
           response.code,
           response.body.to_s
         )
       end
 
-      def on_error(_request, error)
-        raise TimeoutError, error if error.is_a?(::HTTP::TimeoutError)
-        raise NetworkError, error if error.is_a?(::HTTP::Error)
-      end
-
       ::HTTP::Options.register_feature(:better_error, self)
     end
 

data/lib/mihari/rule.rb

@@ -83,6 +83,20 @@ module Mihari
       data[:data_types]
     end
 
+    #
+    # @return [Date, nil]
+    #
+    def created_on
+      data[:created_on]
+    end
+
+    #
+    # @return [Date, nil]
+    #
+    def updated_on
+      data[:updated_on]
+    end
+
     #
     # @return [Array<Mihari::Models::Tag>]
     #
@@ -279,8 +293,8 @@ module Mihari
       # data is serialized as JSON so dates (created_on & updated_on) are stringified in there
       # thus dates & (hash) keys have to be stringified when comparing
       data.deep_dup.tap do |data|
-        data[:created_on] = data[:created_on].to_s
-        data[:updated_on] = data[:updated_on].to_s
+        data[:created_on] = created_on.to_s unless created_on.nil?
+        data[:updated_on] = updated_on.to_s unless updated_on.nil?
       end.deep_stringify_keys
     end
 

data/lib/mihari/services/renderer.rb

@@ -0,0 +1,31 @@
+require "tilt/jbuilder"
+
+module Mihari
+  module Services
+    #
+    # Jbuilder based JSON renderer
+    #
+    class JbuilderRenderer < Service
+      attr_reader :template
+
+      #
+      # @param [String] template
+      # @param [Hash] params
+      #
+      # @return [String]
+      #
+      def call(template, params = {})
+        @template = template
+
+        jbuilder_template = Tilt::JbuilderTemplate.new { template_string }
+        jbuilder_template.render(nil, params)
+      end
+
+      def template_string
+        return File.read(template) if Pathname(template).exist?
+
+        template
+      end
+    end
+  end
+end