mihari 7.1.0 → 7.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6930da0e95068ca8e30d1f226be5692e85375f796b7246cafdd2bec566d00ff7
-  data.tar.gz: 2bf34b1231bffcd88d402ffda335960ad929b1de91cd51b257c9c24f7b2fc16f
+  metadata.gz: 52d1c8320fdb5233c9738f3b4599868260fef892599cdfb42da6c3af17583b75
+  data.tar.gz: 625cd92558eff5a4d5613e588cc5ee85b9b714a9af211788da1a294d8d54ac45
 SHA512:
-  metadata.gz: fd515cdbde67d10c3fcae14c8e45a7929a34931701be176c9f9a294d3db310f522a3e186aefe548d6f119c5c8552723114da4ee6f790dbbe0391138dca9cc00e
-  data.tar.gz: 5f4a3e1049ad55af018cf9ed2186dda49bd18517d37bc84cad6e6f60b10589834ac3bd1f09333e7f9773fab43442400c097c1063f1c304d5be5eb8991af7ec77
+  metadata.gz: 96811d3ebfffcb27a7577b814be280916dd46bd3477233374ffafd2bf784d9e5133f89b1d7650c722c25e2f678f82380f5ced2eff8f9b7e1f3c96583f54a1114
+  data.tar.gz: 5c13581e670c7aff158e93335fe9c626294cad622f153e61a338f31bb5a1e459302f34a3a9ddf18e0bb322e1359c43a6b782387cdc800f9b047c04e5521f4c70

data/.rubocop.yml

@@ -17,6 +17,8 @@ RSpec/MultipleMemoizedHelpers:
   Max: 10
 RSpec/ExampleLength:
   Max: 20
+RSpec/FilePath:
+  SpecSuffixOnly: true
 require:
   - rubocop-factory_bot
   - rubocop-rake

data/README.md

@@ -1,7 +1,8 @@
 # mihari
 
 [![Gem Version](https://badge.fury.io/rb/mihari.svg)](https://badge.fury.io/rb/mihari)
-[![Ruby CI](https://github.com/ninoseki/mihari/actions/workflows/test.yml/badge.svg)](https://github.com/ninoseki/mihari/actions/workflows/test.yml)
+[![Ruby CI](https://github.com/ninoseki/mihari/actions/workflows/ruby.yml/badge.svg)](https://github.com/ninoseki/mihari/actions/workflows/ruby.yml)
+[![Node.js CI](https://github.com/ninoseki/mihari/actions/workflows/node.yml/badge.svg)](https://github.com/ninoseki/mihari/actions/workflows/node.yml)
 [![Coverage Status](https://coveralls.io/repos/github/ninoseki/mihari/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/mihari?branch=master)
 [![CodeFactor](https://www.codefactor.io/repository/github/ninoseki/mihari/badge)](https://www.codefactor.io/repository/github/ninoseki/mihari)
 

data/Rakefile

@@ -7,11 +7,49 @@ RSpec::Core::RakeTask.new(:spec)
 
 task default: :spec
 
-desc "run rackup (via rerun)"
+desc "Run rackup (with rerun)"
 task :rackup do
   sh "rerun --pattern '{Gemfile.lock,lib/**/*.rb,lib/*.rb}' -- rackup config.ru"
 end
 
+def recursive_delete(hash, to_remove)
+  hash.delete(to_remove)
+  hash.each_value do |value|
+    recursive_delete(value, to_remove) if value.is_a? Hash
+  end
+end
+
+def build_swagger_doc(path)
+  require_relative "lib/mihari"
+  require_relative "lib/mihari/web/application"
+
+  require "rack/test"
+
+  app = Mihari::Web::App.new
+  session = Rack::Test::Session.new(app)
+
+  res = session.request("/api/swagger_doc")
+
+  json = JSON.parse(res.body.to_s)
+  # remove host because it can be varied
+  keys_to_remove = %w[host]
+  keys_to_remove.each do |key|
+    recursive_delete json, key
+  end
+
+  f = File.open(path, "w")
+  f.write json.to_yaml
+  f.close
+end
+
+namespace :build do
+  desc "Build Swagger doc"
+  task :swagger, [:path] do |_t, args|
+    args.with_defaults(path: "./frontend/swagger.yaml")
+    build_swagger_doc args.path
+  end
+end
+
 def ci?
   ENV.fetch("CI", false)
 end

data/lib/mihari/actor.rb

@@ -55,7 +55,7 @@ module Mihari
 
       joined = self.class.configuration_keys.join(", ")
       be = (self.class.configuration_keys.length > 1) ? "are" : "is"
-      message = "#{self.class.class_key} is not configured correctly. #{joined} #{be} missing."
+      message = "#{self.class.key} is not configured correctly. #{joined} #{be} missing."
       raise ConfigurationError, message
     end
 
@@ -75,22 +75,22 @@ module Mihari
       #
       # @return [String]
       #
-      def class_key
+      def key
         to_s.split("::").last.downcase
       end
 
       #
       # @return [Array<String>, nil]
       #
-      def class_key_aliases
+      def key_aliases
         nil
       end
 
       #
       # @return [Array<String>]
       #
-      def class_keys
-        ([class_key] + [class_key_aliases]).flatten.compact.map(&:downcase)
+      def keys
+        ([key] + [key_aliases]).flatten.compact.map(&:downcase)
       end
     end
   end

data/lib/mihari/analyzers/base.rb

@@ -65,7 +65,7 @@ module Mihari
           # It is set automatically in #initialize
           artifact = artifact.is_a?(Models::Artifact) ? artifact : Models::Artifact.new(data: artifact)
 
-          artifact.source = self.class.class_key
+          artifact.source = self.class.key
           artifact.query = query
 
           artifact
@@ -80,7 +80,7 @@ module Mihari
       end
 
       def result(...)
-        res = Try[StandardError] do
+        result = Try[StandardError] do
           retry_on_error(
             times: retry_times,
             interval: retry_interval,
@@ -88,19 +88,26 @@ module Mihari
           ) do
             call(...)
           end
-        end
-
-        return res.recover { [] } if ignore_error?
+        end.to_result
 
-        result = res.to_result
         return result if result.success?
 
         # Wrap failure with AnalyzerError to explicitly name a failed analyzer
-        Failure AnalyzerError.new(
-          result.failure.message,
-          self.class.class_key,
-          cause: result.failure
-        )
+        error = AnalyzerError.new(result.failure.message, self.class.key, cause: result.failure)
+        return Failure(error) unless ignore_error?
+
+        # Return Success if ignore_error? is true with logging
+        Mihari.logger.warn("Analyzer:#{self.class.key} with #{truncated_query} failed - #{result.failure}")
+        Success([])
+      end
+
+      #
+      # Truncate query for logging
+      #
+      # @return [String]
+      #
+      def truncated_query
+        query.truncate(32)
       end
 
       class << self

data/lib/mihari/analyzers/passivetotal.rb

@@ -57,7 +57,7 @@ module Mihari
         #
         # @return [Array<String>, nil]
         #
-        def class_key_aliases
+        def key_aliases
           ["pt"]
         end
       end

data/lib/mihari/analyzers/securitytrails.rb

@@ -51,7 +51,7 @@ module Mihari
         #
         # @return [Array<String>, nil]
         #
-        def class_key_aliases
+        def key_aliases
           ["st"]
         end
       end

data/lib/mihari/analyzers/virustotal.rb

@@ -46,7 +46,7 @@ module Mihari
         #
         # @return [Array<String>, nil]
         #
-        def class_key_aliases
+        def key_aliases
           ["vt"]
         end
       end

data/lib/mihari/analyzers/virustotal_intelligence.rb

@@ -34,14 +34,14 @@ module Mihari
         #
         # @return [String]
         #
-        def class_key
+        def key
           "virustotal_intelligence"
         end
 
         #
         # @return [Array<String>, nil]
         #
-        def class_key_aliases
+        def key_aliases
           ["vt_intel"]
         end
       end

data/lib/mihari/cli/application.rb

@@ -38,6 +38,21 @@ module Mihari
       include Concerns::ErrorUnwrappable
 
       no_commands do
+        #
+        # @param [StandardError] error
+        #
+        # @return [String, Hash, nil]
+        #
+        def error_to_detail(error)
+          # Dirty hack to show the DB error message
+          # (NOTE: #safe_execute block suppress #with_db_connection's error message)
+          if error.is_a?(ActiveRecord::StatementInvalid)
+            return "DB migration is not yet complete. Please run 'mihari db migrate'."
+          end
+
+          error.respond_to?(:detail) ? error.detail : nil
+        end
+
         def safe_execute
           yield
         rescue StandardError => e
@@ -48,10 +63,7 @@ module Mihari
           # Raise error if debug is set as true
           raise error if options["debug"]
 
-          data = Entities::ErrorMessage.represent(
-            message: error.message,
-            detail: error.respond_to?(:detail) ? error.detail : nil
-          )
+          data = Entities::ErrorMessage.represent(message: error.message, detail: error_to_detail(error))
           warn JSON.pretty_generate(data.as_json)
 
           Sentry.capture_exception(error) if Sentry.initialized? && !error.is_a?(ValidationError)

data/lib/mihari/commands/alert.rb

@@ -26,7 +26,7 @@ module Mihari
               end
             end
 
-            desc "create [PATH]", "Create an alert"
+            desc "create PATH", "Create an alert"
             around :with_db_connection
             #
             # @param [String] path
@@ -46,7 +46,7 @@ module Mihari
               puts JSON.pretty_generate(data.as_json)
             end
 
-            desc "list [QUERY]", "List/search alerts"
+            desc "list QUERY", "List/search alerts"
             around :with_db_connection
             method_option :page, type: :numeric, default: 1
             method_option :limit, type: :numeric, default: 10
@@ -67,7 +67,7 @@ module Mihari
             desc "list-transform QUERY", "List/search alerts with transformation"
             around :with_db_connection
             method_option :template, type: :string, required: true, aliases: "-t",
-              description: "Jbuilder template itself or a path to a template file"
+              desc: "Jbuilder template stringor a path to a template"
             method_option :page, type: :numeric, default: 1
             method_option :limit, type: :numeric, default: 10
             #
@@ -86,7 +86,7 @@ module Mihari
               )
             end
 
-            desc "get [ID]", "Get an alert"
+            desc "get ID", "Get an alert"
             around :with_db_connection
             #
             # @param [Integer] id
@@ -97,7 +97,7 @@ module Mihari
               puts JSON.pretty_generate(data.as_json)
             end
 
-            desc "delete [ID]", "Delete an alert"
+            desc "delete ID", "Delete an alert"
             around :with_db_connection
             #
             # @param [Integer] id

data/lib/mihari/commands/artifact.rb

@@ -25,7 +25,7 @@ module Mihari
               end
             end
 
-            desc "list [QUERY]", "List/search artifacts"
+            desc "list QUERY", "List/search artifacts"
             around :with_db_connection
             method_option :page, type: :numeric, default: 1
             method_option :limit, type: :numeric, default: 10
@@ -46,7 +46,7 @@ module Mihari
             desc "list-transform QUERY", "List/search artifacts with transformation"
             around :with_db_connection
             method_option :template, type: :string, required: true, aliases: "-t",
-              description: "Jbuilder template itself or a path to a template file"
+              desc: "Jbuilder template stringor a path to a template"
             method_option :page, type: :numeric, default: 1
             method_option :limit, type: :numeric, default: 10
             #
@@ -65,7 +65,7 @@ module Mihari
               )
             end
 
-            desc "get [ID]", "Get an artifact"
+            desc "get ID", "Get an artifact"
             around :with_db_connection
             #
             # @param [Integer] id
@@ -76,7 +76,7 @@ module Mihari
               puts JSON.pretty_generate(data.as_json)
             end
 
-            desc "enrich [ID]", "Enrich an artifact"
+            desc "enrich ID", "Enrich an artifact"
             around :with_db_connection
             #
             # @param [Integer] id
@@ -85,7 +85,7 @@ module Mihari
               Services::ArtifactEnricher.result(id).value!
             end
 
-            desc "delete [ID]", "Delete an artifact"
+            desc "delete ID", "Delete an artifact"
             around :with_db_connection
             #
             # @param [Integer] id

data/lib/mihari/commands/rule.rb

@@ -26,7 +26,7 @@ module Mihari
               end
             end
 
-            desc "validate [PATH]", "Validate a rule file"
+            desc "validate PATH", "Validate a rule"
             #
             # Validate format of a rule
             #
@@ -37,7 +37,7 @@ module Mihari
               puts rule.data.to_yaml
             end
 
-            desc "init [PATH]", "Initialize a new rule file"
+            desc "init PATH", "Initialize a new rule"
             #
             # Initialize a new rule file
             #
@@ -50,7 +50,7 @@ module Mihari
               Services::RuleInitializer.call(path)
             end
 
-            desc "list [QUERY]", "List/search rules"
+            desc "list QUERY", "List/search rules"
             around :with_db_connection
             method_option :page, type: :numeric, default: 1
             method_option :limit, type: :numeric, default: 10
@@ -71,7 +71,7 @@ module Mihari
             desc "list-transform QUERY", "List/search rules with transformation"
             around :with_db_connection
             method_option :template, type: :string, required: true, aliases: "-t",
-              description: "Jbuilder template itself or a path to a template file"
+              desc: "Jbuilder template stringor a path to a template"
             method_option :page, type: :numeric, default: 1
             method_option :limit, type: :numeric, default: 10
             #
@@ -90,7 +90,7 @@ module Mihari
               )
             end
 
-            desc "get [ID]", "Get a rule"
+            desc "get ID", "Get a rule"
             around :with_db_connection
             def get(id)
               value = Services::RuleGetter.result(id).value!
@@ -98,7 +98,7 @@ module Mihari
               puts JSON.pretty_generate(data.as_json)
             end
 
-            desc "delete [ID]", "Delete a rule"
+            desc "delete ID", "Delete a rule"
             around :with_db_connection
             #
             # @param [String] id

data/lib/mihari/commands/search.rb

@@ -11,7 +11,7 @@ module Mihari
           thor.class_eval do
             include Concerns::DatabaseConnectable
 
-            desc "search [PATH_OR_ID]", "Search by a rule"
+            desc "search PATH_OR_ID", "Search by a rule"
             around :with_db_connection
             method_option :force_overwrite, type: :boolean, default: false, aliases: "-f",
               desc: "Force overwriting a rule"

data/lib/mihari/commands/tag.rb

@@ -25,7 +25,7 @@ module Mihari
               end
             end
 
-            desc "list", "List/search tags"
+            desc "list QUERY", "List/search tags"
             around :with_db_connection
             method_option :page, type: :numeric, default: 1
             method_option :limit, type: :numeric, default: 10
@@ -46,7 +46,7 @@ module Mihari
             desc "list-transform QUERY", "List/search tags with transformation"
             around :with_db_connection
             method_option :template, type: :string, required: true, aliases: "-t",
-              description: "Jbuilder template itself or a path to a template file"
+              desc: "Jbuilder template stringor a path to a template"
             method_option :page, type: :numeric, default: 1
             method_option :limit, type: :numeric, default: 10
             #
@@ -65,7 +65,7 @@ module Mihari
               )
             end
 
-            desc "delete [ID]", "Delete a tag"
+            desc "delete ID", "Delete a tag"
             around :with_db_connection
             #
             # @param [Integer] id

data/lib/mihari/concerns/retriable.rb

@@ -21,7 +21,7 @@ module Mihari
 
         case error
         when StatusError
-          error.status_code != 404
+          ![401, 404].include?(error.status_code)
         else
           false
         end

data/lib/mihari/constants.rb

@@ -5,7 +5,7 @@ module Mihari
   DEFAULT_DATA_TYPES = Types::DataTypes.values.freeze
 
   # @return [Array<Hash>]
-  DEFAULT_EMITTERS = Emitters::Database.class_keys.map { |name| { emitter: name.downcase } }.freeze
+  DEFAULT_EMITTERS = Emitters::Database.keys.map { |name| { emitter: name.downcase } }.freeze
 
   # @return [Array<Hash>]
   DEFAULT_ENRICHERS = Mihari.enricher_to_class.keys.map { |name| { enricher: name.downcase } }.freeze

data/lib/mihari/database.rb

@@ -156,7 +156,7 @@ module Mihari
         Mihari::Database.connect unless connected?
         yield
       rescue ActiveRecord::StatementInvalid
-        Mihari.logger.error("The DB migration is not yet complete. Please run 'mihari db migrate'.")
+        Mihari.logger.error("DB migration is not yet complete. Please run 'mihari db migrate'.")
       ensure
         Mihari::Database.close
       end

data/lib/mihari/emitters/base.rb

@@ -19,6 +19,14 @@ module Mihari
         @rule = rule
       end
 
+      # A target to emit the data
+      #
+      # @return [String]
+      #
+      def target
+        raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+      end
+
       #
       # @param [Array<Mihari::Models::Artifact>] artifacts
       #
@@ -30,13 +38,19 @@ module Mihari
       # @return [Dry::Monads::Result::Success<Object>, Dry::Monads::Result::Failure]
       #
       def result(artifacts)
-        Try[StandardError] do
+        result = Try[StandardError] do
           retry_on_error(
             times: retry_times,
             interval: retry_interval,
             exponential_backoff: retry_exponential_backoff
           ) { call(artifacts) }
         end.to_result
+
+        if result.failure?
+          Mihari.logger.warn("Emitter:#{self.class.key} for #{target.truncate(32)} failed - #{result.failure}")
+        end
+
+        result
       end
 
       class << self

data/lib/mihari/emitters/database.rb

@@ -21,6 +21,10 @@ module Mihari
         alert
       end
 
+      def target
+        Mihari.config.database_url.host || Mihari.config.database_url.to_s
+      end
+
       class << self
         def configuration_keys
           %w[database_url]

data/lib/mihari/emitters/misp.rb

@@ -56,6 +56,13 @@ module Mihari
         })
       end
 
+      #
+      # @return [String]
+      #
+      def target
+        URI(url).host || "N/A"
+      end
+
       class << self
         def configuration_keys
           %w[misp_url misp_api_key]

data/lib/mihari/emitters/slack.rb

@@ -165,6 +165,13 @@ module Mihari
         webhook_url?
       end
 
+      #
+      # @return [String]
+      #
+      def target
+        channel
+      end
+
       #
       # @return [::Slack::Notifier]
       #

data/lib/mihari/emitters/the_hive.rb

@@ -33,6 +33,13 @@ module Mihari
         api_key? && url?
       end
 
+      #
+      # @return [String]
+      #
+      def target
+        URI(url).host || "N/A"
+      end
+
       #
       # Create a Hive alert
       #

data/lib/mihari/emitters/webhook.rb

@@ -55,6 +55,13 @@ module Mihari
         %w[http https].include? url.scheme.downcase
       end
 
+      #
+      # @return [String]
+      #
+      def target
+        URI(url).host || "N/A"
+      end
+
       #
       # @param [Array<Mihari::Models::Artifact>] artifacts
       #

data/lib/mihari/enrichers/base.rb

@@ -19,17 +19,25 @@ module Mihari
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
       end
 
+      #
+      # @param [Mihari::Models::Artifact] value
       #
       # @return [Dry::Monads::Result::Success<Object>, Dry::Monads::Result::Failure]
       #
       def result(value)
-        Try[StandardError] do
+        result = Try[StandardError] do
           retry_on_error(
             times: retry_times,
             interval: retry_interval,
             exponential_backoff: retry_exponential_backoff
           ) { call value }
         end.to_result
+
+        if result.failure?
+          Mihari.logger.warn("Enricher:#{self.class.key} for #{value.truncate(32)} failed: #{result.failure}")
+        end
+
+        result
       end
 
       class << self

data/lib/mihari/enrichers/google_public_dns.rb

@@ -21,7 +21,7 @@ module Mihari
         #
         # @return [String]
         #
-        def class_key
+        def key
           "google_public_dns"
         end
       end

data/lib/mihari/rule.rb

@@ -188,20 +188,8 @@ module Mihari
     def bulk_emit
       return [] if enriched_artifacts.empty?
 
-      # NOTE: separate parallel execution and logging
-      #       because the logger does not work along with Parallel
-      results = Parallel.map(emitters) { |emitter| emitter.result enriched_artifacts }
-      results.zip(emitters).map do |result_and_emitter|
-        result, emitter = result_and_emitter
-
-        case result
-        when Success
-          Mihari.logger.info "Emission by #{emitter.class} succeed"
-        else
-          Mihari.logger.info "Emission by #{emitter.class} failed: #{result.failure}"
-        end
-
-        result.value_or nil
+      Parallel.map(emitters) do |emitter|
+        emitter.result(enriched_artifacts).value_or nil
       end.compact
     end