mihari 7.0.5 → 7.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aca048c797a73db1c57f0a5ad5f29066fbfba69451c71df1a26c3ea57853a1e3
-  data.tar.gz: 5ce2d298f325cc9d0ae50a98a6c18a64ec11cc06386504535c8ca647c7c68ad0
+  metadata.gz: 8ea0adf73bba53f264c22f8885d44e24de5657a2a77c2b7ea3533bb5acf6e78b
+  data.tar.gz: 77ab077d9322a22b0e399c81c057485aec1c4cdb1a14f15cbd81f1a3650f37a6
 SHA512:
-  metadata.gz: ac1d6ab49351a2c9c60e703a5fe5fe0f94c84c4294271f297d529a781d3a87a37b515d090a78676bcaf3da225f03617f8900966d108e31557e86319a2407b00b
-  data.tar.gz: f5aad0f8648783f4d3c3cc42a087f646358e5e93586cb394a2644e8f6c32a296b3668ea7758348c0dd55d4868527b4aa6a3d774ee113d5a4a14c39ae3ecd2857
+  metadata.gz: 3b68702c146819189140c0c5626c27f7b53f94021dd21138f0ac2758366486cebb7c450e0168f76cfc3d9940ee86eb2b279eb1725a3922614f51930dcb0e6b71
+  data.tar.gz: e30c01360a6d382e73a8e88a40a5cd3f1dba6bac9ebbbe9b87b9b51cff4a6dc59676d19eec4fe757a3a40d78a560886e434802b74fdf125e663b73ae4c7abd8a

data/.rubocop.yml

@@ -17,6 +17,8 @@ RSpec/MultipleMemoizedHelpers:
   Max: 10
 RSpec/ExampleLength:
   Max: 20
+RSpec/FilePath:
+  SpecSuffixOnly: true
 require:
   - rubocop-factory_bot
   - rubocop-rake

data/README.md

@@ -1,7 +1,8 @@
 # mihari
 
 [![Gem Version](https://badge.fury.io/rb/mihari.svg)](https://badge.fury.io/rb/mihari)
-[![Ruby CI](https://github.com/ninoseki/mihari/actions/workflows/test.yml/badge.svg)](https://github.com/ninoseki/mihari/actions/workflows/test.yml)
+[![Ruby CI](https://github.com/ninoseki/mihari/actions/workflows/ruby.yml/badge.svg)](https://github.com/ninoseki/mihari/actions/workflows/ruby.yml)
+[![Node.js CI](https://github.com/ninoseki/mihari/actions/workflows/node.yml/badge.svg)](https://github.com/ninoseki/mihari/actions/workflows/node.yml)
 [![Coverage Status](https://coveralls.io/repos/github/ninoseki/mihari/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/mihari?branch=master)
 [![CodeFactor](https://www.codefactor.io/repository/github/ninoseki/mihari/badge)](https://www.codefactor.io/repository/github/ninoseki/mihari)
 

data/Rakefile

@@ -7,11 +7,49 @@ RSpec::Core::RakeTask.new(:spec)
 
 task default: :spec
 
-desc "run rackup (via rerun)"
+desc "Run rackup (with rerun)"
 task :rackup do
   sh "rerun --pattern '{Gemfile.lock,lib/**/*.rb,lib/*.rb}' -- rackup config.ru"
 end
 
+def recursive_delete(hash, to_remove)
+  hash.delete(to_remove)
+  hash.each_value do |value|
+    recursive_delete(value, to_remove) if value.is_a? Hash
+  end
+end
+
+def build_swagger_doc(path)
+  require_relative "lib/mihari"
+  require_relative "lib/mihari/web/application"
+
+  require "rack/test"
+
+  app = Mihari::Web::App.new
+  session = Rack::Test::Session.new(app)
+
+  res = session.request("/api/swagger_doc")
+
+  json = JSON.parse(res.body.to_s)
+  # remove host because it can be varied
+  keys_to_remove = %w[host]
+  keys_to_remove.each do |key|
+    recursive_delete json, key
+  end
+
+  f = File.open(path, "w")
+  f.write json.to_yaml
+  f.close
+end
+
+namespace :build do
+  desc "Build Swagger doc"
+  task :swagger, [:path] do |_t, args|
+    args.with_defaults(path: "./frontend/swagger.yaml")
+    build_swagger_doc args.path
+  end
+end
+
 def ci?
   ENV.fetch("CI", false)
 end

data/lib/mihari/analyzers/base.rb

@@ -80,7 +80,7 @@ module Mihari
       end
 
       def result(...)
-        res = Try[StandardError] do
+        result = Try[StandardError] do
           retry_on_error(
             times: retry_times,
             interval: retry_interval,
@@ -88,19 +88,17 @@ module Mihari
           ) do
             call(...)
           end
-        end
-
-        return res.recover { [] } if ignore_error?
+        end.to_result
 
-        result = res.to_result
         return result if result.success?
 
         # Wrap failure with AnalyzerError to explicitly name a failed analyzer
-        Failure AnalyzerError.new(
-          result.failure.message,
-          self.class.class_key,
-          cause: result.failure
-        )
+        error = AnalyzerError.new(result.failure.message, self.class.class_key, cause: result.failure)
+        return Failure(error) unless ignore_error?
+
+        # Return Success if ignore_error? is true with logging
+        Mihari.logger.warn("Analyzer:#{self.class.class_key} failed - #{result.failure}")
+        Success([])
       end
 
       class << self

data/lib/mihari/cli/application.rb

@@ -38,17 +38,32 @@ module Mihari
       include Concerns::ErrorUnwrappable
 
       no_commands do
+        #
+        # @param [StandardError] error
+        #
+        # @return [String, Hash, nil]
+        #
+        def error_to_detail(error)
+          # Dirty hack to show the DB error message
+          # (NOTE: #safe_execute block suppress #with_db_connection's error message)
+          if error.is_a?(ActiveRecord::StatementInvalid)
+            return "DB migration is not yet complete. Please run 'mihari db migrate'."
+          end
+
+          error.respond_to?(:detail) ? error.detail : nil
+        end
+
         def safe_execute
           yield
         rescue StandardError => e
           error = unwrap_error(e)
 
+          # Raise error if it's a Thor::Error to follow Thor's manner
+          raise error if error.is_a?(Thor::Error)
+          # Raise error if debug is set as true
           raise error if options["debug"]
 
-          data = Entities::ErrorMessage.represent(
-            message: error.message,
-            detail: error.respond_to?(:detail) ? error.detail : nil
-          )
+          data = Entities::ErrorMessage.represent(message: error.message, detail: error_to_detail(error))
           warn JSON.pretty_generate(data.as_json)
 
           Sentry.capture_exception(error) if Sentry.initialized? && !error.is_a?(ValidationError)

data/lib/mihari/commands/alert.rb

@@ -12,7 +12,21 @@ module Mihari
           thor.class_eval do
             include Concerns::DatabaseConnectable
 
-            desc "create [PATH]", "Create an alert"
+            no_commands do
+              #
+              # @param [String] q
+              # @param [Integer] page
+              # @param [Integer] limit
+              #
+              # @return [Mihari::Services::ResultValue]
+              #
+              def _search(q, page: 1, limit: 10)
+                filter = Structs::Filters::Search.new(q: q, page: page, limit: limit)
+                Services::AlertSearcher.result(filter).value!
+              end
+            end
+
+            desc "create PATH", "Create an alert"
             around :with_db_connection
             #
             # @param [String] path
@@ -32,7 +46,7 @@ module Mihari
               puts JSON.pretty_generate(data.as_json)
             end
 
-            desc "list [QUERY]", "List/search alerts"
+            desc "list QUERY", "List/search alerts"
             around :with_db_connection
             method_option :page, type: :numeric, default: 1
             method_option :limit, type: :numeric, default: 10
@@ -40,8 +54,7 @@ module Mihari
             # @param [String] q
             #
             def list(q = "")
-              filter = Structs::Filters::Search.new(q: q, page: options["page"], limit: options["limit"])
-              value = Services::AlertSearcher.result(filter).value!
+              value = _search(q, page: options["page"], limit: options["limit"])
               data = Entities::AlertsWithPagination.represent(
                 results: value.results,
                 total: value.total,
@@ -51,7 +64,29 @@ module Mihari
               puts JSON.pretty_generate(data.as_json)
             end
 
-            desc "get [ID]", "Get an alert"
+            desc "list-transform QUERY", "List/search alerts with transformation"
+            around :with_db_connection
+            method_option :template, type: :string, required: true, aliases: "-t",
+              desc: "Jbuilder template stringor a path to a template"
+            method_option :page, type: :numeric, default: 1
+            method_option :limit, type: :numeric, default: 10
+            #
+            # @param [String] q
+            #
+            def list_transform(q = "")
+              value = _search(q, page: options["page"], limit: options["limit"])
+              puts Services::JbuilderRenderer.call(
+                options["template"],
+                {
+                  results: value.results,
+                  total: value.total,
+                  current_page: value.filter[:page].to_i,
+                  page_size: value.filter[:limit].to_i
+                }
+              )
+            end
+
+            desc "get ID", "Get an alert"
             around :with_db_connection
             #
             # @param [Integer] id
@@ -62,7 +97,7 @@ module Mihari
               puts JSON.pretty_generate(data.as_json)
             end
 
-            desc "delete [ID]", "Delete an alert"
+            desc "delete ID", "Delete an alert"
             around :with_db_connection
             #
             # @param [Integer] id

data/lib/mihari/commands/artifact.rb

@@ -11,7 +11,21 @@ module Mihari
           thor.class_eval do
             include Concerns::DatabaseConnectable
 
-            desc "list [QUERY]", "List/search artifacts"
+            no_commands do
+              #
+              # @param [String] q
+              # @param [Integer] page
+              # @param [Integer] limit
+              #
+              # @return [Mihari::Services::ResultValue]
+              #
+              def _search(q, page: 1, limit: 10)
+                filter = Structs::Filters::Search.new(q: q, page: page, limit: limit)
+                Services::ArtifactSearcher.result(filter).value!
+              end
+            end
+
+            desc "list QUERY", "List/search artifacts"
             around :with_db_connection
             method_option :page, type: :numeric, default: 1
             method_option :limit, type: :numeric, default: 10
@@ -19,8 +33,7 @@ module Mihari
             # @param [String] q
             #
             def list(q = "")
-              filter = Structs::Filters::Search.new(q: q, page: options["page"], limit: options["limit"])
-              value = Services::ArtifactSearcher.result(filter).value!
+              value = _search(q, page: options["page"], limit: options["limit"])
               data = Entities::ArtifactsWithPagination.represent(
                 results: value.results,
                 total: value.total,
@@ -30,7 +43,29 @@ module Mihari
               puts JSON.pretty_generate(data.as_json)
             end
 
-            desc "get [ID]", "Get an artifact"
+            desc "list-transform QUERY", "List/search artifacts with transformation"
+            around :with_db_connection
+            method_option :template, type: :string, required: true, aliases: "-t",
+              desc: "Jbuilder template stringor a path to a template"
+            method_option :page, type: :numeric, default: 1
+            method_option :limit, type: :numeric, default: 10
+            #
+            # @param [String] q
+            #
+            def list_transform(q = "")
+              value = _search(q, page: options["page"], limit: options["limit"])
+              puts Services::JbuilderRenderer.call(
+                options["template"],
+                {
+                  results: value.results,
+                  total: value.total,
+                  current_page: value.filter[:page].to_i,
+                  page_size: value.filter[:limit].to_i
+                }
+              )
+            end
+
+            desc "get ID", "Get an artifact"
             around :with_db_connection
             #
             # @param [Integer] id
@@ -41,7 +76,7 @@ module Mihari
               puts JSON.pretty_generate(data.as_json)
             end
 
-            desc "enrich [ID]", "Enrich an artifact"
+            desc "enrich ID", "Enrich an artifact"
             around :with_db_connection
             #
             # @param [Integer] id
@@ -50,7 +85,7 @@ module Mihari
               Services::ArtifactEnricher.result(id).value!
             end
 
-            desc "delete [ID]", "Delete an artifact"
+            desc "delete ID", "Delete an artifact"
             around :with_db_connection
             #
             # @param [Integer] id

data/lib/mihari/commands/config.rb

@@ -9,7 +9,7 @@ module Mihari
       class << self
         def included(thor)
           thor.class_eval do
-            desc "list", "List config"
+            desc "list", "List configs"
             def list
               configs = Services::ConfigSearcher.call
               data = configs.map { |config| Entities::Config.represent(config) }

data/lib/mihari/commands/rule.rb

@@ -12,7 +12,21 @@ module Mihari
           thor.class_eval do
             include Concerns::DatabaseConnectable
 
-            desc "validate [PATH]", "Validate a rule file"
+            no_commands do
+              #
+              # @param [String] q
+              # @param [Integer] page
+              # @param [Integer] limit
+              #
+              # @return [Mihari::Services::ResultValue]
+              #
+              def _search(q, page: 1, limit: 10)
+                filter = Structs::Filters::Search.new(q: q, page: page, limit: limit)
+                Services::RuleSearcher.result(filter).value!
+              end
+            end
+
+            desc "validate PATH", "Validate a rule"
             #
             # Validate format of a rule
             #
@@ -23,7 +37,7 @@ module Mihari
               puts rule.data.to_yaml
             end
 
-            desc "init [PATH]", "Initialize a new rule file"
+            desc "init PATH", "Initialize a new rule"
             #
             # Initialize a new rule file
             #
@@ -36,7 +50,7 @@ module Mihari
               Services::RuleInitializer.call(path)
             end
 
-            desc "list [QUERY]", "List/search rules"
+            desc "list QUERY", "List/search rules"
             around :with_db_connection
             method_option :page, type: :numeric, default: 1
             method_option :limit, type: :numeric, default: 10
@@ -44,8 +58,7 @@ module Mihari
             # @param [String] q
             #
             def list(q = "")
-              filter = Structs::Filters::Search.new(q: q, page: options["page"], limit: options["limit"])
-              value = Services::RuleSearcher.result(filter).value!
+              value = _search(q, page: options["page"], limit: options["limit"])
               data = Entities::RulesWithPagination.represent(
                 results: value.results,
                 total: value.total,
@@ -55,7 +68,29 @@ module Mihari
               puts JSON.pretty_generate(data.as_json)
             end
 
-            desc "get [ID]", "Get a rule"
+            desc "list-transform QUERY", "List/search rules with transformation"
+            around :with_db_connection
+            method_option :template, type: :string, required: true, aliases: "-t",
+              desc: "Jbuilder template stringor a path to a template"
+            method_option :page, type: :numeric, default: 1
+            method_option :limit, type: :numeric, default: 10
+            #
+            # @param [String] q
+            #
+            def list_transform(q = "")
+              value = _search(q, page: options["page"], limit: options["limit"])
+              puts Services::JbuilderRenderer.call(
+                options["template"],
+                {
+                  results: value.results,
+                  total: value.total,
+                  current_page: value.filter[:page].to_i,
+                  page_size: value.filter[:limit].to_i
+                }
+              )
+            end
+
+            desc "get ID", "Get a rule"
             around :with_db_connection
             def get(id)
               value = Services::RuleGetter.result(id).value!
@@ -63,7 +98,7 @@ module Mihari
               puts JSON.pretty_generate(data.as_json)
             end
 
-            desc "delete [ID]", "Delete a rule"
+            desc "delete ID", "Delete a rule"
             around :with_db_connection
             #
             # @param [String] id

data/lib/mihari/commands/search.rb

@@ -11,7 +11,7 @@ module Mihari
           thor.class_eval do
             include Concerns::DatabaseConnectable
 
-            desc "search [PATH_OR_ID]", "Search by a rule"
+            desc "search PATH_OR_ID", "Search by a rule"
             around :with_db_connection
             method_option :force_overwrite, type: :boolean, default: false, aliases: "-f",
               desc: "Force overwriting a rule"

data/lib/mihari/commands/tag.rb

@@ -11,7 +11,21 @@ module Mihari
           thor.class_eval do
             include Concerns::DatabaseConnectable
 
-            desc "list", "List/search tags"
+            no_commands do
+              #
+              # @param [String] q
+              # @param [Integer] page
+              # @param [Integer] limit
+              #
+              # @return [Mihari::Services::ResultValue]
+              #
+              def _search(q, page: 1, limit: 10)
+                filter = Structs::Filters::Search.new(q: q, page: page, limit: limit)
+                Services::TagSearcher.result(filter).value!
+              end
+            end
+
+            desc "list QUERY", "List/search tags"
             around :with_db_connection
             method_option :page, type: :numeric, default: 1
             method_option :limit, type: :numeric, default: 10
@@ -19,8 +33,7 @@ module Mihari
             # @param [String] q
             #
             def list(q = "")
-              filter = Structs::Filters::Search.new(q: q, page: options["page"], limit: options["limit"])
-              value = Services::TagSearcher.result(filter).value!
+              value = _search(q, page: options["page"], limit: options["limit"])
               data = Entities::TagsWithPagination.represent(
                 results: value.results,
                 total: value.total,
@@ -30,7 +43,29 @@ module Mihari
               puts JSON.pretty_generate(data.as_json)
             end
 
-            desc "delete [ID]", "Delete a tag"
+            desc "list-transform QUERY", "List/search tags with transformation"
+            around :with_db_connection
+            method_option :template, type: :string, required: true, aliases: "-t",
+              desc: "Jbuilder template stringor a path to a template"
+            method_option :page, type: :numeric, default: 1
+            method_option :limit, type: :numeric, default: 10
+            #
+            # @param [String] q
+            #
+            def list_transform(q = "")
+              value = _search(q, page: options["page"], limit: options["limit"])
+              puts Services::JbuilderRenderer.call(
+                options["template"],
+                {
+                  results: value.results,
+                  total: value.total,
+                  current_page: value.filter[:page].to_i,
+                  page_size: value.filter[:limit].to_i
+                }
+              )
+            end
+
+            desc "delete ID", "Delete a tag"
             around :with_db_connection
             #
             # @param [Integer] id

data/lib/mihari/concerns/retriable.rb

@@ -20,8 +20,8 @@ module Mihari
         return true if RETRIABLE_ERRORS.any? { |klass| error.is_a? klass }
 
         case error
-        when StatusCodeError
-          error.status_code != 404
+        when StatusError
+          ![401, 404].include?(error.status_code)
         else
           false
         end

data/lib/mihari/data_type.rb

@@ -28,7 +28,7 @@ module Mihari
     def ip?
       Try[IPAddr::InvalidAddressError] do
         IPAddr.new(data).to_s == data
-      end.to_result.value_or(false)
+      end.recover { false }.value!
     end
 
     # @return [Boolean]
@@ -36,7 +36,7 @@ module Mihari
       Try[Addressable::URI::InvalidURIError] do
         uri = Addressable::URI.parse("http://#{data}")
         uri.host == data && PublicSuffix.valid?(uri.host)
-      end.to_result.value_or(false)
+      end.recover { false }.value!
     end
 
     # @return [Boolean]
@@ -44,7 +44,7 @@ module Mihari
       Try[Addressable::URI::InvalidURIError] do
         uri = Addressable::URI.parse(data)
         uri.scheme && uri.host && uri.path && PublicSuffix.valid?(uri.host)
-      end.to_result.value_or(false)
+      end.recover { false }.value!
     end
 
     # @return [Boolean]

data/lib/mihari/database.rb

@@ -156,7 +156,7 @@ module Mihari
         Mihari::Database.connect unless connected?
         yield
       rescue ActiveRecord::StatementInvalid
-        Mihari.logger.error("The DB migration is not yet complete. Please run 'mihari db migrate'.")
+        Mihari.logger.error("DB migration is not yet complete. Please run 'mihari db migrate'.")
       ensure
         Mihari::Database.close
       end

data/lib/mihari/emitters/base.rb

@@ -30,13 +30,17 @@ module Mihari
       # @return [Dry::Monads::Result::Success<Object>, Dry::Monads::Result::Failure]
       #
       def result(artifacts)
-        Try[StandardError] do
+        result = Try[StandardError] do
           retry_on_error(
             times: retry_times,
             interval: retry_interval,
             exponential_backoff: retry_exponential_backoff
           ) { call(artifacts) }
         end.to_result
+
+        Mihari.logger.warn("Emitter:#{self.class.class_key} failed - #{result.failure}") if result.failure?
+
+        result
       end
 
       class << self

data/lib/mihari/emitters/webhook.rb

@@ -1,49 +1,7 @@
 # frozen_string_literal: true
 
-require "erb"
-
 module Mihari
   module Emitters
-    class ERBTemplate < ERB
-      class << self
-        def template
-          %{
-            {
-              "rule": {
-                "id": "<%= @rule.id %>",
-                "title": "<%= @rule.title %>",
-                "description": "<%= @rule.description %>"
-              },
-              "artifacts": [
-                <% @artifacts.each_with_index do |artifact, idx| %>
-                  "<%= artifact.data %>"
-                  <%= ',' if idx < (@artifacts.length - 1) %>
-                <% end %>
-              ],
-              "tags": [
-                <% @rule.tags.each_with_index do |tag, idx| %>
-                  "<%= tag.name %>"
-                  <%= ',' if idx < (@rule.tags.length - 1) %>
-                <% end %>
-              ]
-            }
-          }
-        end
-      end
-
-      def initialize(artifacts:, rule:, options: {})
-        @artifacts = artifacts
-        @rule = rule
-
-        @template = options.fetch(:template, self.class.template)
-        super(@template)
-      end
-
-      def result
-        super(binding)
-      end
-    end
-
     class Webhook < Base
       # @return [Addressable::URI, nil]
       attr_reader :url
@@ -54,12 +12,24 @@ module Mihari
       # @return [String]
       attr_reader :method
 
-      # @return [String, nil]
+      # @return [String]
       attr_reader :template
 
       # @return [Array<Mihari::Models::Artifact>]
       attr_accessor :artifacts
 
+      DEFAULT_TEMPLATE = %{
+          json.rule do
+          json.id rule.id
+          json.title rule.title
+          json.description rule.description
+        end
+
+        json.artifacts artifacts.map(&:data)
+
+        json.tags rule.tags.map(&:name)
+      }
+
       #
       # @param [Mihari::Rule] rule
       # @param [Hash, nil] options
@@ -71,7 +41,7 @@ module Mihari
         @url = Addressable::URI.parse(params[:url])
         @headers = params[:headers] || {}
         @method = params[:method] || "POST"
-        @template = params[:template]
+        @template = params[:template] || DEFAULT_TEMPLATE
 
         @artifacts = []
       end
@@ -114,15 +84,7 @@ module Mihari
       # @return [String]
       #
       def render
-        options = {}
-        options[:template] = File.read(template) unless template.nil?
-
-        erb_template = ERBTemplate.new(
-          artifacts: artifacts,
-          rule: rule,
-          options: options
-        )
-        erb_template.result
+        Services::JbuilderRenderer.call(template, { rule: rule, artifacts: artifacts })
       end
 
       #