mihari 6.3.0 → 7.0.0

data/lib/mihari/web/endpoints/artifacts.rb

@@ -7,117 +7,22 @@ module Mihari
       # Artifact API endpoint
       #
       class Artifacts < Grape::API
-        class ArtifactGetter < Service
-          #
-          # @param [Integer] id
-          #
-          # @return [Mihari::Models::Artifact]
-          #
-          def call(id)
-            artifact = Mihari::Models::Artifact.includes(
-              :autonomous_system,
-              :geolocation,
-              :whois_record,
-              :dns_records,
-              :reverse_dns_names
-            ).find(id)
-            # TODO: improve queries
-            alert_ids = Mihari::Models::Artifact.where(data: artifact.data).pluck(:alert_id)
-            tag_ids = Mihari::Models::Tagging.where(alert_id: alert_ids).pluck(:tag_id)
-            tags = Mihari::Models::Tag.where(id: tag_ids)
-
-            artifact.tags = tags
-
-            artifact
-          end
-        end
-
-        class ArtifactEnricher < Service
-          #
-          # @param [String] id
-          #
-          def call(id)
-            artifact = Mihari::Models::Artifact.includes(
-              :autonomous_system,
-              :geolocation,
-              :whois_record,
-              :dns_records,
-              :reverse_dns_names,
-              :cpes,
-              :ports
-            ).find(id)
-
-            artifact.enrich_all
-            artifact.save
-          end
-        end
-
-        class ArtifactDestroyer < Service
-          #
-          # @param [Integer] id
-          #
-          def call(id)
-            Mihari::Models::Artifact.find(id).destroy
-          end
-        end
-
-        class ArtifactSearcher < Mihari::Service
-          class ResultValue
-            # @return [Array<Mihari::Models::Artifacts>]
-            attr_reader :artifacts
-
-            # @return [Integer]
-            attr_reader :total
-
-            # @return [Mihari::Structs::Filters::Artifact::SearchFilterWithPagination]
-            attr_reader :filter
-
-            #
-            # @param [Array<Mihari::Models::Artifact>] artifacts
-            # @param [Integer] total
-            # @param [Mihari::Structs::Filters::Artifacts::SearchFilterWithPagination] filter
-            #
-            def initialize(artifacts:, total:, filter:)
-              @artifacts = artifacts
-              @total = total
-              @filter = filter
-            end
-          end
-
-          #
-          # @param [Hash] params
-          #
-          # @return [ResultValue]
-          #
-          def call(params)
-            filter = params.to_h.to_snake_keys.symbolize_keys
-            search_filter_with_pagination = Structs::Filters::Artifact::SearchFilterWithPagination.new(**filter)
-            artifacts = Mihari::Models::Artifact.search(search_filter_with_pagination)
-            total = Mihari::Models::Artifact.count(search_filter_with_pagination.without_pagination)
-            ResultValue.new(artifacts: artifacts, total: total, filter: filter)
-          end
-        end
-
         namespace :artifacts do
-          desc "Search artifacts", {
+          desc "List/search artifacts", {
             is_array: true,
             success: Entities::ArtifactsWithPagination,
-            summary: "Search artifacts"
+            summary: "List/search artifacts"
           }
           params do
+            optional :q, type: String, default: ""
             optional :page, type: Integer, default: 1
             optional :limit, type: Integer, default: 10
-            optional :dataType, type: String
-            optional :ruleId, type: String
-            optional :tag, type: String
-            optional :fromAt, type: DateTime
-            optional :toAt, type: DateTime
           end
           get "/" do
-            value = ArtifactSearcher.call(params.to_h)
+            value = Services::ArtifactSearcher.call(params.to_h)
             present(
               {
-                artifacts: value.artifacts,
+                results: value.results,
                 total: value.total,
                 current_page: value.filter[:page].to_i,
                 page_size: value.filter[:limit].to_i
@@ -128,7 +33,7 @@ module Mihari
 
           desc "Get an artifact", {
             success: Entities::Artifact,
-            failure: [{ code: 404, model: Entities::Message }],
+            failure: [{ code: 404, model: Entities::ErrorMessage }],
             summary: "Get an artifact"
           }
           params do
@@ -136,41 +41,52 @@ module Mihari
           end
           get "/:id" do
             id = params[:id].to_i
-            result = ArtifactGetter.result(id)
+            result = Services::ArtifactGetter.result(id)
             return present(result.value!, with: Entities::Artifact) if result.success?
 
             case result.failure
             when ActiveRecord::RecordNotFound
-              error!({ message: "ID:#{id} is not found" }, 404)
+              error!({ message: "ID:#{id} not found" }, 404)
             end
             raise result.failure
           end
 
           desc "Enrich an artifact", {
             success: { code: 201, model: Entities::Message },
-            failure: [{ code: 404, model: Entities::Message }],
+            failure: [{ code: 404, model: Entities::ErrorMessage }],
             summary: "Enrich an artifact"
           }
           params do
             requires :id, type: Integer
           end
-          get "/:id/enrich" do
+          post "/:id/enrich" do
             status 201
 
             id = params["id"].to_i
-            result = ArtifactEnricher.result(id)
-            return present({ message: "#{id} has been enriched" }, with: Entities::Message) if result.success?
+
+            queued = true
+            result = Dry::Monads::Try[StandardError] do
+              if Mihari.sidekiq?
+                Jobs::ArtifactEnrichJob.perform_async id
+              else
+                Services::ArtifactEnricher.call id
+                queued = false
+              end
+            end.to_result
+
+            message = queued ? "ID:#{id}'s enrichment has been queued" : "ID:#{id}'s enrichment has been succeeded"
+            return present({ message: message, queued: queued }, with: Entities::QueueMessage) if result.success?
 
             case result.failure
             when ActiveRecord::RecordNotFound
-              error!({ message: "ID:#{id} is not found" }, 404)
+              error!({ message: "ID:#{id} not found" }, 404)
             end
             raise result.failure
           end
 
           desc "Delete an artifact", {
             success: { code: 204, model: Entities::Message },
-            failure: [{ code: 404, model: Entities::Message }],
+            failure: [{ code: 404, model: Entities::ErrorMessage }],
             summary: "Delete an artifact"
           }
           params do
@@ -180,12 +96,12 @@ module Mihari
             status 204
 
             id = params["id"].to_i
-            result = ArtifactDestroyer.result(id)
+            result = Services::ArtifactDestroyer.result(id)
             return present({ message: "" }, with: Entities::Message) if result.success?
 
             case result.failure
             when ActiveRecord::RecordNotFound
-              error!({ message: "ID:#{id} is not found" }, 404)
+              error!({ message: "ID:#{id} not found" }, 404)
             end
             raise result.failure
           end

data/lib/mihari/web/endpoints/configs.rb

@@ -8,15 +8,13 @@ module Mihari
       #
       class Configs < Grape::API
         namespace :configs do
-          desc "Get configs", {
+          desc "list configs", {
             is_array: true,
             success: Entities::Config,
-            summary: "Get configs"
+            summary: "List configs"
           }
           get "/" do
-            configs = (Mihari.analyzers + Mihari.emitters + Mihari.enrichers).filter_map do |klass|
-              Mihari::Structs::Config.from_class(klass)
-            end
+            configs = Services::ConfigSearcher.call
             present(configs, with: Entities::Config)
           end
         end

data/lib/mihari/web/endpoints/ip_addresses.rb

@@ -7,21 +7,10 @@ module Mihari
       # IP address API endpoint
       #
       class IPAddresses < Grape::API
-        class IPGetter < Service
-          #
-          # @param [String] ip
-          #
-          # @return [Mihari::Structs::IPInfo::Response]
-          #
-          def call(ip)
-            Mihari::Enrichers::IPInfo.new.call ip
-          end
-        end
-
         namespace :ip_addresses do
           desc "Get an IP address", {
             success: Entities::IPAddress,
-            failure: [{ code: 404, model: Entities::Message }],
+            failure: [{ code: 404, model: Entities::ErrorMessage }],
             summary: "Get an IP address"
           }
           params do
@@ -29,13 +18,23 @@ module Mihari
           end
           get "/:ip", requirements: { ip: %r{[^/]+} } do
             ip = params[:ip].to_s
-            result = IPGetter.result(ip)
-            return present(result.value!, with: Entities::IPAddress) if result.success?
+            result = Services::IPGetter.result(ip)
+            if result.success?
+              value = result.value!
+              return present(
+                {
+                  country_code: value.country_code,
+                  asn: value.asn,
+                  loc: value.loc
+                },
+                with: Entities::IPAddress
+              )
+            end
 
             failure = result.failure
             case failure
             when Mihari::StatusCodeError
-              error!({ message: "ID:#{id} is not found" }, 404) if failure.status_code == 404
+              error!({ message: "ID:#{id} not found" }, 404) if failure.status_code == 404
             end
             raise failure
           end

data/lib/mihari/web/endpoints/rules.rb

@@ -7,135 +7,38 @@ module Mihari
       # Rule API endpoint
       #
       class Rules < Grape::API
-        class RuleSearcher < Mihari::Service
-          class ResultValue
-            # @return [Array<Mihari::Models::Rule>]
-            attr_reader :rules
-
-            # @return [Integer]
-            attr_reader :total
-
-            # @return [Mihari::Structs::Filters::Rule::SearchFilterWithPagination]
-            attr_reader :filter
-
-            #
-            # @param [Array<Mihari::Models::Rule>] rules
-            # @param [Integer] total
-            # @param [Mihari::Structs::Filters::Rule::SearchFilterWithPagination] filter
-            #
-            def initialize(rules:, total:, filter:)
-              @rules = rules
-              @total = total
-              @filter = filter
-            end
-          end
-
-          #
-          # @params [Hash]
+        class RuleCreateUpdater < Service
           #
-          # @return [ResultValue]
-          #
-          def call(params)
-            filter = params.to_h.to_snake_keys.symbolize_keys
-            search_filter_with_pagination = Mihari::Structs::Filters::Rule::SearchFilterWithPagination.new(**filter)
-            rules = Mihari::Models::Rule.search(search_filter_with_pagination)
-            total = Mihari::Models::Rule.count(search_filter_with_pagination.without_pagination)
-            ResultValue.new(rules: rules, total: total, filter: filter)
-          end
-        end
-
-        class RuleGetter < Service
-          #
-          # @params [String] id
-          #
-          # @return [Mihari::Models::Rule]
-          #
-          def call(id)
-            Mihari::Models::Rule.find id
-          end
-        end
-
-        class RuleRunner < Service
-          #
-          # @param [String] id
-          #
-          def call(id)
-            rule = Mihari::Rule.from_model(Mihari::Models::Rule.find(id))
-            rule.call
-          end
-        end
-
-        class RuleCreator < Service
-          #
-          # @params [String]
+          # @params [String] yaml
+          # @params [Boolean] overwrite
           #
           # @return [Mihari::Models::Rule]
           #
-          def call(yaml)
+          def call(yaml, overwrite: true)
             rule = Rule.from_yaml(yaml)
 
-            found = Mihari::Models::Rule.find_by_id(rule.id)
-            error!({ message: "ID:#{rule.id} is already registered" }, 400) unless found.nil?
+            raise IntegrityError, "ID:#{rule.id} is already registered" if rule.exists? && !overwrite
 
-            rule.model.save
+            rule.update_or_create
             rule
           end
         end
 
-        class RuleUpdater < Service
-          #
-          # @params [String] id
-          # @params [String] yaml
-          #
-          # @return [Mihari::Models::Rule]
-          #
-          def call(id:, yaml:)
-            Mihari::Models::Rule.find(id)
-
-            rule = Rule.from_yaml(yaml)
-            rule.model.save
-            rule
-          end
-        end
-
-        class RuleDestroyer < Service
-          #
-          # @param [String] id
-          #
-          def call(id)
-            Mihari::Models::Rule.find(id).destroy
-          end
-        end
-
         namespace :rules do
-          desc "Get Rule IDs", {
-            is_array: true,
-            success: Entities::RuleIDs,
-            summary: "Get rule IDs"
-          }
-          get "/ids" do
-            rule_ids = Mihari::Models::Rule.distinct.pluck(:id)
-            present({ rule_ids: rule_ids }, with: Entities::RuleIDs)
-          end
-
-          desc "Search rules", {
+          desc "List/search rules", {
             is_array: true,
             success: Entities::RulesWithPagination,
-            summary: "Search rules"
+            summary: "List/search rules"
           }
           params do
+            optional :q, type: String, default: ""
             optional :page, type: Integer, default: 1
             optional :limit, type: Integer, default: 10
-            optional :title, type: String
-            optional :description, type: String
-            optional :tag, type: String
-            optional :fromAt, type: DateTime
-            optional :toAt, type: DateTime
           end
           get "/" do
-            value = RuleSearcher.call(params.to_h)
+            value = Services::RuleSearcher.call(params.to_h)
             present({
-              rules: value.rules,
+              results: value.results,
               total: value.total,
               current_page: value.filter[:page].to_i,
               page_size: value.filter[:limit].to_i
@@ -145,7 +48,7 @@ module Mihari
 
           desc "Get a rule", {
             success: Entities::Rule,
-            failure: [{ code: 404, model: Entities::Message }],
+            failure: [{ code: 404, model: Entities::ErrorMessage }],
             summary: "Get a rule"
           }
           params do
@@ -153,41 +56,57 @@ module Mihari
           end
           get "/:id" do
             id = params[:id].to_s
-            result = RuleGetter.result(params[:id].to_s)
+            result = Services::RuleGetter.result(params[:id].to_s)
             return present(result.value!, with: Entities::Rule) if result.success?
 
             case result.failure
             when ActiveRecord::RecordNotFound
-              error!({ message: "ID:#{id} is not found" }, 404)
+              error!({ message: "ID:#{id} not found" }, 404)
             end
             raise result.failure
           end
 
-          desc "Run a rule", {
-            success: { code: 201, model: Entities::Message },
-            failure: [{ code: 404, model: Entities::Message }],
+          desc "Search by a rule", {
+            success: { code: 201, model: Entities::QueueMessage },
+            failure: [{ code: 404, model: Entities::ErrorMessage }],
             summary: "Run a rule"
           }
           params do
             requires :id, type: String
           end
-          get "/:id/run" do
+          post "/:id/search" do
             status 201
 
             id = params[:id].to_s
-            result = RuleRunner.result(id)
-            return present({ message: "ID:#{id}} has been ran" }, with: Entities::Message) if result.success?
+
+            queued = true
+            result = Dry::Monads::Try[StandardError] do
+              rule = Mihari::Rule.from_model(Mihari::Models::Rule.find(id))
+
+              if Mihari.sidekiq?
+                Jobs::SearchJob.perform_async(rule.id)
+              else
+                rule.call
+                queued = false
+              end
+            end.to_result
+
+            message = queued ? "ID:#{id}'s search has been queued" : "ID:#{id}'s search has been succeed"
+            return present({ message: message, queued: queued }, with: Entities::QueueMessage) if result.success?
 
             case result.failure
             when ActiveRecord::RecordNotFound
-              error!({ message: "ID:#{id} is not found" }, 404)
+              error!({ message: "ID:#{id} not found" }, 404)
             end
             raise result.failure
           end
 
           desc "Create a rule", {
             success: { code: 201, model: Entities::Rule },
-            failure: [{ code: 404, model: Entities::Message }],
+            failure: [
+              { code: 400, model: Entities::ErrorMessage },
+              { code: 422, model: Entities::ErrorMessage }
+            ],
             summary: "Create a rule"
           }
           params do
@@ -196,22 +115,29 @@ module Mihari
           post "/" do
             status 201
 
-            result = RuleCreator.result(params[:yaml])
+            yaml = params[:yaml].to_s
+
+            result = RuleCreateUpdater.result(yaml, overwrite: false)
             return present(result.value!.model, with: Entities::Rule) if result.success?
 
             failure = result.failure
             case failure
             when Psych::SyntaxError
-              error!({ message: failure.message }, 400)
+              error!({ message: failure.message }, 422)
             when ValidationError
-              error!({ message: "Rule format is invalid", details: failure.errors.to_h }, 400)
+              error!({ message: "Rule format invalid", detail: failure.errors.to_h }, 422)
+            when IntegrityError
+              error!({ message: failure.message }, 400)
             end
             raise failure
           end
 
           desc "Update a rule", {
             success: { code: 201, model: Entities::Rule },
-            failure: [{ code: 404, model: Entities::Message }],
+            failure: [
+              { code: 404, model: Entities::ErrorMessage },
+              { code: 422, model: Entities::ErrorMessage }
+            ],
             summary: "Update a rule"
           }
           params do
@@ -222,24 +148,26 @@ module Mihari
             status 201
 
             id = params[:id].to_s
-            result = RuleUpdater.result(id: id, yaml: params[:yaml].to_s)
+            yaml = params[:yaml].to_s
+
+            result = RuleCreateUpdater.result(yaml, overwrite: true)
             return present(result.value!.model, with: Entities::Rule) if result.success?
 
             failure = result.failure
             case failure
             when ActiveRecord::RecordNotFound
-              error!({ message: "ID:#{id} is not found" }, 404)
+              error!({ message: "ID:#{id} not found" }, 404)
             when Psych::SyntaxError
-              error!({ message: failure.message }, 400)
+              error!({ message: failure.message }, 422)
             when ValidationError
-              error!({ message: "Rule format is invalid", details: failure.errors.to_h }, 400)
+              error!({ message: "Rule format is invalid", detail: failure.errors.to_h }, 422)
             end
             raise failure
           end
 
           desc "Delete a rule", {
             success: { code: 204, model: Entities::Message },
-            failure: [{ code: 404, model: Entities::Message }],
+            failure: [{ code: 404, model: Entities::ErrorMessage }],
             summary: "Delete a rule"
           }
           params do
@@ -249,12 +177,12 @@ module Mihari
             status 204
 
             id = params[:id].to_s
-            result = RuleDestroyer.result(id)
+            result = Services::RuleDestroyer.result(id)
             return present({ message: "ID:#{id} is deleted" }, with: Entities::Message) if result.success?
 
             case result.failure
             when ActiveRecord::RecordNotFound
-              error!({ message: "ID:#{id} is not found" }, 404)
+              error!({ message: "ID:#{id} not found" }, 404)
             end
             raise result.failure
           end

data/lib/mihari/web/endpoints/tags.rb

@@ -7,29 +7,33 @@ module Mihari
       # Tag API endpoint
       #
       class Tags < Grape::API
-        class TagDestroyer < Service
-          #
-          # @param [Integer] id
-          #
-          def call(id)
-            Mihari::Models::Tag.find(id).destroy
-          end
-        end
-
         namespace :tags do
-          desc "Get tags", {
+          desc "List tags", {
             is_array: true,
-            success: Entities::Tags,
-            summary: "Get tags"
+            success: Entities::TagsWithPagination,
+            summary: "List tags"
           }
+          params do
+            optional :q, type: String, default: ""
+            optional :page, type: Integer, default: 1
+            optional :limit, type: Integer, default: 10
+          end
           get "/" do
-            tags = Mihari::Models::Tag.distinct.pluck(:name)
-            present({ tags: tags }, with: Entities::Tags)
+            value = Services::TagSearcher.call(params.to_h)
+            present(
+              {
+                results: value.results,
+                total: value.total,
+                current_page: value.filter[:page].to_i,
+                page_size: value.filter[:limit].to_i
+              },
+              with: Entities::TagsWithPagination
+            )
           end
 
           desc "Delete a tag", {
             success: { code: 204, model: Entities::Message },
-            failure: [{ code: 404, model: Entities::Message }],
+            failure: [{ code: 404, model: Entities::ErrorMessage }],
             summary: "Delete a tag"
           }
           params do
@@ -39,12 +43,12 @@ module Mihari
             status 204
 
             id = params[:id].to_i
-            result = TagDestroyer.result(id)
+            result = Services::TagDestroyer.result(id)
             return present({ message: "" }, with: Entities::Message) if result.success?
 
             case result.failure
             when ActiveRecord::RecordNotFound
-              error!({ message: "ID:#{id} is not found" }, 404)
+              error!({ message: "ID:#{id} not found" }, 404)
             end
             raise result.failure
           end

data/lib/mihari/web/middleware/capture_exceptions.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Web
+    module Middleware
+      #
+      # Customized Sentry::Rack::CaptureExceptions
+      #
+      class CaptureExceptions < Sentry::Rack::CaptureExceptions
+        include Concerns::ErrorUnwrappable
+
+        private
+
+        def capture_exception(exception, env)
+          unwrapped = unwrap_error(exception)
+          Mihari.logger.error unwrapped
+
+          Sentry.capture_exception(unwrapped).tap do |event|
+            env[ERROR_EVENT_ID_KEY] = event.event_id if event
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/web/middleware/{connection_adapter.rb → connection.rb}

@@ -6,7 +6,9 @@ module Mihari
       #
       # DB connection adapter for Rack app
       #
-      class ConnectionAdapter
+      class Connection
+        include Concerns::DatabaseConnectable
+
         attr_reader :app
 
         def initialize(app)
@@ -14,7 +16,7 @@ module Mihari
         end
 
         def call(env)
-          Mihari::Database.with_db_connection { app.call env }
+          with_db_connection { app.call env }
         end
       end
     end