mihari 6.3.0 → 7.0.1

data/lib/mihari/web/{app.rb → application.rb}

@@ -1,21 +1,29 @@
 # frozen_string_literal: true
 
-require "launchy"
-
+# Rack
 require "rack"
-require "rackup"
 require "rack/cors"
+require "rack/session"
+require "rackup"
 
 require "rack/handler/puma"
 
+# Grape
 require "grape-swagger"
 require "grape-swagger-entity"
 
-require "mihari/web/middleware/connection_adapter"
-require "mihari/web/middleware/error_notification_adapter"
+require "launchy"
+
+require "mihari/web/middleware/connection"
+require "mihari/web/middleware/capture_exceptions"
 
 require "mihari/web/api"
 
+# Sidekiq
+require "sidekiq/web"
+
+require "mihari/sidekiq/application"
+
 module Mihari
   module Web
     #
@@ -41,10 +49,10 @@ module Mihari
         status, headers, body = API.call(env)
         return [status, headers, body] unless headers["x-cascade"] == "pass"
 
+        req = Rack::Request.new(env)
         # Check if the App wants us to pass the response along to others
-        request_path = env["PATH_INFO"]
         filenames.each do |path|
-          static_status, static_headers, static_body = rack_static.call(env.merge("PATH_INFO" => request_path + path))
+          static_status, static_headers, static_body = rack_static.call(env.merge("PATH_INFO" => req.path_info + path))
           return [static_status, static_headers, static_body] if static_status != 404
         end
 
@@ -60,24 +68,29 @@ module Mihari
                 resource "*", headers: :any, methods: %i[get post put delete options]
               end
             end
-            use Middleware::ConnectionAdapter
-            use Middleware::ErrorNotificationAdapter
+            use Middleware::Connection
+            use Middleware::CaptureExceptions
+            use BetterErrors::Middleware if Mihari.development? && defined?(BetterErrors::Middleware)
+
+            if Mihari.sidekiq?
+              use Rack::Session::Cookie, secret: SecureRandom.hex(32), same_site: true, max_age: 86_400
 
-            use Sentry::Rack::CaptureExceptions if Sentry.initialized?
-            use BetterErrors::Middleware if ENV["RACK_ENV"] == "development" && defined?(BetterErrors::Middleware)
+              map "/sidekiq" do
+                run Sidekiq::Web
+              end
+            end
 
             run App.new
           end.to_app
         end
 
         def run!(port: 9292, host: "localhost", threads: "0:5", verbose: false, worker_timeout: 60, open: true)
-          url = "http://#{host}:#{port}"
-
           # set maximum number of threads to use as PARALLEL_PROCESSOR_COUNT (if it is not set)
           # ref. https://github.com/grosser/parallel#tips
           # TODO: is this the best way?
           _min_thread, max_thread = threads.split(":")
-          ENV["PARALLEL_PROCESSOR_COUNT"] = max_thread if ENV["PARALLEL_PROCESSOR_COUNT"].nil?
+          ENV["PARALLEL_PROCESSOR_COUNT"] ||= max_thread
+
           Rackup::Handler::Puma.run(
             instance,
             Port: port,
@@ -86,7 +99,7 @@ module Mihari
             Verbose: verbose,
             worker_timeout: worker_timeout
           ) do |_|
-            Launchy.open(url) if ENV["RACK_ENV"] != "development" && open
+            Launchy.open("http://#{host}:#{port}") if !Mihari.development? && open
           rescue Launchy::CommandNotFoundError
             # ref. https://github.com/ninoseki/mihari/issues/477
             # do nothing

data/lib/mihari/web/endpoints/alerts.rb

@@ -7,84 +7,22 @@ module Mihari
       # Alert API endpoint
       #
       class Alerts < Grape::API
-        class AlertSearcher < Mihari::Service
-          class ResultValue
-            # @return [Array<Mihari::Models::Alert>]
-            attr_reader :alerts
-
-            # @return [Integer]
-            attr_reader :total
-
-            # @return [Mihari::Structs::Filters::Alert::SearchFilterWithPagination]
-            attr_reader :filter
-
-            #
-            # @param [Array<Mihari::Models::Alert>] alerts
-            # @param [Integer] total
-            # @param [Mihari::Structs::Filters::Alert::SearchFilterWithPagination] filter
-            #
-            def initialize(alerts:, total:, filter:)
-              @alerts = alerts
-              @total = total
-              @filter = filter
-            end
-          end
-
-          #
-          # @param [Hash] params
-          #
-          # @return [ResultValue]
-          #
-          def call(params)
-            filter = params.to_h.to_snake_keys.symbolize_keys
-            search_filter_with_pagination = Structs::Filters::Alert::SearchFilterWithPagination.new(**filter)
-            alerts = Mihari::Models::Alert.search(search_filter_with_pagination)
-            total = Mihari::Models::Alert.count(search_filter_with_pagination.without_pagination)
-            ResultValue.new(alerts: alerts, total: total, filter: filter)
-          end
-        end
-
-        class AlertCreator < Service
-          #
-          # @param [Hash] params
-          #
-          # @return [Mihari::Models::Alert]
-          #
-          def call(params)
-            proxy = Services::AlertProxy.new(**params.to_snake_keys)
-            Services::AlertRunner.call proxy
-          end
-        end
-
-        class AlertDestroyer < Service
-          #
-          # @param [String] id
-          #
-          def call(id)
-            Mihari::Models::Alert.find(id).destroy
-          end
-        end
-
         namespace :alerts do
-          desc "Search alerts", {
+          desc "List/search alerts", {
             is_array: true,
             success: Entities::AlertsWithPagination,
-            summary: "Search alerts"
+            summary: "List/search alerts"
           }
           params do
+            optional :q, type: String, default: ""
             optional :page, type: Integer, default: 1
             optional :limit, type: Integer, default: 10
-            optional :artifact, type: String
-            optional :ruleId, type: String
-            optional :tag, type: String
-            optional :fromAt, type: DateTime
-            optional :toAt, type: DateTime
           end
           get "/" do
-            value = AlertSearcher.call(params.to_h)
+            value = Services::AlertSearcher.call(params.to_h)
             present(
               {
-                alerts: value.alerts,
+                results: value.results,
                 total: value.total,
                 current_page: value.filter[:page].to_i,
                 page_size: value.filter[:limit].to_i
@@ -93,9 +31,29 @@ module Mihari
             )
           end
 
+          desc "Get an alert", {
+            success: Entities::Alert,
+            failure: [{ code: 404, model: Entities::ErrorMessage }],
+            summary: "Get an alert"
+          }
+          params do
+            requires :id, type: Integer
+          end
+          get "/:id" do
+            id = params[:id].to_i
+            result = Services::AlertGetter.result(id)
+            return present(result.value!, with: Entities::Alert) if result.success?
+
+            case result.failure
+            when ActiveRecord::RecordNotFound
+              error!({ message: "ID:#{id} not found" }, 404)
+            end
+            raise result.failure
+          end
+
           desc "Delete an alert", {
             success: { code: 204, model: Entities::Message },
-            failure: [{ code: 404, model: Entities::Message }],
+            failure: [{ code: 404, model: Entities::ErrorMessage }],
             summary: "Delete an alert"
           }
           params do
@@ -105,34 +63,37 @@ module Mihari
             status 204
 
             id = params["id"].to_i
-            result = AlertDestroyer.result(id)
+            result = Services::AlertDestroyer.result(id)
             return present({ message: "" }, with: Entities::Message) if result.success?
 
             case result.failure
             when ActiveRecord::RecordNotFound
-              error!({ message: "ID:#{id} is not found" }, 404)
+              error!({ message: "ID:#{id} not found" }, 404)
             end
             raise result.failure
           end
 
           desc "Create an alert", {
             success: { code: 201, model: Entities::Alert },
-            failure: [{ code: 404, model: Entities::Message }],
+            failure: [
+              { code: 404, model: Entities::ErrorMessage }
+            ],
             summary: "Create an alert"
           }
           params do
             requires :ruleId, type: String, documentation: { param_type: "body" }
             requires :artifacts, type: Array, documentation: { type: String, is_array: true, param_type: "body" }
+            optional :source, type: String, documentation: { param_type: "body" }
           end
           post "/" do
             status 201
 
-            result = AlertCreator.result(params)
+            result = Services::AlertCreator.result(params)
             return present(result.value!, with: Entities::Alert) if result.success?
 
             case result.failure
             when ActiveRecord::RecordNotFound
-              error!({ message: "Rule:#{params["ruleId"]} is not found" }, 404)
+              error!({ message: "Rule:#{params["ruleId"]} not found" }, 404)
             end
             raise result.failure
           end

data/lib/mihari/web/endpoints/artifacts.rb

@@ -7,117 +7,22 @@ module Mihari
       # Artifact API endpoint
       #
       class Artifacts < Grape::API
-        class ArtifactGetter < Service
-          #
-          # @param [Integer] id
-          #
-          # @return [Mihari::Models::Artifact]
-          #
-          def call(id)
-            artifact = Mihari::Models::Artifact.includes(
-              :autonomous_system,
-              :geolocation,
-              :whois_record,
-              :dns_records,
-              :reverse_dns_names
-            ).find(id)
-            # TODO: improve queries
-            alert_ids = Mihari::Models::Artifact.where(data: artifact.data).pluck(:alert_id)
-            tag_ids = Mihari::Models::Tagging.where(alert_id: alert_ids).pluck(:tag_id)
-            tags = Mihari::Models::Tag.where(id: tag_ids)
-
-            artifact.tags = tags
-
-            artifact
-          end
-        end
-
-        class ArtifactEnricher < Service
-          #
-          # @param [String] id
-          #
-          def call(id)
-            artifact = Mihari::Models::Artifact.includes(
-              :autonomous_system,
-              :geolocation,
-              :whois_record,
-              :dns_records,
-              :reverse_dns_names,
-              :cpes,
-              :ports
-            ).find(id)
-
-            artifact.enrich_all
-            artifact.save
-          end
-        end
-
-        class ArtifactDestroyer < Service
-          #
-          # @param [Integer] id
-          #
-          def call(id)
-            Mihari::Models::Artifact.find(id).destroy
-          end
-        end
-
-        class ArtifactSearcher < Mihari::Service
-          class ResultValue
-            # @return [Array<Mihari::Models::Artifacts>]
-            attr_reader :artifacts
-
-            # @return [Integer]
-            attr_reader :total
-
-            # @return [Mihari::Structs::Filters::Artifact::SearchFilterWithPagination]
-            attr_reader :filter
-
-            #
-            # @param [Array<Mihari::Models::Artifact>] artifacts
-            # @param [Integer] total
-            # @param [Mihari::Structs::Filters::Artifacts::SearchFilterWithPagination] filter
-            #
-            def initialize(artifacts:, total:, filter:)
-              @artifacts = artifacts
-              @total = total
-              @filter = filter
-            end
-          end
-
-          #
-          # @param [Hash] params
-          #
-          # @return [ResultValue]
-          #
-          def call(params)
-            filter = params.to_h.to_snake_keys.symbolize_keys
-            search_filter_with_pagination = Structs::Filters::Artifact::SearchFilterWithPagination.new(**filter)
-            artifacts = Mihari::Models::Artifact.search(search_filter_with_pagination)
-            total = Mihari::Models::Artifact.count(search_filter_with_pagination.without_pagination)
-            ResultValue.new(artifacts: artifacts, total: total, filter: filter)
-          end
-        end
-
         namespace :artifacts do
-          desc "Search artifacts", {
+          desc "List/search artifacts", {
             is_array: true,
             success: Entities::ArtifactsWithPagination,
-            summary: "Search artifacts"
+            summary: "List/search artifacts"
           }
           params do
+            optional :q, type: String, default: ""
             optional :page, type: Integer, default: 1
             optional :limit, type: Integer, default: 10
-            optional :dataType, type: String
-            optional :ruleId, type: String
-            optional :tag, type: String
-            optional :fromAt, type: DateTime
-            optional :toAt, type: DateTime
           end
           get "/" do
-            value = ArtifactSearcher.call(params.to_h)
+            value = Services::ArtifactSearcher.call(params.to_h)
             present(
               {
-                artifacts: value.artifacts,
+                results: value.results,
                 total: value.total,
                 current_page: value.filter[:page].to_i,
                 page_size: value.filter[:limit].to_i
@@ -128,7 +33,7 @@ module Mihari
 
           desc "Get an artifact", {
             success: Entities::Artifact,
-            failure: [{ code: 404, model: Entities::Message }],
+            failure: [{ code: 404, model: Entities::ErrorMessage }],
             summary: "Get an artifact"
           }
           params do
@@ -136,41 +41,52 @@ module Mihari
           end
           get "/:id" do
             id = params[:id].to_i
-            result = ArtifactGetter.result(id)
+            result = Services::ArtifactGetter.result(id)
             return present(result.value!, with: Entities::Artifact) if result.success?
 
             case result.failure
             when ActiveRecord::RecordNotFound
-              error!({ message: "ID:#{id} is not found" }, 404)
+              error!({ message: "ID:#{id} not found" }, 404)
             end
             raise result.failure
           end
 
           desc "Enrich an artifact", {
             success: { code: 201, model: Entities::Message },
-            failure: [{ code: 404, model: Entities::Message }],
+            failure: [{ code: 404, model: Entities::ErrorMessage }],
             summary: "Enrich an artifact"
           }
           params do
             requires :id, type: Integer
           end
-          get "/:id/enrich" do
+          post "/:id/enrich" do
             status 201
 
             id = params["id"].to_i
-            result = ArtifactEnricher.result(id)
-            return present({ message: "#{id} has been enriched" }, with: Entities::Message) if result.success?
+
+            queued = true
+            result = Dry::Monads::Try[StandardError] do
+              if Mihari.sidekiq?
+                Jobs::ArtifactEnrichJob.perform_async id
+              else
+                Services::ArtifactEnricher.call id
+                queued = false
+              end
+            end.to_result
+
+            message = queued ? "ID:#{id}'s enrichment has been queued" : "ID:#{id}'s enrichment has been succeeded"
+            return present({ message: message, queued: queued }, with: Entities::QueueMessage) if result.success?
 
             case result.failure
             when ActiveRecord::RecordNotFound
-              error!({ message: "ID:#{id} is not found" }, 404)
+              error!({ message: "ID:#{id} not found" }, 404)
             end
             raise result.failure
           end
 
           desc "Delete an artifact", {
             success: { code: 204, model: Entities::Message },
-            failure: [{ code: 404, model: Entities::Message }],
+            failure: [{ code: 404, model: Entities::ErrorMessage }],
             summary: "Delete an artifact"
           }
           params do
@@ -180,12 +96,12 @@ module Mihari
             status 204
 
             id = params["id"].to_i
-            result = ArtifactDestroyer.result(id)
+            result = Services::ArtifactDestroyer.result(id)
             return present({ message: "" }, with: Entities::Message) if result.success?
 
             case result.failure
             when ActiveRecord::RecordNotFound
-              error!({ message: "ID:#{id} is not found" }, 404)
+              error!({ message: "ID:#{id} not found" }, 404)
             end
             raise result.failure
           end

data/lib/mihari/web/endpoints/configs.rb

@@ -8,15 +8,13 @@ module Mihari
       #
       class Configs < Grape::API
         namespace :configs do
-          desc "Get configs", {
+          desc "list configs", {
             is_array: true,
             success: Entities::Config,
-            summary: "Get configs"
+            summary: "List configs"
           }
           get "/" do
-            configs = (Mihari.analyzers + Mihari.emitters + Mihari.enrichers).filter_map do |klass|
-              Mihari::Structs::Config.from_class(klass)
-            end
+            configs = Services::ConfigSearcher.call
             present(configs, with: Entities::Config)
           end
         end

data/lib/mihari/web/endpoints/ip_addresses.rb

@@ -7,21 +7,10 @@ module Mihari
       # IP address API endpoint
       #
       class IPAddresses < Grape::API
-        class IPGetter < Service
-          #
-          # @param [String] ip
-          #
-          # @return [Mihari::Structs::IPInfo::Response]
-          #
-          def call(ip)
-            Mihari::Enrichers::IPInfo.new.call ip
-          end
-        end
-
         namespace :ip_addresses do
           desc "Get an IP address", {
             success: Entities::IPAddress,
-            failure: [{ code: 404, model: Entities::Message }],
+            failure: [{ code: 404, model: Entities::ErrorMessage }],
             summary: "Get an IP address"
           }
           params do
@@ -29,13 +18,23 @@ module Mihari
           end
           get "/:ip", requirements: { ip: %r{[^/]+} } do
             ip = params[:ip].to_s
-            result = IPGetter.result(ip)
-            return present(result.value!, with: Entities::IPAddress) if result.success?
+            result = Services::IPGetter.result(ip)
+            if result.success?
+              value = result.value!
+              return present(
+                {
+                  country_code: value.country_code,
+                  asn: value.asn,
+                  loc: value.loc
+                },
+                with: Entities::IPAddress
+              )
+            end
 
             failure = result.failure
             case failure
             when Mihari::StatusCodeError
-              error!({ message: "ID:#{id} is not found" }, 404) if failure.status_code == 404
+              error!({ message: "ID:#{id} not found" }, 404) if failure.status_code == 404
             end
             raise failure
           end