mihari 6.3.0 → 7.0.0

data/lib/mihari/structs/config.rb

@@ -13,11 +13,11 @@ module Mihari
 
       # @!attribute [r] is_configured
       #   @return [Boolean]
-      attribute :is_configured, Types::Bool
+      attribute :configured, Types::Bool
 
       # @!attribute [r] values
       #   @return [Array<Hash>, nil]
-      attribute :values, Types.Array(Types::Hash).optional
+      attribute :items, Types.Array(Types::Hash).optional
 
       class << self
         #
@@ -28,9 +28,9 @@ module Mihari
         # @return [String, nil]
         #
         def get_type(klass)
-          return "Analyzer" if klass.ancestors.include?(Mihari::Analyzers::Base)
-          return "Emitter" if klass.ancestors.include?(Mihari::Emitters::Base)
-          return "Enricher" if klass.ancestors.include?(Mihari::Enrichers::Base)
+          return "analyzer" if klass.ancestors.include?(Mihari::Analyzers::Base)
+          return "emitter" if klass.ancestors.include?(Mihari::Emitters::Base)
+          return "enricher" if klass.ancestors.include?(Mihari::Enrichers::Base)
 
           nil
         end
@@ -45,9 +45,9 @@ module Mihari
         def get_dummy(klass)
           type = get_type(klass)
           case type
-          when "Analyzer"
+          when "analyzer"
             klass.new("dummy")
-          when "Emitter"
+          when "emitter"
             klass.new(rule: nil)
           else
             klass.new
@@ -68,9 +68,9 @@ module Mihari
           begin
             instance = get_dummy(klass)
             new(
-              name: klass.to_s.split("::").last.to_s,
-              values: instance.configuration_values,
-              is_configured: instance.configured?,
+              name: klass.class_key,
+              items: klass.configuration_items,
+              configured: instance.configured?,
               type: type
             )
           rescue ArgumentError

data/lib/mihari/structs/filters.rb

@@ -3,136 +3,18 @@
 module Mihari
   module Structs
     module Filters
-      module Artifact
-        class SearchFilter < Dry::Struct
-          # @!attribute [r] data_type
-          #   @return [String, nil]
-          attribute? :data_type, Types::String.optional
-
-          # @!attribute [r] rule_id
-          #   @return [String, nil]
-          attribute? :rule_id, Types::String.optional
-
-          # @!attribute [r] tag
-          #   @return [String, nil]
-          attribute? :tag, Types::String.optional
-
-          # @!attribute [r] from_at
-          #   @return [DateTime, nil]
-          attribute? :from_at, Types::DateTime.optional
-
-          # @!attribute [r] to_at
-          #   @return [DateTime, nil]
-          attribute? :to_at, Types::DateTime.optional
-        end
-
-        class SearchFilterWithPagination < SearchFilter
-          # @!attribute [r] page
-          #   @return [Integer, nil]
-          attribute? :page, Types::Int.default(1)
-
-          # @!attribute [r] limit
-          #   @return [Integer, nil]
-          attribute? :limit, Types::Int.default(10)
-
-          def without_pagination
-            SearchFilter.new(
-              data_type: data_type,
-              from_at: from_at,
-              rule_id: rule_id,
-              tag: tag,
-              to_at: to_at
-            )
-          end
-        end
-      end
-
-      module Alert
-        class SearchFilter < Dry::Struct
-          # @!attribute [r] artifact
-          #   @return [String, nil]
-          attribute? :artifact, Types::String.optional
-
-          # @!attribute [r] rule_id
-          #   @return [String, nil]
-          attribute? :rule_id, Types::String.optional
-
-          # @!attribute [r] tag
-          #   @return [String, nil]
-          attribute? :tag, Types::String.optional
-
-          # @!attribute [r] from_at
-          #   @return [DateTime, nil]
-          attribute? :from_at, Types::DateTime.optional
-
-          # @!attribute [r] to_at
-          #   @return [DateTime, nil]
-          attribute? :to_at, Types::DateTime.optional
-        end
-
-        class SearchFilterWithPagination < SearchFilter
-          # @!attribute [r] page
-          #   @return [Integer, nil]
-          attribute? :page, Types::Int.default(1)
-
-          # @!attribute [r] limit
-          #   @return [Integer, nil]
-          attribute? :limit, Types::Int.default(10)
-
-          def without_pagination
-            SearchFilter.new(
-              artifact: artifact,
-              from_at: from_at,
-              rule_id: rule_id,
-              tag: tag,
-              to_at: to_at
-            )
-          end
-        end
-      end
-
-      module Rule
-        class SearchFilter < Dry::Struct
-          # @!attribute [r] description
-          #   @return [String, nil]
-          attribute? :description, Types::String.optional
-
-          # @!attribute [r] tag
-          #   @return [String, nil]
-          attribute? :tag, Types::String.optional
-
-          # @!attribute [r] title
-          #   @return [String, nil]
-          attribute? :title, Types::String.optional
-
-          # @!attribute [r] from_at
-          #   @return [DateTime, nil]
-          attribute? :from_at, Types::DateTime.optional
-
-          # @!attribute [r] to_at
-          #   @return [DateTime, nil]
-          attribute? :to_at, Types::DateTime.optional
-        end
-
-        class SearchFilterWithPagination < SearchFilter
-          # @!attribute [r] page
-          #   @return [Integer, nil]
-          attribute? :page, Types::Int.default(1)
-
-          # @!attribute [r] limit
-          #   @return [Integer, nil]
-          attribute? :limit, Types::Int.default(10)
-
-          def without_pagination
-            SearchFilter.new(
-              description: description,
-              from_at: from_at,
-              tag: tag,
-              title: title,
-              to_at: to_at
-            )
-          end
-        end
+      class Search < Dry::Struct
+        # @!attribute [r] q
+        #   @return [String]
+        attribute :q, Types::String
+
+        # @!attribute [r] page
+        #   @return [Integer, nil]
+        attribute? :page, Types::Int.default(1)
+
+        # @!attribute [r] limit
+        #   @return [Integer, nil]
+        attribute? :limit, Types::Int.default(10)
       end
     end
   end

data/lib/mihari/structs/google_public_dns.rb

@@ -5,7 +5,7 @@ module Mihari
     module GooglePublicDNS
       INT_TYPE_TO_TYPE =
         { 1 => :A, 38 => :A6, 28 => :AAAA, 18 => :AFSDB, 255 => :ANY, 42 => :APL, 34 => :ATMA, 252 => :AXFR, 37 => :CERT,
-          5 => :CNAME, 49 => :DHCID, 32_769 => :DLV, 39 => :DNAME, 48 => :DNSKEY, 43 => :DS, 31 => :EID, 102 => :GID, 27 => :GPOS, 13 => :HINFO, 45 => :IPSECKEY, 20 => :ISDN, 251 => :IXFR, 25 => :KEY, 36 => :KX, 29 => :LOC, 254 => :MAILA, 253 => :MAILB, 7 => :MB, 3 => :MD, 4 => :MF, 8 => :MG, 14 => :MINFO, 9 => :MR, 15 => :MX, 35 => :NAPTR, 32 => :NIMLOC, 2 => :NS, 22 => :NSAP, 23 => :NSAP_PTR, 47 => :NSEC, 50 => :NSEC3, 51 => :NSEC3PARAMS, 10 => :NULL, 30 => :NXT, 41 => :OPT, 12 => :PTR, 26 => :PX, 17 => :RP, 46 => :RRSIG, 21 => :RT, 24 => :SIG, 40 => :SINK, 6 => :SOA, 33 => :SRV, 44 => :SSHFP, 250 => :TSIG, 16 => :TXT, 101 => :UID, 100 => :UINFO, 103 => :UNSPEC, 11 => :WKS, 19 => :X25 }
+          5 => :CNAME, 49 => :DHCID, 32_769 => :DLV, 39 => :DNAME, 48 => :DNSKEY, 43 => :DS, 31 => :EID, 102 => :GID, 27 => :GPOS, 13 => :HINFO, 45 => :IPSECKEY, 20 => :ISDN, 251 => :IXFR, 25 => :KEY, 36 => :KX, 29 => :LOC, 254 => :MAILA, 253 => :MAILB, 7 => :MB, 3 => :MD, 4 => :MF, 8 => :MG, 14 => :MINFO, 9 => :MR, 15 => :MX, 35 => :NAPTR, 32 => :NIMLOC, 2 => :NS, 22 => :NSAP, 23 => :NSAP_PTR, 47 => :NSEC, 50 => :NSEC3, 51 => :NSEC3PARAMS, 10 => :NULL, 30 => :NXT, 41 => :OPT, 12 => :PTR, 26 => :PX, 17 => :RP, 46 => :RRSIG, 21 => :RT, 24 => :SIG, 40 => :SINK, 6 => :SOA, 33 => :SRV, 44 => :SSHFP, 250 => :TSIG, 16 => :TXT, 101 => :UID, 100 => :UINFO, 103 => :UNSPEC, 11 => :WKS, 19 => :X25 }.freeze
 
       class Answer < Dry::Struct
         # @!attribute [r] name

data/lib/mihari/structs/greynoise.rb

@@ -4,7 +4,7 @@ module Mihari
   module Structs
     module GreyNoise
       class Metadata < Dry::Struct
-        include Mixins::AutonomousSystem
+        include Concerns::AutonomousSystemNormalizable
 
         # @!attribute [r] country
         #   @return [String]

data/lib/mihari/structs/mmdb.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Structs
+    module MMDB
+      class Country < Dry::Struct
+        # @!attribute [r] iso_code
+        #   @return [String]
+        attribute :iso_code, Types::String
+
+        # @!attribute [r] autonomous_system_number
+        #   @return [String]
+        attribute :autonomous_system_number, Types::String.optional
+
+        class << self
+          def from_dynamic!(d)
+            d = Types::Hash[d]
+            new(
+              iso_code: d.fetch("iso_code"),
+              autonomous_system_number: d["AutonomousSystemNumber"]
+            )
+          end
+        end
+      end
+
+      class CountryInfo < Dry::Struct
+        # @!attribute [r] latitude
+        #   @return [String]
+        attribute :latitude, Types::String
+
+        # @!attribute [r] longitude
+        #   @return [String]
+        attribute :longitude, Types::String
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            latitude: d.fetch("Latitude (average)"),
+            longitude: d.fetch("Longitude (average)")
+          )
+        end
+
+        class << self
+          def from_json!(json)
+            from_dynamic!(JSON.parse(json))
+          end
+        end
+      end
+
+      class Item < Dry::Struct
+        # @!attribute [r] country
+        #   @return [Country]
+        attribute :country, Country
+
+        # @!attribute [r] country_info
+        #   @return [CountryInfo]
+        attribute :country_info, CountryInfo
+
+        class << self
+          def from_dynamic!(d)
+            d = Types::Hash[d]
+            new(
+              country: Country.from_dynamic!(d.fetch("country")),
+              country_info: CountryInfo.from_dynamic!(d.fetch("country_info"))
+            )
+          end
+        end
+      end
+
+      class Response < Dry::Struct
+        # @!attribute [r] items
+        #   @return [Array<Item>]
+        attribute :items, Types.Array(Item)
+
+        #
+        # @return [Item, nil]
+        #
+        def item
+          items.find { |item| item.country.autonomous_system_number }
+        end
+
+        #
+        # @return [Integer, nil]
+        #
+        def asn
+          item&.country&.autonomous_system_number&.to_i
+        end
+
+        #
+        # @return [String, nil]
+        #
+        def country_code
+          item&.country&.iso_code
+        end
+
+        #
+        # @return [String, nil]
+        #
+        def loc
+          return nil if item.nil?
+
+          [item.country_info.latitude, item.country_info.longitude].join(",")
+        end
+
+        class << self
+          def from_dynamic!(d)
+            new(
+              items: d.map { |x| Item.from_dynamic!(x) }
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/structs/onyphe.rb

@@ -4,7 +4,7 @@ module Mihari
   module Structs
     module Onyphe
       class Result < Dry::Struct
-        include Mixins::AutonomousSystem
+        include Concerns::AutonomousSystemNormalizable
 
         # @!attribute [r] asn
         #   @return [String]

data/lib/mihari/structs/shodan.rb

@@ -16,7 +16,7 @@ module Mihari
         # @return [Mihari::Geolocation, nil]
         #
         def geolocation
-          return nil if country_name.nil? && country_code.nil?
+          return nil if country_name.nil? || country_code.nil?
 
           Mihari::Models::Geolocation.new(
             country: country_name,
@@ -41,7 +41,7 @@ module Mihari
       end
 
       class Match < Dry::Struct
-        include Mixins::AutonomousSystem
+        include Concerns::AutonomousSystemNormalizable
 
         # @!attribute [r] asn
         #   @return [String, nil]

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "6.3.0"
+  VERSION = "7.0.0"
 end

data/lib/mihari/web/{app.rb → application.rb}

@@ -1,21 +1,29 @@
 # frozen_string_literal: true
 
-require "launchy"
-
+# Rack
 require "rack"
-require "rackup"
 require "rack/cors"
+require "rack/session"
+require "rackup"
 
 require "rack/handler/puma"
 
+# Grape
 require "grape-swagger"
 require "grape-swagger-entity"
 
-require "mihari/web/middleware/connection_adapter"
-require "mihari/web/middleware/error_notification_adapter"
+require "launchy"
+
+require "mihari/web/middleware/connection"
+require "mihari/web/middleware/capture_exceptions"
 
 require "mihari/web/api"
 
+# Sidekiq
+require "sidekiq/web"
+
+require "mihari/sidekiq/application"
+
 module Mihari
   module Web
     #
@@ -41,10 +49,10 @@ module Mihari
         status, headers, body = API.call(env)
         return [status, headers, body] unless headers["x-cascade"] == "pass"
 
+        req = Rack::Request.new(env)
         # Check if the App wants us to pass the response along to others
-        request_path = env["PATH_INFO"]
         filenames.each do |path|
-          static_status, static_headers, static_body = rack_static.call(env.merge("PATH_INFO" => request_path + path))
+          static_status, static_headers, static_body = rack_static.call(env.merge("PATH_INFO" => req.path_info + path))
           return [static_status, static_headers, static_body] if static_status != 404
         end
 
@@ -60,24 +68,29 @@ module Mihari
                 resource "*", headers: :any, methods: %i[get post put delete options]
               end
             end
-            use Middleware::ConnectionAdapter
-            use Middleware::ErrorNotificationAdapter
+            use Middleware::Connection
+            use Middleware::CaptureExceptions
+            use BetterErrors::Middleware if Mihari.development? && defined?(BetterErrors::Middleware)
+
+            if Mihari.sidekiq?
+              use Rack::Session::Cookie, secret: SecureRandom.hex(32), same_site: true, max_age: 86_400
 
-            use Sentry::Rack::CaptureExceptions if Sentry.initialized?
-            use BetterErrors::Middleware if ENV["RACK_ENV"] == "development" && defined?(BetterErrors::Middleware)
+              map "/sidekiq" do
+                run Sidekiq::Web
+              end
+            end
 
             run App.new
           end.to_app
         end
 
         def run!(port: 9292, host: "localhost", threads: "0:5", verbose: false, worker_timeout: 60, open: true)
-          url = "http://#{host}:#{port}"
-
           # set maximum number of threads to use as PARALLEL_PROCESSOR_COUNT (if it is not set)
           # ref. https://github.com/grosser/parallel#tips
           # TODO: is this the best way?
           _min_thread, max_thread = threads.split(":")
-          ENV["PARALLEL_PROCESSOR_COUNT"] = max_thread if ENV["PARALLEL_PROCESSOR_COUNT"].nil?
+          ENV["PARALLEL_PROCESSOR_COUNT"] ||= max_thread
+
           Rackup::Handler::Puma.run(
             instance,
             Port: port,
@@ -86,7 +99,7 @@ module Mihari
             Verbose: verbose,
             worker_timeout: worker_timeout
           ) do |_|
-            Launchy.open(url) if ENV["RACK_ENV"] != "development" && open
+            Launchy.open("http://#{host}:#{port}") if !Mihari.development? && open
           rescue Launchy::CommandNotFoundError
             # ref. https://github.com/ninoseki/mihari/issues/477
             # do nothing

data/lib/mihari/web/endpoints/alerts.rb

@@ -7,84 +7,22 @@ module Mihari
       # Alert API endpoint
       #
       class Alerts < Grape::API
-        class AlertSearcher < Mihari::Service
-          class ResultValue
-            # @return [Array<Mihari::Models::Alert>]
-            attr_reader :alerts
-
-            # @return [Integer]
-            attr_reader :total
-
-            # @return [Mihari::Structs::Filters::Alert::SearchFilterWithPagination]
-            attr_reader :filter
-
-            #
-            # @param [Array<Mihari::Models::Alert>] alerts
-            # @param [Integer] total
-            # @param [Mihari::Structs::Filters::Alert::SearchFilterWithPagination] filter
-            #
-            def initialize(alerts:, total:, filter:)
-              @alerts = alerts
-              @total = total
-              @filter = filter
-            end
-          end
-
-          #
-          # @param [Hash] params
-          #
-          # @return [ResultValue]
-          #
-          def call(params)
-            filter = params.to_h.to_snake_keys.symbolize_keys
-            search_filter_with_pagination = Structs::Filters::Alert::SearchFilterWithPagination.new(**filter)
-            alerts = Mihari::Models::Alert.search(search_filter_with_pagination)
-            total = Mihari::Models::Alert.count(search_filter_with_pagination.without_pagination)
-            ResultValue.new(alerts: alerts, total: total, filter: filter)
-          end
-        end
-
-        class AlertCreator < Service
-          #
-          # @param [Hash] params
-          #
-          # @return [Mihari::Models::Alert]
-          #
-          def call(params)
-            proxy = Services::AlertProxy.new(**params.to_snake_keys)
-            Services::AlertRunner.call proxy
-          end
-        end
-
-        class AlertDestroyer < Service
-          #
-          # @param [String] id
-          #
-          def call(id)
-            Mihari::Models::Alert.find(id).destroy
-          end
-        end
-
         namespace :alerts do
-          desc "Search alerts", {
+          desc "List/search alerts", {
             is_array: true,
             success: Entities::AlertsWithPagination,
-            summary: "Search alerts"
+            summary: "List/search alerts"
           }
           params do
+            optional :q, type: String, default: ""
             optional :page, type: Integer, default: 1
             optional :limit, type: Integer, default: 10
-            optional :artifact, type: String
-            optional :ruleId, type: String
-            optional :tag, type: String
-            optional :fromAt, type: DateTime
-            optional :toAt, type: DateTime
           end
           get "/" do
-            value = AlertSearcher.call(params.to_h)
+            value = Services::AlertSearcher.call(params.to_h)
             present(
               {
-                alerts: value.alerts,
+                results: value.results,
                 total: value.total,
                 current_page: value.filter[:page].to_i,
                 page_size: value.filter[:limit].to_i
@@ -93,9 +31,29 @@ module Mihari
             )
           end
 
+          desc "Get an alert", {
+            success: Entities::Alert,
+            failure: [{ code: 404, model: Entities::ErrorMessage }],
+            summary: "Get an alert"
+          }
+          params do
+            requires :id, type: Integer
+          end
+          get "/:id" do
+            id = params[:id].to_i
+            result = Services::AlertGetter.result(id)
+            return present(result.value!, with: Entities::Alert) if result.success?
+
+            case result.failure
+            when ActiveRecord::RecordNotFound
+              error!({ message: "ID:#{id} not found" }, 404)
+            end
+            raise result.failure
+          end
+
           desc "Delete an alert", {
             success: { code: 204, model: Entities::Message },
-            failure: [{ code: 404, model: Entities::Message }],
+            failure: [{ code: 404, model: Entities::ErrorMessage }],
             summary: "Delete an alert"
           }
           params do
@@ -105,34 +63,37 @@ module Mihari
             status 204
 
             id = params["id"].to_i
-            result = AlertDestroyer.result(id)
+            result = Services::AlertDestroyer.result(id)
             return present({ message: "" }, with: Entities::Message) if result.success?
 
             case result.failure
             when ActiveRecord::RecordNotFound
-              error!({ message: "ID:#{id} is not found" }, 404)
+              error!({ message: "ID:#{id} not found" }, 404)
             end
             raise result.failure
           end
 
           desc "Create an alert", {
             success: { code: 201, model: Entities::Alert },
-            failure: [{ code: 404, model: Entities::Message }],
+            failure: [
+              { code: 404, model: Entities::ErrorMessage }
+            ],
             summary: "Create an alert"
           }
           params do
             requires :ruleId, type: String, documentation: { param_type: "body" }
             requires :artifacts, type: Array, documentation: { type: String, is_array: true, param_type: "body" }
+            optional :source, type: String, documentation: { param_type: "body" }
           end
           post "/" do
             status 201
 
-            result = AlertCreator.result(params)
+            result = Services::AlertCreator.result(params)
             return present(result.value!, with: Entities::Alert) if result.success?
 
             case result.failure
             when ActiveRecord::RecordNotFound
-              error!({ message: "Rule:#{params["ruleId"]} is not found" }, 404)
+              error!({ message: "Rule:#{params["ruleId"]} not found" }, 404)
             end
             raise result.failure
           end