mihari 6.2.0 → 7.0.0

data/lib/mihari/services/searchers.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Services
+    class ResultValue
+      # @return [Array<Object>]
+      attr_reader :results
+
+      # @return [Integer]
+      attr_reader :total
+
+      # @return [Mihari::Structs::Filters::Search]
+      attr_reader :filter
+
+      #
+      # @param [Array<Object>] results
+      # @param [Integer] total
+      # @param [Mihari::Structs::Filters::Search] filter
+      #
+      def initialize(results:, total:, filter:)
+        @results = results
+        @total = total
+        @filter = filter
+      end
+    end
+
+    class BaseSearcher < Service
+      #
+      # @param [Hash] params
+      #
+      # @return [ResultValue]
+      #
+      def call(params)
+        filter = build_filter(params)
+        ResultValue.new(
+          total: klass.count_by_filter(filter),
+          results: klass.search_by_filter(filter),
+          filter: filter
+        )
+      end
+
+      private
+
+      #
+      # @param [Hash] params
+      #
+      # @return [Mihari::Structs::Filters::Search]
+      #
+      def build_filter(params)
+        normalized = params.to_h.to_snake_keys.symbolize_keys
+        Structs::Filters::Search.new(**normalized)
+      end
+    end
+
+    class AlertSearcher < BaseSearcher
+      def klass
+        Models::Alert
+      end
+    end
+
+    class ArtifactSearcher < BaseSearcher
+      def klass
+        Models::Artifact
+      end
+    end
+
+    class RuleSearcher < BaseSearcher
+      def klass
+        Models::Rule
+      end
+    end
+
+    class TagSearcher < BaseSearcher
+      def klass
+        Models::Tag
+      end
+    end
+
+    class ConfigSearcher < Service
+      #
+      # @return [Array<Mihari::Structs::Config>]
+      #
+      def call
+        # NOTE: literally it does not search but list configs
+        (Mihari.analyzers + Mihari.emitters + Mihari.enrichers).filter_map do |klass|
+          Mihari::Structs::Config.from_class klass
+        end
+      end
+    end
+  end
+end

data/lib/mihari/sidekiq/application.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "sidekiq"
+
+require "mihari/sidekiq/jobs"
+
+Sidekiq.configure_server do |config|
+  config.redis = { url: Mihari.config.sidekiq_redis_url.to_s }
+end
+
+Sidekiq.configure_client do |config|
+  config.redis = { url: Mihari.config.sidekiq_redis_url.to_s }
+end

data/lib/mihari/sidekiq/jobs.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "sidekiq"
+
+module Mihari
+  module Jobs
+    class SearchJob
+      include Sidekiq::Job
+      include Concerns::DatabaseConnectable
+
+      #
+      # @param [String] path_or_id
+      #
+      def perform(path_or_id)
+        with_db_connection do
+          rule = Mihari::Rule.from_model(Mihari::Models::Rule.find(path_or_id))
+          rule.call
+        end
+      end
+    end
+
+    class ArtifactEnrichJob
+      include Sidekiq::Job
+      include Concerns::DatabaseConnectable
+
+      #
+      # @param [Integer] id
+      #
+      def perform(id)
+        with_db_connection do
+          Services::ArtifactEnricher.call id
+        end
+      end
+    end
+  end
+end

data/lib/mihari/structs/censys.rb

@@ -4,7 +4,7 @@ module Mihari
   module Structs
     module Censys
       class AutonomousSystem < Dry::Struct
-        include Mixins::AutonomousSystem
+        include Concerns::AutonomousSystemNormalizable
 
         # @!attribute [r] asn
         #   @return [Integer]

data/lib/mihari/structs/config.rb

@@ -13,11 +13,11 @@ module Mihari
 
       # @!attribute [r] is_configured
       #   @return [Boolean]
-      attribute :is_configured, Types::Bool
+      attribute :configured, Types::Bool
 
       # @!attribute [r] values
       #   @return [Array<Hash>, nil]
-      attribute :values, Types.Array(Types::Hash).optional
+      attribute :items, Types.Array(Types::Hash).optional
 
       class << self
         #
@@ -28,9 +28,9 @@ module Mihari
         # @return [String, nil]
         #
         def get_type(klass)
-          return "Analyzer" if klass.ancestors.include?(Mihari::Analyzers::Base)
-          return "Emitter" if klass.ancestors.include?(Mihari::Emitters::Base)
-          return "Enricher" if klass.ancestors.include?(Mihari::Enrichers::Base)
+          return "analyzer" if klass.ancestors.include?(Mihari::Analyzers::Base)
+          return "emitter" if klass.ancestors.include?(Mihari::Emitters::Base)
+          return "enricher" if klass.ancestors.include?(Mihari::Enrichers::Base)
 
           nil
         end
@@ -45,9 +45,9 @@ module Mihari
         def get_dummy(klass)
           type = get_type(klass)
           case type
-          when "Analyzer"
+          when "analyzer"
             klass.new("dummy")
-          when "Emitter"
+          when "emitter"
             klass.new(rule: nil)
           else
             klass.new
@@ -68,9 +68,9 @@ module Mihari
           begin
             instance = get_dummy(klass)
             new(
-              name: klass.to_s.split("::").last.to_s,
-              values: instance.configuration_values,
-              is_configured: instance.configured?,
+              name: klass.class_key,
+              items: klass.configuration_items,
+              configured: instance.configured?,
               type: type
             )
           rescue ArgumentError

data/lib/mihari/structs/filters.rb

@@ -3,92 +3,18 @@
 module Mihari
   module Structs
     module Filters
-      module Alert
-        class SearchFilter < Dry::Struct
-          # @!attribute [r] artifact_data
-          #   @return [String, nil]
-          attribute? :artifact_data, Types::String.optional
-
-          # @!attribute [r] rule_id
-          #   @return [String, nil]
-          attribute? :rule_id, Types::String.optional
-
-          # @!attribute [r] tag_name
-          #   @return [String, nil]
-          attribute? :tag_name, Types::String.optional
-
-          # @!attribute [r] from_at
-          #   @return [DateTime, nil]
-          attribute? :from_at, Types::DateTime.optional
-
-          # @!attribute [r] to_at
-          #   @return [DateTime, nil]
-          attribute? :to_at, Types::DateTime.optional
-        end
-
-        class SearchFilterWithPagination < SearchFilter
-          # @!attribute [r] page
-          #   @return [Integer, nil]
-          attribute? :page, Types::Int.default(1)
-
-          # @!attribute [r] limit
-          #   @return [Integer, nil]
-          attribute? :limit, Types::Int.default(10)
-
-          def without_pagination
-            SearchFilter.new(
-              artifact_data: artifact_data,
-              from_at: from_at,
-              rule_id: rule_id,
-              tag_name: tag_name,
-              to_at: to_at
-            )
-          end
-        end
-      end
-
-      module Rule
-        class SearchFilter < Dry::Struct
-          # @!attribute [r] description
-          #   @return [String, nil]
-          attribute? :description, Types::String.optional
-
-          # @!attribute [r] tag_name
-          #   @return [String, nil]
-          attribute? :tag_name, Types::String.optional
-
-          # @!attribute [r] title
-          #   @return [String, nil]
-          attribute? :title, Types::String.optional
-
-          # @!attribute [r] from_at
-          #   @return [DateTime, nil]
-          attribute? :from_at, Types::DateTime.optional
-
-          # @!attribute [r] to_at
-          #   @return [DateTime, nil]
-          attribute? :to_at, Types::DateTime.optional
-        end
-
-        class SearchFilterWithPagination < SearchFilter
-          # @!attribute [r] page
-          #   @return [Integer, nil]
-          attribute? :page, Types::Int.default(1)
-
-          # @!attribute [r] limit
-          #   @return [Integer, nil]
-          attribute? :limit, Types::Int.default(10)
-
-          def without_pagination
-            SearchFilter.new(
-              description: description,
-              from_at: from_at,
-              tag_name: tag_name,
-              title: title,
-              to_at: to_at
-            )
-          end
-        end
+      class Search < Dry::Struct
+        # @!attribute [r] q
+        #   @return [String]
+        attribute :q, Types::String
+
+        # @!attribute [r] page
+        #   @return [Integer, nil]
+        attribute? :page, Types::Int.default(1)
+
+        # @!attribute [r] limit
+        #   @return [Integer, nil]
+        attribute? :limit, Types::Int.default(10)
       end
     end
   end

data/lib/mihari/structs/google_public_dns.rb

@@ -5,7 +5,7 @@ module Mihari
     module GooglePublicDNS
       INT_TYPE_TO_TYPE =
         { 1 => :A, 38 => :A6, 28 => :AAAA, 18 => :AFSDB, 255 => :ANY, 42 => :APL, 34 => :ATMA, 252 => :AXFR, 37 => :CERT,
-          5 => :CNAME, 49 => :DHCID, 32_769 => :DLV, 39 => :DNAME, 48 => :DNSKEY, 43 => :DS, 31 => :EID, 102 => :GID, 27 => :GPOS, 13 => :HINFO, 45 => :IPSECKEY, 20 => :ISDN, 251 => :IXFR, 25 => :KEY, 36 => :KX, 29 => :LOC, 254 => :MAILA, 253 => :MAILB, 7 => :MB, 3 => :MD, 4 => :MF, 8 => :MG, 14 => :MINFO, 9 => :MR, 15 => :MX, 35 => :NAPTR, 32 => :NIMLOC, 2 => :NS, 22 => :NSAP, 23 => :NSAP_PTR, 47 => :NSEC, 50 => :NSEC3, 51 => :NSEC3PARAMS, 10 => :NULL, 30 => :NXT, 41 => :OPT, 12 => :PTR, 26 => :PX, 17 => :RP, 46 => :RRSIG, 21 => :RT, 24 => :SIG, 40 => :SINK, 6 => :SOA, 33 => :SRV, 44 => :SSHFP, 250 => :TSIG, 16 => :TXT, 101 => :UID, 100 => :UINFO, 103 => :UNSPEC, 11 => :WKS, 19 => :X25 }
+          5 => :CNAME, 49 => :DHCID, 32_769 => :DLV, 39 => :DNAME, 48 => :DNSKEY, 43 => :DS, 31 => :EID, 102 => :GID, 27 => :GPOS, 13 => :HINFO, 45 => :IPSECKEY, 20 => :ISDN, 251 => :IXFR, 25 => :KEY, 36 => :KX, 29 => :LOC, 254 => :MAILA, 253 => :MAILB, 7 => :MB, 3 => :MD, 4 => :MF, 8 => :MG, 14 => :MINFO, 9 => :MR, 15 => :MX, 35 => :NAPTR, 32 => :NIMLOC, 2 => :NS, 22 => :NSAP, 23 => :NSAP_PTR, 47 => :NSEC, 50 => :NSEC3, 51 => :NSEC3PARAMS, 10 => :NULL, 30 => :NXT, 41 => :OPT, 12 => :PTR, 26 => :PX, 17 => :RP, 46 => :RRSIG, 21 => :RT, 24 => :SIG, 40 => :SINK, 6 => :SOA, 33 => :SRV, 44 => :SSHFP, 250 => :TSIG, 16 => :TXT, 101 => :UID, 100 => :UINFO, 103 => :UNSPEC, 11 => :WKS, 19 => :X25 }.freeze
 
       class Answer < Dry::Struct
         # @!attribute [r] name

data/lib/mihari/structs/greynoise.rb

@@ -4,7 +4,7 @@ module Mihari
   module Structs
     module GreyNoise
       class Metadata < Dry::Struct
-        include Mixins::AutonomousSystem
+        include Concerns::AutonomousSystemNormalizable
 
         # @!attribute [r] country
         #   @return [String]

data/lib/mihari/structs/mmdb.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Structs
+    module MMDB
+      class Country < Dry::Struct
+        # @!attribute [r] iso_code
+        #   @return [String]
+        attribute :iso_code, Types::String
+
+        # @!attribute [r] autonomous_system_number
+        #   @return [String]
+        attribute :autonomous_system_number, Types::String.optional
+
+        class << self
+          def from_dynamic!(d)
+            d = Types::Hash[d]
+            new(
+              iso_code: d.fetch("iso_code"),
+              autonomous_system_number: d["AutonomousSystemNumber"]
+            )
+          end
+        end
+      end
+
+      class CountryInfo < Dry::Struct
+        # @!attribute [r] latitude
+        #   @return [String]
+        attribute :latitude, Types::String
+
+        # @!attribute [r] longitude
+        #   @return [String]
+        attribute :longitude, Types::String
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            latitude: d.fetch("Latitude (average)"),
+            longitude: d.fetch("Longitude (average)")
+          )
+        end
+
+        class << self
+          def from_json!(json)
+            from_dynamic!(JSON.parse(json))
+          end
+        end
+      end
+
+      class Item < Dry::Struct
+        # @!attribute [r] country
+        #   @return [Country]
+        attribute :country, Country
+
+        # @!attribute [r] country_info
+        #   @return [CountryInfo]
+        attribute :country_info, CountryInfo
+
+        class << self
+          def from_dynamic!(d)
+            d = Types::Hash[d]
+            new(
+              country: Country.from_dynamic!(d.fetch("country")),
+              country_info: CountryInfo.from_dynamic!(d.fetch("country_info"))
+            )
+          end
+        end
+      end
+
+      class Response < Dry::Struct
+        # @!attribute [r] items
+        #   @return [Array<Item>]
+        attribute :items, Types.Array(Item)
+
+        #
+        # @return [Item, nil]
+        #
+        def item
+          items.find { |item| item.country.autonomous_system_number }
+        end
+
+        #
+        # @return [Integer, nil]
+        #
+        def asn
+          item&.country&.autonomous_system_number&.to_i
+        end
+
+        #
+        # @return [String, nil]
+        #
+        def country_code
+          item&.country&.iso_code
+        end
+
+        #
+        # @return [String, nil]
+        #
+        def loc
+          return nil if item.nil?
+
+          [item.country_info.latitude, item.country_info.longitude].join(",")
+        end
+
+        class << self
+          def from_dynamic!(d)
+            new(
+              items: d.map { |x| Item.from_dynamic!(x) }
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/structs/onyphe.rb

@@ -4,7 +4,7 @@ module Mihari
   module Structs
     module Onyphe
       class Result < Dry::Struct
-        include Mixins::AutonomousSystem
+        include Concerns::AutonomousSystemNormalizable
 
         # @!attribute [r] asn
         #   @return [String]

data/lib/mihari/structs/shodan.rb

@@ -16,7 +16,7 @@ module Mihari
         # @return [Mihari::Geolocation, nil]
         #
         def geolocation
-          return nil if country_name.nil? && country_code.nil?
+          return nil if country_name.nil? || country_code.nil?
 
           Mihari::Models::Geolocation.new(
             country: country_name,
@@ -41,7 +41,7 @@ module Mihari
       end
 
       class Match < Dry::Struct
-        include Mixins::AutonomousSystem
+        include Concerns::AutonomousSystemNormalizable
 
         # @!attribute [r] asn
         #   @return [String, nil]

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "6.2.0"
+  VERSION = "7.0.0"
 end

data/lib/mihari/web/{app.rb → application.rb}

@@ -1,21 +1,29 @@
 # frozen_string_literal: true
 
-require "launchy"
-
+# Rack
 require "rack"
-require "rackup"
 require "rack/cors"
+require "rack/session"
+require "rackup"
 
 require "rack/handler/puma"
 
+# Grape
 require "grape-swagger"
 require "grape-swagger-entity"
 
-require "mihari/web/middleware/connection_adapter"
-require "mihari/web/middleware/error_notification_adapter"
+require "launchy"
+
+require "mihari/web/middleware/connection"
+require "mihari/web/middleware/capture_exceptions"
 
 require "mihari/web/api"
 
+# Sidekiq
+require "sidekiq/web"
+
+require "mihari/sidekiq/application"
+
 module Mihari
   module Web
     #
@@ -41,10 +49,10 @@ module Mihari
         status, headers, body = API.call(env)
         return [status, headers, body] unless headers["x-cascade"] == "pass"
 
+        req = Rack::Request.new(env)
         # Check if the App wants us to pass the response along to others
-        request_path = env["PATH_INFO"]
         filenames.each do |path|
-          static_status, static_headers, static_body = rack_static.call(env.merge("PATH_INFO" => request_path + path))
+          static_status, static_headers, static_body = rack_static.call(env.merge("PATH_INFO" => req.path_info + path))
           return [static_status, static_headers, static_body] if static_status != 404
         end
 
@@ -60,24 +68,29 @@ module Mihari
                 resource "*", headers: :any, methods: %i[get post put delete options]
               end
             end
-            use Middleware::ConnectionAdapter
-            use Middleware::ErrorNotificationAdapter
+            use Middleware::Connection
+            use Middleware::CaptureExceptions
+            use BetterErrors::Middleware if Mihari.development? && defined?(BetterErrors::Middleware)
+
+            if Mihari.sidekiq?
+              use Rack::Session::Cookie, secret: SecureRandom.hex(32), same_site: true, max_age: 86_400
 
-            use Sentry::Rack::CaptureExceptions if Sentry.initialized?
-            use BetterErrors::Middleware if ENV["RACK_ENV"] == "development" && defined?(BetterErrors::Middleware)
+              map "/sidekiq" do
+                run Sidekiq::Web
+              end
+            end
 
             run App.new
           end.to_app
         end
 
         def run!(port: 9292, host: "localhost", threads: "0:5", verbose: false, worker_timeout: 60, open: true)
-          url = "http://#{host}:#{port}"
-
           # set maximum number of threads to use as PARALLEL_PROCESSOR_COUNT (if it is not set)
           # ref. https://github.com/grosser/parallel#tips
           # TODO: is this the best way?
           _min_thread, max_thread = threads.split(":")
-          ENV["PARALLEL_PROCESSOR_COUNT"] = max_thread if ENV["PARALLEL_PROCESSOR_COUNT"].nil?
+          ENV["PARALLEL_PROCESSOR_COUNT"] ||= max_thread
+
           Rackup::Handler::Puma.run(
             instance,
             Port: port,
@@ -86,7 +99,7 @@ module Mihari
             Verbose: verbose,
             worker_timeout: worker_timeout
           ) do |_|
-            Launchy.open(url) if ENV["RACK_ENV"] != "development" && open
+            Launchy.open("http://#{host}:#{port}") if !Mihari.development? && open
           rescue Launchy::CommandNotFoundError
             # ref. https://github.com/ninoseki/mihari/issues/477
             # do nothing