mihari 8.2.1 → 8.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 194cfb9322a58cdf740fb94c8052b18d4ed100348aea829ce7858d218f8b00f3
-  data.tar.gz: e40c2b89308d2fac55cd13c9cd072392a98768b7458f9097348745f1c69e656c
+  metadata.gz: b337f264e337771e4d543b77153a57cdd13d5ce8960f75f70fc6fd7c70215903
+  data.tar.gz: 5013b97ab578766604d0ce2f1329b2209e1f234d5d018e9b642eda053b8ce4d7
 SHA512:
-  metadata.gz: 99332016111dbcfc9762c38ee92d60cbfd1781a09052703d9df088436840370d9af40a8068d02a52d207d1ec183de21582712b184e81e0578be22ee09f49b92c
-  data.tar.gz: 56391dc4d3e9163919496fe201d851e1ee460d6e8fed7281aae9d762c5a03075d6bf24e4fa2b207159e90bacc91aa477e874c5ff12d1ff78320abc0996a159e1
+  metadata.gz: 942a3f9dab169e653d6c4a75a32e6bda6f54f35207c356c69d2c7b2412031ff7f2ca5f10c92dc8713cc6d567731a8ea420ad2d21f02ecfd1fc1a853996e881b2
+  data.tar.gz: 01e710c03b08ec63f3365a370148e3b1861ea3a961a4a9b53cf5d4eef92b65c7a7afebb623c4f9fb0524c0076b4c0e8c600312fb937b6d2fe30be0c01152418d

data/lib/mihari/analyzers/censys.rb

@@ -12,42 +12,69 @@ module Mihari
       # @return [String, nil]
       attr_reader :secret
 
-      #
-      # @param [String] query
-      # @param [hash, nil] options
-      # @param [String, nil] id
-      # @param [String, nil] secret
-      #
-      def initialize(query, options: nil, id: nil, secret: nil)
+      # @return [String, nil]
+      attr_reader :pat
+
+      # @return [String, nil]
+      attr_reader :organization_id
+
+      # @return [Integer, nil]
+      attr_reader :version
+
+      def initialize(query, version: nil, options: nil, id: nil, secret: nil, pat: nil, organization_id: nil)
         super(query, options:)
 
+        @version = version || Mihari.config.censys_version
+
+        # v2
         @id = id || Mihari.config.censys_id
         @secret = secret || Mihari.config.censys_secret
+        # v3
+        @pat = pat || Mihari.config.censys_pat
+        @organization_id = organization_id || Mihari.config.censys_organization_id
       end
 
-      #
-      # @return [Array<Mihari::Models::Artifact>]
-      #
       def artifacts
-        client.search_with_pagination(query, pagination_limit:).map do |res|
-          res.result.artifacts
-        end.flatten.uniq(&:data)
+        client.search_with_pagination(query, pagination_limit:).flat_map do |res|
+          res.artifacts
+        end.uniq(&:data)
       end
 
-      #
-      # @return [Boolean]
-      #
       def configured?
-        configuration_keys? || (id? && secret?)
+        case version
+        when 2
+          v2_configured?
+        when 3
+          v3_configured?
+        else
+          false
+        end
       end
 
       private
 
-      #
-      # @return [Mihari::Clients::Censys]
-      #
       def client
-        Clients::Censys.new(
+        case version
+        when 2
+          v2_client
+        when 3
+          v3_client
+        else
+          raise "Unsupported Censys version: #{version}"
+        end
+      end
+
+      def v3_client
+        Clients::Censys::V3.new(
+          pat:,
+          organization_id:,
+          pagination_interval:,
+          timeout:
+        )
+      end
+
+      def v2_client
+        Clients::Censys::V2.new(
           id:,
           secret:,
           pagination_interval:,
@@ -55,19 +82,29 @@ module Mihari
         )
       end
 
-      #
-      # @return [Boolean]
-      #
+      def v2_configured?
+        id? && secret?
+      end
+
+      def v3_configured?
+        pat? && organization_id?
+      end
+
       def id?
         !id.nil?
       end
 
-      #
-      # @return [Boolean]
-      #
       def secret?
         !secret.nil?
       end
+
+      def pat?
+        !pat.nil?
+      end
+
+      def organization_id?
+        !organization_id.nil?
+      end
     end
   end
 end

data/lib/mihari/clients/censys.rb

@@ -7,73 +7,106 @@ module Mihari
     #
     # Censys API client
     #
-    class Censys < Base
-      #
-      # @param [String] base_url
-      # @param [String, nil] id
-      # @param [String, nil] secret
-      # @param [Hash] headers
-      # @param [Integer] pagination_interval
-      # @param [Integer, nil] timeout
-      #
-      def initialize(
-        base_url = "https://search.censys.io",
-        id:,
-        secret:,
-        headers: {},
-        pagination_interval: Mihari.config.pagination_interval,
-        timeout: nil
-      )
-        raise(ArgumentError, "id is required") if id.nil?
-        raise(ArgumentError, "secret is required") if secret.nil?
+    module Censys
+      class V2 < Base
+        #
+        # @param [String] base_url
+        # @param [String, nil] id
+        # @param [String, nil] secret
+        # @param [Hash] headers
+        # @param [Integer] pagination_interval
+        # @param [Integer, nil] timeout
+        #
+        def initialize(
+          base_url = "https://search.censys.io",
+          id:,
+          secret:,
+          headers: {},
+          pagination_interval: Mihari.config.pagination_interval,
+          timeout: nil
+        )
+          raise(ArgumentError, "id is required") if id.nil?
+          raise(ArgumentError, "secret is required") if secret.nil?
 
-        headers["authorization"] = "Basic #{Base64.strict_encode64("#{id}:#{secret}")}"
+          headers["authorization"] = "Basic #{Base64.strict_encode64("#{id}:#{secret}")}"
 
-        super(base_url, headers:, pagination_interval:, timeout:)
-      end
+          super(base_url, headers:, pagination_interval:, timeout:)
+        end
+
+        #
+        # Search current index.
+        #
+        # Searches the given index for all records that match the given query.
+        # For more details, see our documentation: https://search.censys.io/api/v2/docs
+        #
+        # @param [String] query the query to be executed.
+        # @param [Integer, nil] per_page the number of results to be returned for each page.
+        # @param [Integer, nil] cursor the cursor of the desired result set.
+        #
+        # @return [Mihari::Structs::Censys::Response]
+        #
+        def search(query, per_page: nil, cursor: nil)
+          params = {q: query, per_page:, cursor:}.compact
+          Structs::Censys::V2::Response.from_dynamic! get_json("/api/v2/hosts/search", params:)
+        end
+
+        #
+        # @param [String] query
+        # @param [Integer, nil] per_page
+        # @param [Integer] pagination_limit
+        #
+        # @return [Enumerable<Mihari::Structs::Censys::Response>]
+        #
+        def search_with_pagination(query, per_page: nil, pagination_limit: Mihari.config.pagination_limit)
+          cursor = nil
+
+          Enumerator.new do |y|
+            pagination_limit.times do
+              res = search(query, per_page:, cursor:)
 
-      #
-      # Search current index.
-      #
-      # Searches the given index for all records that match the given query.
-      # For more details, see our documentation: https://search.censys.io/api/v2/docs
-      #
-      # @param [String] query the query to be executed.
-      # @param [Integer, nil] per_page the number of results to be returned for each page.
-      # @param [Integer, nil] cursor the cursor of the desired result set.
-      #
-      # @return [Mihari::Structs::Censys::Response]
-      #
-      def search(query, per_page: nil, cursor: nil)
-        params = {q: query, per_page:, cursor:}.compact
-        Structs::Censys::Response.from_dynamic! get_json("/api/v2/hosts/search", params:)
+              y.yield res
+
+              cursor = res.result.links.next
+              # NOTE: Censys's search API is unstable recently
+              # it may returns empty links or empty string cursors
+              # - Empty links: "links": {}
+              # - Empty cursors: "links": { "next": "", "prev": "" }
+              # So it needs to check both cases
+              break if cursor.nil? || cursor.empty?
+
+              sleep_pagination_interval
+            end
+          end
+        end
       end
 
-      #
-      # @param [String] query
-      # @param [Integer, nil] per_page
-      # @param [Integer] pagination_limit
-      #
-      # @return [Enumerable<Mihari::Structs::Censys::Response>]
-      #
-      def search_with_pagination(query, per_page: nil, pagination_limit: Mihari.config.pagination_limit)
-        cursor = nil
+      class V3 < Base
+        def initialize(base_url = "https://api.platform.censys.io", pat:, organization_id:, headers: {}, pagination_interval: Mihari.config.pagination_interval, timeout: nil)
+          raise(ArgumentError, "pat is required") if pat.nil?
+          raise(ArgumentError, "organization_id is required") if organization_id.nil?
 
-        Enumerator.new do |y|
-          pagination_limit.times do
-            res = search(query, per_page:, cursor:)
+          headers["Authorization"] = "Bearer #{pat}"
+          headers["Accept"] = "application/vnd.censys.api.v3.host.v1+json"
+          headers["X-Organization-ID"] = organization_id
 
-            y.yield res
+          super(base_url, headers: headers, pagination_interval: pagination_interval, timeout: timeout)
+        end
 
-            cursor = res.result.links.next
-            # NOTE: Censys's search API is unstable recently
-            # it may returns empty links or empty string cursors
-            # - Empty links: "links": {}
-            # - Empty cursors: "links": { "next": "", "prev": "" }
-            # So it needs to check both cases
-            break if cursor.nil? || cursor.empty?
+        def search(query, page_size: nil, page_token: nil)
+          json = {query: query, page_size:, page_token:}.compact
+          Structs::Censys::V3::Response.from_dynamic! post_json("/v3/global/search/query", json:)
+        end
 
-            sleep_pagination_interval
+        def search_with_pagination(query, page_size: nil, pagination_limit: Mihari.config.pagination_limit)
+          page_token = nil
+          Enumerator.new do |y|
+            pagination_limit.times do
+              res = search(query, page_size:, page_token:)
+              y.yield res
+              page_token = res.result&.next_page_token
+              break if page_token.nil? || page_token.empty?
+              sleep_pagination_interval
+            end
           end
         end
       end

data/lib/mihari/config.rb

@@ -11,6 +11,9 @@ module Mihari
       # analyzers, emitters & enrichers
       censys_id: nil,
       censys_secret: nil,
+      censys_pat: nil,
+      censys_version: 2,
+      censys_organization_id: nil,
       circl_passive_password: nil,
       circl_passive_username: nil,
       database_url: URI("sqlite3:mihari.db"),

data/lib/mihari/schemas/analyzer.rb

@@ -50,9 +50,14 @@ module Mihari
       Censys = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::Censys.keys))
         required(:query).filled(:string)
+        optional(:version).value(Types::Coercible::Integer.enum(2, 3)).default(2)
+        optional(:options).hash(AnalyzerPaginationOptions)
+        # v2
         optional(:id).filled(:string)
         optional(:secret).filled(:string)
-        optional(:options).hash(AnalyzerPaginationOptions)
+        # v3
+        optional(:pat).filled(:string)
+        optional(:organization_id).filled(:string)
       end
 
       CIRCL = Dry::Schema.Params do
@@ -82,6 +87,7 @@ module Mihari
       Urlscan = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::Urlscan.keys))
         required(:query).filled(:string)
+        optional(:api_key).filled(:string)
         optional(:data_types).filled(array[Types::NetworkDataTypes]).default(Types::NetworkDataTypes.values)
         optional(:options).hash(AnalyzerPaginationOptions)
       end
@@ -89,6 +95,7 @@ module Mihari
       ZoomEye = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::ZoomEye.keys))
         required(:query).filled(:string)
+        optional(:api_key).filled(:string)
         optional(:data_types).filled(array[Types::NetworkDataTypes]).default(Types::NetworkDataTypes.values)
         optional(:options).hash(AnalyzerPaginationOptions)
       end

data/lib/mihari/sidekiq/application.rb

@@ -6,9 +6,10 @@ require "mihari/sidekiq/jobs"
 
 Sidekiq.configure_server do |config|
   config.redis = {url: Mihari.config.sidekiq_redis_url.to_s}
-  config.default_job_options = {retry: Mihari.config.sidekiq_retry}
 end
 
+Sidekiq.default_job_options = {retry: Mihari.config.sidekiq_retry}
+
 Sidekiq.configure_client do |config|
   config.redis = {url: Mihari.config.sidekiq_redis_url.to_s}
 end