search-engine-for-typesense 30.1.8.16 → 30.1.8.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fd4cd9a02cea20dbf913f71c55c1ae57b77d29245be9e1f02e93a22e791cadcf
-  data.tar.gz: 404f2b78d290227c1fdfcd0b82cfcd7d41f4ec353d5e551a20658cd9a78ff593
+  metadata.gz: 3a268323694adc2817c238b7ae9f522a7685b55b240691fb5891a98b4bc3d371
+  data.tar.gz: 3fcdddf6f9bdd81ecca5aae6755e3d55f7d5a9fd52e50ed3ed2d5bbdaa163b53
 SHA512:
-  metadata.gz: 166cb097643cfb4376a1232eaab0e4f371fd1b3c409a9103f2a3d812dd1ba74f9812e13986e542abcbd086fe4cdcf05db3bca799ddaac2e5ee94215db5d83098
-  data.tar.gz: 5a57d41c35a28022ca8384f8f7924b5d26e7210ac3a3681ccc8a9dcfaf7cba90ba5db6ea3914d09d2f9c19efa6fe43947778a086ecafc13fddc0a4ab1e751d5e
+  metadata.gz: 88f09c6d71f2a6c91505b8d83e57b6e989de2f8fbbc477b7b35185996da5cb0eae918dcf3ed14f8c9f3c69760469b5da9d32da2336b5cd640bffc386e31869b0
+  data.tar.gz: ab29408bb29a73d210aa29c3db62bb3bf684b8105a6dbe5aaf918ed299f8849b3eadb8ecb406d30db11e2353941aafb5ec8cb58ae4c180ca684ad720a5a55697

data/README.md

@@ -222,6 +222,11 @@ SearchEngine.configure do |c|
   c.postgres_outbox.channel = "search_engine_outbox"
   c.postgres_outbox.queue_name = "search_engine"
   c.postgres_outbox.batch_size = 1000
+  c.postgres_outbox.batch_sizes = {
+    product_balances: 10_000,
+    calculated_products: 1_000,
+    products: 2_000
+  }
   c.postgres_outbox.drain_target_parallelism = 1
   c.postgres_outbox.drain_job_max_batches = 1
   c.postgres_outbox.drain_job_max_runtime_s = nil
@@ -241,6 +246,10 @@ SearchEngine.configure do |c|
 end
 ```
 
+`batch_size` is the global fallback for all collections. Use `batch_sizes` when some collections are much
+lighter or heavier than others. Omitted drain limits use the per-collection values; explicit `limit:`
+arguments still override the map and use one global cap for that drain.
+
 Generate and edit the migrations:
 
 ```bash

data/app/search_engine/search_engine/postgres_outbox/drain_job.rb

@@ -22,7 +22,7 @@ module SearchEngine
           return perform_with_drain_slot(limit: limit, target_key: target_key, drain_slot: drain_slot)
         end
 
-        effective_limit = limit || SearchEngine.config.postgres_outbox.batch_size
+        effective_limit = drain_limit(limit)
         drainer = drainer_for(target_key)
         summary = drainer.drain_once(limit: effective_limit)
         enqueue_continuation(limit: limit, target_key: target_key) if continue_draining?(summary, effective_limit)
@@ -40,7 +40,7 @@ module SearchEngine
       def perform_with_drain_slot(limit:, target_key:, drain_slot:)
         target = delivery_target_for!(target_key)
         slot = drain_slot.to_i
-        effective_limit = limit || SearchEngine.config.postgres_outbox.batch_size
+        effective_limit = drain_limit(limit)
         repository = repository_for_slot
         slot_requeued = false
         return stale_slot_summary(target.key, slot) unless repository.start_drain_slot!(
@@ -100,7 +100,7 @@ module SearchEngine
       end
 
       def continue_draining?(summary, effective_limit)
-        summary[:continue] || summary[:claimed].to_i >= effective_limit.to_i
+        summary[:continue] || (!effective_limit.nil? && summary[:claimed].to_i >= effective_limit.to_i)
       end
 
       def drainer_for(target_key)
@@ -184,6 +184,13 @@ module SearchEngine
         [SearchEngine.config.postgres_outbox.drain_job_max_batches.to_i, 1].max
       end
 
+      def drain_limit(limit)
+        return limit unless limit.nil?
+        return nil if SearchEngine.config.postgres_outbox.collection_batch_sizes?
+
+        SearchEngine.config.postgres_outbox.batch_size
+      end
+
       def runtime_budget_exhausted?
         max_runtime_s = SearchEngine.config.postgres_outbox.drain_job_max_runtime_s.to_i
         return false unless max_runtime_s.positive?

data/lib/search_engine/base/creation.rb

@@ -2,8 +2,8 @@
 
 require 'active_support/concern'
 require 'set'
-require 'json'
 require 'search_engine/indexer/batch_planner'
+require 'search_engine/indexer/import_response_parser'
 
 module SearchEngine
   class Base
@@ -531,67 +531,8 @@ module SearchEngine
           result
         end
 
-        def safe_parse_json(str)
-          JSON.parse(str)
-        rescue StandardError
-          nil
-        end
-
         def parse_import_response(raw)
-          return parse_import_response_from_string(raw) if raw.is_a?(String)
-          return parse_import_response_from_array(raw) if raw.is_a?(Array)
-
-          [0, 0, []]
-        end
-
-        def parse_import_response_from_string(str)
-          success = 0
-          failure = 0
-          samples = []
-
-          str.each_line do |line|
-            line = line.strip
-            next if line.empty?
-
-            h = safe_parse_json(line)
-            unless h
-              failure += 1
-              samples << 'invalid-json-line'
-              next
-            end
-
-            if truthy?(h['success'] || h[:success])
-              success += 1
-            else
-              failure += 1
-              msg = h['error'] || h[:error] || h['message'] || h[:message]
-              samples << msg.to_s[0, 200] if msg
-            end
-          end
-
-          [success, failure, samples[0, 5]]
-        end
-
-        def parse_import_response_from_array(arr)
-          success = 0
-          failure = 0
-          samples = []
-
-          arr.each do |h|
-            if h.is_a?(Hash) && truthy?(h['success'] || h[:success])
-              success += 1
-            else
-              failure += 1
-              msg = h.is_a?(Hash) ? (h['error'] || h[:error] || h['message'] || h[:message]) : nil
-              samples << msg.to_s[0, 200] if msg
-            end
-          end
-
-          [success, failure, samples[0, 5]]
-        end
-
-        def truthy?(val)
-          val == true || val.to_s.downcase == 'true'
+          SearchEngine::Indexer::ImportResponseParser.parse(raw)
         end
 
         def normalize_records_input(records)

data/lib/search_engine/config.rb

@@ -280,6 +280,8 @@ module SearchEngine
       attr_accessor :queue_name
       # @return [Integer] maximum events to claim per processing batch
       attr_accessor :batch_size
+      # @return [Hash] optional per-collection processing batch sizes
+      attr_accessor :batch_sizes
       # @return [Integer] maximum processing attempts before leaving an event failed
       attr_accessor :max_attempts
       # @return [Integer] polling interval in seconds
@@ -317,6 +319,7 @@ module SearchEngine
         @channel = 'search_engine_outbox'
         @queue_name = 'search_engine'
         @batch_size = 1000
+        @batch_sizes = {}
         @max_attempts = 10
         @poll_interval_s = 5
         @listener_wait_timeout_s = 30
@@ -332,6 +335,34 @@ module SearchEngine
         @drain_job_max_batches = 1
         @drain_job_max_runtime_s = nil
       end
+
+      # Resolve the processing batch size for a collection.
+      #
+      # @param collection [String, Symbol, nil] collection name
+      # @return [Integer] positive per-collection batch size or global fallback
+      def batch_size_for(collection)
+        configured = normalized_batch_sizes[collection.to_s]
+        return configured if configured&.positive?
+
+        batch_size.to_i
+      end
+
+      # Whether any positive per-collection batch sizes are configured.
+      #
+      # @return [Boolean]
+      def collection_batch_sizes?
+        normalized_batch_sizes.any?
+      end
+
+      # Normalized positive per-collection batch sizes keyed by collection name.
+      #
+      # @return [Hash<String, Integer>]
+      def normalized_batch_sizes
+        Hash(batch_sizes).each_with_object({}) do |(key, value), result|
+          size = value.to_i
+          result[key.to_s] = size if size.positive?
+        end
+      end
     end
 
     # Lightweight nested configuration for observability/logging.
@@ -886,6 +917,7 @@ module SearchEngine
         channel: postgres_outbox.channel,
         queue_name: postgres_outbox.queue_name,
         batch_size: postgres_outbox.batch_size,
+        batch_sizes: postgres_outbox.batch_sizes,
         max_attempts: postgres_outbox.max_attempts,
         poll_interval_s: postgres_outbox.poll_interval_s,
         listener_wait_timeout_s: postgres_outbox.listener_wait_timeout_s,

data/lib/search_engine/indexer/batch_planner.rb

@@ -28,7 +28,7 @@ module SearchEngine
         docs.each_with_index do |raw, idx|
           doc = ensure_hash_document(raw)
           ensure_id!(doc)
-          doc[:doc_updated_at] = now_i if doc.is_a?(Hash)
+          assign_doc_updated_at!(doc, now_i)
           buffer << JSON.generate(doc)
           buffer << "\n" if idx < (size - 1)
           count += 1
@@ -65,6 +65,15 @@ module SearchEngine
           has_id = doc.key?(:id) || doc.key?('id')
           raise SearchEngine::Errors::InvalidParams, 'document is missing required id' unless has_id
         end
+
+        def assign_doc_updated_at!(doc, timestamp)
+          if doc.key?('doc_updated_at')
+            doc.delete(:doc_updated_at)
+            doc['doc_updated_at'] = timestamp
+          else
+            doc[:doc_updated_at] = timestamp
+          end
+        end
       end
     end
   end

data/lib/search_engine/indexer/import_response_parser.rb

@@ -24,8 +24,10 @@ module SearchEngine
       def parse(raw)
         return parse_from_string(raw) if raw.is_a?(String)
         return parse_from_array(raw) if raw.is_a?(Array)
+        return [0, 0, []] if raw.nil?
 
-        [0, 0, []]
+        raise SearchEngine::Errors::InvalidParams,
+              "Unsupported Typesense import response shape: #{raw.class.name}"
       end
 
       def parse_from_string(str)

data/lib/search_engine/postgres_outbox/drain_enqueuer.rb

@@ -69,6 +69,13 @@ module SearchEngine
         kwargs = { target_key: slot.fetch(:target_key), drain_slot: slot.fetch(:slot) }
         kwargs[:limit] = limit unless limit.nil?
         job.perform_later(**kwargs)
+      rescue StandardError => error
+        repository.release_requeued_drain_slot!(
+          target_key: slot.fetch(:target_key),
+          slot: slot.fetch(:slot),
+          error: error
+        )
+        raise
       end
 
       def delivery_targets

data/lib/search_engine/postgres_outbox/drainer.rb

@@ -22,7 +22,7 @@ module SearchEngine
       # Claim, coalesce, order, process, and mark one batch.
       # @param limit [Integer]
       # @return [Hash]
-      def drain_once(limit: SearchEngine.config.postgres_outbox.batch_size)
+      def drain_once(limit: nil)
         SearchEngine::Instrumentation.instrument(
           'search_engine.postgres_outbox.drain',
           drain_payload(limit)
@@ -31,7 +31,7 @@ module SearchEngine
           summary = empty_summary(events)
           next summary if events.empty?
 
-          summary[:continue] = true if continue_after_nonempty_target_batch?
+          summary[:continue] = true if continue_after_nonempty_batch?
 
           kept, superseded_ids = coalesce(events)
           repository.mark_superseded!(superseded_ids)
@@ -61,8 +61,8 @@ module SearchEngine
         summary
       end
 
-      def continue_after_nonempty_target_batch?
-        !target_key.nil?
+      def continue_after_nonempty_batch?
+        !target_key.nil? || SearchEngine.config.postgres_outbox.collection_batch_sizes?
       end
 
       def coalesce(events)

data/lib/search_engine/postgres_outbox/repository.rb

@@ -24,7 +24,7 @@ module SearchEngine
         rows = []
 
         connection.transaction do
-          rows = select_rows(claim_select_sql(limit.to_i))
+          rows = select_rows(claim_select_sql(limit))
           ids = rows.map { |row| row_value(row, :id) }
           execute(supersede_older_pending_sql(rows)) unless rows.empty?
           execute(claim_update_sql(ids, worker_id)) unless ids.empty?
@@ -111,13 +111,13 @@ module SearchEngine
 
       # Create missing delivery rows for all configured delivery targets.
       # @return [void]
-      def materialize_deliveries!(limit: SearchEngine.config.postgres_outbox.batch_size)
+      def materialize_deliveries!(limit: nil)
         targets = materialization_delivery_targets
         return if targets.empty?
 
         rows = []
         connection.transaction do
-          rows = select_rows(delivery_materialization_select_sql(limit.to_i, targets))
+          rows = select_rows(delivery_materialization_select_sql(limit, targets))
           next if rows.empty?
 
           execute(materialization_supersede_older_deliveries_sql(rows, targets))
@@ -267,6 +267,9 @@ module SearchEngine
       end
 
       def delivery_materialization_select_sql(limit, targets)
+        return collection_limited_materialization_select_sql(targets) if collection_limited_batch?(limit)
+
+        limit = global_limit_for(limit)
         <<~SQL
           WITH target(target_key, queue_name) AS (
             VALUES #{delivery_target_values_sql(targets)}
@@ -346,7 +349,7 @@ module SearchEngine
         rows = []
 
         connection.transaction do
-          rows = select_rows(delivery_claim_select_sql(limit.to_i))
+          rows = select_rows(delivery_claim_select_sql(limit))
           delivery_ids = rows.map { |row| row_value(row, :delivery_id) }
           execute(delivery_supersede_older_pending_sql(rows)) unless rows.empty?
           execute(delivery_claim_update_sql(delivery_ids, worker_id)) unless delivery_ids.empty?
@@ -369,6 +372,9 @@ module SearchEngine
       end
 
       def claim_select_sql(limit)
+        return collection_limited_claim_select_sql if collection_limited_batch?(limit)
+
+        limit = global_limit_for(limit)
         <<~SQL
           WITH ranked_pending AS (
             SELECT id,
@@ -410,6 +416,9 @@ module SearchEngine
       end
 
       def delivery_claim_select_sql(limit)
+        return collection_limited_delivery_claim_select_sql if collection_limited_batch?(limit)
+
+        limit = global_limit_for(limit)
         <<~SQL
           WITH ranked_pending AS (
             SELECT deliveries.id AS delivery_id,
@@ -864,6 +873,200 @@ module SearchEngine
         [delay.to_i, 0].max
       end
 
+      def collection_limited_claim_select_sql
+        <<~SQL
+          WITH #{collection_limits_cte_sql},
+          ranked_pending AS (
+            SELECT id,
+                   ROW_NUMBER() OVER (
+                     PARTITION BY collection, document_id
+                     ORDER BY id DESC
+                   ) AS row_number
+            FROM #{quoted_table}
+            WHERE status = 'pending'
+          ),
+          latest_due AS (
+            SELECT outbox.id,
+                   outbox.collection
+            FROM #{quoted_table} outbox
+            INNER JOIN ranked_pending
+              ON ranked_pending.id = outbox.id
+            WHERE ranked_pending.row_number = 1
+              AND (outbox.next_attempt_at IS NULL OR outbox.next_attempt_at <= CURRENT_TIMESTAMP)
+            ORDER BY outbox.id ASC
+            LIMIT #{collection_limited_candidate_limit}
+          ),
+          ranked_by_collection AS (
+            SELECT latest_due.id,
+                   ROW_NUMBER() OVER (
+                     PARTITION BY latest_due.collection
+                     ORDER BY latest_due.id ASC
+                   ) AS collection_row_number,
+                   COALESCE(collection_limits.batch_size, #{global_limit_for(nil)}) AS collection_batch_size
+            FROM latest_due
+            LEFT JOIN collection_limits
+              ON collection_limits.collection = latest_due.collection
+          ),
+          selected_due AS (
+            SELECT id
+            FROM ranked_by_collection
+            WHERE collection_row_number <= collection_batch_size
+          )
+          SELECT outbox.*
+          FROM #{quoted_table} outbox
+          INNER JOIN selected_due
+            ON selected_due.id = outbox.id
+          ORDER BY outbox.id ASC
+          FOR UPDATE SKIP LOCKED
+        SQL
+      end
+
+      def collection_limited_materialization_select_sql(targets)
+        <<~SQL
+          WITH target(target_key, queue_name) AS (
+            VALUES #{delivery_target_values_sql(targets)}
+          ),
+          #{collection_limits_cte_sql},
+          candidate_events AS MATERIALIZED (
+            SELECT outbox.id,
+                   outbox.collection,
+                   outbox.document_id
+            FROM #{quoted_table} outbox
+            WHERE outbox.status IN ('pending', 'processing', 'failed')
+              AND (outbox.next_attempt_at IS NULL OR outbox.next_attempt_at <= CURRENT_TIMESTAMP)
+              AND EXISTS (
+                SELECT 1
+                FROM target
+                WHERE NOT EXISTS (
+                  SELECT 1
+                  FROM #{quoted_delivery_table} deliveries
+                  WHERE deliveries.event_id = outbox.id
+                    AND deliveries.target_key = target.target_key
+                )
+            )
+            ORDER BY outbox.id ASC
+            LIMIT #{collection_limited_candidate_limit}
+            FOR UPDATE SKIP LOCKED
+          ),
+          latest_candidate_ids AS (
+            SELECT id,
+                   collection
+            FROM (
+              SELECT id,
+                     collection,
+                     ROW_NUMBER() OVER (
+                       PARTITION BY collection, document_id
+                       ORDER BY id DESC
+                     ) AS row_number
+              FROM candidate_events
+            ) ranked_candidate_events
+            WHERE row_number = 1
+          ),
+          ranked_by_collection AS (
+            SELECT latest_candidate_ids.id,
+                   ROW_NUMBER() OVER (
+                     PARTITION BY latest_candidate_ids.collection
+                     ORDER BY latest_candidate_ids.id ASC
+                   ) AS collection_row_number,
+                   COALESCE(collection_limits.batch_size, #{global_limit_for(nil)}) AS collection_batch_size
+            FROM latest_candidate_ids
+            LEFT JOIN collection_limits
+              ON collection_limits.collection = latest_candidate_ids.collection
+          ),
+          selected_candidate_ids AS (
+            SELECT id
+            FROM ranked_by_collection
+            WHERE collection_row_number <= collection_batch_size
+          )
+          SELECT outbox.*
+          FROM #{quoted_table} outbox
+          INNER JOIN selected_candidate_ids
+            ON selected_candidate_ids.id = outbox.id
+          ORDER BY outbox.id ASC
+        SQL
+      end
+
+      def collection_limited_delivery_claim_select_sql
+        <<~SQL
+          WITH #{collection_limits_cte_sql},
+          ranked_pending AS (
+            SELECT deliveries.id AS delivery_id,
+                   events.id AS event_id,
+                   events.collection,
+                   ROW_NUMBER() OVER (
+                     PARTITION BY deliveries.target_key, events.collection, events.document_id
+                     ORDER BY events.id DESC, deliveries.id DESC
+                   ) AS row_number
+            FROM #{quoted_delivery_table} deliveries
+            INNER JOIN #{quoted_table} events
+              ON events.id = deliveries.event_id
+            WHERE deliveries.target_key = #{quote(target_key)}
+              AND deliveries.status = 'pending'
+          ),
+          latest_due AS (
+            SELECT deliveries.id,
+                   ranked_pending.collection
+            FROM #{quoted_delivery_table} deliveries
+            INNER JOIN ranked_pending
+              ON ranked_pending.delivery_id = deliveries.id
+            WHERE ranked_pending.row_number = 1
+              AND (deliveries.next_attempt_at IS NULL OR deliveries.next_attempt_at <= CURRENT_TIMESTAMP)
+            ORDER BY deliveries.id ASC
+            LIMIT #{collection_limited_candidate_limit}
+          ),
+          ranked_by_collection AS (
+            SELECT latest_due.id,
+                   ROW_NUMBER() OVER (
+                     PARTITION BY latest_due.collection
+                     ORDER BY latest_due.id ASC
+                   ) AS collection_row_number,
+                   COALESCE(collection_limits.batch_size, #{global_limit_for(nil)}) AS collection_batch_size
+            FROM latest_due
+            LEFT JOIN collection_limits
+              ON collection_limits.collection = latest_due.collection
+          ),
+          selected_due AS (
+            SELECT id
+            FROM ranked_by_collection
+            WHERE collection_row_number <= collection_batch_size
+          )
+          SELECT events.*,
+                 deliveries.id AS delivery_id,
+                 deliveries.target_key,
+                 deliveries.attempts AS delivery_attempts
+          FROM #{quoted_delivery_table} deliveries
+          INNER JOIN #{quoted_table} events
+            ON events.id = deliveries.event_id
+          INNER JOIN selected_due
+            ON selected_due.id = deliveries.id
+          ORDER BY deliveries.id ASC
+          FOR UPDATE SKIP LOCKED
+        SQL
+      end
+
+      def collection_limited_batch?(limit)
+        limit.nil? && SearchEngine.config.postgres_outbox.collection_batch_sizes?
+      end
+
+      def global_limit_for(limit)
+        (limit || SearchEngine.config.postgres_outbox.batch_size).to_i
+      end
+
+      def collection_limited_candidate_limit
+        batch_size_sum = SearchEngine.config.postgres_outbox.normalized_batch_sizes.values.sum
+        [global_limit_for(nil), 1].max + batch_size_sum
+      end
+
+      def collection_limits_cte_sql
+        "collection_limits(collection, batch_size) AS (VALUES #{collection_batch_size_values_sql})"
+      end
+
+      def collection_batch_size_values_sql
+        SearchEngine.config.postgres_outbox.normalized_batch_sizes.sort.map do |collection, size|
+          "(#{quote(collection)}, #{size})"
+        end.join(', ')
+      end
+
       def truncate_error(error)
         message = error.respond_to?(:message) ? error.message : error.to_s
         message.to_s[0, ERROR_LIMIT]

data/lib/search_engine/version.rb

@@ -3,5 +3,5 @@
 module SearchEngine
   # Current gem version.
   # @return [String]
-  VERSION = '30.1.8.16'
+  VERSION = '30.1.8.17'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: search-engine-for-typesense
 version: !ruby/object:Gem::Version
-  version: 30.1.8.16
+  version: 30.1.8.17
 platform: ruby
 authors:
 - Nikita Shkoda