search-engine-for-typesense 30.9.0 → 30.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f8363aa6879873e4b2d897181f0e87494b454b5c1ccf51f19d29f4590e992b24
-  data.tar.gz: bd13ccd5af83d57a4499227629bbc6caa360dd08e15457e8eb80067d174fc20e
+  metadata.gz: 2f5d1b6315a75f241c80d16c1f69727fed0ceef2edabb8ce774d33c348756a59
+  data.tar.gz: fbb2364ac8afc372ff0bbcb8f70174874ab190e226f6d08e1e747b018be8259b
 SHA512:
-  metadata.gz: d8d3847c59ab120f22f076d3c76aeebf7ea0e6b74ccf29771b72f30896f8a7789f213c1d139c40a8565ed4e63612adebafafbe81ae25eb1dd5190a7dc6215e92
-  data.tar.gz: 03700343db8c49a61e5d722992cae22e23ff0dfd51506d98124b2039fd0e7302f0da495914b549e0082718832199778d3ffbedd3d62e89c53f056c56b58c0250
+  metadata.gz: 1db71818517bd5dd234a43ebfe63f1b9affa33b4fcb0f6a15914cb6bcd55a3a6844ba8a87e68ead1bbc81a3a87dbe3dba2d845f9ba8c4da6d3251ed877755c5b
+  data.tar.gz: 56a54cdd5a7a0fdf3566c021788ba9f4d54aa9311f7df508b8b9e99c5d7010c9ecd26cf2d83cc219625a48d7738883e03c49360c9d7c4d256f4fc48d173e315b

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 30.9.1
+
+- Serialize PostgreSQL delivery discovery across schedulers and workers and check delivery membership
+  by event and target. Hosts can enable checkpointed scans to bound discovery on large outboxes.
+- Execute advisory locks and delivery cutover polls on the database when Rails query caching is enabled.
+
 ## 30.9.0
 
 - Qualify geo/vector search and native caching against Typesense 30.2 while keeping the gem major

data/README.md

@@ -565,6 +565,22 @@ target-specific status, retry, lock, and queue state for each logical event. Add
 `create_search_engine_outbox_drain_slots` in that delivery-target migration so the drain enqueuer can cap
 queued and running drain jobs per target.
 
+Concurrent listener, scheduler and worker calls share one nonblocking PostgreSQL transaction lock for
+delivery discovery on each events table. Other callers can keep claiming existing deliveries.
+For large histories, create a checkpoint table in a migration with
+`create_search_engine_outbox_materialization_cursors`, then set
+`c.postgres_outbox.materialization_cursor_table_name = 'search_engine_outbox_materialization_cursors'`.
+Run the migration before starting consumers with this setting.
+
+With checkpoints, each discovery call checks at most 1,000 IDs around the forward position and
+1,000 historical IDs against the requested targets. The forward scan overlaps its previous
+range to catch nearby out-of-order commits. Each target set maintains its own position. A
+transaction that commits farther behind, or an older event that becomes due, can wait until
+the historical scan revisits its ID. Sweep time depends on retained history and polling rate.
+Collection batch quotas and the latest-event rule still govern which deliveries may be created.
+Without a configured checkpoint table the existing full-history discovery uses indexed per-event
+delivery checks and the same concurrency lock.
+
 Drain slots are a generic backpressure mechanism for delivery-target mode. On each listener or polling
 wakeup, the enqueuer materializes missing delivery rows, acquires idle slots from
 `c.postgres_outbox.drain_slot_table_name`, and enqueues one drain job per acquired slot. If all slots for a

data/lib/search_engine/config.rb

@@ -278,6 +278,8 @@ module SearchEngine
       attr_accessor :delivery_table_name
       # @return [String] database table used by host-managed drain slots
       attr_accessor :drain_slot_table_name
+      # @return [String, nil] optional checkpoint table for bounded delivery discovery
+      attr_accessor :materialization_cursor_table_name
       # @return [String] PostgreSQL notification channel for wakeups
       attr_accessor :channel
       # @return [String] queue name used by host app job dispatch
@@ -936,6 +938,7 @@ module SearchEngine
         table_name: postgres_outbox.table_name,
         delivery_table_name: postgres_outbox.delivery_table_name,
         drain_slot_table_name: postgres_outbox.drain_slot_table_name,
+        materialization_cursor_table_name: postgres_outbox.materialization_cursor_table_name,
         channel: postgres_outbox.channel,
         queue_name: postgres_outbox.queue_name,
         batch_size: postgres_outbox.batch_size,

data/lib/search_engine/postgres_outbox/materialization_cursor.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'json'
+
+module SearchEngine
+  module PostgresOutbox
+    # Read under the outbox materialization lock. One page follows new event IDs;
+    # the other sweeps backwards to revisit late commits, due retries and existing events.
+    class MaterializationCursor
+      PAGE_SIZE = 1000
+
+      def initialize(connection, events_table, cursor_table, target_keys)
+        @connection = connection
+        @events = events_table
+        @table = connection.quote_table_name(cursor_table)
+        @scope = connection.quote(Digest::SHA256.hexdigest(JSON.generate([events_table, target_keys.uniq.sort])))
+      end
+
+      # @param limit [Integer] upper bound on the number of event IDs to read in each direction
+      # @return [Array<Integer>] event IDs to check against the configured delivery targets
+      def candidate_ids(limit:)
+        @page_size = [PAGE_SIZE, limit].min
+        state = @connection.select_all(<<~SQL).first
+          SELECT forward_id, backfill_through_id FROM #{@table} WHERE target_scope = #{@scope}
+        SQL
+        if state
+          @forward_id = Integer(state.fetch('forward_id'))
+          @backfill_through_id = Integer(state.fetch('backfill_through_id'))
+        else
+          @forward_id, @backfill_through_id = initialize_positions
+        end
+
+        # Recheck a recent ID range for concurrent inserts that commit behind the watermark.
+        # Even for dense IDs, at least half of the page remains available for new events.
+        @forward_page = event_ids("id > #{@forward_id - @page_size / 2} ORDER BY id ASC")
+        @backfill_page = event_ids("id <= #{@backfill_through_id} ORDER BY id DESC")
+        @forward_page | @backfill_page
+      end
+
+      # Save progress in the same transaction as the new deliveries.
+      def advance!
+        forward = [@forward_page.last || @forward_id, @forward_id].max
+        # Freeze the upper boundary until the backwards sweep reaches the beginning.
+        backfill = @backfill_page.size < @page_size ? forward : @backfill_page.last - 1
+        return if forward == @forward_id && backfill == @backfill_through_id
+
+        @connection.execute(<<~SQL)
+          UPDATE #{@table} SET forward_id = #{forward}, backfill_through_id = #{backfill}
+          WHERE target_scope = #{@scope}
+        SQL
+      end
+
+      private
+
+      def initialize_positions
+        head = Integer(@connection.select_value("SELECT COALESCE(MAX(id), 0) FROM #{@events}"))
+        @connection.execute(<<~SQL)
+          INSERT INTO #{@table} (target_scope, forward_id, backfill_through_id)
+          VALUES (#{@scope}, #{head}, #{head})
+        SQL
+        [head, head]
+      end
+
+      def event_ids(order)
+        @connection.select_all("SELECT id FROM #{@events} WHERE #{order} LIMIT #{@page_size}").map do |row|
+          Integer(row.fetch('id'))
+        end
+      end
+    end
+  end
+end

data/lib/search_engine/postgres_outbox/migration_helpers.rb

@@ -102,6 +102,15 @@ module SearchEngine
                         on_delete: :cascade
       end
 
+      # Create the small checkpoint table used for bounded delivery discovery.
+      # @param table_name [String, Symbol] table configured as materialization_cursor_table_name
+      def create_search_engine_outbox_materialization_cursors(table_name: :search_engine_outbox_materialization_cursors)
+        create_table table_name, id: :string, primary_key: :target_scope do |t|
+          t.bigint :forward_id, null: false
+          t.bigint :backfill_through_id, null: false
+        end
+      end
+
       # Create the SearchEngine outbox drain slots table.
       #
       # @param table_name [String, Symbol] destination drain slots table name

data/lib/search_engine/postgres_outbox/repository.rb

@@ -9,6 +9,7 @@ module SearchEngine
     class Repository
       # Keep expensive duplicate-key ranking bounded while leaving headroom for hot keys.
       CLAIM_CANDIDATE_OVERSAMPLE = 4
+      MATERIALIZATION_LOCK_NAMESPACE = 0x53454F42 # SEOB
       ERROR_LIMIT = 1000
       PROCESSED_AT_ASSIGNMENT = 'processed_at = CURRENT_TIMESTAMP'
       RETIREMENT_BATCH_SIZE = 1000
@@ -181,43 +182,27 @@ module SearchEngine
         with_delivery_target_session_lock(target, mode: :shared, timing: timing, &block)
       end
 
-      # Create missing delivery rows for all configured delivery targets.
-      # @return [void]
+      # Create missing deliveries. Concurrent pollers leave discovery to the current owner.
+      # @return [Array<Hash>, nil] materialized events, or nil when no targets are configured
       def materialize_deliveries!(limit: nil)
         targets = materialization_delivery_targets
         return if targets.empty?
 
-        # Discover work without row locks, then acquire every affected parent before any delivery.
-        # The exact preview is revalidated under those parent locks before mutations begin.
-        preview_rows = select_rows(delivery_materialization_select_sql(limit, targets, lock: false))
-        return preview_rows if preview_rows.empty?
+        scan_limit = collection_limited_batch?(limit) ? collection_limited_candidate_limit : global_limit_for(limit)
+        return [] if scan_limit.zero?
 
-        supersede_parents = select_rows(supersede_parent_ids_sql(preview_rows))
-        rows = []
         connection.transaction do
-          parent_ids = preview_rows.map { |row| row_value(row, :id) }
-          parent_ids.concat(supersede_parents.map { |row| row_value(row, :event_id) })
-          lock_event_rows!(parent_ids)
-
-          rows = select_rows(
-            delivery_materialization_select_sql(
-              limit,
-              targets,
-              event_ids: preview_rows.map { |row| row_value(row, :id) }
-            )
-          )
-          next if rows.empty?
-
-          supersede_candidates = select_rows(materialization_supersede_parent_candidates_sql(rows, targets))
-          older_event_ids = supersede_candidates.map { |candidate| row_value(candidate, :event_id) }
-          older_delivery_ids = supersede_candidates.map { |candidate| row_value(candidate, :delivery_id) }
-          lock_delivery_rows!(older_delivery_ids)
-          supersede_pending_events_by_id!(older_event_ids)
-          supersede_pending_deliveries!(older_delivery_ids, event_ids: older_event_ids)
-          execute(delivery_materialization_insert_sql(rows, targets))
+          next [] unless materialization_lock_acquired?
+
+          cursor = materialization_cursor(targets)
+          event_ids = cursor&.candidate_ids(limit: scan_limit)
+          preview = select_rows(delivery_materialization_select_sql(limit, targets, event_ids: event_ids, lock: false))
+          rows = preview.empty? ? [] : materialize_batch(limit, targets, preview)
+          if cursor && rows.size == preview.size && materialization_page_drained?(rows, event_ids, limit)
+            cursor.advance!
+          end
+          rows
         end
-
-        rows
       end
 
       # Refresh cleanup-eligible parent event statuses from terminal delivery rows.
@@ -415,6 +400,49 @@ module SearchEngine
         end
       end
 
+      # PostgreSQL's two-integer lock space is separate from the single-key delivery cutover locks.
+      def materialization_lock_acquired?
+        sql = <<~SQL
+          SELECT pg_try_advisory_xact_lock(#{MATERIALIZATION_LOCK_NAMESPACE}, #{quote(quoted_table)}::regclass::integer)
+        SQL
+        advisory_lock_truthy?(select_scalar(sql))
+      end
+
+      def materialization_cursor(targets)
+        table = SearchEngine.config.postgres_outbox.materialization_cursor_table_name
+        MaterializationCursor.new(connection, quoted_table, table, targets.map(&:key)) if table
+      end
+
+      # When a quota may have omitted matches, revisit the page after these deliveries commit.
+      def materialization_page_drained?(rows, event_ids, limit)
+        return true if rows.empty? || rows.size == event_ids.size
+
+        batch_size = global_limit_for(limit)
+        return rows.size < batch_size unless collection_limited_batch?(limit)
+
+        per_collection = SearchEngine.config.postgres_outbox.normalized_batch_sizes.values
+        rows.size < [batch_size, *per_collection].min
+      end
+
+      # Revalidate the exact preview under parent locks before locking any deliveries.
+      def materialize_batch(limit, targets, preview_rows)
+        supersede_parents = select_rows(supersede_parent_ids_sql(preview_rows))
+        preview_ids = preview_rows.map { |row| row_value(row, :id) }
+        lock_event_rows!(preview_ids + supersede_parents.map { |row| row_value(row, :event_id) })
+
+        rows = select_rows(delivery_materialization_select_sql(limit, targets, event_ids: preview_ids))
+        return rows if rows.empty?
+
+        supersede_candidates = select_rows(materialization_supersede_parent_candidates_sql(rows, targets))
+        older_event_ids = supersede_candidates.map { |candidate| row_value(candidate, :event_id) }
+        older_delivery_ids = supersede_candidates.map { |candidate| row_value(candidate, :delivery_id) }
+        lock_delivery_rows!(older_delivery_ids)
+        supersede_pending_events_by_id!(older_event_ids)
+        supersede_pending_deliveries!(older_delivery_ids, event_ids: older_event_ids)
+        execute(delivery_materialization_insert_sql(rows, targets))
+        rows
+      end
+
       def claim_pending_deliveries(limit:, worker_id:)
         rows = claim_pending_delivery_rows(limit: limit, worker_id: worker_id)
 
@@ -431,51 +459,34 @@ module SearchEngine
           return collection_limited_materialization_select_sql(targets, event_ids: event_ids, lock: lock)
         end
 
-        limit = global_limit_for(limit)
         <<~SQL
-          WITH target(target_key, queue_name) AS (
-            VALUES #{delivery_target_values_sql(targets)}
-          ),
-          candidate_events AS MATERIALIZED (
-            SELECT outbox.id,
-                   outbox.collection,
-                   outbox.document_id
-            FROM #{quoted_table} outbox
-            WHERE outbox.status IN ('pending', 'processing', 'failed')
-              AND (outbox.next_attempt_at IS NULL OR outbox.next_attempt_at <= CURRENT_TIMESTAMP)
-              #{newer_event_exclusion_sql('outbox')}
-              #{event_id_filter_sql('outbox', event_ids)}
-              AND EXISTS (
-                SELECT 1
-                FROM target
-                WHERE NOT EXISTS (
-                  SELECT 1
-                  FROM #{quoted_delivery_table} deliveries
-                  WHERE deliveries.event_id = outbox.id
-                    AND deliveries.target_key = target.target_key
-                )
-              )
-            ORDER BY outbox.id ASC
-            LIMIT #{limit}
-            #{row_lock_sql(lock)}
-          ),
-          latest_candidate_ids AS (
-            SELECT id
-            FROM (
-              SELECT id,
-                     ROW_NUMBER() OVER (
-                       PARTITION BY collection, document_id
-                       ORDER BY id DESC
-                     ) AS row_number
-              FROM candidate_events
-            ) ranked_candidate_events
-            WHERE row_number = 1
-          )
-          SELECT outbox.*
+          SELECT #{lock ? 'outbox.*' : 'outbox.id, outbox.collection, outbox.document_id'}
           FROM #{quoted_table} outbox
-          INNER JOIN latest_candidate_ids
-            ON latest_candidate_ids.id = outbox.id
+          WHERE #{materialization_filter_sql(targets, event_ids)}
           ORDER BY outbox.id ASC
+          LIMIT #{global_limit_for(limit)}
+          #{row_lock_sql(lock)}
+        SQL
+      end
+
+      # Each membership check reads at most one unique delivery per requested key.
+      # The scalar newer-event probe also prevents a hash over the entire parent history.
+      def materialization_filter_sql(targets, event_ids)
+        keys = targets.map(&:key).uniq
+        <<~SQL.chomp
+          outbox.status IN ('pending', 'processing', 'failed')
+          AND (outbox.next_attempt_at IS NULL OR outbox.next_attempt_at <= CURRENT_TIMESTAMP)
+          #{event_id_filter_sql('outbox', event_ids)}
+          AND (
+            SELECT newer.id FROM #{quoted_table} newer
+            WHERE newer.collection = outbox.collection AND newer.document_id = outbox.document_id
+              AND newer.id > outbox.id
+            LIMIT 1
+          ) IS NULL
+          AND (
+            SELECT COUNT(*) FROM #{quoted_delivery_table} deliveries
+            WHERE deliveries.event_id = outbox.id AND deliveries.target_key IN (#{ids_sql(keys)})
+          ) < #{keys.size}
         SQL
       end
 
@@ -683,6 +694,8 @@ module SearchEngine
       end
 
       def select_scalar(sql)
+        # Advisory locks and quiescence polls must reach the current database session on every attempt.
+        return connection.exec_query(sql).rows.first&.first if connection.respond_to?(:exec_query)
         return connection.select_value(sql) if connection.respond_to?(:select_value)
 
         result = connection.execute(sql)
@@ -1594,64 +1607,32 @@ module SearchEngine
 
       def collection_limited_materialization_select_sql(targets, event_ids: nil, lock: true)
         <<~SQL
-          WITH target(target_key, queue_name) AS (
-            VALUES #{delivery_target_values_sql(targets)}
-          ),
-          #{collection_limits_cte_sql},
+          WITH #{collection_limits_cte_sql},
           candidate_events AS MATERIALIZED (
-            SELECT outbox.id,
-                   outbox.collection,
-                   outbox.document_id
+            SELECT outbox.id, outbox.collection
             FROM #{quoted_table} outbox
-            WHERE outbox.status IN ('pending', 'processing', 'failed')
-              AND (outbox.next_attempt_at IS NULL OR outbox.next_attempt_at <= CURRENT_TIMESTAMP)
-              #{newer_event_exclusion_sql('outbox')}
-              #{event_id_filter_sql('outbox', event_ids)}
-              AND EXISTS (
-                SELECT 1
-                FROM target
-                WHERE NOT EXISTS (
-                  SELECT 1
-                  FROM #{quoted_delivery_table} deliveries
-                  WHERE deliveries.event_id = outbox.id
-                    AND deliveries.target_key = target.target_key
-                )
-            )
+            WHERE #{materialization_filter_sql(targets, event_ids)}
             ORDER BY outbox.id ASC
             LIMIT #{collection_limited_candidate_limit}
             #{row_lock_sql(lock)}
           ),
-          latest_candidate_ids AS (
-            SELECT id,
-                   collection
-            FROM (
-              SELECT id,
-                     collection,
-                     ROW_NUMBER() OVER (
-                       PARTITION BY collection, document_id
-                       ORDER BY id DESC
-                     ) AS row_number
-              FROM candidate_events
-            ) ranked_candidate_events
-            WHERE row_number = 1
-          ),
           ranked_by_collection AS (
-            SELECT latest_candidate_ids.id,
+            SELECT candidate_events.id,
                    ROW_NUMBER() OVER (
-                     PARTITION BY latest_candidate_ids.collection
-                     ORDER BY latest_candidate_ids.id ASC
+                     PARTITION BY candidate_events.collection
+                     ORDER BY candidate_events.id ASC
                    ) AS collection_row_number,
                    COALESCE(collection_limits.batch_size, #{global_limit_for(nil)}) AS collection_batch_size
-            FROM latest_candidate_ids
+            FROM candidate_events
             LEFT JOIN collection_limits
-              ON collection_limits.collection = latest_candidate_ids.collection
+              ON collection_limits.collection = candidate_events.collection
           ),
           selected_candidate_ids AS (
             SELECT id
             FROM ranked_by_collection
             WHERE collection_row_number <= collection_batch_size
           )
-          SELECT outbox.*
+          SELECT #{lock ? 'outbox.*' : 'outbox.id, outbox.collection, outbox.document_id'}
           FROM #{quoted_table} outbox
           INNER JOIN selected_candidate_ids
             ON selected_candidate_ids.id = outbox.id
@@ -1784,18 +1765,6 @@ module SearchEngine
         Array(ids).compact.uniq(&:to_s)
       end
 
-      def newer_event_exclusion_sql(event_relation)
-        <<~SQL.chomp
-          AND NOT EXISTS (
-            SELECT 1
-            FROM #{quoted_table} newer_events
-            WHERE newer_events.collection = #{event_relation}.collection
-              AND newer_events.document_id = #{event_relation}.document_id
-              AND newer_events.id > #{event_relation}.id
-          )
-        SQL
-      end
-
       def processing_event_exclusion_sql(event_relation)
         <<~SQL.chomp
           AND NOT EXISTS (

data/lib/search_engine/postgres_outbox.rb

@@ -3,6 +3,7 @@
 require 'search_engine/postgres_outbox/event'
 require 'search_engine/postgres_outbox/delivery_target'
 require 'search_engine/postgres_outbox/processor_result'
+require 'search_engine/postgres_outbox/materialization_cursor'
 require 'search_engine/postgres_outbox/repository'
 require 'search_engine/postgres_outbox/event_processor'
 require 'search_engine/postgres_outbox/drainer'

data/lib/search_engine/version.rb

@@ -3,5 +3,5 @@
 module SearchEngine
   # Current gem version.
   # @return [String]
-  VERSION = '30.9.0'
+  VERSION = '30.9.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: search-engine-for-typesense
 version: !ruby/object:Gem::Version
-  version: 30.9.0
+  version: 30.9.1
 platform: ruby
 authors:
 - Nikita Shkoda
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-11-02 00:00:00.000000000 Z
+date: 2026-11-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -189,6 +189,7 @@ files:
 - lib/search_engine/postgres_outbox/event.rb
 - lib/search_engine/postgres_outbox/event_processor.rb
 - lib/search_engine/postgres_outbox/listener.rb
+- lib/search_engine/postgres_outbox/materialization_cursor.rb
 - lib/search_engine/postgres_outbox/migration_helpers.rb
 - lib/search_engine/postgres_outbox/processor_result.rb
 - lib/search_engine/postgres_outbox/repository.rb