search-engine-for-typesense 30.9.1 → 30.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2f5d1b6315a75f241c80d16c1f69727fed0ceef2edabb8ce774d33c348756a59
-  data.tar.gz: fbb2364ac8afc372ff0bbcb8f70174874ab190e226f6d08e1e747b018be8259b
+  metadata.gz: e2a45aebc5484036b192915f9de295af3efbce20c7831c3e301dd9608b68a26d
+  data.tar.gz: e7274a561a6f852f5daa8489f33349bbdf430da811f8e0db3556c36547a1098e
 SHA512:
-  metadata.gz: 1db71818517bd5dd234a43ebfe63f1b9affa33b4fcb0f6a15914cb6bcd55a3a6844ba8a87e68ead1bbc81a3a87dbe3dba2d845f9ba8c4da6d3251ed877755c5b
-  data.tar.gz: 56a54cdd5a7a0fdf3566c021788ba9f4d54aa9311f7df508b8b9e99c5d7010c9ecd26cf2d83cc219625a48d7738883e03c49360c9d7c4d256f4fc48d173e315b
+  metadata.gz: 38f920eb23837108385ad80e09607d6e8f221847660af0870de49552110305d5b1a8d8a4186d0f9635b609ccafdaf0817127fa911cf5f3a9f9d17f1d6ea3d032
+  data.tar.gz: c89b82cbb8575cc4608bbeab0536599aa0b5d37f26198117e6f519df40fd0b588919297c7fa26b0466efaa370fce9df56cd196e9a21cfcb15a7cebd679bd01a8

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## 30.9.2
+
+- Check recent event IDs during catch-up so a returning delivery target can discover new work
+  while its saved position moves through older events. Collection quotas still apply.
+- Read outbox checkpoints, delivery state and row locks directly from the database on every poll,
+  including when Active Record query caching is enabled on Rails 6.1 and 7.0.
+
 ## 30.9.1
 
 - Serialize PostgreSQL delivery discovery across schedulers and workers and check delivery membership

data/README.md

@@ -572,9 +572,10 @@ For large histories, create a checkpoint table in a migration with
 `c.postgres_outbox.materialization_cursor_table_name = 'search_engine_outbox_materialization_cursors'`.
 Run the migration before starting consumers with this setting.
 
-With checkpoints, each discovery call checks at most 1,000 IDs around the forward position and
-1,000 historical IDs against the requested targets. The forward scan overlaps its previous
-range to catch nearby out-of-order commits. Each target set maintains its own position. A
+With checkpoints, each discovery call checks at most 2,000 event IDs. Each target set maintains
+its own forward position and a historical sweep. The forward scan overlaps its previous range
+to catch nearby out-of-order commits. When it falls behind, part of the historical scan budget
+checks the newest IDs while the forward scan continues through the intervening events. A
 transaction that commits farther behind, or an older event that becomes due, can wait until
 the historical scan revisits its ID. Sweep time depends on retained history and polling rate.
 Collection batch quotas and the latest-event rule still govern which deliveries may be created.

data/lib/search_engine/postgres_outbox/database_queries.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module SearchEngine
+  module PostgresOutbox
+    # Operational reads must observe current delivery state and acquire row locks on each poll.
+    module DatabaseQueries
+      private
+
+      def select_rows(sql)
+        result = connection.respond_to?(:exec_query) ? connection.exec_query(sql) : connection.select_all(sql)
+        return result.to_a if result.respond_to?(:to_a)
+
+        Array(result)
+      end
+    end
+  end
+end

data/lib/search_engine/postgres_outbox/materialization_cursor.rb

@@ -5,11 +5,14 @@ require 'json'
 
 module SearchEngine
   module PostgresOutbox
-    # Read under the outbox materialization lock. One page follows new event IDs;
-    # the other sweeps backwards to revisit late commits, due retries and existing events.
+    # Scan under the materialization lock. Keep moving through history even when new IDs arrive faster.
     class MaterializationCursor
+      include DatabaseQueries
+
       PAGE_SIZE = 1000
 
+      attr_reader :checkpoint_ids
+
       def initialize(connection, events_table, cursor_table, target_keys)
         @connection = connection
         @events = events_table
@@ -21,7 +24,7 @@ module SearchEngine
       # @return [Array<Integer>] event IDs to check against the configured delivery targets
       def candidate_ids(limit:)
         @page_size = [PAGE_SIZE, limit].min
-        state = @connection.select_all(<<~SQL).first
+        state = select_rows(<<~SQL).first
           SELECT forward_id, backfill_through_id FROM #{@table} WHERE target_scope = #{@scope}
         SQL
         if state
@@ -34,18 +37,27 @@ module SearchEngine
         # Recheck a recent ID range for concurrent inserts that commit behind the watermark.
         # Even for dense IDs, at least half of the page remains available for new events.
         @forward_page = event_ids("id > #{@forward_id - @page_size / 2} ORDER BY id ASC")
-        @backfill_page = event_ids("id <= #{@backfill_through_id} ORDER BY id DESC")
-        @forward_page | @backfill_page
+        # A returning target can be far behind. Reserve some of the second page for the current head.
+        @recent_page = if @forward_page.size == @page_size
+                         event_ids("id > #{@forward_page.last} ORDER BY id DESC", [@page_size / 2, 1].max)
+                       else
+                         []
+                       end
+        @backfill_limit = [@page_size - @recent_page.size, 1].max
+        @backfill_page = event_ids("id <= #{@backfill_through_id} ORDER BY id DESC", @backfill_limit)
+        @checkpoint_ids = @forward_page | @backfill_page
+        @checkpoint_ids | @recent_page
       end
 
       # Save progress in the same transaction as the new deliveries.
       def advance!
         forward = [@forward_page.last || @forward_id, @forward_id].max
         # Freeze the upper boundary until the backwards sweep reaches the beginning.
-        backfill = @backfill_page.size < @page_size ? forward : @backfill_page.last - 1
+        head = [forward, @recent_page.first || forward].max
+        backfill = @backfill_page.size < @backfill_limit ? head : @backfill_page.last - 1
         return if forward == @forward_id && backfill == @backfill_through_id
 
-        @connection.execute(<<~SQL)
+        connection.execute(<<~SQL)
           UPDATE #{@table} SET forward_id = #{forward}, backfill_through_id = #{backfill}
           WHERE target_scope = #{@scope}
         SQL
@@ -53,17 +65,19 @@ module SearchEngine
 
       private
 
+      attr_reader :connection
+
       def initialize_positions
-        head = Integer(@connection.select_value("SELECT COALESCE(MAX(id), 0) FROM #{@events}"))
-        @connection.execute(<<~SQL)
+        head = Integer(select_rows("SELECT COALESCE(MAX(id), 0) AS head FROM #{@events}").first.fetch('head'))
+        connection.execute(<<~SQL)
           INSERT INTO #{@table} (target_scope, forward_id, backfill_through_id)
           VALUES (#{@scope}, #{head}, #{head})
         SQL
         [head, head]
       end
 
-      def event_ids(order)
-        @connection.select_all("SELECT id FROM #{@events} WHERE #{order} LIMIT #{@page_size}").map do |row|
+      def event_ids(order, limit = @page_size)
+        select_rows("SELECT id FROM #{@events} WHERE #{order} LIMIT #{limit}").map do |row|
           Integer(row.fetch('id'))
         end
       end

data/lib/search_engine/postgres_outbox/repository.rb

@@ -7,6 +7,8 @@ module SearchEngine
   module PostgresOutbox
     # Raw SQL repository for host-managed PostgreSQL outbox rows.
     class Repository
+      include DatabaseQueries
+
       # Keep expensive duplicate-key ranking bounded while leaving headroom for hot keys.
       CLAIM_CANDIDATE_OVERSAMPLE = 4
       MATERIALIZATION_LOCK_NAMESPACE = 0x53454F42 # SEOB
@@ -198,7 +200,7 @@ module SearchEngine
           event_ids = cursor&.candidate_ids(limit: scan_limit)
           preview = select_rows(delivery_materialization_select_sql(limit, targets, event_ids: event_ids, lock: false))
           rows = preview.empty? ? [] : materialize_batch(limit, targets, preview)
-          if cursor && rows.size == preview.size && materialization_page_drained?(rows, event_ids, limit)
+          if cursor && rows.size == preview.size && page_drained?(rows, event_ids, limit, cursor, targets)
             cursor.advance!
           end
           rows
@@ -414,14 +416,23 @@ module SearchEngine
       end
 
       # When a quota may have omitted matches, revisit the page after these deliveries commit.
-      def materialization_page_drained?(rows, event_ids, limit)
+      def page_drained?(rows, event_ids, limit, cursor, targets)
         return true if rows.empty? || rows.size == event_ids.size
 
         batch_size = global_limit_for(limit)
-        return rows.size < batch_size unless collection_limited_batch?(limit)
-
-        per_collection = SearchEngine.config.postgres_outbox.normalized_batch_sizes.values
-        rows.size < [batch_size, *per_collection].min
+        if collection_limited_batch?(limit)
+          quotas = SearchEngine.config.postgres_outbox.normalized_batch_sizes.values
+          batch_size = [batch_size, *quotas].min
+        end
+        return true if rows.size < batch_size
+        return false if event_ids.size == cursor.checkpoint_ids.size
+
+        # A moving head can fill a quota on every poll. Only work on the saved pages must hold progress.
+        select_rows(<<~SQL).empty?
+          SELECT 1 FROM #{quoted_table} outbox
+          WHERE #{materialization_filter_sql(targets, cursor.checkpoint_ids)}
+          LIMIT 1
+        SQL
       end
 
       # Revalidate the exact preview under parent locks before locking any deliveries.
@@ -1289,13 +1300,6 @@ module SearchEngine
           'THEN COALESCE(aggregate.terminal_processed_at, CURRENT_TIMESTAMP) ELSE NULL END'
       end
 
-      def select_rows(sql)
-        result = connection.select_all(sql)
-        return result.to_a if result.respond_to?(:to_a)
-
-        Array(result)
-      end
-
       def execute(sql)
         connection.execute(sql)
       end

data/lib/search_engine/postgres_outbox.rb

@@ -3,6 +3,7 @@
 require 'search_engine/postgres_outbox/event'
 require 'search_engine/postgres_outbox/delivery_target'
 require 'search_engine/postgres_outbox/processor_result'
+require 'search_engine/postgres_outbox/database_queries'
 require 'search_engine/postgres_outbox/materialization_cursor'
 require 'search_engine/postgres_outbox/repository'
 require 'search_engine/postgres_outbox/event_processor'

data/lib/search_engine/version.rb

@@ -3,5 +3,5 @@
 module SearchEngine
   # Current gem version.
   # @return [String]
-  VERSION = '30.9.1'
+  VERSION = '30.9.2'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: search-engine-for-typesense
 version: !ruby/object:Gem::Version
-  version: 30.9.1
+  version: 30.9.2
 platform: ruby
 authors:
 - Nikita Shkoda
@@ -183,6 +183,7 @@ files:
 - lib/search_engine/otel.rb
 - lib/search_engine/partitioner.rb
 - lib/search_engine/postgres_outbox.rb
+- lib/search_engine/postgres_outbox/database_queries.rb
 - lib/search_engine/postgres_outbox/delivery_target.rb
 - lib/search_engine/postgres_outbox/drain_enqueuer.rb
 - lib/search_engine/postgres_outbox/drainer.rb