search-engine-for-typesense 30.1.8.5 → 30.1.8.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 88a738bfc2dcfe706d2c67c70b4f4923859ca808a50364b4b97ec93f1ecfdc66
-  data.tar.gz: dcb84555d2106d27c5bff18e31b0d9f20af7b2584ec2740ebe32cf4f07e842d2
+  metadata.gz: 77958fc8a7cb84effc7e3a4ca73b2f01e88eacc9ddead6e888c746e17b7e28d5
+  data.tar.gz: 74a046ef9f7cbdddeeed1c9fb7d9749d501b71300adc30bd0d2d771088aae189
 SHA512:
-  metadata.gz: db614f2ad79a4ed16424721b06b990a6b523f98bbc0e3e054fbf7f7ad1939c9282e948c70db257b40623865337b94f88c6614845a8088b048904c2baf105762d
-  data.tar.gz: 98d27347d7e98ee4f0380884331464e280d003af96fc5cd87c5278efca443ce88d8047f2652055e111621bbb10808f93f6fb8d9fec3000f8f475d3a73128b3b7
+  metadata.gz: 9e6bc4e2159d41cd66f0f4f9568d503c3b40ebbbe59be2cb6709ce008589d2e2caa3028f54f3de180a1dc9ff5c37c37cab5ba049780b4583a4974018eaf7ca39
+  data.tar.gz: a478abb0104cf415d31bcd069d93eb89039acff25c081423117e965fed6d6cd2ffbc859a88601988036246e4b343633e454fda25e27999013c5363e4f824d0c9

data/app/search_engine/search_engine/postgres_outbox/drain_job.rb

@@ -14,10 +14,20 @@ module SearchEngine
       def perform(limit: nil)
         return nil unless SearchEngine.config.postgres_outbox.enabled
 
+        effective_limit = limit || SearchEngine.config.postgres_outbox.batch_size
         drainer = SearchEngine::PostgresOutbox::Drainer.new
-        return drainer.drain_once if limit.nil?
+        summary = drainer.drain_once(limit: effective_limit)
+        enqueue_continuation(limit: limit) if summary[:claimed].to_i >= effective_limit.to_i
 
-        drainer.drain_once(limit: limit)
+        summary
+      end
+
+      private
+
+      def enqueue_continuation(limit:)
+        return self.class.perform_later if limit.nil?
+
+        self.class.perform_later(limit: limit)
       end
     end
   end

data/lib/search_engine/postgres_outbox/listener.rb

@@ -37,6 +37,7 @@ module SearchEngine
         @mutex = Mutex.new
         @stop_requested = false
         @thread = nil
+        @last_enqueue_at = nil
       end
 
       # Start the listener thread.
@@ -120,7 +121,7 @@ module SearchEngine
             next if notified || stop_requested?
 
             instrument('search_engine.postgres_outbox.listener_fallback') {}
-            enqueue_drain
+            enqueue_drain(force: true)
           end
         ensure
           connection.execute("UNLISTEN #{quoted_channel}")
@@ -187,10 +188,33 @@ module SearchEngine
         end
       end
 
-      def enqueue_drain
+      def enqueue_drain(force: false)
+        return if enqueue_throttled?(force: force)
+
         drain_job.perform_later
       end
 
+      def enqueue_throttled?(force:)
+        @mutex.synchronize do
+          now = monotonic_time
+          if force
+            @last_enqueue_at = now
+            next false
+          end
+
+          if @last_enqueue_at && (now - @last_enqueue_at) < poll_interval_s
+            true
+          else
+            @last_enqueue_at = now
+            false
+          end
+        end
+      end
+
+      def monotonic_time
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
       def drain_job
         @drain_job ||= SearchEngine::PostgresOutbox::DrainJob
       end

data/lib/search_engine/postgres_outbox/repository.rb

@@ -22,6 +22,7 @@ module SearchEngine
         connection.transaction do
           rows = select_rows(claim_select_sql(limit.to_i))
           ids = rows.map { |row| row_value(row, :id) }
+          execute(supersede_older_pending_sql(rows)) unless rows.empty?
           execute(claim_update_sql(ids, worker_id)) unless ids.empty?
         end
 
@@ -103,12 +104,30 @@ module SearchEngine
 
       def claim_select_sql(limit)
         <<~SQL
-          SELECT *
-          FROM #{quoted_table}
-          WHERE status = 'pending'
-            AND (next_attempt_at IS NULL OR next_attempt_at <= CURRENT_TIMESTAMP)
-          ORDER BY id ASC
-          LIMIT #{limit}
+          WITH ranked_pending AS (
+            SELECT id,
+                   ROW_NUMBER() OVER (
+                     PARTITION BY collection, document_id
+                     ORDER BY id DESC
+                   ) AS row_number
+            FROM #{quoted_table}
+            WHERE status = 'pending'
+          ),
+          latest_due AS (
+            SELECT outbox.id
+            FROM #{quoted_table} outbox
+            INNER JOIN ranked_pending
+              ON ranked_pending.id = outbox.id
+            WHERE ranked_pending.row_number = 1
+              AND (outbox.next_attempt_at IS NULL OR outbox.next_attempt_at <= CURRENT_TIMESTAMP)
+            ORDER BY outbox.id ASC
+            LIMIT #{limit}
+          )
+          SELECT outbox.*
+          FROM #{quoted_table} outbox
+          INNER JOIN latest_due
+            ON latest_due.id = outbox.id
+          ORDER BY outbox.id ASC
           FOR UPDATE SKIP LOCKED
         SQL
       end
@@ -124,6 +143,24 @@ module SearchEngine
         SQL
       end
 
+      def supersede_older_pending_sql(rows)
+        <<~SQL
+          UPDATE #{quoted_table} older
+          SET status = 'superseded',
+              processed_at = CURRENT_TIMESTAMP,
+              locked_at = NULL,
+              locked_by = NULL,
+              updated_at = CURRENT_TIMESTAMP
+          FROM (
+            VALUES #{coalesce_values_sql(rows)}
+          ) AS latest(collection, document_id, id)
+          WHERE older.status = 'pending'
+            AND older.collection = latest.collection
+            AND older.document_id = latest.document_id
+            AND older.id < latest.id
+        SQL
+      end
+
       def update_status!(event_ids, status, extra:)
         ids = Array(event_ids).compact
         return if ids.empty?
@@ -154,6 +191,16 @@ module SearchEngine
         ids.map { |id| quote(id) }.join(', ')
       end
 
+      def coalesce_values_sql(rows)
+        rows.map do |row|
+          collection = row_value(row, :collection)
+          document_id = row_value(row, :document_id)
+          id = row_value(row, :id)
+
+          "(#{quote(collection)}, #{quote(document_id)}, #{quote(id)})"
+        end.join(', ')
+      end
+
       def quoted_table
         connection.quote_table_name(SearchEngine.config.postgres_outbox.table_name)
       end

data/lib/search_engine/version.rb

@@ -3,5 +3,5 @@
 module SearchEngine
   # Current gem version.
   # @return [String]
-  VERSION = '30.1.8.5'
+  VERSION = '30.1.8.7'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: search-engine-for-typesense
 version: !ruby/object:Gem::Version
-  version: 30.1.8.5
+  version: 30.1.8.7
 platform: ruby
 authors:
 - Nikita Shkoda