search-engine-for-typesense 30.1.8.11 → 30.1.8.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 95396b4ac85d2dfc142b05cf38aea45e73fd551ec7acb6915da3d7df7ffde9dd
-  data.tar.gz: 644a13a4a7ec1c8a841b19b6e8b59b88762b36db235428c13b5483a1082996c4
+  metadata.gz: e12c8943b56f8054dbb1cfff9c52f17f0a926664eeb77436ee711cfba06e9d0c
+  data.tar.gz: 833df3fbf7c7ff98576c29e218a15dea996230e1a1a624f63fdc896a3728d2ed
 SHA512:
-  metadata.gz: a59fc89cefc6c5a320616737159e6adbeebad8dac16886dea3b9d4e25e8124028469fc62b30c1f45bf53c1e6c2addad54aee694c40bfbe14cc399a93a9732625
-  data.tar.gz: eaa3305204a97489af5bf1d2e89062f0be63c922425061c9705b05cb44f16035dd8bc1eac7740fc8146c8a0ad38ecdeec2a3a3af564cec39188371828988d302
+  metadata.gz: 65dcbf97c8b91ae1f85836dcb69a955fa6cf63a52bf71063b8ebd53a8ec7c82eac0a8df211fc9e2e4fc8da0f488b5c8f3ee442dbb870d9002987c959e990f689
+  data.tar.gz: 78fbe84b31b3fa457144ce7c81466a1b074a25c4218705f19d4ce3875f0e7cd6fedfbd32f72658396eca255db0aee5554693c2df078590d65bd04765489a274f

data/lib/search_engine/postgres_outbox/drain_enqueuer.rb

@@ -27,7 +27,7 @@ module SearchEngine
         targets = delivery_targets
         return enqueue_legacy(limit: limit) if targets.empty?
 
-        repository.materialize_deliveries!
+        materialize_deliveries(limit: limit)
         targets.each { |target| enqueue_target(target, limit: limit) }
         nil
       end
@@ -36,6 +36,12 @@ module SearchEngine
 
       attr_reader :repository, :targets_resolver
 
+      def materialize_deliveries(limit:)
+        return repository.materialize_deliveries! if limit.nil?
+
+        repository.materialize_deliveries!(limit: limit)
+      end
+
       def enqueue_legacy(limit:)
         return drain_job.perform_later if limit.nil?
 

data/lib/search_engine/postgres_outbox/repository.rb

@@ -111,11 +111,78 @@ module SearchEngine
 
       # Create missing delivery rows for all configured delivery targets.
       # @return [void]
-      def materialize_deliveries!
+      def materialize_deliveries!(limit: SearchEngine.config.postgres_outbox.batch_size)
         targets = delivery_targets
         return if targets.empty?
 
-        execute(<<~SQL)
+        rows = []
+        connection.transaction do
+          rows = select_rows(delivery_materialization_select_sql(limit.to_i, targets))
+          next if rows.empty?
+
+          execute(materialization_supersede_older_deliveries_sql(rows, targets))
+          execute(supersede_older_pending_sql(rows))
+          execute(delivery_materialization_insert_sql(rows, targets))
+        end
+
+        rows
+      end
+
+      private
+
+      attr_reader :target_key
+
+      def connection
+        @connection ||= begin
+          require 'active_record'
+          ActiveRecord::Base.connection
+        end
+      end
+
+      def claim_pending_deliveries(limit:, worker_id:)
+        reset_stale_delivery_processing!
+        rows = claim_pending_delivery_rows(limit: limit, worker_id: worker_id)
+
+        if rows.empty?
+          materialize_deliveries!(limit: limit)
+          rows = claim_pending_delivery_rows(limit: limit, worker_id: worker_id)
+        end
+
+        rows.map { |row| Event.new(row) }
+      end
+
+      def delivery_materialization_select_sql(limit, targets)
+        <<~SQL
+          WITH target(target_key, queue_name) AS (
+            VALUES #{delivery_target_values_sql(targets)}
+          ),
+          candidate_events AS (
+            SELECT outbox.*
+            FROM #{quoted_table} outbox
+            WHERE outbox.status IN ('pending', 'processing', 'failed')
+              AND (outbox.next_attempt_at IS NULL OR outbox.next_attempt_at <= CURRENT_TIMESTAMP)
+              AND EXISTS (
+                SELECT 1
+                FROM target
+                WHERE NOT EXISTS (
+                  SELECT 1
+                  FROM #{quoted_delivery_table} deliveries
+                  WHERE deliveries.event_id = outbox.id
+                    AND deliveries.target_key = target.target_key
+                )
+              )
+            ORDER BY outbox.id ASC
+            LIMIT #{limit}
+            FOR UPDATE SKIP LOCKED
+          )
+          SELECT DISTINCT ON (collection, document_id) *
+          FROM candidate_events
+          ORDER BY collection, document_id, id DESC
+        SQL
+      end
+
+      def delivery_materialization_insert_sql(rows, targets)
+        <<~SQL
           INSERT INTO #{quoted_delivery_table} (
             event_id,
             target_key,
@@ -132,29 +199,19 @@ module SearchEngine
                  0,
                  CURRENT_TIMESTAMP,
                  CURRENT_TIMESTAMP
-          FROM #{quoted_table} outbox
+          FROM (
+            VALUES #{materialization_event_values_sql(rows)}
+          ) AS selected_events(event_id)
+          INNER JOIN #{quoted_table} outbox
+            ON outbox.id = selected_events.event_id
           CROSS JOIN (
             VALUES #{delivery_target_values_sql(targets)}
           ) AS target(target_key, queue_name)
-          WHERE outbox.status IN ('pending', 'processing', 'failed')
           ON CONFLICT (event_id, target_key) DO NOTHING
         SQL
       end
 
-      private
-
-      attr_reader :target_key
-
-      def connection
-        @connection ||= begin
-          require 'active_record'
-          ActiveRecord::Base.connection
-        end
-      end
-
-      def claim_pending_deliveries(limit:, worker_id:)
-        materialize_deliveries!
-        reset_stale_delivery_processing!
+      def claim_pending_delivery_rows(limit:, worker_id:)
         rows = []
 
         connection.transaction do
@@ -164,7 +221,7 @@ module SearchEngine
           execute(delivery_claim_update_sql(delivery_ids, worker_id)) unless delivery_ids.empty?
         end
 
-        rows.map { |row| Event.new(row) }
+        rows
       end
 
       def reset_stale_delivery_processing!
@@ -271,6 +328,46 @@ module SearchEngine
         SQL
       end
 
+      def materialization_supersede_older_deliveries_sql(rows, targets)
+        <<~SQL
+          WITH updated_deliveries AS (
+            UPDATE #{quoted_delivery_table} older_deliveries
+            SET status = 'superseded',
+                processed_at = CURRENT_TIMESTAMP,
+                locked_at = NULL,
+                locked_by = NULL,
+                updated_at = CURRENT_TIMESTAMP
+            FROM #{quoted_table} older_events,
+                 (
+                   VALUES #{coalesce_values_sql(rows)}
+                 ) AS latest(collection, document_id, id),
+                 (
+                   VALUES #{delivery_target_values_sql(targets)}
+                 ) AS target(target_key, queue_name)
+            WHERE older_deliveries.event_id = older_events.id
+              AND older_deliveries.status = 'pending'
+              AND older_deliveries.target_key = target.target_key
+              AND older_events.collection = latest.collection
+              AND older_events.document_id = latest.document_id
+              AND older_events.id < latest.id
+            RETURNING older_deliveries.event_id
+          ),
+          aggregate AS (
+            #{event_status_aggregate_sql('SELECT event_id FROM updated_deliveries')}
+          )
+          UPDATE #{quoted_table} events
+          SET status = aggregate.status,
+              processed_at = CASE
+                WHEN aggregate.status IN ('processed', 'superseded') THEN CURRENT_TIMESTAMP
+                ELSE NULL
+              END,
+              last_error = aggregate.last_error,
+              updated_at = CURRENT_TIMESTAMP
+          FROM aggregate
+          WHERE events.id = aggregate.event_id
+        SQL
+      end
+
       def supersede_older_pending_sql(rows)
         <<~SQL
           UPDATE #{quoted_table} older
@@ -468,6 +565,12 @@ module SearchEngine
         end.join(', ')
       end
 
+      def materialization_event_values_sql(rows)
+        rows.map do |row|
+          "(#{quote(row_value(row, :id))})"
+        end.join(', ')
+      end
+
       def quoted_table
         connection.quote_table_name(SearchEngine.config.postgres_outbox.table_name)
       end

data/lib/search_engine/version.rb

@@ -3,5 +3,5 @@
 module SearchEngine
   # Current gem version.
   # @return [String]
-  VERSION = '30.1.8.11'
+  VERSION = '30.1.8.12'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: search-engine-for-typesense
 version: !ruby/object:Gem::Version
-  version: 30.1.8.11
+  version: 30.1.8.12
 platform: ruby
 authors:
 - Nikita Shkoda