search-engine-for-typesense 30.1.8.17 → 30.1.8.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3a268323694adc2817c238b7ae9f522a7685b55b240691fb5891a98b4bc3d371
-  data.tar.gz: 3fcdddf6f9bdd81ecca5aae6755e3d55f7d5a9fd52e50ed3ed2d5bbdaa163b53
+  metadata.gz: 20f330c7ec29b4f4ea6366c48d23f05d5a089e47eaeba79f0ad741c2a83b09ec
+  data.tar.gz: 1359ad4c68d304d334c238bae7beddeee57a6350565b8b8e5b6b8d3f97c75e82
 SHA512:
-  metadata.gz: 88f09c6d71f2a6c91505b8d83e57b6e989de2f8fbbc477b7b35185996da5cb0eae918dcf3ed14f8c9f3c69760469b5da9d32da2336b5cd640bffc386e31869b0
-  data.tar.gz: ab29408bb29a73d210aa29c3db62bb3bf684b8105a6dbe5aaf918ed299f8849b3eadb8ecb406d30db11e2353941aafb5ec8cb58ae4c180ca684ad720a5a55697
+  metadata.gz: 55191671cd8fb5dc620d25b605715b7608db8f8de48c220e7fe3d59727c239a9dd386574a421fd4477b6170f71c010575acf8b5c3e961a4fe2a455a568fe3e1c
+  data.tar.gz: f74ac6499069ed09918eac51273bcfcd5e82e0d91295001a9f3c5505b1ad69159f3c92f75c796d0cef43af29ee3159ecc79438c285fd15b2a4b0cb666d4618e1

data/README.md

@@ -405,6 +405,11 @@ inspected before deletion because they contain the last error and retry state. A
 only rows with `status IN ('processed', 'superseded')` and `processed_at` older than
 `c.postgres_outbox.retention_s`.
 
+In delivery-target mode, cleanup can first call
+`SearchEngine::PostgresOutbox::Repository#refresh_terminal_delivery_event_statuses!`. This bounded helper
+repairs parent events whose delivery rows are all terminal but whose parent status is still non-terminal,
+then normal retention cleanup can delete the repaired parent rows and cascade their deliveries.
+
 ## Example app
 
 See `examples/demo_shop` — demonstrates single/multi search, JOINs, grouping, presets/curation, and DX/observability. Supports offline mode via the stub client (see [Testing](https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/testing)).

data/lib/search_engine/postgres_outbox/migration_helpers.rb

@@ -42,6 +42,9 @@ module SearchEngine
         add_index table_name,
                   %i[collection document_id status id],
                   name: 'idx_search_engine_outbox_coalescing'
+        add_index table_name,
+                  %i[collection document_id id],
+                  name: 'idx_search_engine_outbox_coalesce_lookup'
         add_index table_name,
                   :locked_at,
                   name: 'idx_search_engine_outbox_processing',

data/lib/search_engine/postgres_outbox/repository.rb

@@ -128,6 +128,24 @@ module SearchEngine
         rows
       end
 
+      # Refresh cleanup-eligible parent event statuses from terminal delivery rows.
+      #
+      # This repairs historical residue where every delivery row for an event is
+      # terminal but the parent event still has a non-terminal status. The
+      # candidate set is bounded before aggregate work to keep cleanup safe on
+      # large outbox tables.
+      #
+      # @param retention_s [Integer] retention window in seconds
+      # @param limit [Integer, nil] maximum parent events to refresh
+      # @return [Integer] refreshed parent event count
+      def refresh_terminal_delivery_event_statuses!(retention_s:, limit: nil)
+        return 0 unless delivery_table_exists?
+
+        rows = select_rows(terminal_delivery_event_status_refresh_sql(retention_s: retention_s, limit: limit))
+
+        rows.size
+      end
+
       # Check whether the optional drain slot table exists.
       #
       # @return [Boolean]
@@ -360,14 +378,27 @@ module SearchEngine
 
       def reset_stale_delivery_processing!
         execute(<<~SQL)
-          UPDATE #{quoted_delivery_table}
-          SET status = 'pending',
-              locked_at = NULL,
-              locked_by = NULL,
+          WITH reset_deliveries AS (
+            UPDATE #{quoted_delivery_table}
+            SET status = 'pending',
+                locked_at = NULL,
+                locked_by = NULL,
+                updated_at = CURRENT_TIMESTAMP
+            WHERE target_key = #{quote(target_key)}
+              AND status = 'processing'
+              AND locked_at < (CURRENT_TIMESTAMP - interval '#{processing_timeout_s} seconds')
+            RETURNING event_id
+          ),
+          aggregate AS (
+            #{event_status_aggregate_sql('SELECT event_id FROM reset_deliveries')}
+          )
+          UPDATE #{quoted_table} events
+          SET status = aggregate.status,
+              processed_at = #{aggregate_processed_at_sql},
+              last_error = aggregate.last_error,
               updated_at = CURRENT_TIMESTAMP
-          WHERE target_key = #{quote(target_key)}
-            AND status = 'processing'
-            AND locked_at < (CURRENT_TIMESTAMP - interval '#{processing_timeout_s} seconds')
+          FROM aggregate
+          WHERE events.id = aggregate.event_id
         SQL
       end
 
@@ -470,26 +501,33 @@ module SearchEngine
 
       def materialization_supersede_older_deliveries_sql(rows, targets)
         <<~SQL
-          WITH updated_deliveries AS (
+          WITH latest(collection, document_id, id) AS (
+            VALUES #{coalesce_values_sql(rows)}
+          ),
+          target(target_key, queue_name) AS (
+            VALUES #{delivery_target_values_sql(targets)}
+          ),
+          older_event_targets AS MATERIALIZED (
+            SELECT older_events.id AS event_id,
+                   target.target_key
+            FROM latest
+            CROSS JOIN target
+            INNER JOIN #{quoted_table} older_events
+              ON older_events.collection = latest.collection
+             AND older_events.document_id = latest.document_id
+             AND older_events.id < latest.id
+          ),
+          updated_deliveries AS (
             UPDATE #{quoted_delivery_table} older_deliveries
             SET status = 'superseded',
                 processed_at = CURRENT_TIMESTAMP,
                 locked_at = NULL,
                 locked_by = NULL,
                 updated_at = CURRENT_TIMESTAMP
-            FROM #{quoted_table} older_events,
-                 (
-                   VALUES #{coalesce_values_sql(rows)}
-                 ) AS latest(collection, document_id, id),
-                 (
-                   VALUES #{delivery_target_values_sql(targets)}
-                 ) AS target(target_key, queue_name)
-            WHERE older_deliveries.event_id = older_events.id
+            FROM older_event_targets
+            WHERE older_deliveries.event_id = older_event_targets.event_id
               AND older_deliveries.status = 'pending'
-              AND older_deliveries.target_key = target.target_key
-              AND older_events.collection = latest.collection
-              AND older_events.document_id = latest.document_id
-              AND older_events.id < latest.id
+              AND older_deliveries.target_key = older_event_targets.target_key
             RETURNING older_deliveries.event_id
           ),
           aggregate AS (
@@ -497,10 +535,7 @@ module SearchEngine
           )
           UPDATE #{quoted_table} events
           SET status = aggregate.status,
-              processed_at = CASE
-                WHEN aggregate.status IN ('processed', 'superseded') THEN CURRENT_TIMESTAMP
-                ELSE NULL
-              END,
+              processed_at = #{aggregate_processed_at_sql},
               last_error = aggregate.last_error,
               updated_at = CURRENT_TIMESTAMP
           FROM aggregate
@@ -528,23 +563,29 @@ module SearchEngine
 
       def delivery_supersede_older_pending_sql(rows)
         <<~SQL
-          WITH updated_deliveries AS (
+          WITH latest(target_key, collection, document_id, event_id, delivery_id) AS (
+            VALUES #{delivery_coalesce_values_sql(rows)}
+          ),
+          older_event_targets AS MATERIALIZED (
+            SELECT older_events.id AS event_id,
+                   latest.target_key
+            FROM latest
+            INNER JOIN #{quoted_table} older_events
+              ON older_events.collection = latest.collection
+             AND older_events.document_id = latest.document_id
+             AND older_events.id < latest.event_id
+          ),
+          updated_deliveries AS (
             UPDATE #{quoted_delivery_table} older_deliveries
             SET status = 'superseded',
                 processed_at = CURRENT_TIMESTAMP,
                 locked_at = NULL,
                 locked_by = NULL,
                 updated_at = CURRENT_TIMESTAMP
-            FROM #{quoted_table} older_events,
-                 (
-                   VALUES #{delivery_coalesce_values_sql(rows)}
-                 ) AS latest(target_key, collection, document_id, event_id, delivery_id)
-            WHERE older_deliveries.event_id = older_events.id
+            FROM older_event_targets
+            WHERE older_deliveries.event_id = older_event_targets.event_id
               AND older_deliveries.status = 'pending'
-              AND older_deliveries.target_key = latest.target_key
-              AND older_events.collection = latest.collection
-              AND older_events.document_id = latest.document_id
-              AND older_events.id < latest.event_id
+              AND older_deliveries.target_key = older_event_targets.target_key
             RETURNING older_deliveries.event_id
           ),
           aggregate AS (
@@ -552,10 +593,7 @@ module SearchEngine
           )
           UPDATE #{quoted_table} events
           SET status = aggregate.status,
-              processed_at = CASE
-                WHEN aggregate.status IN ('processed', 'superseded') THEN CURRENT_TIMESTAMP
-                ELSE NULL
-              END,
+              processed_at = #{aggregate_processed_at_sql},
               last_error = aggregate.last_error,
               updated_at = CURRENT_TIMESTAMP
           FROM aggregate
@@ -582,47 +620,60 @@ module SearchEngine
         ids = Array(event_ids).compact
         return if ids.empty?
 
-        execute(<<~SQL)
-          UPDATE #{quoted_delivery_table}
-          SET status = #{quote(status)},
-              #{extra},
-              locked_at = NULL,
-              locked_by = NULL,
-              updated_at = CURRENT_TIMESTAMP
-          WHERE target_key = #{quote(target_key)}
-            AND event_id IN (#{ids_sql(ids)})
-        SQL
-        refresh_event_statuses!(ids)
+        with_delivery_status_refresh(ids) do
+          execute(<<~SQL)
+            UPDATE #{quoted_delivery_table}
+            SET status = #{quote(status)},
+                #{extra},
+                locked_at = NULL,
+                locked_by = NULL,
+                updated_at = CURRENT_TIMESTAMP
+            WHERE target_key = #{quote(target_key)}
+              AND event_id IN (#{ids_sql(ids)})
+          SQL
+        end
       end
 
       def mark_delivery_retryable!(event_ids, error:)
-        execute(<<~SQL)
-          UPDATE #{quoted_delivery_table}
-          SET attempts = attempts + 1,
-              status = CASE WHEN attempts + 1 >= #{max_attempts} THEN 'failed' ELSE 'pending' END,
-              next_attempt_at = CURRENT_TIMESTAMP + #{retry_interval_case_sql},
-              locked_at = NULL,
-              locked_by = NULL,
-              last_error = #{quote(truncate_error(error))},
-              updated_at = CURRENT_TIMESTAMP
-          WHERE target_key = #{quote(target_key)}
-            AND event_id IN (#{ids_sql(event_ids)})
-        SQL
-        refresh_event_statuses!(event_ids)
+        with_delivery_status_refresh(event_ids) do
+          execute(<<~SQL)
+            UPDATE #{quoted_delivery_table}
+            SET attempts = attempts + 1,
+                status = CASE WHEN attempts + 1 >= #{max_attempts} THEN 'failed' ELSE 'pending' END,
+                next_attempt_at = CURRENT_TIMESTAMP + #{retry_interval_case_sql},
+                locked_at = NULL,
+                locked_by = NULL,
+                last_error = #{quote(truncate_error(error))},
+                updated_at = CURRENT_TIMESTAMP
+            WHERE target_key = #{quote(target_key)}
+              AND event_id IN (#{ids_sql(event_ids)})
+          SQL
+        end
       end
 
       def mark_delivery_failed!(event_ids, error:)
-        execute(<<~SQL)
-          UPDATE #{quoted_delivery_table}
-          SET status = 'failed',
-              locked_at = NULL,
-              locked_by = NULL,
-              last_error = #{quote(truncate_error(error))},
-              updated_at = CURRENT_TIMESTAMP
-          WHERE target_key = #{quote(target_key)}
-            AND event_id IN (#{ids_sql(event_ids)})
-        SQL
-        refresh_event_statuses!(event_ids)
+        with_delivery_status_refresh(event_ids) do
+          execute(<<~SQL)
+            UPDATE #{quoted_delivery_table}
+            SET status = 'failed',
+                locked_at = NULL,
+                locked_by = NULL,
+                last_error = #{quote(truncate_error(error))},
+                updated_at = CURRENT_TIMESTAMP
+            WHERE target_key = #{quote(target_key)}
+              AND event_id IN (#{ids_sql(event_ids)})
+          SQL
+        end
+      end
+
+      def with_delivery_status_refresh(event_ids)
+        ids = Array(event_ids).compact
+        return if ids.empty?
+
+        connection.transaction do
+          yield
+          refresh_event_statuses!(ids)
+        end
       end
 
       def refresh_event_statuses!(event_ids)
@@ -632,10 +683,7 @@ module SearchEngine
         execute(<<~SQL)
           UPDATE #{quoted_table} events
           SET status = aggregate.status,
-              processed_at = CASE
-                WHEN aggregate.status IN ('processed', 'superseded') THEN CURRENT_TIMESTAMP
-                ELSE NULL
-              END,
+              processed_at = #{aggregate_processed_at_sql},
               last_error = aggregate.last_error,
               updated_at = CURRENT_TIMESTAMP
           FROM (
@@ -655,6 +703,7 @@ module SearchEngine
                    WHEN COUNT(*) FILTER (WHERE status = 'processed') > 0 THEN 'processed'
                    ELSE 'pending'
                  END AS status,
+                 MAX(processed_at) FILTER (WHERE status IN ('processed', 'superseded')) AS terminal_processed_at,
                  (ARRAY_AGG(last_error ORDER BY updated_at DESC) FILTER (WHERE last_error IS NOT NULL))[1] AS last_error
           FROM #{quoted_delivery_table}
           WHERE event_id IN (#{event_ids_sql})
@@ -662,6 +711,63 @@ module SearchEngine
         SQL
       end
 
+      def terminal_delivery_event_status_refresh_sql(retention_s:, limit:)
+        retention_seconds = [retention_s.to_i, 0].max
+        batch_limit = global_limit_for(limit)
+
+        <<~SQL
+          WITH candidate_events AS MATERIALIZED (
+            SELECT events.id
+            FROM #{quoted_table} events
+            WHERE events.status NOT IN ('processed', 'superseded')
+              AND EXISTS (
+                SELECT 1
+                FROM #{quoted_delivery_table} deliveries
+                WHERE deliveries.event_id = events.id
+              )
+              AND NOT EXISTS (
+                SELECT 1
+                FROM #{quoted_delivery_table} deliveries
+                WHERE deliveries.event_id = events.id
+                  AND deliveries.status IN ('pending', 'processing', 'failed')
+              )
+            ORDER BY events.id ASC
+            LIMIT #{batch_limit}
+            FOR UPDATE SKIP LOCKED
+          ),
+          eligible_events AS (
+            SELECT candidate_events.id
+            FROM candidate_events
+            WHERE (
+              SELECT MAX(deliveries.processed_at)
+              FROM #{quoted_delivery_table} deliveries
+              WHERE deliveries.event_id = candidate_events.id
+                AND deliveries.status IN ('processed', 'superseded')
+            ) < (CURRENT_TIMESTAMP - interval '#{retention_seconds} seconds')
+          ),
+          aggregate AS (
+            #{event_status_aggregate_sql('SELECT id FROM eligible_events')}
+          ),
+          updated_events AS (
+            UPDATE #{quoted_table} events
+            SET status = aggregate.status,
+                processed_at = #{aggregate_processed_at_sql},
+                last_error = aggregate.last_error,
+                updated_at = CURRENT_TIMESTAMP
+            FROM aggregate
+            WHERE events.id = aggregate.event_id
+            RETURNING events.id
+          )
+          SELECT id
+          FROM updated_events
+        SQL
+      end
+
+      def aggregate_processed_at_sql
+        "CASE WHEN aggregate.status IN ('processed', 'superseded') " \
+          'THEN COALESCE(aggregate.terminal_processed_at, CURRENT_TIMESTAMP) ELSE NULL END'
+      end
+
       def select_rows(sql)
         result = connection.select_all(sql)
         return result.to_a if result.respond_to?(:to_a)
@@ -819,6 +925,15 @@ module SearchEngine
         SearchEngine.config.postgres_outbox.drain_slot_table_name
       end
 
+      def delivery_table_exists?
+        table_name = SearchEngine.config.postgres_outbox.delivery_table_name
+        if connection.respond_to?(:data_source_exists?)
+          connection.data_source_exists?(table_name)
+        else
+          connection.table_exists?(table_name)
+        end
+      end
+
       def quote(value)
         connection.quote(value)
       end

data/lib/search_engine/version.rb

@@ -3,5 +3,5 @@
 module SearchEngine
   # Current gem version.
   # @return [String]
-  VERSION = '30.1.8.17'
+  VERSION = '30.1.8.19'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: search-engine-for-typesense
 version: !ruby/object:Gem::Version
-  version: 30.1.8.17
+  version: 30.1.8.19
 platform: ruby
 authors:
 - Nikita Shkoda
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-06-08 00:00:00.000000000 Z
+date: 2026-07-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby