search-engine-for-typesense 30.1.8.7 → 30.1.8.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 77958fc8a7cb84effc7e3a4ca73b2f01e88eacc9ddead6e888c746e17b7e28d5
-  data.tar.gz: 74a046ef9f7cbdddeeed1c9fb7d9749d501b71300adc30bd0d2d771088aae189
+  metadata.gz: 41043de36398de61b1587b3beb737cebbb47712254860e8694220583e08ff4d8
+  data.tar.gz: 9d33237f590d99d65a3f457c9a69e10f39184414308d76a6c05dcd58f1c89b77
 SHA512:
-  metadata.gz: 9e6bc4e2159d41cd66f0f4f9568d503c3b40ebbbe59be2cb6709ce008589d2e2caa3028f54f3de180a1dc9ff5c37c37cab5ba049780b4583a4974018eaf7ca39
-  data.tar.gz: a478abb0104cf415d31bcd069d93eb89039acff25c081423117e965fed6d6cd2ffbc859a88601988036246e4b343633e454fda25e27999013c5363e4f824d0c9
+  metadata.gz: 19718f2365aee9d80cebd2b282e81c084e4d343177fae938ecf3c817a5cba9ee407058d690d3fe73ffd3216f451821c8492d0a68927234c10739c671396cecc6
+  data.tar.gz: 7baabe836610680f7c5cf9b06224fc50dad9f19661ccf3769d2ed732d84669045e180a93d47dbd1240e597bec10f9cd001a497124364c9595286906b8ca84215

data/README.md

@@ -217,6 +217,7 @@ SearchEngine.configure do |c|
   c.postgres_outbox.enabled = true
   c.postgres_outbox.listener_enabled = -> { Rails.env.production? }
   c.postgres_outbox.table_name = "search_engine_outbox_events"
+  c.postgres_outbox.delivery_table_name = "search_engine_outbox_deliveries"
   c.postgres_outbox.channel = "search_engine_outbox"
   c.postgres_outbox.queue_name = "search_engine"
   c.postgres_outbox.batch_size = 1000
@@ -225,6 +226,14 @@ SearchEngine.configure do |c|
 
   # Optional. Leave off when your deployment already guarantees one listener.
   c.postgres_outbox.advisory_lock = false
+
+  # Optional. Leave empty for the default single-target flow.
+  c.postgres_outbox.delivery_targets = lambda do
+    [
+      { key: :mirror_a, queue_name: :search_engine_mirror_a },
+      { key: :mirror_b, queue_name: :search_engine_mirror_b }
+    ]
+  end
 end
 ```
 
@@ -242,6 +251,8 @@ class CreateSearchEngineOutboxEvents < ActiveRecord::Migration[7.1]
 
   def change
     create_search_engine_outbox_events
+    # Required only when c.postgres_outbox.delivery_targets is configured.
+    create_search_engine_outbox_deliveries
   end
 end
 ```
@@ -278,6 +289,13 @@ end
 `record_id_sql` and `document_id_sql` are trusted migration SQL expressions. They may refer to the
 PL/pgSQL `record_data` variable, which is `NEW` for inserts/updates and `OLD` for deletes.
 
+The event table stores logical changes. When `delivery_targets` is empty, drain jobs claim those event rows
+directly and existing single-target setups do not need to create or use delivery rows. When you configure
+delivery targets, `search_engine_outbox_deliveries` stores target-specific status, retry, lock, and queue
+state for each logical event. The listener uses the drain enqueuer to materialize missing delivery rows and
+enqueue one drain job per target queue. Processors still receive event objects and return event IDs; the
+parent event status is refreshed from the aggregate delivery states.
+
 Pair triggered source models with `sync_strategy: :postgres_outbox` so Active Record callbacks do not also
 write to Typesense for the same changes:
 

data/app/search_engine/search_engine/postgres_outbox/drain_job.rb

@@ -10,25 +10,55 @@ module SearchEngine
 
       # Drain pending outbox events once when PostgreSQL outbox processing is enabled.
       # @param limit [Integer, nil] optional maximum number of events to claim
+      # @param target_key [String, Symbol, nil] optional delivery target scope
       # @return [Hash, nil]
-      def perform(limit: nil)
+      def perform(limit: nil, target_key: nil)
         return nil unless SearchEngine.config.postgres_outbox.enabled
 
         effective_limit = limit || SearchEngine.config.postgres_outbox.batch_size
-        drainer = SearchEngine::PostgresOutbox::Drainer.new
+        drainer = drainer_for(target_key)
         summary = drainer.drain_once(limit: effective_limit)
-        enqueue_continuation(limit: limit) if summary[:claimed].to_i >= effective_limit.to_i
+        enqueue_continuation(limit: limit, target_key: target_key) if summary[:claimed].to_i >= effective_limit.to_i
 
         summary
       end
 
       private
 
-      def enqueue_continuation(limit:)
+      def drainer_for(target_key)
+        return SearchEngine::PostgresOutbox::Drainer.new if target_key.nil?
+
+        SearchEngine::PostgresOutbox::Drainer.new(target_key: target_key)
+      end
+
+      def enqueue_continuation(limit:, target_key:)
+        return enqueue_target_continuation(limit: limit, target_key: target_key) unless target_key.nil?
         return self.class.perform_later if limit.nil?
 
         self.class.perform_later(limit: limit)
       end
+
+      def enqueue_target_continuation(limit:, target_key:)
+        target = delivery_target_for!(target_key)
+        job = self.class.set(queue: target.queue_name)
+        return job.perform_later(target_key: target.key) if limit.nil?
+
+        job.perform_later(target_key: target.key, limit: limit)
+      end
+
+      def delivery_target_for!(target_key)
+        normalized_key = target_key.to_s
+        target = delivery_targets.find { |candidate| candidate.key == normalized_key }
+        return target if target
+
+        raise ArgumentError, "unknown postgres outbox delivery target: #{normalized_key}"
+      end
+
+      def delivery_targets
+        configured = SearchEngine.config.postgres_outbox.delivery_targets
+        raw_targets = configured.respond_to?(:call) ? configured.call : configured
+        Array(raw_targets).map { |target| DeliveryTarget.normalize(target) }
+      end
     end
   end
 end

data/lib/generators/search_engine/postgres_outbox/templates/create_outbox_events.rb.tt

@@ -5,5 +5,6 @@ class CreateSearchEngineOutboxEvents < ActiveRecord::Migration[<%= ActiveRecord:
 
   def change
     create_search_engine_outbox_events
+    create_search_engine_outbox_deliveries
   end
 end

data/lib/search_engine/config.rb

@@ -270,6 +270,8 @@ module SearchEngine
       attr_accessor :enabled
       # @return [String] database table used by host-managed outbox events
       attr_accessor :table_name
+      # @return [String] database table used by host-managed outbox deliveries
+      attr_accessor :delivery_table_name
       # @return [String] PostgreSQL notification channel for wakeups
       attr_accessor :channel
       # @return [String] queue name used by host app job dispatch
@@ -296,10 +298,13 @@ module SearchEngine
       attr_accessor :collection_processors
       # @return [#call] retry backoff calculator receiving the attempt number
       attr_accessor :retry_backoff
+      # @return [#call] resolver returning configured delivery targets
+      attr_accessor :delivery_targets
 
       def initialize
         @enabled = false
         @table_name = 'search_engine_outbox_events'
+        @delivery_table_name = 'search_engine_outbox_deliveries'
         @channel = 'search_engine_outbox'
         @queue_name = 'search_engine'
         @batch_size = 1000
@@ -313,6 +318,7 @@ module SearchEngine
         @listener_enabled = -> { false }
         @collection_processors = {}
         @retry_backoff = ->(attempt) { [attempt.to_i, 1].max * 5 }
+        @delivery_targets = -> { [] }
       end
     end
 
@@ -863,6 +869,7 @@ module SearchEngine
       {
         enabled: postgres_outbox.enabled ? true : false,
         table_name: postgres_outbox.table_name,
+        delivery_table_name: postgres_outbox.delivery_table_name,
         channel: postgres_outbox.channel,
         queue_name: postgres_outbox.queue_name,
         batch_size: postgres_outbox.batch_size,
@@ -875,7 +882,8 @@ module SearchEngine
         advisory_lock_key: postgres_outbox.advisory_lock_key,
         listener_enabled: postgres_outbox.listener_enabled,
         collection_processors: postgres_outbox.collection_processors,
-        retry_backoff: postgres_outbox.retry_backoff
+        retry_backoff: postgres_outbox.retry_backoff,
+        delivery_targets: postgres_outbox.delivery_targets
       }
     end
 

data/lib/search_engine/postgres_outbox/delivery_target.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module SearchEngine
+  module PostgresOutbox
+    # Generic destination for PostgreSQL outbox delivery processing.
+    class DeliveryTarget
+      # @return [String] stable target identifier stored in delivery rows
+      attr_reader :key
+      # @return [String] ActiveJob queue name used to process this target
+      attr_reader :queue_name
+
+      # @param key [String, Symbol] stable target identifier
+      # @param queue_name [String, Symbol] queue name for target-specific drain jobs
+      def initialize(key:, queue_name:)
+        @key = normalize_value(key, 'key')
+        @queue_name = normalize_value(queue_name, 'queue_name')
+      end
+
+      # Normalize a configured target into a DeliveryTarget.
+      #
+      # @param value [DeliveryTarget, Hash, #key] target-like object
+      # @return [DeliveryTarget]
+      def self.normalize(value)
+        return value if value.is_a?(self)
+
+        if value.respond_to?(:to_hash)
+          hash = value.to_hash
+          return new(key: fetch_hash_value(hash, :key), queue_name: fetch_hash_value(hash, :queue_name))
+        end
+
+        if value.respond_to?(:key) && value.respond_to?(:queue_name)
+          return new(key: value.key, queue_name: value.queue_name)
+        end
+
+        raise ArgumentError, 'delivery target must be a DeliveryTarget, Hash, or target-like object'
+      end
+
+      class << self
+        private
+
+        def fetch_hash_value(hash, key)
+          return hash[key] if hash.key?(key)
+
+          hash[key.to_s]
+        end
+      end
+
+      private
+
+      def normalize_value(value, name)
+        normalized = value.to_s
+        raise ArgumentError, "#{name} must be present" if normalized.strip.empty?
+
+        normalized
+      end
+    end
+  end
+end

data/lib/search_engine/postgres_outbox/drain_enqueuer.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module SearchEngine
+  module PostgresOutbox
+    # Enqueues PostgreSQL outbox drain jobs for legacy or target-aware delivery mode.
+    class DrainEnqueuer
+      # Enqueue drain jobs for all configured targets or the legacy queue.
+      # @param limit [Integer, nil] optional maximum number of events to claim
+      # @return [void]
+      def self.enqueue_all(limit: nil)
+        new.enqueue_all(limit: limit)
+      end
+
+      # @param repository [SearchEngine::PostgresOutbox::Repository]
+      # @param drain_job [#perform_later, #set, nil] ActiveJob-compatible drain job class
+      # @param targets_resolver [#call, nil] optional delivery targets resolver
+      def initialize(repository: Repository.new, drain_job: nil, targets_resolver: nil)
+        @repository = repository
+        @drain_job = drain_job
+        @targets_resolver = targets_resolver
+      end
+
+      # Enqueue drain jobs for all configured targets or the legacy queue.
+      # @param limit [Integer, nil] optional maximum number of events to claim
+      # @return [void]
+      def enqueue_all(limit: nil)
+        targets = delivery_targets
+        return enqueue_legacy(limit: limit) if targets.empty?
+
+        repository.materialize_deliveries!
+        targets.each { |target| enqueue_target(target, limit: limit) }
+        nil
+      end
+
+      private
+
+      attr_reader :repository, :targets_resolver
+
+      def enqueue_legacy(limit:)
+        return drain_job.perform_later if limit.nil?
+
+        drain_job.perform_later(limit: limit)
+      end
+
+      def enqueue_target(target, limit:)
+        job = drain_job.set(queue: target.queue_name)
+        return job.perform_later(target_key: target.key) if limit.nil?
+
+        job.perform_later(target_key: target.key, limit: limit)
+      end
+
+      def delivery_targets
+        raw_targets = targets_resolver ? targets_resolver.call : configured_delivery_targets
+        Array(raw_targets).map { |target| DeliveryTarget.normalize(target) }
+      end
+
+      def configured_delivery_targets
+        configured = SearchEngine.config.postgres_outbox.delivery_targets
+        configured.respond_to?(:call) ? configured.call : configured
+      end
+
+      def drain_job
+        @drain_job ||= SearchEngine::PostgresOutbox::DrainJob
+      end
+    end
+  end
+end

data/lib/search_engine/postgres_outbox/drainer.rb

@@ -11,8 +11,10 @@ module SearchEngine
       # @param repository [SearchEngine::PostgresOutbox::Repository]
       # @param processor [#call]
       # @param worker_id [String, nil]
-      def initialize(repository: Repository.new, processor: EventProcessor, worker_id: nil)
-        @repository = repository
+      # @param target_key [String, Symbol, nil] optional delivery target scope
+      def initialize(repository: nil, processor: EventProcessor, worker_id: nil, target_key: nil)
+        @target_key = normalize_optional_target_key(target_key)
+        @repository = repository || Repository.new(target_key: @target_key)
         @processor = processor
         @worker_id = worker_id || default_worker_id
       end
@@ -21,7 +23,10 @@ module SearchEngine
       # @param limit [Integer]
       # @return [Hash]
       def drain_once(limit: SearchEngine.config.postgres_outbox.batch_size)
-        SearchEngine::Instrumentation.instrument('search_engine.postgres_outbox.drain', limit: limit) do |payload|
+        SearchEngine::Instrumentation.instrument(
+          'search_engine.postgres_outbox.drain',
+          drain_payload(limit)
+        ) do |payload|
           events = repository.claim_pending(limit: limit, worker_id: worker_id)
           summary = empty_summary(events)
           next summary if events.empty?
@@ -39,10 +44,10 @@ module SearchEngine
 
       private
 
-      attr_reader :repository, :processor, :worker_id
+      attr_reader :repository, :processor, :worker_id, :target_key
 
       def empty_summary(events)
-        {
+        summary = {
           claimed: events.size,
           processed: 0,
           superseded: 0,
@@ -50,6 +55,8 @@ module SearchEngine
           failed: 0,
           collections: []
         }
+        summary[:target_key] = target_key if target_key
+        summary
       end
 
       def coalesce(events)
@@ -109,7 +116,7 @@ module SearchEngine
       end
 
       def call_processor(collection, events)
-        result = processor_for(collection).call(events: events, context: { worker_id: worker_id })
+        result = processor_for(collection).call(events: events, context: processor_context)
         normalize_result(result, events)
       end
 
@@ -171,6 +178,25 @@ module SearchEngine
       def default_worker_id
         "#{Socket.gethostname}:#{$PROCESS_ID}:#{Thread.current.object_id}"
       end
+
+      def processor_context
+        context = { worker_id: worker_id }
+        context[:target_key] = target_key if target_key
+        context
+      end
+
+      def drain_payload(limit)
+        payload = { limit: limit }
+        payload[:target_key] = target_key if target_key
+        payload
+      end
+
+      def normalize_optional_target_key(value)
+        normalized = value&.to_s
+        return nil if normalized.nil? || normalized.strip.empty?
+
+        normalized
+      end
     end
   end
 end

data/lib/search_engine/postgres_outbox/event.rb

@@ -15,7 +15,9 @@ module SearchEngine
                   :operation,
                   :attempts,
                   :payload,
-                  :created_at
+                  :created_at,
+                  :delivery_id,
+                  :target_key
 
       # @param row [Hash] outbox row with string or symbol keys
       # @raise [ArgumentError] when operation is not upsert/delete
@@ -27,9 +29,11 @@ module SearchEngine
         @record_id = string_value(row, :record_id)
         @document_id = string_value(row, :document_id)
         @operation = normalize_operation(value(row, :operation))
-        @attempts = value(row, :attempts).to_i
+        @attempts = (value(row, :delivery_attempts) || value(row, :attempts)).to_i
         @payload = value(row, :payload) || {}
         @created_at = value(row, :created_at)
+        @delivery_id = value(row, :delivery_id)
+        @target_key = string_value(row, :target_key)
       end
 
       # @return [Array<String>] key used by drainer coalescing

data/lib/search_engine/postgres_outbox/listener.rb

@@ -10,6 +10,7 @@ module SearchEngine
 
       # @param connection_pool [#with_connection, nil] ActiveRecord connection pool
       # @param drain_job [#perform_later] job class used to nudge outbox draining
+      # @param drain_enqueuer [#enqueue_all, nil] enqueue coordinator for drain jobs
       # @param channel [String] PostgreSQL notification channel
       # @param wait_timeout_s [Numeric] wait timeout for LISTEN notifications
       # @param poll_interval_s [Numeric] fallback/retry polling interval
@@ -19,6 +20,7 @@ module SearchEngine
       def initialize(
         connection_pool: nil,
         drain_job: nil,
+        drain_enqueuer: nil,
         channel: SearchEngine.config.postgres_outbox.channel,
         wait_timeout_s: SearchEngine.config.postgres_outbox.listener_wait_timeout_s,
         poll_interval_s: SearchEngine.config.postgres_outbox.poll_interval_s,
@@ -28,6 +30,7 @@ module SearchEngine
       )
         @connection_pool = connection_pool
         @drain_job = drain_job
+        @drain_enqueuer = drain_enqueuer
         @channel = channel.to_s
         @wait_timeout_s = wait_timeout_s.to_f
         @poll_interval_s = poll_interval_s.to_f
@@ -191,7 +194,7 @@ module SearchEngine
       def enqueue_drain(force: false)
         return if enqueue_throttled?(force: force)
 
-        drain_job.perform_later
+        drain_enqueuer.enqueue_all
       end
 
       def enqueue_throttled?(force:)
@@ -219,6 +222,10 @@ module SearchEngine
         @drain_job ||= SearchEngine::PostgresOutbox::DrainJob
       end
 
+      def drain_enqueuer
+        @drain_enqueuer ||= SearchEngine::PostgresOutbox::DrainEnqueuer.new(drain_job: drain_job)
+      end
+
       def connection_pool
         @connection_pool ||= ::ActiveRecord::Base.connection_pool
       end

data/lib/search_engine/postgres_outbox/migration_helpers.rb

@@ -52,6 +52,53 @@ module SearchEngine
                   where: "status IN ('processed', 'superseded')"
       end
 
+      # Create the durable SearchEngine outbox deliveries table.
+      #
+      # @param table_name [String, Symbol] destination delivery table name
+      # @param events_table_name [String, Symbol] source events table name
+      # @return [void]
+      def create_search_engine_outbox_deliveries(
+        table_name: SearchEngine.config.postgres_outbox.delivery_table_name,
+        events_table_name: SearchEngine.config.postgres_outbox.table_name
+      )
+        create_table table_name do |t|
+          t.bigint :event_id, null: false
+          t.string :target_key, null: false
+          t.string :queue_name, null: false
+          t.string :status, null: false, default: 'pending'
+          t.integer :attempts, null: false, default: 0
+          t.datetime :locked_at
+          t.datetime :next_attempt_at
+          t.datetime :processed_at
+          t.string :locked_by
+          t.text :last_error
+          t.timestamps
+        end
+
+        add_index table_name,
+                  %i[event_id target_key],
+                  name: 'idx_se_outbox_deliveries_unique',
+                  unique: true
+        add_index table_name,
+                  %i[target_key status next_attempt_at id],
+                  name: 'idx_se_outbox_deliveries_pending'
+        add_index table_name,
+                  %i[target_key status event_id],
+                  name: 'idx_se_outbox_deliveries_coalescing'
+        add_index table_name,
+                  :locked_at,
+                  name: 'idx_se_outbox_deliveries_processing',
+                  where: "status = 'processing'"
+        add_index table_name,
+                  :processed_at,
+                  name: 'idx_se_outbox_deliveries_cleanup',
+                  where: "status IN ('processed', 'superseded')"
+        add_foreign_key table_name,
+                        events_table_name,
+                        column: :event_id,
+                        on_delete: :cascade
+      end
+
       # Create or replace a row-level PostgreSQL trigger that writes outbox events.
       #
       # @param table_name [String, Symbol] source table name

data/lib/search_engine/postgres_outbox/repository.rb

@@ -7,8 +7,10 @@ module SearchEngine
       ERROR_LIMIT = 1000
 
       # @param connection [Object, nil] ActiveRecord-compatible connection
-      def initialize(connection: nil)
+      # @param target_key [String, Symbol, nil] optional delivery target scope
+      def initialize(connection: nil, target_key: nil)
         @connection = connection
+        @target_key = normalize_optional_target_key(target_key)
       end
 
       # Claim pending rows for one worker and return event objects.
@@ -16,6 +18,8 @@ module SearchEngine
       # @param worker_id [String]
       # @return [Array<SearchEngine::PostgresOutbox::Event>]
       def claim_pending(limit:, worker_id:)
+        return claim_pending_deliveries(limit: limit, worker_id: worker_id) if delivery_mode?
+
         reset_stale_processing!
         rows = []
 
@@ -32,6 +36,8 @@ module SearchEngine
       # Reset timed-out processing rows to pending.
       # @return [void]
       def reset_stale_processing!
+        return reset_stale_delivery_processing! if delivery_mode?
+
         execute(<<~SQL)
           UPDATE #{quoted_table}
           SET status = 'pending',
@@ -46,12 +52,20 @@ module SearchEngine
       # @param event_ids [Array<Integer, String>]
       # @return [void]
       def mark_processed!(event_ids)
+        if delivery_mode?
+          return update_delivery_status!(event_ids, 'processed', extra: 'processed_at = CURRENT_TIMESTAMP')
+        end
+
         update_status!(event_ids, 'processed', extra: 'processed_at = CURRENT_TIMESTAMP')
       end
 
       # @param event_ids [Array<Integer, String>]
       # @return [void]
       def mark_superseded!(event_ids)
+        if delivery_mode?
+          return update_delivery_status!(event_ids, 'superseded', extra: 'processed_at = CURRENT_TIMESTAMP')
+        end
+
         update_status!(event_ids, 'superseded', extra: 'processed_at = CURRENT_TIMESTAMP')
       end
 
@@ -61,6 +75,7 @@ module SearchEngine
       def mark_retryable!(event_ids, error:)
         ids = Array(event_ids).compact
         return if ids.empty?
+        return mark_delivery_retryable!(ids, error: error) if delivery_mode?
 
         execute(<<~SQL)
           UPDATE #{quoted_table}
@@ -81,6 +96,7 @@ module SearchEngine
       def mark_failed!(event_ids, error:)
         ids = Array(event_ids).compact
         return if ids.empty?
+        return mark_delivery_failed!(ids, error: error) if delivery_mode?
 
         execute(<<~SQL)
           UPDATE #{quoted_table}
@@ -93,8 +109,42 @@ module SearchEngine
         SQL
       end
 
+      # Create missing delivery rows for all configured delivery targets.
+      # @return [void]
+      def materialize_deliveries!
+        targets = delivery_targets
+        return if targets.empty?
+
+        execute(<<~SQL)
+          INSERT INTO #{quoted_delivery_table} (
+            event_id,
+            target_key,
+            queue_name,
+            status,
+            attempts,
+            created_at,
+            updated_at
+          )
+          SELECT outbox.id,
+                 target.target_key,
+                 target.queue_name,
+                 'pending',
+                 0,
+                 CURRENT_TIMESTAMP,
+                 CURRENT_TIMESTAMP
+          FROM #{quoted_table} outbox
+          CROSS JOIN (
+            VALUES #{delivery_target_values_sql(targets)}
+          ) AS target(target_key, queue_name)
+          WHERE outbox.status IN ('pending', 'processing', 'failed')
+          ON CONFLICT (event_id, target_key) DO NOTHING
+        SQL
+      end
+
       private
 
+      attr_reader :target_key
+
       def connection
         @connection ||= begin
           require 'active_record'
@@ -102,6 +152,34 @@ module SearchEngine
         end
       end
 
+      def claim_pending_deliveries(limit:, worker_id:)
+        materialize_deliveries!
+        reset_stale_delivery_processing!
+        rows = []
+
+        connection.transaction do
+          rows = select_rows(delivery_claim_select_sql(limit.to_i))
+          delivery_ids = rows.map { |row| row_value(row, :delivery_id) }
+          execute(delivery_supersede_older_pending_sql(rows)) unless rows.empty?
+          execute(delivery_claim_update_sql(delivery_ids, worker_id)) unless delivery_ids.empty?
+        end
+
+        rows.map { |row| Event.new(row) }
+      end
+
+      def reset_stale_delivery_processing!
+        execute(<<~SQL)
+          UPDATE #{quoted_delivery_table}
+          SET status = 'pending',
+              locked_at = NULL,
+              locked_by = NULL,
+              updated_at = CURRENT_TIMESTAMP
+          WHERE target_key = #{quote(target_key)}
+            AND status = 'processing'
+            AND locked_at < (CURRENT_TIMESTAMP - interval '#{processing_timeout_s} seconds')
+        SQL
+      end
+
       def claim_select_sql(limit)
         <<~SQL
           WITH ranked_pending AS (
@@ -143,6 +221,56 @@ module SearchEngine
         SQL
       end
 
+      def delivery_claim_select_sql(limit)
+        <<~SQL
+          WITH ranked_pending AS (
+            SELECT deliveries.id AS delivery_id,
+                   events.id AS event_id,
+                   ROW_NUMBER() OVER (
+                     PARTITION BY deliveries.target_key, events.collection, events.document_id
+                     ORDER BY events.id DESC, deliveries.id DESC
+                   ) AS row_number
+            FROM #{quoted_delivery_table} deliveries
+            INNER JOIN #{quoted_table} events
+              ON events.id = deliveries.event_id
+            WHERE deliveries.target_key = #{quote(target_key)}
+              AND deliveries.status = 'pending'
+          ),
+          latest_due AS (
+            SELECT deliveries.id
+            FROM #{quoted_delivery_table} deliveries
+            INNER JOIN ranked_pending
+              ON ranked_pending.delivery_id = deliveries.id
+            WHERE ranked_pending.row_number = 1
+              AND (deliveries.next_attempt_at IS NULL OR deliveries.next_attempt_at <= CURRENT_TIMESTAMP)
+            ORDER BY deliveries.id ASC
+            LIMIT #{limit}
+          )
+          SELECT events.*,
+                 deliveries.id AS delivery_id,
+                 deliveries.target_key,
+                 deliveries.attempts AS delivery_attempts
+          FROM #{quoted_delivery_table} deliveries
+          INNER JOIN #{quoted_table} events
+            ON events.id = deliveries.event_id
+          INNER JOIN latest_due
+            ON latest_due.id = deliveries.id
+          ORDER BY deliveries.id ASC
+          FOR UPDATE SKIP LOCKED
+        SQL
+      end
+
+      def delivery_claim_update_sql(delivery_ids, worker_id)
+        <<~SQL
+          UPDATE #{quoted_delivery_table}
+          SET status = 'processing',
+              locked_at = CURRENT_TIMESTAMP,
+              locked_by = #{quote(worker_id)},
+              updated_at = CURRENT_TIMESTAMP
+          WHERE id IN (#{ids_sql(delivery_ids)})
+        SQL
+      end
+
       def supersede_older_pending_sql(rows)
         <<~SQL
           UPDATE #{quoted_table} older
@@ -161,6 +289,43 @@ module SearchEngine
         SQL
       end
 
+      def delivery_supersede_older_pending_sql(rows)
+        <<~SQL
+          WITH updated_deliveries AS (
+            UPDATE #{quoted_delivery_table} older_deliveries
+            SET status = 'superseded',
+                processed_at = CURRENT_TIMESTAMP,
+                locked_at = NULL,
+                locked_by = NULL,
+                updated_at = CURRENT_TIMESTAMP
+            FROM #{quoted_table} older_events,
+                 (
+                   VALUES #{delivery_coalesce_values_sql(rows)}
+                 ) AS latest(target_key, collection, document_id, event_id, delivery_id)
+            WHERE older_deliveries.event_id = older_events.id
+              AND older_deliveries.status = 'pending'
+              AND older_deliveries.target_key = latest.target_key
+              AND older_events.collection = latest.collection
+              AND older_events.document_id = latest.document_id
+              AND older_events.id < latest.event_id
+            RETURNING older_deliveries.event_id
+          ),
+          aggregate AS (
+            #{event_status_aggregate_sql('SELECT event_id FROM updated_deliveries')}
+          )
+          UPDATE #{quoted_table} events
+          SET status = aggregate.status,
+              processed_at = CASE
+                WHEN aggregate.status IN ('processed', 'superseded') THEN CURRENT_TIMESTAMP
+                ELSE NULL
+              END,
+              last_error = aggregate.last_error,
+              updated_at = CURRENT_TIMESTAMP
+          FROM aggregate
+          WHERE events.id = aggregate.event_id
+        SQL
+      end
+
       def update_status!(event_ids, status, extra:)
         ids = Array(event_ids).compact
         return if ids.empty?
@@ -176,6 +341,90 @@ module SearchEngine
         SQL
       end
 
+      def update_delivery_status!(event_ids, status, extra:)
+        ids = Array(event_ids).compact
+        return if ids.empty?
+
+        execute(<<~SQL)
+          UPDATE #{quoted_delivery_table}
+          SET status = #{quote(status)},
+              #{extra},
+              locked_at = NULL,
+              locked_by = NULL,
+              updated_at = CURRENT_TIMESTAMP
+          WHERE target_key = #{quote(target_key)}
+            AND event_id IN (#{ids_sql(ids)})
+        SQL
+        refresh_event_statuses!(ids)
+      end
+
+      def mark_delivery_retryable!(event_ids, error:)
+        execute(<<~SQL)
+          UPDATE #{quoted_delivery_table}
+          SET attempts = attempts + 1,
+              status = CASE WHEN attempts + 1 >= #{max_attempts} THEN 'failed' ELSE 'pending' END,
+              next_attempt_at = CURRENT_TIMESTAMP + #{retry_interval_case_sql},
+              locked_at = NULL,
+              locked_by = NULL,
+              last_error = #{quote(truncate_error(error))},
+              updated_at = CURRENT_TIMESTAMP
+          WHERE target_key = #{quote(target_key)}
+            AND event_id IN (#{ids_sql(event_ids)})
+        SQL
+        refresh_event_statuses!(event_ids)
+      end
+
+      def mark_delivery_failed!(event_ids, error:)
+        execute(<<~SQL)
+          UPDATE #{quoted_delivery_table}
+          SET status = 'failed',
+              locked_at = NULL,
+              locked_by = NULL,
+              last_error = #{quote(truncate_error(error))},
+              updated_at = CURRENT_TIMESTAMP
+          WHERE target_key = #{quote(target_key)}
+            AND event_id IN (#{ids_sql(event_ids)})
+        SQL
+        refresh_event_statuses!(event_ids)
+      end
+
+      def refresh_event_statuses!(event_ids)
+        ids = Array(event_ids).compact
+        return if ids.empty?
+
+        execute(<<~SQL)
+          UPDATE #{quoted_table} events
+          SET status = aggregate.status,
+              processed_at = CASE
+                WHEN aggregate.status IN ('processed', 'superseded') THEN CURRENT_TIMESTAMP
+                ELSE NULL
+              END,
+              last_error = aggregate.last_error,
+              updated_at = CURRENT_TIMESTAMP
+          FROM (
+            #{event_status_aggregate_sql(ids_sql(ids))}
+          ) aggregate
+          WHERE events.id = aggregate.event_id
+        SQL
+      end
+
+      def event_status_aggregate_sql(event_ids_sql)
+        <<~SQL.chomp
+          SELECT event_id,
+                 CASE
+                   WHEN COUNT(*) FILTER (WHERE status = 'failed') > 0 THEN 'failed'
+                   WHEN COUNT(*) FILTER (WHERE status IN ('pending', 'processing')) > 0 THEN 'pending'
+                   WHEN COUNT(*) FILTER (WHERE status = 'superseded') = COUNT(*) THEN 'superseded'
+                   WHEN COUNT(*) FILTER (WHERE status = 'processed') > 0 THEN 'processed'
+                   ELSE 'pending'
+                 END AS status,
+                 (ARRAY_AGG(last_error ORDER BY updated_at DESC) FILTER (WHERE last_error IS NOT NULL))[1] AS last_error
+          FROM #{quoted_delivery_table}
+          WHERE event_id IN (#{event_ids_sql})
+          GROUP BY event_id
+        SQL
+      end
+
       def select_rows(sql)
         result = connection.select_all(sql)
         return result.to_a if result.respond_to?(:to_a)
@@ -201,10 +450,32 @@ module SearchEngine
         end.join(', ')
       end
 
+      def delivery_coalesce_values_sql(rows)
+        rows.map do |row|
+          target = row_value(row, :target_key)
+          collection = row_value(row, :collection)
+          document_id = row_value(row, :document_id)
+          event_id = row_value(row, :id)
+          delivery_id = row_value(row, :delivery_id)
+
+          "(#{quote(target)}, #{quote(collection)}, #{quote(document_id)}, #{quote(event_id)}, #{quote(delivery_id)})"
+        end.join(', ')
+      end
+
+      def delivery_target_values_sql(targets)
+        targets.map do |target|
+          "(#{quote(target.key)}, #{quote(target.queue_name)})"
+        end.join(', ')
+      end
+
       def quoted_table
         connection.quote_table_name(SearchEngine.config.postgres_outbox.table_name)
       end
 
+      def quoted_delivery_table
+        connection.quote_table_name(SearchEngine.config.postgres_outbox.delivery_table_name)
+      end
+
       def quote(value)
         connection.quote(value)
       end
@@ -213,6 +484,23 @@ module SearchEngine
         row[key] || row[key.to_s]
       end
 
+      def delivery_mode?
+        !target_key.nil?
+      end
+
+      def normalize_optional_target_key(value)
+        normalized = value&.to_s
+        return nil if normalized.nil? || normalized.strip.empty?
+
+        normalized
+      end
+
+      def delivery_targets
+        configured = SearchEngine.config.postgres_outbox.delivery_targets
+        raw_targets = configured.respond_to?(:call) ? configured.call : configured
+        Array(raw_targets).map { |target| DeliveryTarget.normalize(target) }
+      end
+
       def max_attempts
         SearchEngine.config.postgres_outbox.max_attempts.to_i
       end

data/lib/search_engine/postgres_outbox.rb

@@ -1,10 +1,12 @@
 # frozen_string_literal: true
 
 require 'search_engine/postgres_outbox/event'
+require 'search_engine/postgres_outbox/delivery_target'
 require 'search_engine/postgres_outbox/processor_result'
 require 'search_engine/postgres_outbox/repository'
 require 'search_engine/postgres_outbox/event_processor'
 require 'search_engine/postgres_outbox/drainer'
+require 'search_engine/postgres_outbox/drain_enqueuer'
 require 'search_engine/postgres_outbox/listener'
 require 'search_engine/postgres_outbox/migration_helpers'
 

data/lib/search_engine/result.rb

@@ -93,6 +93,7 @@ module SearchEngine
           next unless entry[:document]
 
           obj = hydrate(entry[:document])
+          attach_curation!(obj, entry)
           attach_highlighting!(obj, entry)
           attach_geo_distance!(obj, entry)
           hydrated << obj
@@ -295,6 +296,15 @@ module SearchEngine
       end
     end
 
+    # Per-hit curation mixin: added onto hydrated objects when Typesense
+    # returns curated metadata for the hit.
+    module HitCuration
+      # @return [Boolean] whether Typesense marked this search hit as curated
+      def curated_hit?
+        instance_variable_get(:@__se_curated_hit__) == true
+      end
+    end
+
     def parse_facets
       @__facets_parsed_memo || {}.freeze
     end
@@ -401,6 +411,7 @@ module SearchEngine
 
           obj = hydrate(doc)
           sym_sub = symbolize_hit(sub)
+          attach_curation!(obj, sym_sub)
           attach_highlighting!(obj, sym_sub)
           attach_geo_distance!(obj, sym_sub)
           hydrated << obj
@@ -546,6 +557,18 @@ module SearchEngine
       obj
     end
 
+    def attach_curation!(obj, hit_entry)
+      return obj unless hit_entry.key?(:curated)
+
+      value = hit_entry[:curated]
+      curated = value == true || value.to_s == 'true'
+      obj.extend(HitCuration) unless obj.singleton_class.included_modules.include?(HitCuration)
+      obj.instance_variable_set(:@__se_curated_hit__, curated)
+      obj
+    rescue StandardError
+      obj
+    end
+
     def attach_geo_distance!(obj, hit_entry)
       raw_geo = hit_entry[:geo_distance_meters]
       return obj unless raw_geo.is_a?(Hash) && !raw_geo.empty?

data/lib/search_engine/version.rb

@@ -3,5 +3,5 @@
 module SearchEngine
   # Current gem version.
   # @return [String]
-  VERSION = '30.1.8.7'
+  VERSION = '30.1.8.9'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: search-engine-for-typesense
 version: !ruby/object:Gem::Version
-  version: 30.1.8.7
+  version: 30.1.8.9
 platform: ruby
 authors:
 - Nikita Shkoda
@@ -182,6 +182,8 @@ files:
 - lib/search_engine/otel.rb
 - lib/search_engine/partitioner.rb
 - lib/search_engine/postgres_outbox.rb
+- lib/search_engine/postgres_outbox/delivery_target.rb
+- lib/search_engine/postgres_outbox/drain_enqueuer.rb
 - lib/search_engine/postgres_outbox/drainer.rb
 - lib/search_engine/postgres_outbox/event.rb
 - lib/search_engine/postgres_outbox/event_processor.rb