search-engine-for-typesense 30.9.1 → 30.9.3

data/lib/search_engine/postgres_outbox/materialization_cursor.rb

@@ -5,11 +5,14 @@ require 'json'
 
 module SearchEngine
   module PostgresOutbox
-    # Read under the outbox materialization lock. One page follows new event IDs;
-    # the other sweeps backwards to revisit late commits, due retries and existing events.
+    # Scan under the materialization lock. Keep moving through history even when new IDs arrive faster.
     class MaterializationCursor
+      include DatabaseQueries
+
       PAGE_SIZE = 1000
 
+      attr_reader :checkpoint_ids
+
       def initialize(connection, events_table, cursor_table, target_keys)
         @connection = connection
         @events = events_table
@@ -21,7 +24,7 @@ module SearchEngine
       # @return [Array<Integer>] event IDs to check against the configured delivery targets
       def candidate_ids(limit:)
         @page_size = [PAGE_SIZE, limit].min
-        state = @connection.select_all(<<~SQL).first
+        state = select_rows(<<~SQL).first
           SELECT forward_id, backfill_through_id FROM #{@table} WHERE target_scope = #{@scope}
         SQL
         if state
@@ -34,18 +37,27 @@ module SearchEngine
         # Recheck a recent ID range for concurrent inserts that commit behind the watermark.
         # Even for dense IDs, at least half of the page remains available for new events.
         @forward_page = event_ids("id > #{@forward_id - @page_size / 2} ORDER BY id ASC")
-        @backfill_page = event_ids("id <= #{@backfill_through_id} ORDER BY id DESC")
-        @forward_page | @backfill_page
+        # A returning target can be far behind. Reserve some of the second page for the current head.
+        @recent_page = if @forward_page.size == @page_size
+                         event_ids("id > #{@forward_page.last} ORDER BY id DESC", [@page_size / 2, 1].max)
+                       else
+                         []
+                       end
+        @backfill_limit = [@page_size - @recent_page.size, 1].max
+        @backfill_page = event_ids("id <= #{@backfill_through_id} ORDER BY id DESC", @backfill_limit)
+        @checkpoint_ids = @forward_page | @backfill_page
+        @checkpoint_ids | @recent_page
       end
 
       # Save progress in the same transaction as the new deliveries.
       def advance!
         forward = [@forward_page.last || @forward_id, @forward_id].max
         # Freeze the upper boundary until the backwards sweep reaches the beginning.
-        backfill = @backfill_page.size < @page_size ? forward : @backfill_page.last - 1
+        head = [forward, @recent_page.first || forward].max
+        backfill = @backfill_page.size < @backfill_limit ? head : @backfill_page.last - 1
         return if forward == @forward_id && backfill == @backfill_through_id
 
-        @connection.execute(<<~SQL)
+        connection.execute(<<~SQL)
           UPDATE #{@table} SET forward_id = #{forward}, backfill_through_id = #{backfill}
           WHERE target_scope = #{@scope}
         SQL
@@ -53,17 +65,19 @@ module SearchEngine
 
       private
 
+      attr_reader :connection
+
       def initialize_positions
-        head = Integer(@connection.select_value("SELECT COALESCE(MAX(id), 0) FROM #{@events}"))
-        @connection.execute(<<~SQL)
+        head = Integer(select_rows("SELECT COALESCE(MAX(id), 0) AS head FROM #{@events}").first.fetch('head'))
+        connection.execute(<<~SQL)
           INSERT INTO #{@table} (target_scope, forward_id, backfill_through_id)
           VALUES (#{@scope}, #{head}, #{head})
         SQL
         [head, head]
       end
 
-      def event_ids(order)
-        @connection.select_all("SELECT id FROM #{@events} WHERE #{order} LIMIT #{@page_size}").map do |row|
+      def event_ids(order, limit = @page_size)
+        select_rows("SELECT id FROM #{@events} WHERE #{order} LIMIT #{limit}").map do |row|
           Integer(row.fetch('id'))
         end
       end

data/lib/search_engine/postgres_outbox/repository.rb

@@ -7,6 +7,8 @@ module SearchEngine
   module PostgresOutbox
     # Raw SQL repository for host-managed PostgreSQL outbox rows.
     class Repository
+      include DatabaseQueries
+
       # Keep expensive duplicate-key ranking bounded while leaving headroom for hot keys.
       CLAIM_CANDIDATE_OVERSAMPLE = 4
       MATERIALIZATION_LOCK_NAMESPACE = 0x53454F42 # SEOB
@@ -28,6 +30,12 @@ module SearchEngine
         @monotonic_clock = monotonic_clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
       end
 
+      # Single-target claims issue a fresh owner even when a caller reuses its worker id.
+      # Target-scoped repositories accept the caller's per-attempt owner.
+      def assigns_event_lease_owner?
+        !delivery_mode?
+      end
+
       # Claim pending rows for one worker and return event objects.
       # @param limit [Integer]
       # @param worker_id [String]
@@ -36,16 +44,15 @@ module SearchEngine
         return claim_pending_deliveries(limit: limit, worker_id: worker_id) if delivery_mode?
 
         reset_stale_processing!
-        rows = []
-
         connection.transaction do
           rows = select_rows(claim_select_sql(limit))
+          next [] if rows.empty?
+
+          execute(supersede_older_pending_sql(rows))
           ids = rows.map { |row| row_value(row, :id) }
-          execute(supersede_older_pending_sql(rows)) unless rows.empty?
-          execute(claim_update_sql(ids, worker_id)) unless ids.empty?
+          owner = Event.new_lease_owner(worker_id)
+          select_rows(claim_update_sql(ids, owner)).map { |row| Event.new(row) }.sort_by { |event| event.id.to_i }
         end
-
-        rows.map { |row| Event.new(row) }
       end
 
       # Reset timed-out processing rows to pending.
@@ -71,62 +78,37 @@ module SearchEngine
         SQL
       end
 
-      # @param events [Array<SearchEngine::PostgresOutbox::Event, Integer, String>]
+      # @param events [Array<SearchEngine::PostgresOutbox::Event>]
       # @return [Array<Integer, String>] event ids whose current lease was acknowledged
       def mark_processed!(events)
-        return update_delivery_status!(events, 'processed', extra: PROCESSED_AT_ASSIGNMENT) if delivery_mode?
-
-        update_status!(events, 'processed', extra: PROCESSED_AT_ASSIGNMENT)
+        acknowledge!(events) { "status = 'processed', #{PROCESSED_AT_ASSIGNMENT}" }
       end
 
-      # @param events [Array<SearchEngine::PostgresOutbox::Event, Integer, String>]
+      # @param events [Array<SearchEngine::PostgresOutbox::Event>]
       # @return [Array<Integer, String>] event ids whose current lease was acknowledged
       def mark_superseded!(events)
-        return update_delivery_status!(events, 'superseded', extra: PROCESSED_AT_ASSIGNMENT) if delivery_mode?
-
-        update_status!(events, 'superseded', extra: PROCESSED_AT_ASSIGNMENT)
+        acknowledge!(events) { "status = 'superseded', #{PROCESSED_AT_ASSIGNMENT}" }
       end
 
-      # @param events [Array<SearchEngine::PostgresOutbox::Event, Integer, String>]
+      # @param events [Array<SearchEngine::PostgresOutbox::Event>]
       # @param error [Exception, String]
       # @return [Array<Integer, String>] event ids whose current lease was acknowledged
       def mark_retryable!(events, error:)
-        ids = event_ids(events)
-        return [] if ids.empty?
-        return mark_delivery_retryable!(events, error: error) if delivery_mode?
-
-        execute(<<~SQL)
-          UPDATE #{quoted_table}
-          SET attempts = attempts + 1,
-              status = CASE WHEN attempts + 1 >= #{max_attempts} THEN 'failed' ELSE 'pending' END,
-              next_attempt_at = CURRENT_TIMESTAMP + #{retry_interval_case_sql},
-              locked_at = NULL,
-              locked_by = NULL,
-              last_error = #{quote(truncate_error(error))},
-              updated_at = CURRENT_TIMESTAMP
-          WHERE id IN (#{ids_sql(ids)})
-        SQL
-        ids
+        acknowledge!(events) do
+          <<~SQL.chomp
+            attempts = attempts + 1,
+            status = CASE WHEN attempts + 1 >= #{max_attempts} THEN 'failed' ELSE 'pending' END,
+            next_attempt_at = CURRENT_TIMESTAMP + #{retry_interval_case_sql},
+            last_error = #{quote(truncate_error(error))}
+          SQL
+        end
       end
 
-      # @param events [Array<SearchEngine::PostgresOutbox::Event, Integer, String>]
+      # @param events [Array<SearchEngine::PostgresOutbox::Event>]
       # @param error [Exception, String]
       # @return [Array<Integer, String>] event ids whose current lease was acknowledged
       def mark_failed!(events, error:)
-        ids = event_ids(events)
-        return [] if ids.empty?
-        return mark_delivery_failed!(events, error: error) if delivery_mode?
-
-        execute(<<~SQL)
-          UPDATE #{quoted_table}
-          SET status = 'failed',
-              locked_at = NULL,
-              locked_by = NULL,
-              last_error = #{quote(truncate_error(error))},
-              updated_at = CURRENT_TIMESTAMP
-          WHERE id IN (#{ids_sql(ids)})
-        SQL
-        ids
+        acknowledge!(events) { "status = 'failed', last_error = #{quote(truncate_error(error))}" }
       end
 
       # Refresh the lock timestamp only for deliveries still owned by these claims.
@@ -198,7 +180,7 @@ module SearchEngine
           event_ids = cursor&.candidate_ids(limit: scan_limit)
           preview = select_rows(delivery_materialization_select_sql(limit, targets, event_ids: event_ids, lock: false))
           rows = preview.empty? ? [] : materialize_batch(limit, targets, preview)
-          if cursor && rows.size == preview.size && materialization_page_drained?(rows, event_ids, limit)
+          if cursor && rows.size == preview.size && page_drained?(rows, event_ids, limit, cursor, targets)
             cursor.advance!
           end
           rows
@@ -414,14 +396,23 @@ module SearchEngine
       end
 
       # When a quota may have omitted matches, revisit the page after these deliveries commit.
-      def materialization_page_drained?(rows, event_ids, limit)
+      def page_drained?(rows, event_ids, limit, cursor, targets)
         return true if rows.empty? || rows.size == event_ids.size
 
         batch_size = global_limit_for(limit)
-        return rows.size < batch_size unless collection_limited_batch?(limit)
-
-        per_collection = SearchEngine.config.postgres_outbox.normalized_batch_sizes.values
-        rows.size < [batch_size, *per_collection].min
+        if collection_limited_batch?(limit)
+          quotas = SearchEngine.config.postgres_outbox.normalized_batch_sizes.values
+          batch_size = [batch_size, *quotas].min
+        end
+        return true if rows.size < batch_size
+        return false if event_ids.size == cursor.checkpoint_ids.size
+
+        # A moving head can fill a quota on every poll. Only work on the saved pages must hold progress.
+        select_rows(<<~SQL).empty?
+          SELECT 1 FROM #{quoted_table} outbox
+          WHERE #{materialization_filter_sql(targets, cursor.checkpoint_ids)}
+          LIMIT 1
+        SQL
       end
 
       # Revalidate the exact preview under parent locks before locking any deliveries.
@@ -874,7 +865,7 @@ module SearchEngine
             ORDER BY ranked_candidate_events.id ASC
             LIMIT #{limit}
           )
-          SELECT outbox.*
+          SELECT outbox.id, outbox.collection, outbox.document_id
           FROM #{quoted_table} outbox
           INNER JOIN latest_due
             ON latest_due.id = outbox.id
@@ -892,6 +883,7 @@ module SearchEngine
               updated_at = CURRENT_TIMESTAMP
           WHERE id IN (#{ids_sql(ids)})
             AND status = 'pending'
+          RETURNING *, locked_by AS event_lease_owner
         SQL
       end
 
@@ -1087,58 +1079,29 @@ module SearchEngine
         refresh_event_statuses!(parent_ids)
       end
 
-      def update_status!(events, status, extra:)
-        ids = event_ids(events)
-        return [] if ids.empty?
+      def acknowledge!(events)
+        events = Array(events).compact
+        return [] if events.empty?
 
-        execute(<<~SQL)
-          UPDATE #{quoted_table}
-          SET status = #{quote(status)},
-              #{extra},
-              locked_at = NULL,
-              locked_by = NULL,
-              updated_at = CURRENT_TIMESTAMP
-          WHERE id IN (#{ids_sql(ids)})
-        SQL
-        ids
-      end
+        assignments = "#{yield}, locked_at = NULL, locked_by = NULL"
+        return mutate_claimed_deliveries!(events, assignments: assignments) if delivery_mode?
 
-      def update_delivery_status!(events, status, extra:)
-        mutate_claimed_deliveries!(
-          events,
-          assignments: <<~SQL.chomp
-            status = #{quote(status)},
-            #{extra},
-            locked_at = NULL,
-            locked_by = NULL
-          SQL
-        )
-      end
+        claims = events.map do |event|
+          unless event.is_a?(Event) && event.id && event.delivery_id.nil? && !event.event_lease_owner.to_s.empty?
+            raise ArgumentError, 'single-target mutations require Event objects returned by claim_pending'
+          end
 
-      def mark_delivery_retryable!(events, error:)
-        mutate_claimed_deliveries!(
-          events,
-          assignments: <<~SQL.chomp
-            attempts = attempts + 1,
-            status = CASE WHEN attempts + 1 >= #{max_attempts} THEN 'failed' ELSE 'pending' END,
-            next_attempt_at = CURRENT_TIMESTAMP + #{retry_interval_case_sql},
-            locked_at = NULL,
-            locked_by = NULL,
-            last_error = #{quote(truncate_error(error))}
-          SQL
-        )
-      end
+          "(CAST(#{quote(event.id)} AS bigint), #{quote(event.event_lease_owner)})"
+        end
 
-      def mark_delivery_failed!(events, error:)
-        mutate_claimed_deliveries!(
-          events,
-          assignments: <<~SQL.chomp
-            status = 'failed',
-            locked_at = NULL,
-            locked_by = NULL,
-            last_error = #{quote(truncate_error(error))}
-          SQL
-        )
+        rows = select_rows(<<~SQL)
+          UPDATE #{quoted_table}
+          SET #{assignments}, updated_at = CURRENT_TIMESTAMP
+          WHERE status = 'processing'
+            AND (id, locked_by) IN (VALUES #{claims.join(', ')})
+          RETURNING id
+        SQL
+        rows.map { |row| row_value(row, :id) }
       end
 
       def mutate_claimed_deliveries!(events, assignments:, refresh_parent: true)
@@ -1289,13 +1252,6 @@ module SearchEngine
           'THEN COALESCE(aggregate.terminal_processed_at, CURRENT_TIMESTAMP) ELSE NULL END'
       end
 
-      def select_rows(sql)
-        result = connection.select_all(sql)
-        return result.to_a if result.respond_to?(:to_a)
-
-        Array(result)
-      end
-
       def execute(sql)
         connection.execute(sql)
       end
@@ -1304,12 +1260,6 @@ module SearchEngine
         ids.map { |id| quote(id) }.join(', ')
       end
 
-      def event_ids(events)
-        Array(events).compact.map do |event|
-          event.respond_to?(:id) ? event.id : event
-        end
-      end
-
       def delivery_claims(events)
         Array(events).compact.map do |event|
           unless event.respond_to?(:delivery_id) && event.respond_to?(:delivery_lease_owner)
@@ -1596,7 +1546,7 @@ module SearchEngine
             ORDER BY id ASC
             LIMIT #{collection_limited_candidate_limit}
           )
-          SELECT outbox.*
+          SELECT outbox.id, outbox.collection, outbox.document_id
           FROM #{quoted_table} outbox
           INNER JOIN selected_due
             ON selected_due.id = outbox.id

data/lib/search_engine/postgres_outbox.rb

@@ -3,6 +3,7 @@
 require 'search_engine/postgres_outbox/event'
 require 'search_engine/postgres_outbox/delivery_target'
 require 'search_engine/postgres_outbox/processor_result'
+require 'search_engine/postgres_outbox/database_queries'
 require 'search_engine/postgres_outbox/materialization_cursor'
 require 'search_engine/postgres_outbox/repository'
 require 'search_engine/postgres_outbox/event_processor'
@@ -14,5 +15,6 @@ require 'search_engine/postgres_outbox/migration_helpers'
 module SearchEngine
   # PostgreSQL-backed outbox integration points.
   module PostgresOutbox
+    autoload :DrainJob, 'search_engine/postgres_outbox/drain_job'
   end
 end

data/lib/search_engine/registry.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'active_support/inflector'
+
 # Internal SearchEngine registry and APIs for model mapping.
 module SearchEngine
   # Internal registry for mapping Typesense collection names to model classes.
@@ -73,40 +75,22 @@ module SearchEngine
     # Resolve the model class for a given Typesense collection name.
     #
     # @param name [#to_s]
-    # @return [Class]
-    # @raise [ArgumentError] when the collection is not registered
+    # @yield when no model is registered after autoload
+    # @return [Class, Object] registered model or the value of the supplied block
+    # @raise [ArgumentError] when the collection is not registered and no block is supplied
     def collection_for(name)
       normalized_name = name.to_s
       klass = Registry.mapping[normalized_name]
-      return klass if klass
-
-      # Lazy autoload fallback: try to resolve a namespaced model constant
-      # based on the collection name and trigger its registration.
-      begin
-        begin
-          require 'active_support/inflector'
-        rescue StandardError
-          nil
-        end
-
-        demod = if defined?(ActiveSupport::Inflector)
-                  ActiveSupport::Inflector.classify(normalized_name)
-                else
-                  # Minimal classify fallback: singularize by dropping trailing 's' and camelize tokens
-                  base = normalized_name.end_with?('s') ? normalized_name[0..-2] : normalized_name
-                  base.split('_').map { |p| p[0] ? p[0].upcase + p[1..] : '' }.join
-                end
-
-        const_name = "SearchEngine::#{demod}"
-        # Trigger autoload; ignore when constant is missing
-        Object.const_get(const_name)
-      rescue NameError
-        # constant not found; proceed to error
+      if klass
+        # A class can call `collection` before the rest of its file has loaded.
+        # Let Ruby finish (or retry) its autoload before using its identity strategy.
+        load_model_constant(Object, klass.name) if klass.name
+      else
+        load_model_constant(self, ActiveSupport::Inflector.classify(normalized_name))
       end
-
-      # Re-check after potential autoload/registration
       klass = Registry.mapping[normalized_name]
       return klass if klass
+      return yield if block_given?
 
       message = 'Unregistered collection: ' \
                 "'#{normalized_name}'. " \
@@ -117,6 +101,21 @@ module SearchEngine
 
     private
 
+    def load_model_constant(namespace, name)
+      name.split('::').each do |part|
+        break unless namespace.is_a?(Module) && collection_constant_defined?(namespace, part)
+
+        namespace = namespace.const_get(part, false)
+      end
+    end
+
+    def collection_constant_defined?(namespace, part)
+      namespace.const_defined?(part, false)
+    rescue NameError
+      # Physical collection names can contain characters that cannot appear in Ruby constants.
+      false
+    end
+
     def mapping_idempotent?(current, new_klass)
       return false unless current
 

data/lib/search_engine/relation/compiler.rb

@@ -91,13 +91,13 @@ module SearchEngine
           compiled = SearchEngine::Compiler.compile(ast_nodes, klass: @klass)
           return nil if compiled.to_s.empty?
 
-          return compiled
+          return map_join_tokens(compiled)
         end
 
         fragments = Array(@state[:filters])
         return nil if fragments.empty?
 
-        fragments.join(' && ')
+        map_join_tokens(fragments.join(' && '))
       end
 
       # Compile sort_by from normalized order entries.
@@ -276,16 +276,10 @@ module SearchEngine
       end
 
       def add_pagination_line!(lines, params)
-        page = params[:page]
-        per = params[:per_page]
-        return unless page || per
-
-        if page && per
-          lines << "  page/per: #{page}/#{per}"
-        elsif page
-          lines << "  page/per: #{page}/"
-        elsif per
-          lines << "  page/per: /#{per}"
+        if params.key?(:offset)
+          lines << "  offset/limit: #{params[:offset]}/#{params[:limit]}"
+        elsif params.key?(:page) || params.key?(:per_page)
+          lines << "  page/per: #{params[:page]}/#{params[:per_page]}"
         end
       end
 
@@ -384,7 +378,6 @@ module SearchEngine
 
       def assign_filter_by!(params, ast_nodes)
         filter_str = compiled_filter_by(ast_nodes)
-        filter_str = map_join_tokens(filter_str)
         params[:filter_by] = filter_str if filter_str
         filter_str
       end
@@ -417,17 +410,12 @@ module SearchEngine
         rewrites = join_token_rewrites
         return value if rewrites.empty?
 
-        mapped = str.dup
-        rewrites.each do |assoc, collection|
-          coll = collection.to_s
-          assoc_token = assoc.to_s
-          next if coll.strip.empty? || assoc_token.empty? || coll == assoc_token
-
-          pattern = /\$#{Regexp.escape(assoc_token)}(?=[.(])/
-          mapped = mapped.gsub(pattern, "$#{coll}")
+        names = Regexp.union(rewrites.keys.map(&:to_s))
+        # Backtick-delimited values can contain literal reference syntax.
+        str.gsub(/`[^`]*`|\$(#{names})(?=[.(])/) do |match|
+          name = Regexp.last_match(1)
+          name ? "$#{rewrites.fetch(name.to_sym)}" : match
         end
-
-        mapped
       end
 
       def join_token_rewrites
@@ -687,17 +675,14 @@ module SearchEngine
       def apply_pagination_and_hit_limits!(params)
         hits_info = {}
         pagination = compute_pagination
-        if (hl = @state[:hit_limits]) && hl[:early_limit]
-          if pagination.key?(:per_page) && pagination[:per_page].to_i > hl[:early_limit].to_i
-            pagination = pagination.merge(per_page: hl[:early_limit].to_i)
-            hits_info[:per_adjusted] = true
-          else
-            hits_info[:per_adjusted] = false
-          end
-          hits_info[:early_limit] = hl[:early_limit].to_i
+        if (early_limit = @state.dig(:hit_limits, :early_limit))
+          size_key = pagination.key?(:limit) ? :limit : :per_page
+          size = pagination[size_key]
+          hits_info[:per_adjusted] = size.to_i > early_limit
+          pagination[size_key] = early_limit if hits_info[:per_adjusted]
+          hits_info[:early_limit] = early_limit
         end
-        params[:page] = pagination[:page] if pagination.key?(:page)
-        params[:per_page] = pagination[:per_page] if pagination.key?(:per_page)
+        params.merge!(pagination)
         hits_info
       end