search-engine-for-typesense 30.1.7.0 → 30.1.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9eea6beb086957bbee9df77008a70750be3a74f6f39a53be361a3f1ae8295219
-  data.tar.gz: 0eaf79b97f1c68b7bdab350f11cbfe09eecfb4944c76fd5c49528b68e63a5f09
+  metadata.gz: '0852da3905487de90a84633634760fba6d4806d236e6738a8404d70e9122eef4'
+  data.tar.gz: fe62daa0161f71c0b5bc24da4195708a96b1687bc096383a12056aa346178710
 SHA512:
-  metadata.gz: ab9dd851d14c4a7019902afad59af281623576952ade57dd2a7df7d458caf68a3f432ac742873b52bc94fd8a88b14ef8715807d7ab3d7d7fd44cc05765424489
-  data.tar.gz: 546d394202e194705c24f1c2584b57239bca5e1084d299a3c801fc2781c480d666e47e2906992e6885c788a12ec9162c57e18b1e7eb88e65e93d2a661a2d27f3
+  metadata.gz: 6c12c7c80626ac24ea4fd5515559d354632a42a0bcd2381d61ed577aa3e24e6e60448cdf810defdf00d819e8960c93a69ce5129e27225a76c6a476e0b62115c4
+  data.tar.gz: 405565a325c099df28ba1d54d85a308f5297299a59eb40cd9f03a24ebb8306f588becc7c31be5d82cb3be05216cb443dd39a1152d417e07da2b64b97162cbe2a

data/README.md

@@ -164,6 +164,30 @@ You can control this explicitly with:
 
 If you set `SearchEngine.configure { |c| c.client = ... }`, the custom client is always used.
 
+## Async partition indexing
+
+Async partition indexing is an advanced opt-in mode for partitioned full indexing. The default remains
+inline execution. Apps with a real ActiveJob backend can use queue-backed partition execution so each
+search/index partition imports into the same blue/green physical collection before the alias is swapped.
+This mode does not require Sidekiq; use any ActiveJob backend that can run the partition jobs.
+
+```ruby
+SearchEngine.configure do |c|
+  c.indexer.partition_execution = :active_job
+  c.indexer.partition_queue_name = "search_index_partitions"
+  c.indexer.partition_timeout_s = 7_200
+end
+```
+
+Async mode is partition-based, not app-domain based. The gem enqueues one
+`SearchEngine::IndexPartitionJob` per configured partition, waits for every partition to finish, and
+only then lets the schema lifecycle swap the alias. If any partition fails or times out, the previous
+alias target remains active.
+
+Use a shared `Rails.cache` backend, or provide `c.indexer.partition_run_store`, so worker processes and
+the parent indexing process can see the same run metadata. Size the queue carefully: worker concurrency
+multiplies with any per-partition `max_parallel` setting.
+
 ## Example app
 
 See `examples/demo_shop` — demonstrates single/multi search, JOINs, grouping, presets/curation, and DX/observability. Supports offline mode via the stub client (see [Testing](https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/testing)).

data/app/search_engine/search_engine/index_partition_job.rb

@@ -8,6 +8,8 @@ module SearchEngine
   # - partition [Object] (JSON-serializable)
   # - into [String, nil]
   # - metadata [Hash]
+  # - run_id [String, nil]
+  # - partition_key [String, nil]
   class IndexPartitionJob < ::ActiveJob::Base
     queue_as do
       cfg = SearchEngine.config.indexer
@@ -31,11 +33,16 @@ module SearchEngine
     # @param partition [Object]
     # @param into [String, nil]
     # @param metadata [Hash]
+    # @param run_id [String, nil]
+    # @param partition_key [String, nil]
     # @return [void]
-    def perform(collection_class_name, partition, into: nil, metadata: {})
+    def perform(collection_class_name, partition, into: nil, metadata: {}, run_id: nil, partition_key: nil)
       payload = nil
       klass = constantize_collection!(collection_class_name)
       payload = base_payload(klass, partition: partition, into: into)
+      run_store = indexing_run_store(run_id)
+      partition_key ||= SearchEngine::IndexingRun.partition_key(partition) if run_id
+      run_store&.mark_started(run_id: run_id, partition_key: partition_key, job_id: job_id)
       instrument('search_engine.dispatcher.job_started',
                  payload.merge(queue: queue_name, job_id: job_id, metadata: metadata)
                 )
@@ -46,6 +53,7 @@ module SearchEngine
         summary = SearchEngine::Indexer.rebuild_partition!(klass, partition: partition, into: into)
       end
       duration = (monotonic_ms - started).round(1)
+      run_store&.mark_succeeded(run_id: run_id, partition_key: partition_key, summary: summary)
 
       instrument(
         'search_engine.dispatcher.job_finished',
@@ -56,6 +64,7 @@ module SearchEngine
       nil
     rescue StandardError => error
       safe_payload = payload || error_payload(error)
+      run_store&.mark_failed(run_id: run_id, partition_key: partition_key, error: error) unless retryable_error?(error)
       instrument_error(error, payload: safe_payload.merge(metadata: metadata || {}))
       raise
     end
@@ -99,6 +108,24 @@ module SearchEngine
       retry_job wait: wait_seconds
     end
 
+    def indexing_run_store(run_id)
+      return nil if run_id.nil?
+
+      SearchEngine::IndexingRunStore.resolve
+    end
+
+    def retryable_error?(error)
+      transient_error?(error) && executions.to_i < retry_policy.attempts
+    end
+
+    def transient_error?(error)
+      return true if error.is_a?(SearchEngine::Errors::Timeout)
+      return true if error.is_a?(SearchEngine::Errors::Connection)
+
+      error.is_a?(SearchEngine::Errors::Api) &&
+        SearchEngine::Indexer::RetryPolicy.transient_status?(error.status.to_i)
+    end
+
     def error_payload(error)
       {
         collection: arguments_dig_collection,

data/lib/generators/search_engine/install/templates/initializer.rb.tt

@@ -181,6 +181,24 @@ SearchEngine.configure do |c|
   # Queue name for ActiveJob dispatch. Default: 'search_index'
   # c.indexer.queue_name = 'search_index'
 
+  # Advanced partition execution for full indexing. Default: :inline
+  # Use :active_job with a real ActiveJob backend to enqueue one job per
+  # partition during blue/green indexing. Does not require Sidekiq.
+  # c.indexer.partition_execution = :inline
+
+  # Optional queue override for partition jobs. Falls back to c.indexer.queue_name.
+  # Size worker concurrency carefully: partition jobs can multiply partition max_parallel.
+  # c.indexer.partition_queue_name = nil
+
+  # Parent wait settings for async partition runs. Defaults shown.
+  # c.indexer.partition_poll_interval_s = 2
+  # c.indexer.partition_timeout_s = nil
+
+  # Operational metadata for async partition runs. Defaults shown.
+  # Use shared Rails.cache or provide a custom store visible to workers and parent.
+  # c.indexer.partition_run_store = nil
+  # c.indexer.partition_run_ttl_s = 86_400
+
   # --- Sources -------------------------------------------------------------
 
   # ActiveRecord source: default ORM batch size. Default: 2000

data/lib/search_engine/async_partition_coordinator.rb

@@ -0,0 +1,208 @@
+# frozen_string_literal: true
+
+module SearchEngine
+  # Coordinates ActiveJob-backed partition indexing and waits for run completion.
+  module AsyncPartitionCoordinator
+    module_function
+
+    # Enqueue one partition job per partition and return a lifecycle-compatible aggregate.
+    # @param klass [Class] SearchEngine collection class
+    # @param partitions [Array<Object>] logical partition values
+    # @param into [String] target physical collection name
+    # @param queue [String, nil] optional ActiveJob queue override
+    # @param timeout_s [Numeric, nil] maximum wait time in seconds
+    # @param poll_interval_s [Numeric, nil] polling interval in seconds
+    # @param store [Object, nil] optional run store
+    # @param ttl_s [Numeric, nil] run metadata TTL in seconds
+    # @return [Hash] lifecycle result with :status, :docs_total, :success_total, :failed_total, :sample_error
+    def call(klass:, partitions:, into:, queue: nil, timeout_s: nil, poll_interval_s: nil, store: nil, ttl_s: nil)
+      cfg = SearchEngine.config.indexer
+      partition_list = Array(partitions)
+      run_id = SearchEngine::IndexingRun.generate_id
+      queue_name = resolve_queue_name(queue, cfg)
+      run_store = SearchEngine::IndexingRunStore.validate!(store || SearchEngine::IndexingRunStore.resolve)
+      ttl = ttl_s || cfg.partition_run_ttl_s
+
+      snapshot = create_run(
+        store: run_store,
+        run_id: run_id,
+        klass: klass,
+        into: into,
+        partitions: partition_list,
+        ttl_s: ttl
+      )
+      instrument('search_engine.indexing_run.started', event_payload(snapshot, queue: queue_name))
+      enqueue_partitions(klass, partition_list, into: into, queue_name: queue_name, run_id: run_id)
+      instrument('search_engine.indexing_run.enqueued', event_payload(snapshot, queue: queue_name))
+
+      wait_for_completion(
+        store: run_store,
+        run_id: run_id,
+        timeout_s: timeout_s.nil? ? cfg.partition_timeout_s : timeout_s,
+        poll_interval_s: poll_interval_s.nil? ? cfg.partition_poll_interval_s : poll_interval_s
+      )
+    end
+
+    def create_run(store:, run_id:, klass:, into:, partitions:, ttl_s:)
+      store.create_run(
+        run_id: run_id,
+        collection: collection_name(klass),
+        collection_class_name: klass.name,
+        into: into,
+        partitions: partitions,
+        ttl_s: ttl_s
+      )
+    end
+    private_class_method :create_run
+
+    def enqueue_partitions(klass, partitions, into:, queue_name:, run_id:)
+      partitions.each do |partition|
+        SearchEngine::IndexPartitionJob
+          .set(queue: queue_name)
+          .perform_later(
+            klass.name,
+            partition,
+            into: into,
+            metadata: {},
+            run_id: run_id,
+            partition_key: SearchEngine::IndexingRun.partition_key(partition)
+          )
+      end
+    end
+    private_class_method :enqueue_partitions
+
+    def wait_for_completion(store:, run_id:, timeout_s:, poll_interval_s:)
+      deadline = timeout_s.nil? ? nil : monotonic_seconds + timeout_s.to_f
+
+      loop do
+        snapshot = store.snapshot(run_id: run_id)
+        result = result_for_snapshot(snapshot)
+        return finish_failure(snapshot, result) if invalid_snapshot_result?(result)
+        return finish_success(snapshot, result) if result[:status] == :ok
+        return finish_failure(snapshot, result) if failure_terminal?(snapshot)
+        return timeout_result(store, run_id, snapshot) if deadline && monotonic_seconds >= deadline
+
+        sleep([poll_interval_s.to_f, 0].max)
+      end
+    end
+    private_class_method :wait_for_completion
+
+    def result_for_snapshot(snapshot)
+      SearchEngine::IndexingRun.aggregate_result(snapshot)
+    end
+    private_class_method :result_for_snapshot
+
+    def invalid_snapshot_result?(result)
+      result[:status] == :failed && result[:sample_error].to_s.start_with?('async partition indexing run snapshot')
+    end
+    private_class_method :invalid_snapshot_result?
+
+    def finish_success(snapshot, result)
+      instrument('search_engine.indexing_run.finished', event_payload(snapshot, result: result))
+      result
+    end
+    private_class_method :finish_success
+
+    def finish_failure(snapshot, result)
+      result = failed_result(snapshot, result)
+      instrument('search_engine.indexing_run.failed', event_payload(snapshot, result: result))
+      result
+    end
+    private_class_method :finish_failure
+
+    def timeout_result(store, run_id, snapshot)
+      mark_non_terminal_failed(store, run_id, snapshot)
+      snapshot = store.snapshot(run_id: run_id) || snapshot
+      result = failed_result(
+        snapshot,
+        SearchEngine::IndexingRun.aggregate_result(snapshot),
+        sample_error: "SearchEngine async partition indexing timed out for run #{run_id}"
+      )
+      instrument('search_engine.indexing_run.failed', event_payload(snapshot, result: result))
+      result
+    end
+    private_class_method :timeout_result
+
+    def mark_non_terminal_failed(store, run_id, snapshot)
+      partitions = snapshot && snapshot[:partitions]
+      return unless partitions.is_a?(Hash)
+
+      partitions.each do |partition_key, entry|
+        status = entry[:status].to_s
+        next if %w[succeeded failed].include?(status)
+
+        store.mark_failed(
+          run_id: run_id,
+          partition_key: partition_key,
+          error: 'partition did not finish before timeout'
+        )
+      end
+    end
+    private_class_method :mark_non_terminal_failed
+
+    def failed_result(snapshot, result, sample_error: nil)
+      statuses = partition_statuses(snapshot)
+      success_total = result[:success_total].to_i
+      status = success_total.positive? || statuses.include?('succeeded') ? :partial : :failed
+
+      result.merge(
+        status: status,
+        failed_total: [result[:failed_total].to_i, failed_partition_count(statuses)].max,
+        sample_error: sample_error || result[:sample_error] || 'async partition indexing failed'
+      )
+    end
+    private_class_method :failed_result
+
+    def failure_terminal?(snapshot)
+      partition_statuses(snapshot).include?('failed')
+    end
+    private_class_method :failure_terminal?
+
+    def partition_statuses(snapshot)
+      partitions = snapshot && snapshot[:partitions]
+      return [] unless partitions.is_a?(Hash)
+
+      partitions.values.map { |entry| entry[:status].to_s }
+    end
+    private_class_method :partition_statuses
+
+    def failed_partition_count(statuses)
+      statuses.count { |status| status == 'failed' }
+    end
+    private_class_method :failed_partition_count
+
+    def resolve_queue_name(queue, cfg)
+      (queue || cfg.partition_queue_name || cfg.queue_name || 'search_index').to_s
+    end
+    private_class_method :resolve_queue_name
+
+    def collection_name(klass)
+      klass.respond_to?(:collection) ? klass.collection.to_s : klass.name.to_s
+    end
+    private_class_method :collection_name
+
+    def event_payload(snapshot, queue: nil, result: nil)
+      payload = {
+        run_id: snapshot && snapshot[:run_id],
+        collection: snapshot && snapshot[:collection],
+        collection_class_name: snapshot && snapshot[:collection_class_name],
+        into: snapshot && snapshot[:into],
+        total_partitions: snapshot && snapshot[:total_partitions],
+        queue: queue
+      }
+      payload.merge!(result) if result
+      payload
+    end
+    private_class_method :event_payload
+
+    def instrument(event, payload)
+      SearchEngine::Instrumentation.instrument(event, payload) {}
+    end
+    private_class_method :instrument
+
+    def monotonic_seconds
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+    private_class_method :monotonic_seconds
+  end
+end

data/lib/search_engine/base/index_maintenance.rb

@@ -281,6 +281,11 @@ module SearchEngine
           if compiled
             parts = Array(compiled.partitions)
             max_p = compiled.max_parallel.to_i
+            if __se_async_partition_execution_requested?(parts)
+              return __se_async_partition_result_unless_available unless __se_async_partition_execution_available?
+
+              return SearchEngine::AsyncPartitionCoordinator.call(klass: self, partitions: parts, into: into)
+            end
             return __se_index_partitions_seq!(parts, into, compiled) if max_p <= 1 || parts.size <= 1
 
             __se_index_partitions_parallel!(parts, into, max_p, compiled)
@@ -292,6 +297,32 @@ module SearchEngine
             sample_error: "#{error.class}: #{error.message.to_s[0, 200]}" }
         end
 
+        def __se_async_partition_execution_requested?(parts)
+          parts.size > 1 && SearchEngine.config.indexer.partition_execution.to_sym == :active_job
+        end
+
+        def __se_async_partition_execution_available?
+          defined?(::ActiveJob::Base) && defined?(SearchEngine::IndexPartitionJob)
+        end
+
+        def __se_async_partition_result_unless_available
+          missing = []
+          missing << 'ActiveJob::Base' unless defined?(::ActiveJob::Base)
+          missing << 'SearchEngine::IndexPartitionJob' unless defined?(SearchEngine::IndexPartitionJob)
+
+          {
+            status: :failed,
+            docs_total: 0,
+            success_total: 0,
+            failed_total: 0,
+            sample_error: "async partition indexing requires #{missing.join(' and ')}"
+          }
+        end
+
+        private :__se_async_partition_execution_requested?,
+                :__se_async_partition_execution_available?,
+                :__se_async_partition_result_unless_available
+
         def __se_index_single_with_renderer!(into)
           docs_estimate = __se_heuristic_docs_estimate(1)
           renderer = SearchEngine::Logging::LiveRenderer.new(

data/lib/search_engine/config.rb

@@ -108,6 +108,18 @@ module SearchEngine
       attr_accessor :dispatch
       # @return [String] queue name for ActiveJob enqueues
       attr_accessor :queue_name
+      # @return [Symbol] partition execution mode: :inline or :active_job
+      attr_accessor :partition_execution
+      # @return [String, nil] optional queue override for async partition jobs
+      attr_accessor :partition_queue_name
+      # @return [Integer] parent wait polling interval for async partition runs (seconds)
+      attr_accessor :partition_poll_interval_s
+      # @return [Integer, nil] maximum parent wait budget for async partition runs (seconds)
+      attr_accessor :partition_timeout_s
+      # @return [Integer] TTL for async partition run metadata (seconds)
+      attr_accessor :partition_run_ttl_s
+      # @return [Object, nil] custom async partition run store
+      attr_accessor :partition_run_store
       # @return [Boolean] whether to run model.count for progress bar estimates (default true)
       attr_accessor :estimate_progress
       # @return [Integer, nil] graceful-shutdown timeout (seconds) for the parallel
@@ -121,6 +133,12 @@ module SearchEngine
         @gzip = false
         @dispatch = active_job_available? ? :active_job : :inline
         @queue_name = 'search_index'
+        @partition_execution = :inline
+        @partition_queue_name = nil
+        @partition_poll_interval_s = 2
+        @partition_timeout_s = nil
+        @partition_run_ttl_s = 86_400
+        @partition_run_store = nil
         @estimate_progress = true
         @pool_timeout = nil
       end
@@ -737,6 +755,12 @@ module SearchEngine
         gzip: indexer.gzip ? true : false,
         dispatch: indexer.dispatch,
         queue_name: indexer.queue_name,
+        partition_execution: indexer.partition_execution,
+        partition_queue_name: indexer.partition_queue_name,
+        partition_poll_interval_s: indexer.partition_poll_interval_s,
+        partition_timeout_s: indexer.partition_timeout_s,
+        partition_run_ttl_s: indexer.partition_run_ttl_s,
+        partition_run_store: indexer.partition_run_store,
         estimate_progress: indexer.estimate_progress
       }
     end

data/lib/search_engine/indexing_run.rb

@@ -0,0 +1,191 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'date'
+require 'json'
+require 'securerandom'
+require 'time'
+
+module SearchEngine
+  # Helpers for async partition indexing run metadata.
+  module IndexingRun
+    module_function
+
+    # Generate a unique run id suitable for operational metadata keys.
+    # @return [String]
+    def generate_id
+      "se-run-#{Time.now.utc.strftime('%Y%m%d%H%M%S')}-#{SecureRandom.hex(8)}"
+    end
+
+    # Return a stable primitive key for a partition value.
+    # @param partition [Object]
+    # @return [String]
+    def partition_key(partition)
+      "p-#{Digest::SHA256.hexdigest(normalized_partition_json(partition))}"
+    end
+
+    # Convert a partition value into ActiveJob-friendly primitives when possible.
+    # @param value [Object]
+    # @return [Object]
+    def serialize_partition(value)
+      normalize(value)
+    end
+
+    # Return a compact partition string for logs and snapshots.
+    # @param partition [Object]
+    # @param max [Integer]
+    # @return [String]
+    def partition_display(partition, max: 160)
+      text = begin
+        JSON.generate(serialize_partition(partition))
+      rescue StandardError
+        partition.to_s
+      end
+      compact(text, max)
+    end
+
+    # Build lifecycle-compatible aggregate counters from a run snapshot.
+    # @param snapshot [Hash, nil]
+    # @return [Hash] result with :status, :docs_total, :success_total, :failed_total, and :sample_error
+    def aggregate_result(snapshot)
+      unless valid_snapshot?(snapshot)
+        return {
+          status: :failed,
+          docs_total: 0,
+          success_total: 0,
+          failed_total: 1,
+          sample_error: invalid_snapshot_error(snapshot)
+        }
+      end
+
+      partitions = snapshot && snapshot[:partitions]
+      values = partitions.is_a?(Hash) ? partitions.values : []
+
+      docs_total = 0
+      success_total = 0
+      failed_total = 0
+      sample_error = nil
+      statuses = []
+
+      values.each do |entry|
+        data = symbolize_keys(entry)
+        statuses << data[:status].to_s
+        docs_total += data[:docs_total].to_i
+        success_total += data[:success_total].to_i
+        failed_total += data[:failed_total].to_i
+        sample_error ||= present_string(data[:sample_error] || data[:error])
+      end
+
+      {
+        status: aggregate_status(statuses, success_total, failed_total),
+        docs_total: docs_total,
+        success_total: success_total,
+        failed_total: failed_total,
+        sample_error: sample_error
+      }
+    end
+
+    # Build the stored metadata for one partition.
+    # @param partition [Object]
+    # @return [Hash]
+    def partition_entry(partition)
+      serialized = serialize_partition(partition)
+      {
+        partition: serialized,
+        partition_display: partition_display(serialized),
+        status: 'pending',
+        docs_total: 0,
+        success_total: 0,
+        failed_total: 0,
+        sample_error: nil,
+        job_id: nil,
+        updated_at: iso8601_now
+      }
+    end
+
+    # @return [String]
+    def iso8601_now
+      Time.now.utc.iso8601(6)
+    end
+
+    def normalized_partition_json(value)
+      JSON.generate(serialize_partition(value))
+    rescue StandardError
+      JSON.generate('__search_engine_fallback__' => value.to_s)
+    end
+
+    def normalize(value)
+      case value
+      when NilClass, TrueClass, FalseClass, Numeric, String
+        value
+      when Symbol
+        value.to_s
+      when Time
+        value.utc.iso8601(6)
+      when Date
+        value.iso8601
+      when Array
+        value.map { |item| normalize(item) }
+      when Hash
+        value.keys.map(&:to_s).sort.each_with_object({}) do |key, hash|
+          original_key = value.key?(key) ? key : value.keys.find { |candidate| candidate.to_s == key }
+          hash[key] = normalize(value[original_key])
+        end
+      else
+        value.to_s
+      end
+    end
+    private_class_method :normalized_partition_json, :normalize
+
+    def aggregate_status(statuses, success_total, failed_total)
+      return :failed if statuses.empty?
+      return :failed if failed_total.positive? && success_total.zero?
+      return :partial if failed_total.positive?
+      return :running if statuses.any? { |status| %w[pending running].include?(status) }
+
+      :ok
+    end
+    private_class_method :aggregate_status
+
+    def valid_snapshot?(snapshot)
+      return false unless snapshot.is_a?(Hash)
+
+      partitions = snapshot[:partitions]
+      return false unless partitions.is_a?(Hash)
+
+      expected_total = snapshot[:total_partitions].to_i
+      expected_total.positive? && partitions.size == expected_total
+    end
+    private_class_method :valid_snapshot?
+
+    def invalid_snapshot_error(snapshot)
+      return 'async partition indexing run snapshot is missing' unless snapshot.is_a?(Hash)
+      return 'async partition indexing run snapshot has no partition metadata' unless snapshot[:partitions].is_a?(Hash)
+
+      expected_total = snapshot[:total_partitions].to_i
+      actual_total = snapshot[:partitions].size
+      "async partition indexing run snapshot has #{actual_total}/#{expected_total} partitions"
+    end
+    private_class_method :invalid_snapshot_error
+
+    def symbolize_keys(hash)
+      return {} unless hash.is_a?(Hash)
+
+      hash.transform_keys(&:to_sym)
+    end
+    private_class_method :symbolize_keys
+
+    def present_string(value)
+      text = value.to_s
+      text.strip.empty? ? nil : text
+    end
+    private_class_method :present_string
+
+    def compact(text, max)
+      return text if text.length <= max
+
+      "#{text[0, max - 3]}..."
+    end
+    private_class_method :compact
+  end
+end

data/lib/search_engine/indexing_run_store/rails_cache.rb

@@ -0,0 +1,231 @@
+# frozen_string_literal: true
+
+require 'search_engine/indexing_run'
+
+module SearchEngine
+  module IndexingRunStore
+    # Rails.cache-backed implementation for async partition indexing run metadata.
+    class RailsCache
+      # @param cache [Object, nil] cache object responding to read/write/delete
+      def initialize(cache: nil)
+        @cache = cache || resolve_rails_cache!
+        validate_cache!
+      end
+
+      # Create a run snapshot with pending partition entries.
+      # @return [Hash]
+      def create_run(run_id:, collection:, collection_class_name:, into:, partitions:, ttl_s:)
+        entries = build_partitions(partitions)
+        run = {
+          run_id: run_id.to_s,
+          collection: collection.to_s,
+          collection_class_name: collection_class_name.to_s,
+          into: into.to_s,
+          status: 'running',
+          total_partitions: Array(partitions).size,
+          partition_keys: entries.keys,
+          ttl_s: ttl_s,
+          created_at: SearchEngine::IndexingRun.iso8601_now,
+          updated_at: SearchEngine::IndexingRun.iso8601_now
+        }
+        write_meta(run, ttl_s: ttl_s)
+        entries.each do |partition_key, entry|
+          @cache.write(partition_cache_key(run_id, partition_key), entry, expires_in: ttl_s)
+        end
+        snapshot(run_id: run_id)
+      end
+
+      # Mark a partition as started.
+      # @return [Hash]
+      def mark_started(run_id:, partition_key:, job_id: nil)
+        update_partition(run_id, partition_key) do |entry|
+          entry[:status] = 'running'
+          entry[:job_id] = job_id.to_s unless job_id.nil?
+        end
+      end
+
+      # Mark a partition as succeeded with import counters.
+      # @return [Hash]
+      def mark_succeeded(run_id:, partition_key:, summary:)
+        update_partition(run_id, partition_key) do |entry|
+          assign_summary!(entry, summary)
+          entry[:status] = 'succeeded'
+          entry[:sample_error] = nil
+        end
+      end
+
+      # Mark a partition as failed.
+      # @return [Hash]
+      def mark_failed(run_id:, partition_key:, error:)
+        update_partition(run_id, partition_key) do |entry|
+          entry[:status] = 'failed'
+          entry[:sample_error] = error_message(error)
+          entry[:failed_total] = [entry[:failed_total].to_i, 1].max
+        end
+      end
+
+      # Read the current run snapshot.
+      # @return [Hash, nil]
+      def snapshot(run_id:)
+        meta = symbolize_meta(@cache.read(meta_cache_key(run_id)))
+        return nil unless meta
+
+        partition_keys = Array(meta[:partition_keys]).map(&:to_s)
+        partitions = partition_keys.each_with_object({}) do |partition_key, hash|
+          entry = @cache.read(partition_cache_key(run_id, partition_key))
+          hash[partition_key] = if entry.is_a?(Hash)
+                                  deep_symbolize(entry)
+                                else
+                                  missing_partition_entry(partition_key)
+                                end
+        end
+        meta[:partitions] = partitions
+        meta[:status] = run_status(partitions)
+        meta
+      end
+
+      # Expire a run immediately.
+      # @return [Object]
+      def expire(run_id:)
+        meta = symbolize_meta(@cache.read(meta_cache_key(run_id)))
+        Array(meta && meta[:partition_keys]).each do |partition_key|
+          @cache.delete(partition_cache_key(run_id, partition_key))
+        end
+        @cache.delete(meta_cache_key(run_id))
+      end
+
+      private
+
+      def resolve_rails_cache!
+        rails = Object.const_get(:Rails) if Object.const_defined?(:Rails)
+        cache = rails.cache if rails.respond_to?(:cache)
+        return cache if cache
+
+        raise ArgumentError,
+              'SearchEngine async partition indexing requires Rails.cache or a custom partition_run_store'
+      rescue NameError
+        raise ArgumentError,
+              'SearchEngine async partition indexing requires Rails.cache or a custom partition_run_store'
+      end
+
+      def validate_cache!
+        missing = %i[read write delete].reject { |method_name| @cache.respond_to?(method_name) }
+        unless missing.empty?
+          raise ArgumentError, "Rails.cache is unusable for indexing runs; missing: #{missing.join(', ')}"
+        end
+
+        probe_key = meta_cache_key("__probe__#{object_id}")
+        @cache.write(probe_key, { ok: true }, expires_in: 1)
+        value = @cache.read(probe_key)
+        @cache.delete(probe_key)
+        return if value
+
+        raise ArgumentError, 'Rails.cache is unusable for indexing runs; probe read returned nil'
+      rescue ArgumentError
+        raise
+      rescue StandardError => error
+        raise ArgumentError, "Rails.cache is unusable for indexing runs: #{error.class}: #{error.message}"
+      end
+
+      def build_partitions(partitions)
+        Array(partitions).each_with_object({}) do |partition, hash|
+          hash[SearchEngine::IndexingRun.partition_key(partition)] = SearchEngine::IndexingRun.partition_entry(partition)
+        end
+      end
+
+      def write_meta(run, ttl_s:)
+        @cache.write(meta_cache_key(run[:run_id]), run, expires_in: ttl_s)
+      end
+
+      def update_partition(run_id, partition_key)
+        meta = symbolize_meta(@cache.read(meta_cache_key(run_id)))
+        raise KeyError, "indexing run not found: #{run_id}" unless meta
+
+        partition_key = partition_key.to_s
+        unless Array(meta[:partition_keys]).map(&:to_s).include?(partition_key)
+          raise KeyError, "indexing run partition not found: #{partition_key}"
+        end
+
+        entry = @cache.read(partition_cache_key(run_id, partition_key))
+        raise KeyError, "indexing run partition not found: #{partition_key}" unless entry
+
+        entry = deep_symbolize(entry)
+        yield entry
+        now = SearchEngine::IndexingRun.iso8601_now
+        entry[:updated_at] = now
+        @cache.write(partition_cache_key(run_id, partition_key), entry, expires_in: meta[:ttl_s])
+        meta[:updated_at] = now
+        write_meta(meta, ttl_s: meta[:ttl_s])
+        snapshot(run_id: run_id)
+      end
+
+      def run_status(partitions)
+        statuses = partitions.values.map { |entry| entry[:status].to_s }
+        return 'failed' if statuses.include?('failed')
+        return 'succeeded' if statuses.all? { |status| status == 'succeeded' }
+
+        'running'
+      end
+
+      def assign_summary!(entry, summary)
+        entry[:docs_total] = summary_value(summary, :docs_total).to_i
+        entry[:success_total] = summary_value(summary, :success_total).to_i
+        entry[:failed_total] = summary_value(summary, :failed_total).to_i
+        entry[:sample_error] = summary_value(summary, :sample_error)
+      end
+
+      def summary_value(summary, key)
+        return summary.public_send(key) if summary.respond_to?(key)
+        return summary[key] if summary.is_a?(Hash) && summary.key?(key)
+        return summary[key.to_s] if summary.is_a?(Hash) && summary.key?(key.to_s)
+
+        nil
+      end
+
+      def error_message(error)
+        return error if error.is_a?(String)
+
+        "#{error.class}: #{error.message.to_s[0, 200]}"
+      end
+
+      def meta_cache_key(run_id)
+        "search_engine:indexing_run:#{run_id}:meta"
+      end
+
+      def partition_cache_key(run_id, partition_key)
+        "search_engine:indexing_run:#{run_id}:partition:#{partition_key}"
+      end
+
+      def missing_partition_entry(partition_key)
+        {
+          partition: nil,
+          partition_display: partition_key,
+          status: 'failed',
+          docs_total: 0,
+          success_total: 0,
+          failed_total: 1,
+          sample_error: "partition metadata missing for #{partition_key}",
+          job_id: nil,
+          updated_at: SearchEngine::IndexingRun.iso8601_now
+        }
+      end
+
+      def symbolize_meta(meta)
+        return nil unless meta.is_a?(Hash)
+
+        deep_symbolize(meta)
+      end
+
+      def deep_symbolize(value)
+        case value
+        when Hash
+          value.each_with_object({}) { |(key, item), hash| hash[key.to_sym] = deep_symbolize(item) }
+        when Array
+          value.map { |item| deep_symbolize(item) }
+        else
+          value
+        end
+      end
+    end
+  end
+end

data/lib/search_engine/indexing_run_store.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module SearchEngine
+  # Resolver and interface contract for async partition indexing run stores.
+  module IndexingRunStore
+    module_function
+
+    # Resolve a configured store or the Rails.cache-backed default.
+    # @param config [SearchEngine::Config, nil]
+    # @return [Object]
+    def resolve(config: nil)
+      cfg = config || SearchEngine.config
+      configured = configured_store(cfg)
+      return configured if configured
+
+      default
+    end
+
+    # Build the default run store.
+    # @return [SearchEngine::IndexingRunStore::RailsCache]
+    def default
+      require 'search_engine/indexing_run_store/rails_cache'
+      SearchEngine::IndexingRunStore::RailsCache.new
+    end
+
+    # Validate that an object implements the run-store contract.
+    # @param store [Object]
+    # @return [Object]
+    # @raise [ArgumentError]
+    def validate!(store)
+      missing = required_methods.reject { |method_name| store.respond_to?(method_name) }
+      return store if missing.empty?
+
+      raise ArgumentError, "indexing run store missing methods: #{missing.join(', ')}"
+    end
+
+    # @return [Array<Symbol>]
+    def required_methods
+      %i[create_run mark_started mark_succeeded mark_failed snapshot expire]
+    end
+
+    def configured_store(config)
+      indexer = config.respond_to?(:indexer) ? config.indexer : nil
+      return nil unless indexer.respond_to?(:partition_run_store)
+
+      store = indexer.partition_run_store
+      store ? validate!(store) : nil
+    end
+    private_class_method :configured_store
+  end
+end

data/lib/search_engine/version.rb

@@ -3,5 +3,5 @@
 module SearchEngine
   # Current gem version.
   # @return [String]
-  VERSION = '30.1.7.0'
+  VERSION = '30.1.8.0'
 end

data/lib/search_engine.rb

@@ -30,6 +30,10 @@ require 'search_engine/indexer/bulk_import'
 require 'search_engine/mapper'
 require 'search_engine/sources'
 require 'search_engine/partitioner'
+require 'search_engine/indexing_run'
+require 'search_engine/indexing_run_store'
+require 'search_engine/indexing_run_store/rails_cache'
+require 'search_engine/async_partition_coordinator'
 require 'search_engine/dispatcher'
 require 'search_engine/joins/guard'
 require 'search_engine/admin'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: search-engine-for-typesense
 version: !ruby/object:Gem::Version
-  version: 30.1.7.0
+  version: 30.1.8.0
 platform: ruby
 authors:
 - Nikita Shkoda
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-05-13 00:00:00.000000000 Z
+date: 2026-06-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -92,6 +92,7 @@ files:
 - lib/search_engine/ast/prefix.rb
 - lib/search_engine/ast/raw.rb
 - lib/search_engine/ast/unary_op.rb
+- lib/search_engine/async_partition_coordinator.rb
 - lib/search_engine/base.rb
 - lib/search_engine/base/creation.rb
 - lib/search_engine/base/deletion.rb
@@ -150,6 +151,9 @@ files:
 - lib/search_engine/indexer/import_dispatcher.rb
 - lib/search_engine/indexer/import_response_parser.rb
 - lib/search_engine/indexer/retry_policy.rb
+- lib/search_engine/indexing_run.rb
+- lib/search_engine/indexing_run_store.rb
+- lib/search_engine/indexing_run_store/rails_cache.rb
 - lib/search_engine/instrumentation.rb
 - lib/search_engine/interruptible_pool.rb
 - lib/search_engine/joins/guard.rb