search-engine-for-typesense 30.1.6.18 → 30.1.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 01b787168b6ef3694f27552da317fa390eaee171bf0734d94725b4527a883968
-  data.tar.gz: 28b30b42639215b992ce6b152bd9c076dc5dc221f6c9a18be5f2355aa7c63865
+  metadata.gz: '0852da3905487de90a84633634760fba6d4806d236e6738a8404d70e9122eef4'
+  data.tar.gz: fe62daa0161f71c0b5bc24da4195708a96b1687bc096383a12056aa346178710
 SHA512:
-  metadata.gz: f5229d42405c66d0811102f31654631d8654bd52bdd55605f12f6baf446c2d61428d95a3f6a0c56a05b6dc4b404a68d3772ee13d4a1ca6a1739efd3de2f93390
-  data.tar.gz: faf42770cec8639077c6efae5e0f1599d1d9eabba0b3fdab23149a5a334ba3a3f50cc1156940bbb8b373f95b695f3e70457370687a774e39c0d2934b9f4cc07a
+  metadata.gz: 6c12c7c80626ac24ea4fd5515559d354632a42a0bcd2381d61ed577aa3e24e6e60448cdf810defdf00d819e8960c93a69ce5129e27225a76c6a476e0b62115c4
+  data.tar.gz: 405565a325c099df28ba1d54d85a308f5297299a59eb40cd9f03a24ebb8306f588becc7c31be5d82cb3be05216cb443dd39a1152d417e07da2b64b97162cbe2a

data/README.md

@@ -116,6 +116,36 @@ SearchEngine::Product.upsert_bulk(records: Product.limit(2))
 
 # Bulk upsert mapped payloads
 SearchEngine::Product.upsert_bulk(data: [mapped])
+
+# Geo search
+class SearchEngine::Venue < SearchEngine::Base
+  collection :venues
+  identify_by :id
+
+  attribute :name, :string
+  attribute :location, :geopoint
+end
+
+# Filter by radius
+SearchEngine::Venue
+  .where_geo(:location, within_radius: { lat: 54.69, lng: 25.28, radius: "10 km" })
+  .order_geo(:location, from: { lat: 54.69, lng: 25.28 })
+  .to_a
+
+# Filter by polygon (viewport)
+SearchEngine::Venue
+  .where_geo(:location, within_polygon: [[54.72, 25.35], [54.72, 25.22], [54.67, 25.22], [54.67, 25.35]])
+  .to_a
+
+# Viewport boost with _eval() + distance tiebreaker
+SearchEngine::Venue
+  .order_eval("location:(54.72,25.35, 54.72,25.22, 54.67,25.22, 54.67,25.35)", direction: :desc)
+  .order_geo(:location, from: { lat: 54.69, lng: 25.28 })
+  .to_a
+
+# Access geo distance on results (present when order_geo is used)
+result = SearchEngine::Venue.all.order_geo(:location, from: { lat: 54.69, lng: 25.28 }).execute
+result.hits.first.geo_distance_meters # => { "location" => 1234 }
 ```
 
 ## Documentation
@@ -134,6 +164,30 @@ You can control this explicitly with:
 
 If you set `SearchEngine.configure { |c| c.client = ... }`, the custom client is always used.
 
+## Async partition indexing
+
+Async partition indexing is an advanced opt-in mode for partitioned full indexing. The default remains
+inline execution. Apps with a real ActiveJob backend can use queue-backed partition execution so each
+search/index partition imports into the same blue/green physical collection before the alias is swapped.
+This mode does not require Sidekiq; use any ActiveJob backend that can run the partition jobs.
+
+```ruby
+SearchEngine.configure do |c|
+  c.indexer.partition_execution = :active_job
+  c.indexer.partition_queue_name = "search_index_partitions"
+  c.indexer.partition_timeout_s = 7_200
+end
+```
+
+Async mode is partition-based, not app-domain based. The gem enqueues one
+`SearchEngine::IndexPartitionJob` per configured partition, waits for every partition to finish, and
+only then lets the schema lifecycle swap the alias. If any partition fails or times out, the previous
+alias target remains active.
+
+Use a shared `Rails.cache` backend, or provide `c.indexer.partition_run_store`, so worker processes and
+the parent indexing process can see the same run metadata. Size the queue carefully: worker concurrency
+multiplies with any per-partition `max_parallel` setting.
+
 ## Example app
 
 See `examples/demo_shop` — demonstrates single/multi search, JOINs, grouping, presets/curation, and DX/observability. Supports offline mode via the stub client (see [Testing](https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/testing)).

data/app/search_engine/search_engine/index_partition_job.rb

@@ -8,6 +8,8 @@ module SearchEngine
   # - partition [Object] (JSON-serializable)
   # - into [String, nil]
   # - metadata [Hash]
+  # - run_id [String, nil]
+  # - partition_key [String, nil]
   class IndexPartitionJob < ::ActiveJob::Base
     queue_as do
       cfg = SearchEngine.config.indexer
@@ -31,11 +33,16 @@ module SearchEngine
     # @param partition [Object]
     # @param into [String, nil]
     # @param metadata [Hash]
+    # @param run_id [String, nil]
+    # @param partition_key [String, nil]
     # @return [void]
-    def perform(collection_class_name, partition, into: nil, metadata: {})
+    def perform(collection_class_name, partition, into: nil, metadata: {}, run_id: nil, partition_key: nil)
       payload = nil
       klass = constantize_collection!(collection_class_name)
       payload = base_payload(klass, partition: partition, into: into)
+      run_store = indexing_run_store(run_id)
+      partition_key ||= SearchEngine::IndexingRun.partition_key(partition) if run_id
+      run_store&.mark_started(run_id: run_id, partition_key: partition_key, job_id: job_id)
       instrument('search_engine.dispatcher.job_started',
                  payload.merge(queue: queue_name, job_id: job_id, metadata: metadata)
                 )
@@ -46,6 +53,7 @@ module SearchEngine
         summary = SearchEngine::Indexer.rebuild_partition!(klass, partition: partition, into: into)
       end
       duration = (monotonic_ms - started).round(1)
+      run_store&.mark_succeeded(run_id: run_id, partition_key: partition_key, summary: summary)
 
       instrument(
         'search_engine.dispatcher.job_finished',
@@ -56,6 +64,7 @@ module SearchEngine
       nil
     rescue StandardError => error
       safe_payload = payload || error_payload(error)
+      run_store&.mark_failed(run_id: run_id, partition_key: partition_key, error: error) unless retryable_error?(error)
       instrument_error(error, payload: safe_payload.merge(metadata: metadata || {}))
       raise
     end
@@ -99,6 +108,24 @@ module SearchEngine
       retry_job wait: wait_seconds
     end
 
+    def indexing_run_store(run_id)
+      return nil if run_id.nil?
+
+      SearchEngine::IndexingRunStore.resolve
+    end
+
+    def retryable_error?(error)
+      transient_error?(error) && executions.to_i < retry_policy.attempts
+    end
+
+    def transient_error?(error)
+      return true if error.is_a?(SearchEngine::Errors::Timeout)
+      return true if error.is_a?(SearchEngine::Errors::Connection)
+
+      error.is_a?(SearchEngine::Errors::Api) &&
+        SearchEngine::Indexer::RetryPolicy.transient_status?(error.status.to_i)
+    end
+
     def error_payload(error)
       {
         collection: arguments_dig_collection,

data/lib/generators/search_engine/install/templates/initializer.rb.tt

@@ -181,6 +181,24 @@ SearchEngine.configure do |c|
   # Queue name for ActiveJob dispatch. Default: 'search_index'
   # c.indexer.queue_name = 'search_index'
 
+  # Advanced partition execution for full indexing. Default: :inline
+  # Use :active_job with a real ActiveJob backend to enqueue one job per
+  # partition during blue/green indexing. Does not require Sidekiq.
+  # c.indexer.partition_execution = :inline
+
+  # Optional queue override for partition jobs. Falls back to c.indexer.queue_name.
+  # Size worker concurrency carefully: partition jobs can multiply partition max_parallel.
+  # c.indexer.partition_queue_name = nil
+
+  # Parent wait settings for async partition runs. Defaults shown.
+  # c.indexer.partition_poll_interval_s = 2
+  # c.indexer.partition_timeout_s = nil
+
+  # Operational metadata for async partition runs. Defaults shown.
+  # Use shared Rails.cache or provide a custom store visible to workers and parent.
+  # c.indexer.partition_run_store = nil
+  # c.indexer.partition_run_ttl_s = 86_400
+
   # --- Sources -------------------------------------------------------------
 
   # ActiveRecord source: default ORM batch size. Default: 2000

data/lib/search_engine/active_record_syncable.rb

@@ -235,10 +235,21 @@ module SearchEngine
       end
 
       actions = cfg[:actions]
+      timing = begin
+        SearchEngine.config.syncable_callback_timing
+      rescue StandardError
+        :after_commit
+      end
 
-      ar_klass.after_create :__se_syncable_upsert! if actions.include?(:create)
-      ar_klass.after_update :__se_syncable_upsert! if actions.include?(:update)
-      ar_klass.after_destroy :__se_syncable_delete! if actions.include?(:destroy)
+      if timing == :after_commit
+        ar_klass.after_create_commit :__se_syncable_upsert! if actions.include?(:create)
+        ar_klass.after_update_commit :__se_syncable_upsert! if actions.include?(:update)
+        ar_klass.after_destroy_commit :__se_syncable_delete! if actions.include?(:destroy)
+      else
+        ar_klass.after_create :__se_syncable_upsert! if actions.include?(:create)
+        ar_klass.after_update :__se_syncable_upsert! if actions.include?(:update)
+        ar_klass.after_destroy :__se_syncable_delete! if actions.include?(:destroy)
+      end
 
       ar_klass.instance_variable_set(:@__se_syncable_callbacks_installed__, true)
       nil

data/lib/search_engine/async_partition_coordinator.rb

@@ -0,0 +1,208 @@
+# frozen_string_literal: true
+
+module SearchEngine
+  # Coordinates ActiveJob-backed partition indexing and waits for run completion.
+  module AsyncPartitionCoordinator
+    module_function
+
+    # Enqueue one partition job per partition and return a lifecycle-compatible aggregate.
+    # @param klass [Class] SearchEngine collection class
+    # @param partitions [Array<Object>] logical partition values
+    # @param into [String] target physical collection name
+    # @param queue [String, nil] optional ActiveJob queue override
+    # @param timeout_s [Numeric, nil] maximum wait time in seconds
+    # @param poll_interval_s [Numeric, nil] polling interval in seconds
+    # @param store [Object, nil] optional run store
+    # @param ttl_s [Numeric, nil] run metadata TTL in seconds
+    # @return [Hash] lifecycle result with :status, :docs_total, :success_total, :failed_total, :sample_error
+    def call(klass:, partitions:, into:, queue: nil, timeout_s: nil, poll_interval_s: nil, store: nil, ttl_s: nil)
+      cfg = SearchEngine.config.indexer
+      partition_list = Array(partitions)
+      run_id = SearchEngine::IndexingRun.generate_id
+      queue_name = resolve_queue_name(queue, cfg)
+      run_store = SearchEngine::IndexingRunStore.validate!(store || SearchEngine::IndexingRunStore.resolve)
+      ttl = ttl_s || cfg.partition_run_ttl_s
+
+      snapshot = create_run(
+        store: run_store,
+        run_id: run_id,
+        klass: klass,
+        into: into,
+        partitions: partition_list,
+        ttl_s: ttl
+      )
+      instrument('search_engine.indexing_run.started', event_payload(snapshot, queue: queue_name))
+      enqueue_partitions(klass, partition_list, into: into, queue_name: queue_name, run_id: run_id)
+      instrument('search_engine.indexing_run.enqueued', event_payload(snapshot, queue: queue_name))
+
+      wait_for_completion(
+        store: run_store,
+        run_id: run_id,
+        timeout_s: timeout_s.nil? ? cfg.partition_timeout_s : timeout_s,
+        poll_interval_s: poll_interval_s.nil? ? cfg.partition_poll_interval_s : poll_interval_s
+      )
+    end
+
+    def create_run(store:, run_id:, klass:, into:, partitions:, ttl_s:)
+      store.create_run(
+        run_id: run_id,
+        collection: collection_name(klass),
+        collection_class_name: klass.name,
+        into: into,
+        partitions: partitions,
+        ttl_s: ttl_s
+      )
+    end
+    private_class_method :create_run
+
+    def enqueue_partitions(klass, partitions, into:, queue_name:, run_id:)
+      partitions.each do |partition|
+        SearchEngine::IndexPartitionJob
+          .set(queue: queue_name)
+          .perform_later(
+            klass.name,
+            partition,
+            into: into,
+            metadata: {},
+            run_id: run_id,
+            partition_key: SearchEngine::IndexingRun.partition_key(partition)
+          )
+      end
+    end
+    private_class_method :enqueue_partitions
+
+    def wait_for_completion(store:, run_id:, timeout_s:, poll_interval_s:)
+      deadline = timeout_s.nil? ? nil : monotonic_seconds + timeout_s.to_f
+
+      loop do
+        snapshot = store.snapshot(run_id: run_id)
+        result = result_for_snapshot(snapshot)
+        return finish_failure(snapshot, result) if invalid_snapshot_result?(result)
+        return finish_success(snapshot, result) if result[:status] == :ok
+        return finish_failure(snapshot, result) if failure_terminal?(snapshot)
+        return timeout_result(store, run_id, snapshot) if deadline && monotonic_seconds >= deadline
+
+        sleep([poll_interval_s.to_f, 0].max)
+      end
+    end
+    private_class_method :wait_for_completion
+
+    def result_for_snapshot(snapshot)
+      SearchEngine::IndexingRun.aggregate_result(snapshot)
+    end
+    private_class_method :result_for_snapshot
+
+    def invalid_snapshot_result?(result)
+      result[:status] == :failed && result[:sample_error].to_s.start_with?('async partition indexing run snapshot')
+    end
+    private_class_method :invalid_snapshot_result?
+
+    def finish_success(snapshot, result)
+      instrument('search_engine.indexing_run.finished', event_payload(snapshot, result: result))
+      result
+    end
+    private_class_method :finish_success
+
+    def finish_failure(snapshot, result)
+      result = failed_result(snapshot, result)
+      instrument('search_engine.indexing_run.failed', event_payload(snapshot, result: result))
+      result
+    end
+    private_class_method :finish_failure
+
+    def timeout_result(store, run_id, snapshot)
+      mark_non_terminal_failed(store, run_id, snapshot)
+      snapshot = store.snapshot(run_id: run_id) || snapshot
+      result = failed_result(
+        snapshot,
+        SearchEngine::IndexingRun.aggregate_result(snapshot),
+        sample_error: "SearchEngine async partition indexing timed out for run #{run_id}"
+      )
+      instrument('search_engine.indexing_run.failed', event_payload(snapshot, result: result))
+      result
+    end
+    private_class_method :timeout_result
+
+    def mark_non_terminal_failed(store, run_id, snapshot)
+      partitions = snapshot && snapshot[:partitions]
+      return unless partitions.is_a?(Hash)
+
+      partitions.each do |partition_key, entry|
+        status = entry[:status].to_s
+        next if %w[succeeded failed].include?(status)
+
+        store.mark_failed(
+          run_id: run_id,
+          partition_key: partition_key,
+          error: 'partition did not finish before timeout'
+        )
+      end
+    end
+    private_class_method :mark_non_terminal_failed
+
+    def failed_result(snapshot, result, sample_error: nil)
+      statuses = partition_statuses(snapshot)
+      success_total = result[:success_total].to_i
+      status = success_total.positive? || statuses.include?('succeeded') ? :partial : :failed
+
+      result.merge(
+        status: status,
+        failed_total: [result[:failed_total].to_i, failed_partition_count(statuses)].max,
+        sample_error: sample_error || result[:sample_error] || 'async partition indexing failed'
+      )
+    end
+    private_class_method :failed_result
+
+    def failure_terminal?(snapshot)
+      partition_statuses(snapshot).include?('failed')
+    end
+    private_class_method :failure_terminal?
+
+    def partition_statuses(snapshot)
+      partitions = snapshot && snapshot[:partitions]
+      return [] unless partitions.is_a?(Hash)
+
+      partitions.values.map { |entry| entry[:status].to_s }
+    end
+    private_class_method :partition_statuses
+
+    def failed_partition_count(statuses)
+      statuses.count { |status| status == 'failed' }
+    end
+    private_class_method :failed_partition_count
+
+    def resolve_queue_name(queue, cfg)
+      (queue || cfg.partition_queue_name || cfg.queue_name || 'search_index').to_s
+    end
+    private_class_method :resolve_queue_name
+
+    def collection_name(klass)
+      klass.respond_to?(:collection) ? klass.collection.to_s : klass.name.to_s
+    end
+    private_class_method :collection_name
+
+    def event_payload(snapshot, queue: nil, result: nil)
+      payload = {
+        run_id: snapshot && snapshot[:run_id],
+        collection: snapshot && snapshot[:collection],
+        collection_class_name: snapshot && snapshot[:collection_class_name],
+        into: snapshot && snapshot[:into],
+        total_partitions: snapshot && snapshot[:total_partitions],
+        queue: queue
+      }
+      payload.merge!(result) if result
+      payload
+    end
+    private_class_method :event_payload
+
+    def instrument(event, payload)
+      SearchEngine::Instrumentation.instrument(event, payload) {}
+    end
+    private_class_method :instrument
+
+    def monotonic_seconds
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+    private_class_method :monotonic_seconds
+  end
+end

data/lib/search_engine/base/index_maintenance.rb

@@ -281,6 +281,11 @@ module SearchEngine
           if compiled
             parts = Array(compiled.partitions)
             max_p = compiled.max_parallel.to_i
+            if __se_async_partition_execution_requested?(parts)
+              return __se_async_partition_result_unless_available unless __se_async_partition_execution_available?
+
+              return SearchEngine::AsyncPartitionCoordinator.call(klass: self, partitions: parts, into: into)
+            end
             return __se_index_partitions_seq!(parts, into, compiled) if max_p <= 1 || parts.size <= 1
 
             __se_index_partitions_parallel!(parts, into, max_p, compiled)
@@ -292,6 +297,32 @@ module SearchEngine
             sample_error: "#{error.class}: #{error.message.to_s[0, 200]}" }
         end
 
+        def __se_async_partition_execution_requested?(parts)
+          parts.size > 1 && SearchEngine.config.indexer.partition_execution.to_sym == :active_job
+        end
+
+        def __se_async_partition_execution_available?
+          defined?(::ActiveJob::Base) && defined?(SearchEngine::IndexPartitionJob)
+        end
+
+        def __se_async_partition_result_unless_available
+          missing = []
+          missing << 'ActiveJob::Base' unless defined?(::ActiveJob::Base)
+          missing << 'SearchEngine::IndexPartitionJob' unless defined?(SearchEngine::IndexPartitionJob)
+
+          {
+            status: :failed,
+            docs_total: 0,
+            success_total: 0,
+            failed_total: 0,
+            sample_error: "async partition indexing requires #{missing.join(' and ')}"
+          }
+        end
+
+        private :__se_async_partition_execution_requested?,
+                :__se_async_partition_execution_available?,
+                :__se_async_partition_result_unless_available
+
         def __se_index_single_with_renderer!(into)
           docs_estimate = __se_heuristic_docs_estimate(1)
           renderer = SearchEngine::Logging::LiveRenderer.new(

data/lib/search_engine/config.rb

@@ -56,6 +56,10 @@ module SearchEngine
     #   @return [String, nil, false] path to host app SearchEngine models directory. May be
     #     relative to `Rails.root` (e.g., "app/search_engine") or absolute. When `nil` or
     #     `false`, gem-managed loading of host SearchEngine models is disabled.
+    # @!attribute [rw] syncable_callback_timing
+    #   @return [Symbol] controls ActiveRecordSyncable callback timing.
+    #     +:after_commit+ (default) uses +after_*_commit+ callbacks (safe, post-transaction).
+    #     +:after_save+ uses legacy +after_*+ callbacks (in-transaction).
     attr_accessor :logger,
                   :default_query_by,
                   :default_infix,
@@ -67,7 +71,8 @@ module SearchEngine
                   :client,
                   :default_console_model,
                   :search_engine_models,
-                  :relation_print_materializes
+                  :relation_print_materializes,
+                  :syncable_callback_timing
 
     # Lightweight nested configuration for schema lifecycle.
     class SchemaConfig
@@ -103,6 +108,18 @@ module SearchEngine
       attr_accessor :dispatch
       # @return [String] queue name for ActiveJob enqueues
       attr_accessor :queue_name
+      # @return [Symbol] partition execution mode: :inline or :active_job
+      attr_accessor :partition_execution
+      # @return [String, nil] optional queue override for async partition jobs
+      attr_accessor :partition_queue_name
+      # @return [Integer] parent wait polling interval for async partition runs (seconds)
+      attr_accessor :partition_poll_interval_s
+      # @return [Integer, nil] maximum parent wait budget for async partition runs (seconds)
+      attr_accessor :partition_timeout_s
+      # @return [Integer] TTL for async partition run metadata (seconds)
+      attr_accessor :partition_run_ttl_s
+      # @return [Object, nil] custom async partition run store
+      attr_accessor :partition_run_store
       # @return [Boolean] whether to run model.count for progress bar estimates (default true)
       attr_accessor :estimate_progress
       # @return [Integer, nil] graceful-shutdown timeout (seconds) for the parallel
@@ -116,6 +133,12 @@ module SearchEngine
         @gzip = false
         @dispatch = active_job_available? ? :active_job : :inline
         @queue_name = 'search_index'
+        @partition_execution = :inline
+        @partition_queue_name = nil
+        @partition_poll_interval_s = 2
+        @partition_timeout_s = nil
+        @partition_run_ttl_s = 86_400
+        @partition_run_store = nil
         @estimate_progress = true
         @pool_timeout = nil
       end
@@ -402,6 +425,9 @@ module SearchEngine
       @search_engine_models = 'app/search_engine'
       # When true, Relation#inspect/pretty_print materialize a preview (AR-like).
       @relation_print_materializes = true
+      # Controls whether ActiveRecordSyncable uses after_*_commit (safe, default)
+      # or after_* (legacy in-transaction) callbacks. Values: :after_commit, :after_save.
+      @syncable_callback_timing = :after_commit
     end
 
     # Whether the engine should avoid network I/O and use an offline client.
@@ -701,7 +727,8 @@ module SearchEngine
         presets: presets_hash_for_to_h,
         curation: curation_hash_for_to_h,
         embedding: embedding_hash_for_to_h,
-        relation_print_materializes: relation_print_materializes ? true : false
+        relation_print_materializes: relation_print_materializes ? true : false,
+        syncable_callback_timing: syncable_callback_timing
       }
     end
 
@@ -728,6 +755,12 @@ module SearchEngine
         gzip: indexer.gzip ? true : false,
         dispatch: indexer.dispatch,
         queue_name: indexer.queue_name,
+        partition_execution: indexer.partition_execution,
+        partition_queue_name: indexer.partition_queue_name,
+        partition_poll_interval_s: indexer.partition_poll_interval_s,
+        partition_timeout_s: indexer.partition_timeout_s,
+        partition_run_ttl_s: indexer.partition_run_ttl_s,
+        partition_run_store: indexer.partition_run_store,
         estimate_progress: indexer.estimate_progress
       }
     end