search-engine-for-typesense 30.1.8.23 → 30.8.24

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dd7e450cd267dab4d557451ce255f20707b07d2a97cf42b143dfdc601561668f
-  data.tar.gz: 29e7f4f858407039f8e477a8d0f0fc1b9985e3922214bae03def126065e31106
+  metadata.gz: db1ad7dbacfe019861ce7dbe7f3511326d67f0afc8bbf22ed2929cdb300797ad
+  data.tar.gz: 5bfafebc3623ee781a1c519c799621bfb8ca62b19f21e6370e79d4d165bc3999
 SHA512:
-  metadata.gz: 8ab3eda607afe100090554542a0c74a89d1418da5bf8c0034c35fbd4b26ae1c2b6dfbeb8789d4a16c313b34179f64b0c031a61bb463133c14a51087e5f1fe83a
-  data.tar.gz: a3a21521da9b613004e0883142d6c9ec63e3711d573be90b28db7eeb8dfcf391872876d182f2bd2c21e08d56963a2e746415bd6c4cc0eddf4166bf67b2cffc70
+  metadata.gz: b1ce3108ba2a8ef685803cf06264e88a9565dfe7e640f983465f198a03ba2f5b4cf059555c00b2e1016fa38f49aadb6be6bc7bcc538b5564a46c922e103f6db5
+  data.tar.gz: 9a2ddb0b81c46d8c81ce06e622f4b16adf1f81f50a4f4ccf2ff756c3d2542cf8d90111d31b7412f77e88dd441d7429106779457033e30bfc0c8d6b12e169342f

data/CHANGELOG.md

@@ -1,5 +1,22 @@
 # Changelog
 
+## 30.8.24
+
+- Adopt `{typesense_major}.{gem_minor}.{gem_patch}` versioning, omitting the Typesense minor component.
+- Include the same runtime fixes as 30.1.8.24; no additional runtime behavior changes.
+
+## 30.1.8.24
+
+- Preserve caller transactions during PostgreSQL streaming by rejecting active transactions before opening a cursor.
+  Apply statement timeouts inside the streaming transaction and restore connection settings on completion or failure.
+- Synchronize both record creation and updates after commit, and propagate failed custom document identity calculations.
+- Propagate parallel import worker failures and timeouts instead of returning successful summaries for incomplete imports.
+- Execute ID-filtered queries through normal search materialization so text filters, selection, and memoization apply.
+- Use native Typesense joins without incomplete client-side key lookups. Missing references and invalid search
+  configurations now raise errors instead of returning empty results. Existing joins require valid server references.
+- Allow upserts to omit server-generated embeddings and preserve boolean coercions that produce false.
+- Correct generated models, documented declarations, and demo collection references and timestamp mappings.
+
 ## 30.1.8.23
 
 - Run retained-physical alias rollbacks through the same cutover guard. Rollback accepts explicit destination

data/README.md

@@ -8,9 +8,9 @@ Mountless Rails::Engine for [Typesense](https://typesense.org). Expressive Relat
 
 ## Versioning
 
-The gem version mirrors the Typesense server major/minor it targets. Patch releases are reserved for gem-only fixes and enhancements.
+Versions use `{typesense_major}.{gem_minor}.{gem_patch}`. The first component identifies the targeted Typesense server major version; the remaining components track gem enhancements and fixes independently of Typesense minor releases.
 
-Example: `30.1.x` targets Typesense `30.1`.
+Example: `30.8.24` targets Typesense `30.x`. Typesense minor versions are not encoded in the gem version.
 
 ## Quickstart
 
@@ -26,6 +26,7 @@ SearchEngine.configure do |c|
   c.port = 8108
   c.protocol = "http"
   c.api_key = ENV.fetch("TYPESENSE_API_KEY")
+  c.default_infix = "off"
 end
 ```
 
@@ -33,10 +34,10 @@ end
 class SearchEngine::Product < SearchEngine::Base
   collection :products
 
-  attribute :id, :integer
+  identify_by :id
   attribute :name, :string
 
-  query_by %i[name brand description]
+  query_by :name
 end
 
 SearchEngine::Product.where(name: "milk").select(:id, :name).limit(5).to_a
@@ -58,22 +59,50 @@ SearchEngine.configure do |c|
 end
 ```
 
+SQL sources stream PostgreSQL queries in their own read-only transaction. Start streaming
+outside an existing transaction; an active transaction is rejected before cursor SQL runs.
+The configured statement timeout applies only to the streaming transaction.
+
+Joined filters execute as native Typesense joins and require valid collection references.
+Search configuration and missing-reference errors propagate to callers; they do not
+produce empty results or trigger client-side joins.
+
 ## Usage examples
 
 ```ruby
-# Model
+# Models (the source objects below are the host application's ActiveRecord models)
+class SearchEngine::Brand < SearchEngine::Base
+  collection "brands"
+  identify_by :id
+  attribute :name, :string
+  query_by :name
+end
+
 class SearchEngine::Product < SearchEngine::Base
   collection "products"
 
-  attribute :id, :integer
+  identify_by :id
   attribute :name, :string
+  attribute :price_cents, :integer, sort: true
+  attribute :brand_id, :string, facet: true
+  attribute :category, :string, facet: true
+  belongs_to :brands, collection: "brands", local_key: :brand_id, foreign_key: :id
+  query_by :name
+
+  index do
+    source :active_record, model: ::Product
+    map do |record|
+      { name: record.name, price_cents: record.price_cents,
+        brand_id: record.brand_id.to_s, category: record.category }
+    end
+  end
 end
 
 # Basic query
 SearchEngine::Product
   .where(name: "milk")
   # Explicit query_by always wins over model/global defaults
-  .options(query_by: 'name,brand')
+  .options(query_by: 'name')
   .select(:id, :name)
   .order(price_cents: :asc)
   .limit(5)
@@ -95,7 +124,7 @@ rel = SearchEngine::Product
 params = rel.to_h # compiled Typesense params
 
 # Multi-search
-result_set = SearchEngine.multi_search(common: { query_by: SearchEngine.config.default_query_by }) do |m|
+result_set = SearchEngine.multi_search(common: { query_by: "name" }) do |m|
   m.add :products, SearchEngine::Product.where("name:~rud").per(10)
   m.add :brands,   SearchEngine::Brand.all.per(5)
 end

data/lib/generators/search_engine/model/model_generator.rb

@@ -48,12 +48,14 @@ module SearchEngine
         return [] if raw.nil?
 
         tokens = raw.split(/[\s,]+/).map(&:strip).reject(&:empty?)
-        tokens.map do |pair|
+        tokens.filter_map do |pair|
           name, type = pair.split(':', 2)
           raise Thor::Error, "invalid attribute token: #{pair.inspect} (expected name:type)" unless name
 
           type = (type || 'string').to_s
           normalized = normalize_type(type)
+          next if name.to_s.underscore == 'id'
+
           [name.to_s.underscore, normalized]
         end
       end

data/lib/generators/search_engine/model/templates/model.rb.tt

@@ -7,6 +7,7 @@
 # - https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/vector-search
 class SearchEngine::<%= class_name %> < SearchEngine::Base
   collection "<%= @collection_name %>"
+  identify_by :id
 <% Array(@attributes).each do |(name, type)| -%>
   attribute :<%= name %>, :<%= type %>
 <% end -%>

data/lib/search_engine/active_record_syncable.rb

@@ -265,8 +265,8 @@ module SearchEngine
       end
 
       if timing == :after_commit
-        ar_klass.after_create_commit :__se_syncable_upsert! if actions.include?(:create)
-        ar_klass.after_update_commit :__se_syncable_upsert! if actions.include?(:update)
+        upsert_actions = actions & %i[create update]
+        ar_klass.after_commit :__se_syncable_upsert!, on: upsert_actions unless upsert_actions.empty?
         ar_klass.after_destroy_commit :__se_syncable_delete! if actions.include?(:destroy)
       else
         ar_klass.after_create :__se_syncable_upsert! if actions.include?(:create)

data/lib/search_engine/base/creation.rb

@@ -111,7 +111,8 @@ module SearchEngine
         end
 
         def compute_required_keys_from_schema(klass, compiled_schema)
-          fields = Array(compiled_schema[:fields]).map { |f| (f[:name] || f['name']).to_s }
+          fields = Array(compiled_schema[:fields]).reject { |f| f[:embed] || f['embed'] }
+                                                  .map { |f| (f[:name] || f['name']).to_s }
           base = fields.reject { |fname| fname.include?('.') }.to_set
           begin
             opts = klass.respond_to?(:attribute_options) ? (klass.attribute_options || {}) : {}
@@ -229,7 +230,7 @@ module SearchEngine
             next if value.nil? && optional_fields.include?(key.to_s)
 
             valid, coerced, err = validate_value_for_type(expected, value, coercions_enabled: coercions_enabled)
-            if coerced
+            if !coerced.nil?
               document[key.to_s] = coerced
             elsif !valid
               raise SearchEngine::Errors::InvalidParams.new(

data/lib/search_engine/hydration/materializers.rb

@@ -28,43 +28,9 @@ module SearchEngine
           params = SearchEngine::CompiledParams.from(relation.to_typesense_params)
           url_opts = relation.send(:build_url_opts)
 
-          raw_result = nil
-
-          # Preflight client-side fallback (extracted for readability)
-          preflight_raw, relation, params = preflight_join_fallback_if_needed(relation, params)
-
-          begin
-            raw_result = preflight_raw || relation.send(:client).search(
-              collection: collection,
-              params: params,
-              url_opts: url_opts
-            )
-          rescue SearchEngine::Errors::Api => error
-            # Graceful empty fallback for infix/prefix configuration errors
-            if infix_missing_error?(error)
-              empty = { 'hits' => [], 'found' => 0, 'out_of' => 0 }
-              raw_result = SearchEngine::Result.new(empty, klass: relation.klass)
-            else
-              # Client-side join fallback: handle missing Typesense reference for joined filters
-              raise unless join_reference_missing_error?(error) && Array(relation.joins_list).any?
-
-              fallback_rel = build_client_side_join_fallback_relation(relation)
-              if fallback_rel.equal?(:__empty__)
-                # Short-circuit: no matches
-                empty = { 'hits' => [], 'found' => 0, 'out_of' => 0 }
-                raw_result = SearchEngine::Result.new(empty, klass: relation.klass)
-              else
-                # Retry with rewritten base relation
-                new_params = SearchEngine::CompiledParams.from(fallback_rel.to_typesense_params)
-                raw_result = relation.send(:client).search(collection: collection, params: new_params,
-                                                           url_opts: url_opts
-                )
-                instrument_client_side_fallback(relation)
-                # Replace relation for selection/facets context below
-                relation = fallback_rel
-              end
-            end
-          end
+          raw_result = relation.send(:client).search(
+            collection: collection, params: params, url_opts: url_opts
+          )
 
           selection_ctx = SearchEngine::Hydration::SelectionContext.build(relation)
           facets_ctx = build_facets_context_from_state(relation)
@@ -132,7 +98,7 @@ module SearchEngine
         params = SearchEngine::CompiledParams.from(preview_relation.to_typesense_params)
         url_opts = preview_relation.send(:build_url_opts)
 
-        raw_result = perform_preview_search_with_fallback(preview_relation, collection, params, url_opts)
+        raw_result = preview_relation.send(:client).search(collection: collection, params: params, url_opts: url_opts)
 
         selection_ctx = SearchEngine::Hydration::SelectionContext.build(preview_relation)
         facets_ctx = build_facets_context_from_state(preview_relation)
@@ -153,58 +119,12 @@ module SearchEngine
         array
       end
 
-      # Internal: perform preview search, applying client-side fallback when Typesense
-      # reports a missing reference for joined filters.
-      # @param preview_relation [SearchEngine::Relation]
-      # @param collection [String]
-      # @param params [SearchEngine::CompiledParams]
-      # @param url_opts [Hash]
-      # @return [Object] raw result from client
-      def perform_preview_search_with_fallback(preview_relation, collection, params, url_opts)
-        preview_relation.send(:client).search(
-          collection: collection,
-          params: params,
-          url_opts: url_opts
-        )
-      rescue SearchEngine::Errors::Api => error
-        # Graceful empty fallback for infix/prefix configuration errors
-        if infix_missing_error?(error)
-          empty_raw = { 'hits' => [], 'found' => 0, 'out_of' => 0 }
-          return SearchEngine::Result.new(empty_raw, klass: preview_relation.klass)
-        end
-        raise unless join_reference_missing_error?(error) && Array(preview_relation.joins_list).any?
-
-        fallback_rel = build_client_side_join_fallback_relation(preview_relation)
-        if fallback_rel.equal?(:__empty__)
-          empty_raw = { 'hits' => [], 'found' => 0, 'out_of' => 0 }
-          return SearchEngine::Result.new(empty_raw, klass: preview_relation.klass)
-        end
-
-        new_params = SearchEngine::CompiledParams.from(fallback_rel.to_typesense_params)
-        res = preview_relation.send(:client).search(
-          collection: collection,
-          params: new_params,
-          url_opts: url_opts
-        )
-        instrument_client_side_fallback(preview_relation)
-        res
-      end
-
       def each(relation, &block)
         arr = to_a(relation)
         block_given? ? arr.each(&block) : arr.each
       end
 
       def first(relation, n = nil)
-        # Fast path: when relation has a single equality predicate on id and n=nil, use document GET
-        if n.nil?
-          id_value = detect_equality_id_predicate_value(relation)
-          if id_value
-            doc = retrieve_document_by_id(relation, id_value)
-            return doc unless doc.nil?
-          end
-        end
-
         arr = to_a(relation)
         return arr.first if n.nil?
 
@@ -340,25 +260,7 @@ module SearchEngine
         minimal[:include_fields] = 'id'
 
         url_opts = relation.send(:build_url_opts)
-        begin
-          result = relation.send(:client).search(collection: collection, params: minimal, url_opts: url_opts)
-        rescue SearchEngine::Errors::Api => error
-          return 0 if infix_missing_error?(error)
-          # Client-side join fallback: handle missing Typesense reference for joined filters in count path
-          raise unless join_reference_missing_error?(error) && Array(relation.joins_list).any?
-
-          fallback_rel = build_client_side_join_fallback_relation(relation)
-          return 0 if fallback_rel.equal?(:__empty__)
-
-          new_params = SearchEngine::CompiledParams.from(fallback_rel.to_typesense_params).to_h
-          new_minimal = new_params.dup
-          new_minimal[:per_page] = 1
-          new_minimal[:page] = 1
-          new_minimal[:include_fields] = 'id'
-
-          result = relation.send(:client).search(collection: collection, params: new_minimal, url_opts: url_opts)
-          instrument_client_side_fallback(relation)
-        end
+        result = relation.send(:client).search(collection: collection, params: minimal, url_opts: url_opts)
 
         count = result.found.to_i
         relation.send(:enforce_hit_validator_if_needed!, count, collection: collection)
@@ -381,296 +283,6 @@ module SearchEngine
       end
       module_function :curated_hits_count
 
-      # Detect Typesense 400 errors caused by missing infix/prefix configuration
-      # e.g., "Could not find `name` in the infix index. Make sure to enable infix search by specifying `infix: true` in the schema."
-      def infix_missing_error?(error)
-        return false unless error.is_a?(SearchEngine::Errors::Api)
-
-        status = error.status.to_i
-        return false unless status == 400
-
-        body = error.body
-        msg = error.message.to_s
-        # Match common phrasing from Typesense for missing infix/prefix index
-        need_infix = 'infix index'
-        enable_infix = 'enable infix'
-        could_not_find = 'Could not find'
-        missing_prefix = 'prefix index'
-        (
-          body.is_a?(String) && (body.include?(need_infix) ||
-          body.include?(enable_infix) || body.include?(missing_prefix))
-        ) ||
-          msg.include?(need_infix) || msg.include?(enable_infix) ||
-          msg.include?(missing_prefix) || msg.include?(could_not_find)
-      rescue StandardError
-        false
-      end
-
-      # --- client-side join fallback helpers ---------------------------------
-
-      # True when the relation uses joined fields in filters and the base
-      # collection schema lacks a matching reference for at least one of
-      # those associations.
-      def join_fallback_preflight_required?(relation)
-        state = relation.instance_variable_get(:@state) || {}
-        ast_nodes = Array(state[:ast]).flatten.compact
-        assocs = extract_join_assocs_from_ast(ast_nodes)
-        return false if assocs.empty?
-
-        base_klass = relation.klass
-        compiled = SearchEngine::Schema.compile(base_klass)
-        fields = Array(compiled[:fields])
-        by_name = {}
-        fields.each do |f|
-          name = (f[:name] || f['name']).to_s
-          by_name[name] = f
-        end
-
-        assocs.any? do |assoc|
-          begin
-            cfg = base_klass.join_for(assoc)
-            lk = (cfg[:local_key] || '').to_s
-            fk = (cfg[:foreign_key] || '').to_s
-            coll = (cfg[:collection] || '').to_s
-            expected = "#{coll}.#{fk}"
-            entry = by_name[lk]
-            actual = entry && (entry[:reference] || entry['reference'])
-            actual_str = actual.to_s
-            # Accept async suffix on actual
-            next true if actual_str.empty? || !actual_str.start_with?(expected)
-          rescue StandardError
-            next true
-          end
-          false
-        end
-      rescue StandardError
-        false
-      end
-
-      # Walk AST nodes and collect association names used via "$assoc.field".
-      def extract_join_assocs_from_ast(nodes)
-        list = Array(nodes).flatten.compact
-        return [] if list.empty?
-
-        seen = []
-        walker = lambda do |node|
-          return unless node.is_a?(SearchEngine::AST::Node)
-
-          if node.respond_to?(:field)
-            field = node.field.to_s
-            if field.start_with?('$')
-              m = field.match(/^\$(\w+)\./)
-              if m
-                name = m[1].to_sym
-                seen << name unless seen.include?(name)
-              end
-            end
-          end
-
-          Array(node.children).each { |child| walker.call(child) }
-        end
-        list.each { |n| walker.call(n) }
-        seen
-      end
-
-      # Attempt a client-side fallback rewrite before making a request when
-      # joined filters are present but the base schema lacks the needed reference.
-      # Returns [raw_result_or_nil, relation, params]
-      def preflight_join_fallback_if_needed(relation, params)
-        raw_result = nil
-        try_fallback = begin
-          join_fallback_preflight_required?(relation)
-        rescue StandardError
-          false
-        end
-
-        if try_fallback
-          begin
-            fallback_rel = build_client_side_join_fallback_relation(relation)
-            if fallback_rel.equal?(:__empty__)
-              empty = { 'hits' => [], 'found' => 0, 'out_of' => 0 }
-              raw_result = SearchEngine::Result.new(empty, klass: relation.klass)
-            else
-              relation = fallback_rel
-              params = SearchEngine::CompiledParams.from(relation.to_typesense_params)
-              instrument_client_side_fallback(relation)
-            end
-          rescue StandardError
-            # ignore and proceed
-          end
-        end
-        [raw_result, relation, params]
-      end
-
-      def join_reference_missing_error?(error)
-        return false unless error.is_a?(SearchEngine::Errors::Api)
-
-        body = error.body
-        msg = error.message.to_s
-        needle = 'No reference field found'
-        (body.is_a?(String) && body.include?(needle)) || msg.include?(needle)
-      rescue StandardError
-        false
-      end
-
-      def build_client_side_join_fallback_relation(relation)
-        state = relation.instance_variable_get(:@state) || {}
-        ast_nodes = Array(state[:ast]).flatten.compact
-        joins = Array(state[:joins]).flatten.compact
-        return relation if joins.empty?
-
-        # Guard: sorting or selection on joined fields not supported by client-side join fallback
-        orders = Array(state[:orders]).map(&:to_s)
-        if orders.any? { |o| o.start_with?('$') }
-          raise SearchEngine::Errors::InvalidOption.new(
-            'Sorting by joined fields is not supported by client-side join fallback',
-            doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/joins#client-side-fallback'
-          )
-        end
-        include_str = begin
-          relation.send(:compile_include_fields_string)
-        rescue StandardError
-          nil
-        end
-        if include_str&.split(',')&.any? { |seg| seg.strip.start_with?('$') }
-          raise SearchEngine::Errors::InvalidOption.new(
-            'Selecting joined fields is not supported by client-side join fallback',
-            doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/joins#client-side-fallback'
-          )
-        end
-
-        # For each applied assoc, collect inner predicates (Eq/In only)
-        per_assoc_inners = extract_join_inners(ast_nodes)
-        return relation if per_assoc_inners.empty?
-
-        # Resolve keys and fetch key sets via pre-query
-        key_sets = {}
-        base_klass = relation.klass
-        joins.each do |assoc|
-          cfg = base_klass.join_for(assoc)
-          inners = per_assoc_inners[assoc] || []
-          next if inners.empty?
-
-          keys = fetch_keys_for_assoc(base_klass, cfg, inners)
-          key_sets[assoc] = keys
-        end
-
-        # If any assoc produced no keys, the AND semantics imply empty
-        return :__empty__ if key_sets.values.any? { |arr| Array(arr).empty? }
-
-        # Rewrite AST: remove joined nodes and add base IN(local_key, keys) per assoc
-        rewritten_ast = rewrite_ast_with_key_sets(ast_nodes, key_sets, base_klass)
-
-        relation.send(:spawn) do |s|
-          s[:ast] = rewritten_ast
-          # NOTE: s[:filters] retained (base fragments only); joins preserved for DX
-        end
-      end
-
-      def extract_join_inners(ast_nodes)
-        map = {}
-        walker = lambda do |node|
-          return unless node.is_a?(SearchEngine::AST::Node)
-
-          node.children.each { |ch| walker.call(ch) } if node.respond_to?(:children) && node.children
-
-          if node.respond_to?(:field)
-            field = node.field.to_s
-            if (m = field.match(/^\$(\w+)\.(.+)$/))
-              assoc = m[1].to_sym
-              inner_field = m[2]
-              case node
-              when SearchEngine::AST::Eq
-                (map[assoc] ||= []) << [:eq, inner_field, node.value]
-              when SearchEngine::AST::In
-                (map[assoc] ||= []) << [:in, inner_field, node.values]
-              else
-                # Unsupported node type for fallback (e.g., ranges, not_eq, etc.)
-                raise SearchEngine::Errors::InvalidOption.new(
-                  'Only equality and IN predicates on joined fields are supported by client-side join fallback',
-                  doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/joins#client-side-fallback'
-                )
-              end
-            end
-          end
-        end
-
-        Array(ast_nodes).each { |n| walker.call(n) }
-        map
-      end
-
-      def fetch_keys_for_assoc(base_klass, assoc_cfg, inners)
-        require 'search_engine/joins/resolver'
-        keys = SearchEngine::Joins::Resolver.resolve_keys(base_klass, assoc_cfg)
-        collection = assoc_cfg[:collection]
-        target_klass = SearchEngine.collection_for(collection)
-
-        # Build target relation by AND-ing inner predicates
-        rel = target_klass.all
-        inners.each do |(op, field, value)|
-          case op
-          when :eq
-            rel = rel.where(field.to_s => value)
-          when :in
-            rel = rel.where(field.to_s => Array(value))
-          end
-        end
-
-        vals = rel.pluck(keys[:foreign_key])
-        Array(vals).flatten.compact.uniq
-      end
-
-      def rewrite_ast_with_key_sets(ast_nodes, key_sets, base_klass)
-        # Remove joined predicates and append base IN(local_key, keys) for each assoc
-        stripped = strip_join_nodes(ast_nodes)
-        added = []
-        key_sets.each do |assoc, keys|
-          cfg = base_klass.join_for(assoc)
-          require 'search_engine/joins/resolver'
-          lk = SearchEngine::Joins::Resolver.resolve_keys(base_klass, cfg)[:local_key]
-          added << SearchEngine::AST.in_(lk.to_sym, keys)
-        end
-        (Array(stripped) + added).flatten.compact
-      end
-
-      def strip_join_nodes(nodes)
-        out = []
-        Array(nodes).each do |node|
-          next unless node.is_a?(SearchEngine::AST::Node)
-
-          case node
-          when SearchEngine::AST::And
-            children = strip_join_nodes(node.children)
-            out.concat(Array(children))
-          when SearchEngine::AST::Or
-            # Fallback does not support OR with joined nodes; reject early
-            raise SearchEngine::Errors::InvalidOption.new(
-              'OR with joined predicates is not supported by client-side join fallback',
-              doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/joins#client-side-fallback'
-            )
-          else
-            if node.respond_to?(:field)
-              f = node.field.to_s
-              next if f.start_with?('$')
-            end
-            out << node
-          end
-        end
-        out
-      end
-
-      def instrument_client_side_fallback(relation)
-        return unless defined?(SearchEngine::Instrumentation)
-
-        SearchEngine::Instrumentation.instrument(
-          'search_engine.joins.client_side_fallback',
-          collection: (relation.klass.respond_to?(:collection) ? relation.klass.collection : nil),
-          joins: Array(relation.joins_list).map(&:to_s)
-        )
-      rescue StandardError
-        nil
-      end
-
       def effective_per_page(relation)
         state = relation.instance_variable_get(:@state) || {}
         per_page = state[:per_page]
@@ -799,33 +411,6 @@ module SearchEngine
         { fields: fields.freeze, queries: queries.freeze }.freeze
       end
       module_function :build_facets_context_from_state
-
-      # Detect a simple AST eq(:id, value) predicate with no other filters.
-      def detect_equality_id_predicate_value(relation)
-        state = relation.instance_variable_get(:@state) || {}
-        ast_nodes = Array(state[:ast]).flatten.compact
-        return nil unless ast_nodes.size == 1
-
-        node = ast_nodes.first
-        return nil unless node.is_a?(SearchEngine::AST::Eq)
-        return nil unless node.field.to_s == 'id'
-
-        node.value
-      rescue StandardError
-        nil
-      end
-
-      # Use the Typesense retrieve-by-id endpoint and hydrate the document.
-      def retrieve_document_by_id(relation, id_value)
-        collection = relation.send(:collection_name_for_klass)
-        client = relation.send(:client)
-        raw = client.retrieve_document(collection: collection, id: id_value)
-        return nil unless raw.is_a?(Hash)
-
-        SearchEngine::Base::Creation::Helpers.hydrate_from_document(relation.klass, raw)
-      rescue StandardError
-        nil
-      end
     end
   end
 end

data/lib/search_engine/indexer/bulk_import.rb

@@ -161,6 +161,7 @@ module SearchEngine
         # @return [SearchEngine::Indexer::Summary]
         def call_parallel(klass:, into:, enum:, batch_size:, action:, log_batches:, max_parallel:, on_batch: nil)
           require 'concurrent-ruby'
+          require 'search_engine/interruptible_pool'
 
           docs_enum = normalize_enum(enum)
           total_batches_estimate = estimate_total_batches(klass)
@@ -186,7 +187,7 @@ module SearchEngine
             log_batches: log_batches, started_at: started_at, total_estimate: total_batches_estimate
           ) { |err| producer_error = err }
 
-          SearchEngine::InterruptiblePool.run(
+          pool_status = SearchEngine::InterruptiblePool.run(
             pool,
             on_interrupt: -> { interrupt_parallel!(shared_state[:cancelled], batch_queue) }
           ) do
@@ -196,17 +197,18 @@ module SearchEngine
               pool: pool, shared_state: shared_state, max_parallel: max_parallel
             )
           end
-          producer_thread.join if producer_thread.alive?
+          raise SearchEngine::Errors::InvalidParams, 'Parallel import workers timed out' if pool_status == :timed_out
+
+          producer_thread.join
+          raise shared_state[:worker_error] if shared_state[:worker_error]
 
           raise producer_error if producer_error
 
           build_summary(klass, shared_state)
         ensure
-          begin
-            producer_thread&.join(5) if producer_thread&.alive?
-          rescue StandardError
-            nil
-          end
+          shared_state[:cancelled].make_true if shared_state
+          batch_queue&.close
+          producer_thread&.join
         end
 
         # Initialize shared state hash for parallel batch processing.
@@ -267,6 +269,8 @@ module SearchEngine
                          end
               SearchEngine::Logging::Output.puts(SearchEngine::Logging::Color.dim(progress))
             end
+          rescue ClosedQueueError
+            raise unless cancelled.true?
           rescue StandardError => error
             yield error if block_given?
             err_msg = "  Producer failed at batch #{batch_count}: #{error.class}: #{error.message.to_s[0, 200]}"
@@ -327,6 +331,10 @@ module SearchEngine
                   shared_state: shared_state
                 )
               end
+            rescue StandardError => error
+              shared_state[:mtx].synchronize { shared_state[:worker_error] ||= error }
+              cancelled.make_true
+              batch_queue.close
             end
           end
         end

data/lib/search_engine/mapper.rb

@@ -355,13 +355,7 @@ module SearchEngine
           hash = normalize_document(@map_proc.call(row))
           hash.delete(:id)
           hash.delete('id')
-          begin
-            computed_id = @klass.compute_document_id(row)
-          rescue NoMethodError
-            rid = row.respond_to?(:id) ? row.id : nil
-            computed_id = rid.is_a?(String) ? rid : rid.to_s
-          end
-          hash[:id] = computed_id
+          hash[:id] = @klass.compute_document_id(row)
           hash[:doc_updated_at] = now_i
 
           strip_auto_embedding_fields!(hash)
@@ -441,7 +435,7 @@ module SearchEngine
           next if value.nil? && @__optional_blank_targets__.include?(fname)
 
           valid, coerced, err = validate_value(expected, value, field: fname)
-          if coerced
+          if !coerced.nil?
             stats[:coerced_count] += 1
             hash[key] = coerced
           elsif !valid && stats[:invalid_type_samples].size < @options[:max_error_samples]

data/lib/search_engine/sources/sql_source.rb

@@ -26,8 +26,8 @@ module SearchEngine
         @binds = binds
         cfg = SearchEngine.config.sources.sql
         @fetch_size = (fetch_size || cfg.fetch_size).to_i
-        @row_shape = row_shape || :auto
-        @statement_timeout_ms = statement_timeout_ms
+        @row_shape = row_shape || cfg.row_shape
+        @statement_timeout_ms = statement_timeout_ms || cfg.statement_timeout_ms
       end
 
       # Iterate over batches produced by the SQL query.
@@ -38,9 +38,9 @@ module SearchEngine
       def each_batch(partition: nil, cursor: nil, &block)
         return enum_for(:each_batch, partition: partition, cursor: cursor) unless block_given?
 
-        run_with_connection do
-          if postgres_connection?(ActiveRecord::Base.connection.raw_connection)
-            stream_postgres(ActiveRecord::Base.connection.raw_connection, partition: partition, cursor: cursor, &block)
+        run_with_connection do |conn|
+          if postgres_connection?(conn)
+            stream_postgres(conn, partition: partition, cursor: cursor, &block)
           else
             stream_generic(nil, partition: partition, cursor: cursor, &block)
           end
@@ -78,13 +78,17 @@ module SearchEngine
       end
 
       def stream_postgres(conn, partition:, cursor:)
+        unless conn.transaction_status.zero?
+          raise SearchEngine::Errors::InvalidParams,
+                'SqlSource streaming requires a connection without an active transaction'
+        end
+
         cursor_name = "se_cursor_#{object_id}"
         sql, params = build_sql_and_params(partition: partition, cursor: cursor)
         started = monotonic_ms
+        conn.exec('BEGIN READ ONLY')
         begin
           set_statement_timeout(conn, @statement_timeout_ms) if @statement_timeout_ms
-          # Use unnamed prepared statement + DECLARE CURSOR for streaming
-          conn.exec('BEGIN READ ONLY')
           conn.exec_params("DECLARE #{cursor_name} NO SCROLL CURSOR FOR #{sql}", params)
           idx = 0
           loop do
@@ -101,18 +105,10 @@ module SearchEngine
             idx += 1
             started = monotonic_ms
           end
+          conn.exec("CLOSE #{cursor_name}")
+          conn.exec('COMMIT')
         ensure
-          begin
-            conn.exec("CLOSE #{cursor_name}")
-          rescue StandardError
-            # ignore
-          end
-          begin
-            conn.exec('COMMIT')
-          rescue StandardError
-            # ignore
-          end
-          reset_statement_timeout(conn) if @statement_timeout_ms
+          conn.exec('ROLLBACK') unless conn.transaction_status.zero?
         end
       end
 
@@ -181,15 +177,7 @@ module SearchEngine
       end
 
       def set_statement_timeout(conn, ms)
-        conn.exec_params('SET LOCAL statement_timeout = $1', [Integer(ms)])
-      rescue StandardError
-        # ignore
-      end
-
-      def reset_statement_timeout(conn)
-        conn.exec('RESET statement_timeout')
-      rescue StandardError
-        # ignore
+        conn.exec_params("SELECT set_config('statement_timeout', $1, true)", [Integer(ms).to_s])
       end
     end
   end

data/lib/search_engine/version.rb

@@ -3,5 +3,5 @@
 module SearchEngine
   # Current gem version.
   # @return [String]
-  VERSION = '30.1.8.23'
+  VERSION = '30.8.24'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: search-engine-for-typesense
 version: !ruby/object:Gem::Version
-  version: 30.1.8.23
+  version: 30.8.24
 platform: ruby
 authors:
 - Nikita Shkoda
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-07-11 00:00:00.000000000 Z
+date: 2026-09-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby