search-engine-for-typesense 30.9.1 → 30.9.3

data/lib/search_engine/filters/sanitizer.rb

@@ -1,168 +1,204 @@
 # frozen_string_literal: true
 
+require 'date'
+require 'time'
+require 'search_engine/joins/guard'
+
 module SearchEngine
   module Filters
-    # Sanitizer utilities for Typesense-compatible filters.
-    #
-    # Provides quoting/escaping and helpers to build normalized filter strings
-    # from hashes and templates with placeholders.
+    # Encodes structured predicates for the Typesense filter_by grammar.
     module Sanitizer
       module_function
 
-      # Quote a Ruby value into a Typesense filter literal.
-      #
-      # - NilClass => "null"
-      # - TrueClass/FalseClass => "true"/"false"
-      # - Numeric => as-is (stringified)
-      # - String => double-quoted with minimal escaping for \ and "
-      # - Time/DateTime/Date => ISO8601 string, then quoted as a string
-      # - Array => one-level flatten, each element quoted, wrapped with [ ... ]
-      #
-      # @param value [Object]
-      # @return [String]
-      def quote(value)
-        case value
-        when String
-          %("#{escape_string(value)}")
-        when Array
-          elements = value.flatten(1).map { |el| quote(el) }
-          "[#{elements.join(', ')}]"
-        else
-          quote_non_string(value)
+      # A field path is data too. Only AST predicates may use the internal
+      # "$association.field" notation, which the compiler turns into a join.
+      def field_name(field, joined: false)
+        raise ArgumentError, 'filter field must be a String or Symbol' unless field.is_a?(String) || field.is_a?(Symbol)
+
+        name = field.to_s
+        path = joined && name.start_with?('$') && name.include?('.') ? name[1..] : name
+        raise ArgumentError, 'invalid filter field name' unless path.match?(/\A[[:alnum:]_-]+(?:\.[[:alnum:]_-]+)*\z/)
+
+        name
+      end
+
+      # Backticks protect operators during expression parsing. A list (even
+      # with a single item) also removes the delimiters before tokenization,
+      # including on collections that explicitly index the backtick symbol.
+      def quote(value, field: nil, klass: nil, joined: false)
+        if value.is_a?(Array)
+          items = value.flatten(1).map { |item| literal(item, field: field, klass: klass, joined: joined) }
+          return "[#{items.join(', ')}]"
         end
+
+        item = literal(value, field: field, klass: klass, joined: joined)
+        item.start_with?('`') ? "[#{item}]" : item
       end
 
-      # Quote a scalar Ruby value for Typesense filters with conditional quoting for strings.
-      #
-      # Rules (based on Typesense filter_by syntax):
-      # - Strings that match a safe token pattern (e.g., Active, ACTIVE_1, foo-bar) are emitted bare
-      # - Reserved words true/false/null remain bare only when actually booleans/nil; string forms are quoted
-      # - Strings with any other characters are double-quoted with escaping
-      # - Arrays are delegated to +quote+ to preserve element quoting rules
-      #
-      # @param value [Object]
-      # @return [String]
       def quote_scalar_for_filter(value)
-        return quote(value) if value.is_a?(Array)
+        quote(value)
+      end
 
+      def literal(value, field:, klass:, joined:)
         case value
-        when String, Symbol
-          str = value.to_s
-          lc = str.strip.downcase
-          return %("#{escape_string(str)}") if %w[true false null].include?(lc)
+        when nil then 'null'
+        when true then 'true'
+        when false then 'false'
+        when Numeric
+          number = value.to_s
+          unless number.match?(/\A[+-]?[0-9]+(?:\.[0-9]+)?(?:e[+-]?[0-9]+)?\z/i)
+            raise ArgumentError, 'invalid numeric filter value'
+          end
 
-          safe_bare_string?(str) ? str : %("#{escape_string(str)}")
+          number
+        when Array
+          raise ArgumentError, 'expected a scalar filter list item'
         else
-          quote_non_string(value)
+          string = if value.respond_to?(:to_time) && !value.is_a?(String)
+                     type = Array(klass.attributes[field.to_sym]).first if field && klass.respond_to?(:attributes)
+                     if joined && type.nil? && field.to_s != 'id'
+                       raise ArgumentError, 'joined time filters require a declared type or a string or epoch value'
+                     end
+
+                     time = value.to_time
+                     return time.to_i.to_s if %i[time datetime integer].include?(type)
+
+                     value.respond_to?(:iso8601) ? value.iso8601 : time.iso8601
+                   else
+                     value.to_s
+                   end
+          string_literal(string, field: field, joined: joined)
+        end
+      end
+      private_class_method :literal
+
+      def string_literal(string, field:, joined:)
+        # The expression scanner and list splitter disagree about escaped
+        # backticks. The splitter also trims spaces, recognizes phrases, and
+        # gives a trailing asterisk special meaning after removing delimiters.
+        id_field = field.to_s == 'id'
+        if string.empty? || string.match?(/[`[:cntrl:]]/) || string.start_with?(' ') || string.end_with?(' ') ||
+           (string == '*' && (id_field || field.nil?)) || (!id_field && string.length > 1 && string.end_with?('*')) ||
+           (!id_field && string.length > 1 && string.start_with?('"') && string.end_with?('"'))
+          raise ArgumentError, 'filter value cannot be represented as an exact Typesense literal'
+        end
+
+        # Typesense recognizes geopoint syntax even inside a backtick pair.
+        # Its outer join scanner also counts parentheses inside quoted values.
+        if string.match?(/: *(?:\[ *)?\(/) || (joined && !balanced_parentheses?(string))
+          raise ArgumentError, 'filter value contains unrepresentable Typesense delimiters'
+        end
+
+        return string if string.match?(/\A[[:alnum:]_\\-]+\z/)
+        raise ArgumentError, 'quoted filter value cannot end in a backslash' if string.end_with?('\\')
+
+        "`#{string}`"
+      end
+      private_class_method :string_literal
+
+      def balanced_parentheses?(string)
+        depth = 0
+        string.each_char do |char|
+          depth += 1 if char == '('
+          depth -= 1 if char == ')'
+          return false if depth.negative?
         end
+        depth.zero?
       end
+      private_class_method :balanced_parentheses?
 
-      # Build normalized filter fragments from a Hash.
-      # Scalars become "field:=<quoted>", arrays become "field:=<quoted_list>".
-      #
-      # @param hash [Hash{#to_sym=>Object}]
-      # @param _attributes_map [Hash] (ignored here; validation should be done by caller)
-      # @return [Array<String>]
-      def build_from_hash(hash, _attributes_map = nil)
+      def build_from_hash(hash, _attributes_map = nil, klass: nil)
         raise ArgumentError, 'filters hash must be a Hash' unless hash.is_a?(Hash)
 
-        hash.map do |key, raw|
-          field = key.to_sym.to_s
-          if array_like?(raw)
-            "#{field}:=#{quote(Array(raw))}"
-          else
-            "#{field}:=#{quote_scalar_for_filter(raw)}"
-          end
+        hash.map do |key, value|
+          field = field_name(key)
+          "#{field}:=#{quote(value, field: field, klass: klass)}"
         end
       end
 
-      # Apply placeholder substitution for templates with '?' markers.
-      #
-      # Each unescaped '?' is replaced with a quoted argument from +args+ in order.
-      #
-      # @param template [String]
-      # @param args [Array<Object>]
-      # @return [String]
-      def apply_placeholders(template, args)
+      # Consume a quoted literal or an escape before looking for structural
+      # tokens. Typesense only uses backticks to protect filter operators.
+      TEMPLATE_TOKEN = /`[^`]*(?:`|\z)|(?<![[:alnum:]])(?:"[^"]*(?:"|\z)|'[^']*(?:'|\z))|\\[?\\]|&&|\|\||[(){}\[\]?]/
+      BOUND_FIELD = /\A\s*([[:alnum:]_.-]+)\s*(?::\s*(?:!=|>=|<=|[!<>=])?|!=|>=|<=|[=<>]|NOT\s+IN|IN)\s*\z/i
+      CLOSING_DELIMITERS = { '(' => ')', '[' => ']', '{' => '}' }.freeze
+      private_constant :TEMPLATE_TOKEN, :BOUND_FIELD, :CLOSING_DELIMITERS
+
+      def apply_placeholders(template, args, klass: nil)
         raise ArgumentError, 'template must be a String' unless template.is_a?(String)
         raise ArgumentError, 'args must be an Array' unless args.is_a?(Array)
 
         needed = count_placeholders(template)
-        provided = args.length
-        raise ArgumentError, "expected #{needed} args for #{needed} placeholders, got #{provided}" if needed != provided
-
-        idx = -1
-        template.gsub(/(?<!\\)\?/) do
-          idx += 1
-          val = args[idx]
-          val.is_a?(Array) ? quote(val) : quote_scalar_for_filter(val)
+        unless needed == args.length
+          raise ArgumentError, "expected #{needed} args for #{needed} placeholders, got #{args.length}"
         end
-      end
-
-      # Count unescaped '?' placeholders.
-      # @param template [String]
-      # @return [Integer]
-      def count_placeholders(template)
-        count = 0
-        escaped = false
-        template.each_char do |ch|
-          if escaped
-            escaped = false
-            next
-          end
-          if ch == '\\'
-            escaped = true
-          elsif ch == '?'
-            count += 1
+        return template if needed.zero?
+
+        # Keep trusted fixed expressions verbatim. Track groups and reference
+        # filters so only a complete, unquoted comparison operand accepts data.
+        scopes = [[nil, klass, false]]
+        start = 0
+        binds = args.each
+        result = template.gsub(TEMPLATE_TOKEN) do |token|
+          match = Regexp.last_match
+          case token
+          when '?'
+            prefix = template[start...match.begin(0)]
+            field = prefix[BOUND_FIELD, 1]
+            ends_operand = template.match?(/\G\s*(?:&&|\|\||\)|\z)/, match.end(0))
+            _, model, joined = scopes.last
+            unless field && ends_operand && model != :opaque
+              raise ArgumentError, 'placeholder must be a complete filter operand'
+            end
+
+            next quote(binds.next, field: field_name(field), klass: model, joined: joined)
+          when '(', '[', '{'
+            prefix = template[start...match.begin(0)].strip
+            scopes << template_scope(token, prefix, scopes.last)
+          when ')', ']', '}'
+            raise ArgumentError, 'unbalanced filter template' unless scopes.last.first == token
+
+            scopes.pop
+          when /\A[`"'\\]/
+            validate_template_literal!(token, joined: scopes.last.last)
+            next token
           end
+          start = match.end(0)
+          token
         end
-        count
+        raise ArgumentError, 'unbalanced filter template' unless scopes.length == 1
+
+        result
       end
 
-      # Shared quoting logic for non-String, non-Array values.
-      # @param value [Object]
-      # @return [String]
-      # @api private
-      def quote_non_string(value)
-        case value
-        when NilClass then 'null'
-        when TrueClass then 'true'
-        when FalseClass then 'false'
-        when Numeric then value.to_s
-        when Time, DateTime then %("#{value.iso8601}")
-        when Date then %("#{value.iso8601}")
-        else
-          if value.respond_to?(:to_time)
-            %("#{value.to_time.iso8601}")
-          else
-            %("#{escape_string(value.to_s)}")
-          end
+      def validate_template_literal!(token, joined:)
+        raise ArgumentError, 'placeholder must be a complete filter operand' if count_placeholders(token).positive?
+        if (token.start_with?('"', "'") && token.include?('`')) || token.match?(/: *(?:\[ *)?\(/)
+          raise ArgumentError, 'ambiguous delimiters in quoted filter template'
         end
-      end
+        return unless joined && !balanced_parentheses?(token)
 
-      # Escape a raw string for inclusion inside double quotes.
-      # @param str [String]
-      # @return [String]
-      def escape_string(str)
-        str.gsub('\\', '\\\\').gsub('"', '\\"')
+        raise ArgumentError, 'unbalanced parentheses in quoted reference filter'
       end
+      private_class_method :validate_template_literal!
 
-      # Determine whether a string can be emitted bare without quotes in filter_by.
-      # Safe if it matches: starts with a letter or underscore; then letters/digits/underscore/hyphen.
-      # This avoids ambiguity with numbers/booleans/null and special characters.
-      def safe_bare_string?(str)
-        return false if str.nil? || str.empty?
-
-        # Disallow surrounding/backtick/dquote characters quickly
-        return false if str.include?('"') || str.include?('`') || str.include?(',') || str.include?(' ')
+      def template_scope(token, prefix, parent)
+        _, model, joined = parent
+        if token == '(' && model != :opaque
+          return [')', model, joined] if prefix.empty?
 
-        # Must start with a letter or underscore; subsequent chars may include digits or hyphens/underscores
-        !!(str =~ /^[A-Za-z_][A-Za-z0-9_-]*$/)
+          if (name = prefix[/\A!?\$([[:alnum:]_-]+)\z/, 1])
+            return [')', SearchEngine::Joins::Guard.target_model(model, name), true]
+          end
+        end
+        # Lists, object filters and geopoint tuples use their own grammars.
+        [CLOSING_DELIMITERS.fetch(token), :opaque, joined]
       end
+      private_class_method :template_scope
+
+      def count_placeholders(template)
+        return 0 unless template.include?('?')
 
-      # @api private
-      def array_like?(value)
-        value.is_a?(Array)
+        template.scan(/\\.|(\?)/m).count(&:first)
       end
     end
   end

data/lib/search_engine/hydration/materializers.rb

@@ -24,29 +24,7 @@ module SearchEngine
           memo = relation.instance_variable_get(:@__result_memo)
           return memo if loaded && memo
 
-          collection = relation.send(:collection_name_for_klass)
-          params = SearchEngine::CompiledParams.from(relation.to_typesense_params)
-          url_opts = relation.send(:build_url_opts)
-
-          raw_result = relation.send(:client).search(
-            collection: collection, params: params, url_opts: url_opts
-          )
-
-          selection_ctx = SearchEngine::Hydration::SelectionContext.build(relation)
-          facets_ctx = build_facets_context_from_state(relation)
-
-          result = if selection_ctx || facets_ctx
-                     SearchEngine::Result.new(
-                       raw_result.raw,
-                       klass: relation.klass,
-                       selection: selection_ctx,
-                       facets: facets_ctx
-                     )
-                   else
-                     raw_result
-                   end
-
-          relation.send(:enforce_hit_validator_if_needed!, result.found, collection: collection)
+          result = search_result(relation)
           relation.instance_variable_set(:@__result_memo, result)
           relation.instance_variable_set(:@__loaded, true)
           result
@@ -84,39 +62,10 @@ module SearchEngine
           return Array(cached).first(limit)
         end
 
-        # Use the relation's effective per_page for the underlying request to keep
-        # ordering consistent with to_a; slice to the preview limit locally.
-        per_for_request = effective_per_page(relation)
-        per_for_request = limit if per_for_request.to_i <= 0
-
-        preview_relation = relation.send(:spawn) do |state|
-          state[:page] = 1
-          state[:per_page] = per_for_request
-        end
-
-        collection = preview_relation.send(:collection_name_for_klass)
-        params = SearchEngine::CompiledParams.from(preview_relation.to_typesense_params)
-        url_opts = preview_relation.send(:build_url_opts)
-
-        raw_result = preview_relation.send(:client).search(collection: collection, params: params, url_opts: url_opts)
-
-        selection_ctx = SearchEngine::Hydration::SelectionContext.build(preview_relation)
-        facets_ctx = build_facets_context_from_state(preview_relation)
-
-        result = if selection_ctx || facets_ctx
-                   SearchEngine::Result.new(
-                     raw_result.raw,
-                     klass: preview_relation.klass,
-                     selection: selection_ctx,
-                     facets: facets_ctx
-                   )
-                 else
-                   raw_result
-                 end
-
-        array = Array(result.respond_to?(:to_a) ? result.to_a : result).first(limit)
+        result = search_result(relation)
+        array = result.to_a
         relation.instance_variable_set(:@__preview_memo, array)
-        array
+        array.first(limit)
       end
 
       def each(relation, &block)
@@ -172,44 +121,20 @@ module SearchEngine
         validate_pluck_fields_allowed!(relation, names)
 
         result = execute(relation)
-        raw_hits = Array(result.raw['hits'])
-        objects = result.to_a
-
-        enforce_strict_for_pluck_row = lambda do |doc, requested|
-          present_keys = doc.keys.map(&:to_s)
-          if result.respond_to?(:send)
-            ctx = result.instance_variable_get(:@selection_ctx) || {}
-            if ctx[:strict_missing] == true
-              result.send(:enforce_strict_missing_if_needed!, present_keys, requested_override: requested)
-            end
+        documents = Result.documents(result.raw)
+        result.to_a.each_with_index.map do |object, index|
+          doc = documents.fetch(index)
+          if result.is_a?(Result)
+            result.send(:enforce_strict_missing_if_needed!, doc.keys.map(&:to_s), requested_override: names)
           end
-        end
-
-        if names.length == 1
-          field = names.first
-          return objects.each_with_index.map do |obj, idx|
-            doc = (raw_hits[idx] && raw_hits[idx]['document']) || {}
-            # Enforce strict missing for the requested field against present keys
-            enforce_strict_for_pluck_row.call(doc, [field])
-            if obj.respond_to?(field)
-              obj.public_send(field)
-            else
-              doc[field]
-            end
-          end
-        end
-
-        objects.each_with_index.map do |obj, idx|
-          doc = (raw_hits[idx] && raw_hits[idx]['document']) || {}
-          # Enforce strict missing for all requested fields against present keys
-          enforce_strict_for_pluck_row.call(doc, names)
-          names.map do |field|
-            if obj.respond_to?(field)
-              obj.public_send(field)
+          values = names.map do |field|
+            if object.respond_to?(field)
+              object.public_send(field)
             else
-              doc[field]
+              doc.fetch(field) { doc[field.to_sym] }
             end
           end
+          names.one? ? values.first : values
         end
       end
 
@@ -250,11 +175,38 @@ module SearchEngine
 
       # --- internals --------------------------------------------------------
 
+      def result_options(relation)
+        {
+          klass: relation.klass,
+          selection: SelectionContext.build(relation),
+          facets: build_facets_context_from_state(relation)
+        }
+      end
+
+      def search_result(relation, client: relation.send(:client))
+        collection = relation.send(:collection_name_for_klass)
+        params = SearchEngine::CompiledParams.from(relation.to_typesense_params)
+        options = result_options(relation)
+        built = false
+        url_opts = relation.send(:build_url_opts)
+        result = client.search(collection: collection, params: params, url_opts: url_opts) do |raw|
+          built = true
+          Result.new(raw, **options)
+        end
+
+        # Legacy clients return an already wrapped result without yielding the response.
+        if !built && (options[:selection] || options[:facets])
+          result = result.is_a?(Result) ? result.with_context(**options) : Result.new(result.raw, **options)
+        end
+        relation.send(:enforce_hit_validator_if_needed!, result.found, collection: collection)
+        result
+      end
+
       def fetch_found_only(relation)
         collection = relation.send(:collection_name_for_klass)
         base = SearchEngine::CompiledParams.from(relation.to_typesense_params).to_h
 
-        minimal = base.dup
+        minimal = base.except(:offset, :limit)
         minimal[:per_page] = 1
         minimal[:page] = 1
         minimal[:include_fields] = 'id'
@@ -285,8 +237,8 @@ module SearchEngine
 
       def effective_per_page(relation)
         state = relation.instance_variable_get(:@state) || {}
-        per_page = state[:per_page]
-        return per_page.to_i if per_page.to_i.positive?
+        per_page = state[:per_page] || (state[:limit] unless state[:page])
+        return [per_page, state.dig(:hit_limits, :early_limit)].compact.min if per_page
 
         loaded = relation.instance_variable_get(:@__loaded)
         memo = relation.instance_variable_get(:@__result_memo)
@@ -297,8 +249,6 @@ module SearchEngine
             per = request_params['per_page'] || request_params[:per_page]
             return per.to_i if per.to_i.positive?
           end
-
-          return memo.hits.size if memo.respond_to?(:hits)
         end
 
         10
@@ -313,35 +263,20 @@ module SearchEngine
         total = count(relation)
         return [] if total.to_i <= 0
 
-        per = 250
+        per = [250, relation.hit_limits&.dig(:early_limit)].compact.min
         pages = (total.to_f / per).ceil
 
-        # Fast path for a single page
-        return relation.page(1).per(per).to_a if pages <= 1
+        client = relation.send(:client)
+        return search_result(relation.page(1).per(per), client: client).to_a if pages == 1
 
-        limit = begin
-          SearchEngine.config.multi_search_limit.to_i
-        rescue StandardError
-          50
-        end
-        limit = 1 if limit <= 0
-
-        page_numbers = (1..pages).to_a
+        url_opts = relation.send(:build_url_opts)
         out = []
-
-        page_numbers.each_slice(limit) do |batch|
-          mr = SearchEngine.multi_search_result do |m|
-            batch.each do |p|
-              m.add("p#{p}", relation.page(p).per(per))
-            end
-          end
-
-          batch.each do |p|
-            res = mr["p#{p}"]
-            out.concat(Array(res&.to_a))
-          end
+        (1..pages).each_slice([SearchEngine.config.multi_search_limit, 1].max) do |batch|
+          builder = SearchEngine::Multi.new
+          batch.each { |page| builder.add("p#{page}", relation.page(page).per(per)) }
+          results = builder.execute(client: client, url_opts: url_opts)
+          results.each_label { |_, result| out.concat(result.hits) }
         end
-
         out
       end
 

data/{app/search_engine → lib}/search_engine/index_partition_job.rb

@@ -139,7 +139,7 @@ module SearchEngine
     def indexing_run_store(run_id)
       return nil if run_id.nil?
 
-      SearchEngine::IndexingRunStore.resolve
+      SearchEngine::IndexingRunStore.partition_writer
     end
 
     def retryable_error?(error)