parse-stack-next 5.7.6 → 5.8.1

data/lib/parse/vector_search/index_definition.rb

@@ -0,0 +1,237 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+
+module Parse
+  module VectorSearch
+    # Derives an Atlas `vectorSearch` index definition from a model's
+    # declarations, so the index an operator deploys matches what the SDK
+    # will query instead of being written by hand.
+    #
+    # Sources, all read from the model:
+    #
+    # * the `:vector` property: path, `numDimensions` (`dimensions:`),
+    #   `similarity` (`similarity:`, default `cosine`), and the optional
+    #   index-side `quantization:` (`:scalar` / `:binary`);
+    # * `agent_searchable filter_fields:`, the fields the agent tool lets a
+    #   caller pass as `vector_filter:` (pointer fields map to their
+    #   `_p_<column>` storage path, matching the pointer translation
+    #   {Parse::Retrieval.retrieve} applies);
+    # * the `agent_tenant_scope` field, which retrieval folds into
+    #   `$vectorSearch.filter` on every scoped query.
+    #
+    # Output is deterministic: the vector entry first, then filter entries
+    # sorted by path, keys in a fixed order. Generating is side-effect free;
+    # applying an index stays explicit through
+    # {Parse::Schema::SearchIndexMigrator} (declare it with the
+    # `vector_search_index` model macro and run `apply_search_indexes!`).
+    #
+    # @example
+    #   class Article < Parse::Object
+    #     property :embedding, :vector, dimensions: 1024, similarity: :dotProduct,
+    #                                   quantization: :scalar
+    #     agent_searchable field: :embedding, filter_fields: %i[category]
+    #     vector_search_index "article_vec"   # generated; applied explicitly
+    #   end
+    #
+    #   Parse::VectorSearch::IndexDefinition.build(Article)
+    #   # => { "fields" => [
+    #   #      { "type" => "vector", "path" => "embedding", "numDimensions" => 1024,
+    #   #        "similarity" => "dotProduct", "quantization" => "scalar" },
+    #   #      { "type" => "filter", "path" => "category" } ] }
+    module IndexDefinition
+      # Similarity emitted when the property declares none. Atlas requires
+      # one; cosine is the safe choice for unit-normalized embeddings.
+      DEFAULT_SIMILARITY = "cosine"
+
+      # Index-side quantization values accepted on a `:vector` property.
+      QUANTIZATIONS = %w[scalar binary].freeze
+
+      # Key order for each field entry, so serialized output is stable.
+      VECTOR_KEY_ORDER = %w[type path numDimensions similarity quantization].freeze
+
+      module_function
+
+      # Build the definition for one `:vector` field of `model_class`.
+      #
+      # @param model_class [Class] a Parse::Object subclass.
+      # @param field [Symbol, String, nil] the `:vector` property; may be
+      #   omitted when the class declares exactly one searchable vector.
+      # @return [Hash] a string-keyed `{ "fields" => [...] }` definition.
+      # @raise [ArgumentError] when the field cannot be resolved.
+      def build(model_class, field: nil)
+        field_sym = resolve_field!(model_class, field)
+        meta = model_class.vector_properties.fetch(field_sym)
+
+        vector = {
+          "type" => "vector",
+          # The stored column (field_map), which is what $vectorSearch
+          # queries: `body_embedding` is saved as `bodyEmbedding`.
+          "path" => (model_class.respond_to?(:vector_storage_path) ? model_class.vector_storage_path(field_sym) : field_sym.to_s),
+          "numDimensions" => meta[:dimensions],
+          "similarity" => (meta[:similarity] || DEFAULT_SIMILARITY).to_s,
+        }
+        vector["quantization"] = meta[:quantization].to_s if meta[:quantization]
+
+        filters = filter_paths(model_class).map { |path| { "type" => "filter", "path" => path } }
+        { "fields" => [order_keys(vector)] + filters }
+      end
+
+      # Preview the declaration a `vector_search_index` macro would apply.
+      #
+      # @return [Hash] `{ name:, type: "vectorSearch", definition: }`.
+      def preview(model_class, name:, field: nil)
+        { name: name.to_s, type: "vectorSearch", definition: build(model_class, field: field) }
+      end
+
+      # Structured, deterministic comparison of a declared definition with a
+      # live one. Accepts either definitions or index documents carrying
+      # `latestDefinition` (the shape `$listSearchIndexes` returns).
+      #
+      # @param declared [Hash] the generated (or declared) definition.
+      # @param live [Hash, nil] the live definition or index document.
+      # @return [Hash] `{ in_sync:, vector: { key => { declared:, live: } },
+      #   filters_missing: [...], filters_extra: [...] }`. `vector` lists
+      #   only differing keys; `filters_missing` are declared filter paths
+      #   absent from the live index (scoped queries on them fail
+      #   Atlas-side), `filters_extra` are live paths not declared.
+      def diff(declared, live)
+        declared_defn = definition_of(declared)
+        live_defn = definition_of(live)
+
+        d_vec = vector_entry(declared_defn)
+        # Compare against the live vector entry for the SAME path, so an
+        # index with several vector fields does not report false drift.
+        l_vec = vector_entry(live_defn, path: d_vec["path"])
+        vector_changes = {}
+        (VECTOR_KEY_ORDER - %w[type]).each do |key|
+          dv = normalize_vector_value(key, d_vec[key])
+          lv = normalize_vector_value(key, l_vec[key])
+          vector_changes[key] = { declared: dv, live: lv } unless dv == lv
+        end
+
+        d_filters = filter_paths_of(declared_defn)
+        l_filters = filter_paths_of(live_defn)
+        missing = (d_filters - l_filters).sort
+        extra = (l_filters - d_filters).sort
+
+        {
+          in_sync: vector_changes.empty? && missing.empty? && extra.empty?,
+          vector: vector_changes,
+          filters_missing: missing,
+          filters_extra: extra,
+        }
+      end
+
+      # @!visibility private
+      def resolve_field!(model_class, field)
+        unless model_class.respond_to?(:vector_properties)
+          raise ArgumentError, "#{model_class.inspect} declares no :vector properties."
+        end
+        props = model_class.vector_properties
+        searchable = props.keys.reject { |k| props[k][:searchable] == false }
+        if field
+          sym = field.to_sym
+          unless searchable.include?(sym)
+            raise ArgumentError,
+                  "#{model_class}: :#{sym} is not a searchable :vector property " \
+                  "(searchable: #{searchable.inspect})."
+          end
+          return sym
+        end
+        return searchable.first if searchable.length == 1
+        raise ArgumentError,
+              "#{model_class}: cannot infer the vector field (searchable: #{searchable.inspect}); " \
+              "pass field:."
+      end
+
+      # @!visibility private
+      # Filter paths the SDK pre-filters on for this class, deduplicated and
+      # sorted. Tenant path matches the resolution the migrator's
+      # augmentation and first-query drift check use.
+      def filter_paths(model_class)
+        paths = []
+        class_name = model_class.parse_class
+        if defined?(Parse::Agent::MetadataRegistry)
+          Parse::Agent::MetadataRegistry.searchable_filter_fields(class_name).each do |f|
+            paths << storage_path(model_class, f)
+          end
+          rule = Parse::Agent::MetadataRegistry.tenant_scope_rule(class_name)
+          paths << wire_name(model_class, rule[:field]) if rule
+        end
+        paths.compact.uniq.sort
+      end
+
+      # @!visibility private
+      def storage_path(model_class, field)
+        wire = wire_name(model_class, field)
+        type = model_class.respond_to?(:fields) ? model_class.fields[field.to_sym] : nil
+        type == :pointer ? "_p_#{wire}" : wire
+      end
+
+      # @!visibility private
+      def wire_name(model_class, field)
+        sym = field.to_sym
+        fmap = model_class.respond_to?(:field_map) ? model_class.field_map : {}
+        (fmap[sym] || sym.to_s.columnize).to_s
+      end
+
+      # @!visibility private
+      def order_keys(entry)
+        VECTOR_KEY_ORDER.each_with_object({}) { |k, h| h[k] = entry[k] if entry.key?(k) }
+      end
+
+      # @!visibility private
+      def definition_of(value)
+        return {} unless value.is_a?(Hash)
+        inner = value["latestDefinition"] || value[:latestDefinition] ||
+                value[:definition] || value["definition"]
+        inner.is_a?(Hash) ? inner : value
+      end
+
+      # @!visibility private
+      def fields_of(defn)
+        Array(defn["fields"] || defn[:fields]).select { |f| f.is_a?(Hash) }
+      end
+
+      # @!visibility private
+      def vector_entry(defn, path: nil)
+        vectors = fields_of(defn).select { |f| (f["type"] || f[:type]).to_s == "vector" }
+        entry = (path && vectors.find { |f| (f["path"] || f[:path]).to_s == path.to_s }) || vectors.first || {}
+        entry.each_with_object({}) { |(k, v), h| h[k.to_s] = v }
+      end
+
+      # @!visibility private
+      def filter_paths_of(defn)
+        fields_of(defn).select { |f| (f["type"] || f[:type]).to_s == "filter" }
+                       .map { |f| (f["path"] || f[:path]).to_s }.uniq.sort
+      end
+
+      # @!visibility private
+      # An absent `quantization` means none on both sides; numbers compare
+      # as integers; everything else as strings.
+      def normalize_vector_value(key, value)
+        case key
+        when "quantization"
+          value.nil? || value.to_s.empty? ? "none" : value.to_s
+        when "numDimensions"
+          # A malformed live value is reported as drift, not raised.
+          value.nil? ? nil : (Integer(value) rescue value.to_s)
+        else
+          value&.to_s
+        end
+      end
+    end
+  end
+
+  module Schema
+    class << self
+      # Generate the Atlas `vectorSearch` index definition for a model's
+      # `:vector` field. See {Parse::VectorSearch::IndexDefinition.build}.
+      #
+      # @return [Hash]
+      def vector_index_definition(model_class, field: nil)
+        Parse::VectorSearch::IndexDefinition.build(model_class, field: field)
+      end
+    end
+  end
+end

data/lib/parse/vector_search.rb

@@ -5,6 +5,7 @@ require_relative "pipeline_security"
 require_relative "acl_scope"
 require_relative "clp_scope"
 require_relative "mongodb"
+require_relative "atlas_search/protected_paths"
 
 module Parse
   # Atlas Vector Search entry point. Routes through `Parse::MongoDB`
@@ -46,8 +47,10 @@ module Parse
   #
   # * CLP `find` boundary check — refuses calls the equivalent REST
   #   find would refuse.
-  # * Optional `pointerFields` post-filter — drops rows that don't
-  #   name the current user_id in the configured pointer fields.
+  # * `pointerFields` / `readUserFields` ownership `$match` after
+  #   `$vectorSearch`, pushed into the `$vectorSearch.filter` prefilter
+  #   when every owner pointer (`_p_<field>`) is declared as a
+  #   `type: "filter"` field of the index.
   # * Post-`$vectorSearch` ACL `$match` injection (Parse Server's
   #   `_rperm` predicate).
   # * Post-fetch `protectedFields` redaction.
@@ -123,9 +126,10 @@ module Parse
     #
     # * `candidate_limit` / `num_candidates` — the requested window.
     # * `post_filter_count` — rows returned by the pipeline, i.e. after
-    #   the server-side ACL `$match` and any caller `filter`.
+    #   the server-side ACL `$match`, the pointerFields `$match`, and any
+    #   caller `filter`.
     # * `post_pointer_count` — rows left after client-side redaction and
-    #   pointer-field filtering.
+    #   the defense-in-depth pointer-field re-check.
     # * `returned_count` — rows handed back, after trimming to `k`.
     # * `underfilled` — the caller received fewer than `k`.
     #
@@ -136,6 +140,10 @@ module Parse
     # Accepted {.index_drift_policy} values.
     INDEX_DRIFT_POLICIES = %i[warn raise ignore].freeze
 
+    # Guards the lazy creation of the owner-prefilter negative cache mutex.
+    PREFILTER_MUTEX_INIT = Mutex.new
+    private_constant :PREFILTER_MUTEX_INIT
+
     class << self
       # Policy applied when first-query index verification (see
       # {Parse::Core::VectorSearchable}) finds the deployed Atlas
@@ -306,6 +314,8 @@ module Parse
         protected_fields = Parse::CLPScope.protected_fields_for(
           collection_name, resolution.permission_strings,
         )
+        assert_protected_fields_untouched!(collection_name, path, filter, vector_filter,
+                                           protected_fields, resolution)
 
         vs_stage = {
           "index" => index_name.to_s,
@@ -316,7 +326,14 @@ module Parse
           # result set is trimmed to `k` after enforcement runs.
           "limit" => candidate_limit,
         }
-        vs_stage["filter"] = vector_filter if vector_filter && !vector_filter.empty?
+        caller_prefilter = vector_filter if vector_filter && !vector_filter.empty?
+        owner_prefilter = unless resolution.master?
+            owner_vector_prefilter(collection_name, index_name, pointer_fields, resolution)
+          end
+        prefilter = [caller_prefilter, owner_prefilter].compact
+        unless prefilter.empty?
+          vs_stage["filter"] = prefilter.size == 1 ? prefilter.first : { "$and" => prefilter }
+        end
         pipeline = [{ "$vectorSearch" => vs_stage }]
 
         pipeline << {
@@ -334,12 +351,39 @@ module Parse
         unless resolution.master?
           acl_match = Parse::ACLScope.match_stage_for(resolution)
           pipeline << acl_match if acl_match
+          # pointerFields / readUserFields ownership, server-side so it
+          # runs on the whole candidate window before the trim to `k`.
+          # When every owner pointer is filter-indexed it also ran as the
+          # `$vectorSearch.filter` prefilter above; this stage then drops
+          # nothing but still covers the array-of-pointers storage form.
+          if pointer_fields
+            pipeline << {
+              "$match" => Parse::CLPScope.pointer_fields_predicate(pointer_fields, resolution.user_id),
+            }
+          end
         end
 
         pipeline << { "$match" => filter } if filter
 
-        raw_results = run_pipeline!(collection_name, pipeline, max_time_ms: max_time_ms,
-                                                               authorizing_client: Parse::ACLScope.client_of(resolution))
+        authorizing_client = Parse::ACLScope.client_of(resolution)
+        raw_results = begin
+            run_pipeline!(collection_name, pipeline, max_time_ms: max_time_ms,
+                                                     authorizing_client: authorizing_client)
+          rescue StandardError => e
+            raise unless owner_prefilter && owner_prefilter_rejected?(e, owner_prefilter)
+            # Atlas is serving an index version without the owner filter
+            # path (a rebuild in progress, or a stale cached definition).
+            # Stop pushing it down and rerun once without it; the
+            # ownership `$match` after `$vectorSearch` still enforces it.
+            owner_prefilter_unavailable!(collection_name, index_name)
+            if caller_prefilter
+              vs_stage["filter"] = caller_prefilter
+            else
+              vs_stage.delete("filter")
+            end
+            run_pipeline!(collection_name, pipeline, max_time_ms: max_time_ms,
+                                                     authorizing_client: authorizing_client)
+          end
         # Already past the server-side ACL `$match` and any caller
         # `filter` — NOT the number $vectorSearch emitted.
         post_filter_count = raw_results.length
@@ -350,6 +394,8 @@ module Parse
         unless resolution.master?
           Parse::ACLScope.redact_results!(raw_results, resolution)
           Parse::CLPScope.redact_protected_fields!(raw_results, protected_fields) if protected_fields.any?
+          # Defense in depth: the pointerFields `$match` already ran in
+          # the pipeline, so this should drop nothing.
           if pointer_fields
             raw_results = Parse::CLPScope.filter_by_pointer_fields(
               raw_results, pointer_fields, resolution.user_id,
@@ -439,6 +485,52 @@ module Parse
       #   @return [String, nil]
       attr_accessor :default_index
 
+      # @!visibility private
+      # Whether an error is Atlas refusing one of the owner prefilter's
+      # paths because the served index does not declare it as
+      # `type: "filter"`. Atlas words the refusal the same way for any
+      # path, so the message must name an owner path: a caller
+      # `vector_filter` on an unindexed field raises as it always did and
+      # does not switch the owner prefilter off for other callers.
+      # @param error [Exception]
+      # @param owner_prefilter [Hash] the owner clause that was pushed down.
+      # @return [Boolean]
+      def owner_prefilter_rejected?(error, owner_prefilter)
+        message = error.message.to_s
+        return false unless message.match?(PREFILTER_REJECTED_PATTERN)
+        owner_prefilter_paths(owner_prefilter).any? { |path| message.include?(path) }
+      end
+
+      # @!visibility private
+      # The `_p_<field>` paths an owner prefilter clause names.
+      # @param owner_prefilter [Hash]
+      # @return [Array<String>]
+      def owner_prefilter_paths(owner_prefilter)
+        return [] unless owner_prefilter.is_a?(Hash)
+        clauses = owner_prefilter.key?("$or") ? Array(owner_prefilter["$or"]) : [owner_prefilter]
+        clauses.flat_map { |c| c.is_a?(Hash) ? c.keys.map(&:to_s) : [] }.select { |k| k.start_with?("_p_") }
+      end
+
+      # @!visibility private
+      # Stop pushing the owner prefilter down for this index until the
+      # index cache TTL passes, and drop the cached index definition so
+      # the next lookup reads what Atlas now reports.
+      # @param collection_name [String]
+      # @param index_name [String, Symbol]
+      def owner_prefilter_unavailable!(collection_name, index_name)
+        require_relative "atlas_search"
+        Parse::AtlasSearch::IndexManager.clear_cache(collection_name)
+        mark_prefilter_unavailable(prefilter_cache_key(collection_name, index_name))
+      end
+
+      # @!visibility private
+      # Forget every negative prefilter lookup (tests, or after an
+      # operator fixes the index and does not want to wait out the TTL).
+      def clear_prefilter_cache!
+        prefilter_mutex.synchronize { @prefilter_unavailable = {} }
+        nil
+      end
+
       private
 
       def require_available!
@@ -453,31 +545,173 @@ module Parse
       # scope, refuse the call when the resolved claim set can't
       # `find` on the collection. Mirrors `Parse::AtlasSearch.search`.
       def assert_clp_find!(collection_name, resolution)
-        return if resolution.nil? || resolution.master?
-        unless Parse::CLPScope.permits?(collection_name, :find, resolution.permission_strings)
-          raise Parse::CLPScope::Denied.new(
-            collection_name, :find,
-            "CLP refuses find on '#{collection_name}' for the current VectorSearch scope.",
+        # Same CLP branch evaluation as Parse::MongoDB.aggregate (public,
+        # user, and role grants first; then pointerFields / readUserFields).
+        # Raises Parse::CLPScope::Denied when the scope cannot find at all.
+        Parse::CLPScope.row_constraint_for!(collection_name, :find, resolution,
+                                            label: "VectorSearch")
+        nil
+      end
+
+      # Resolve and return pointerFields for `find` on the collection.
+      # Refuse a scoped vector search that lets a protected field decide
+      # which rows match or how they rank: a protected vector `field:`, a
+      # `filter:` / `vector_filter:` predicate keyed on a protected field
+      # (top level or under $and/$or/$nor/$not, dotted or `_p_` form), or
+      # an `$expr` reference to one. The output strip alone does not close
+      # that oracle. Master scopes and classes with nothing protected are
+      # unaffected.
+      #
+      # @raise [Parse::CLPScope::Denied]
+      def assert_protected_fields_untouched!(collection_name, path, filter, vector_filter,
+                                             protected_fields, resolution)
+        paths = Parse::AtlasSearch::ProtectedPaths
+        return unless paths.enforce?(resolution, protected_fields)
+        paths.assert_paths_allowed!(path, protected_fields, resolution,
+                                    collection_name: collection_name,
+                                    method_name: "Parse::VectorSearch.search",
+                                    what: "vector field")
+        [filter, vector_filter].each do |f|
+          next if f.nil? || f.empty?
+          Parse::PipelineSecurity.refuse_protected_field_references!(
+            [{ "$match" => f }], collection_name, resolution,
           )
+          paths.assert_filter_allowed!(f, protected_fields, resolution,
+                                       collection_name: collection_name,
+                                       method_name: "Parse::VectorSearch.search")
         end
+        nil
       end
 
-      # Resolve and return pointerFields for `find` on the collection.
       # Raises CLPScope::Denied when pointerFields is set but the
       # current scope has no user_id (acl_role-only / public agents).
       # Returns nil when master-mode or no pointerFields entry exists.
       def resolve_pointer_fields!(collection_name, resolution)
-        return nil if resolution.nil? || resolution.master?
-        pointer_fields = Parse::CLPScope.pointer_fields_for(collection_name, :find)
-        return nil if pointer_fields.nil?
-        if resolution.user_id.nil?
-          raise Parse::CLPScope::Denied.new(
-            collection_name, :find,
-            "CLP requires user identity (pointerFields=#{pointer_fields.inspect}) " \
-            "but the current VectorSearch scope has no user_id.",
-          )
+        # nil when a public, user, or role grant already permits every row
+        # (Parse Server ignores pointer permissions then); otherwise the
+        # pointerFields plus readUserFields the rows must match. The older
+        # permits? / pointer_fields_for pair missed readUserFields entirely
+        # and over-restricted a public grant that also listed pointerFields.
+        Parse::CLPScope.row_constraint_for!(collection_name, :find, resolution,
+                                            label: "VectorSearch")
+      end
+
+      # The pointerFields ownership constraint as a `$vectorSearch.filter`
+      # clause, or nil when it cannot be pushed down. Atlas only accepts a
+      # prefilter on paths declared `type: "filter"` in the index, so this
+      # applies only when every owner pointer's storage path
+      # (`_p_<field>`) is filter-indexed. A prefilter keeps other users'
+      # rows out of the candidate window entirely, so an owner-scoped
+      # caller is not underfilled by higher-ranked rows they cannot read.
+      # Any lookup failure returns nil: the post-`$vectorSearch` `$match`
+      # still enforces ownership, only the fill is weaker.
+      #
+      # The prefilter matches only the scalar `_p_<field>` storage form.
+      # CLP ownership also accepts an array of pointers stored under the
+      # bare field name, so pushing down on such a field would drop rows
+      # the caller owns. It is therefore used only when every owner field
+      # is declared as a scalar pointer on the local model.
+      def owner_vector_prefilter(collection_name, index_name, pointer_fields, resolution)
+        return nil if pointer_fields.nil? || pointer_fields.empty?
+        uid = resolution.user_id.to_s
+        return nil if uid.empty?
+        return nil unless scalar_pointer_fields?(collection_name, pointer_fields)
+        paths = pointer_fields.map { |f| "_p_#{f}" }
+        indexed = vector_filter_paths(collection_name, index_name)
+        return nil unless paths.all? { |path| indexed.include?(path) }
+        storage = "#{Parse::Model::CLASS_USER}$#{uid}"
+        clauses = paths.map { |path| { path => { "$eq" => storage } } }
+        clauses.size == 1 ? clauses.first : { "$or" => clauses }
+      end
+
+      # Whether every owner field is declared as a scalar pointer
+      # (`belongs_to`) on the local model for the collection. Read from the
+      # model's declared fields, so no schema request is made. An unknown
+      # class, an undeclared field, or an array field all answer false.
+      def scalar_pointer_fields?(collection_name, pointer_fields)
+        klass = Parse::Model.find_class(collection_name.to_s)
+        return false unless klass.respond_to?(:fields)
+        fields = klass.fields
+        pointer_fields.all? { |f| fields[f.to_s.to_sym] == :pointer }
+      rescue StandardError
+        false
+      end
+
+      # Paths the named vectorSearch index declares as `type: "filter"`.
+      # Read through {Parse::AtlasSearch::IndexManager}, which caches the
+      # `$listSearchIndexes` result. Empty unless the index is READY and
+      # serving its latest definition: during a rebuild
+      # `latestDefinition` already lists the new filter path while Atlas
+      # still serves the old version, and a prefilter on that path is
+      # refused. A lookup error is remembered for the index cache TTL so a
+      # deployment without the `listSearchIndexes` privilege does not pay
+      # a failing round trip on every search.
+      def vector_filter_paths(collection_name, index_name)
+        key = prefilter_cache_key(collection_name, index_name)
+        return Set.new if prefilter_unavailable?(key)
+        require_relative "atlas_search"
+        idx = begin
+            Parse::AtlasSearch::IndexManager.get_index(collection_name, index_name.to_s)
+          rescue StandardError
+            mark_prefilter_unavailable(key)
+            return Set.new
+          end
+        return Set.new unless idx && index_serving_latest?(idx)
+        defn = idx["latestDefinition"] || idx[:latestDefinition] || {}
+        Array(defn["fields"] || defn[:fields]).each_with_object(Set.new) do |f, set|
+          next unless (f["type"] || f[:type]).to_s == "filter"
+          set << (f["path"] || f[:path]).to_s
+        end
+      end
+
+      # Whether Atlas is serving the index's latest definition: status
+      # READY and, when per-host `statusDetail` is reported, every host's
+      # main index at `latestDefinitionVersion`. Missing status fails
+      # toward "not serving" so the prefilter is simply skipped.
+      def index_serving_latest?(idx)
+        return false unless (idx["status"] || idx[:status]).to_s.upcase == "READY"
+        latest = definition_version(idx["latestDefinitionVersion"] || idx[:latestDefinitionVersion])
+        details = Array(idx["statusDetail"] || idx[:statusDetail])
+        return true if latest.nil? || details.empty?
+        details.all? do |detail|
+          main = detail["mainIndex"] || detail[:mainIndex]
+          main && definition_version(main["definitionVersion"] || main[:definitionVersion]) == latest
+        end
+      end
+
+      def definition_version(value)
+        return nil unless value.is_a?(Hash)
+        value["version"] || value[:version]
+      end
+
+      PREFILTER_REJECTED_PATTERN = /indexed as (?:a )?filter/i
+      private_constant :PREFILTER_REJECTED_PATTERN
+
+      def prefilter_cache_key(collection_name, index_name)
+        "#{collection_name}\x1f#{index_name}"
+      end
+
+      def prefilter_mutex
+        @prefilter_mutex ||= PREFILTER_MUTEX_INIT.synchronize { @prefilter_mutex ||= Mutex.new }
+      end
+
+      def prefilter_unavailable?(key)
+        prefilter_mutex.synchronize do
+          expires = (@prefilter_unavailable ||= {})[key]
+          next false unless expires
+          next true if Process.clock_gettime(Process::CLOCK_MONOTONIC) < expires
+          @prefilter_unavailable.delete(key)
+          false
+        end
+      end
+
+      def mark_prefilter_unavailable(key)
+        require_relative "atlas_search"
+        ttl = Parse::AtlasSearch::IndexManager.cache_ttl.to_f
+        return if ttl <= 0
+        prefilter_mutex.synchronize do
+          (@prefilter_unavailable ||= {})[key] = Process.clock_gettime(Process::CLOCK_MONOTONIC) + ttl
         end
-        pointer_fields
       end
 
       # Execute the pipeline directly against the MongoDB collection.
@@ -501,3 +735,5 @@ module Parse
     @default_index = nil
   end
 end
+
+require_relative "vector_search/index_definition"