parse-stack-next 5.5.6 → 5.6.0

data/lib/parse/embeddings.rb

@@ -247,6 +247,9 @@ module Parse
           @configuration = nil
           @allowed_image_hosts = nil
           @allowed_image_types = nil
+          @allowed_video_types = nil
+          @max_media_bytes = nil
+          BindingAudit.reset!
           @trust_provider_url_fetch = nil
         end
       end
@@ -323,6 +326,54 @@ module Parse
         @allowed_image_types ||= ImageFetch::DEFAULT_ALLOWED_IMAGE_TYPES
       end
 
+      # Configure the MIME types {VideoSource.verify!} accepts after
+      # magic-byte sniffing. Defaults to
+      # {VideoSource::DEFAULT_ALLOWED_VIDEO_TYPES}, which is MP4 only —
+      # the sole container Voyage accepts. As with images, the sniffed
+      # type — never the `Content-Type` header — is what gets checked.
+      #
+      # Widening this does NOT widen what a provider accepts: the
+      # Voyage adapter still enforces its own format and size limits.
+      #
+      # @param types [Array<String>] MIME type strings.
+      # @return [Array<String>]
+      def allowed_video_types=(types)
+        unless types.is_a?(Array) && !types.empty? &&
+               types.all? { |t| t.is_a?(String) && t.include?("/") }
+          raise ArgumentError,
+                "Parse::Embeddings.allowed_video_types= expects a non-empty Array of " \
+                "MIME type Strings (got #{types.inspect})."
+        end
+        CONFIG_MUTEX.synchronize { @allowed_video_types = types.dup.freeze }
+      end
+
+      # @return [Array<String>] MIME allowlist for video bytes (frozen).
+      def allowed_video_types
+        @allowed_video_types ||= VideoSource::DEFAULT_ALLOWED_VIDEO_TYPES
+      end
+
+      # Per-file ceiling for streamed media, checked by
+      # {MediaFile}. Voyage documents 20 MB per image and 20 MB per
+      # video; the default matches. Streaming keeps an oversized file
+      # from exhausting memory, but the provider still rejects it, so
+      # failing locally turns a wasted upload into an immediate error.
+      #
+      # @param bytes [Integer] positive byte count.
+      # @return [Integer]
+      def max_media_bytes=(bytes)
+        unless bytes.is_a?(Integer) && bytes.positive?
+          raise ArgumentError,
+                "Parse::Embeddings.max_media_bytes= expects a positive Integer " \
+                "(got #{bytes.inspect})."
+        end
+        CONFIG_MUTEX.synchronize { @max_media_bytes = bytes }
+      end
+
+      # @return [Integer] per-file ceiling for {MediaFile}, in bytes.
+      def max_media_bytes
+        @max_media_bytes ||= MediaFile::DEFAULT_MAX_MEDIA_BYTES
+      end
+
       # Sentinel-gated opt-in for forwarding image URLs to embedding
       # providers. Assign the exact {TRUST_PROVIDER_URL_FETCH_SENTINEL}
       # String to unlock; any other value (including `true`, `1`,
@@ -590,5 +641,9 @@ require_relative "embeddings/qwen"
 require_relative "embeddings/local_http"
 require_relative "embeddings/spend_cap"
 require_relative "embeddings/image_fetch"
+require_relative "embeddings/video_source"
+require_relative "embeddings/streaming_body"
+require_relative "embeddings/media_file"
+require_relative "embeddings/binding_audit"
 require_relative "embeddings/cache"
 require_relative "embeddings/batch_embedder"

data/lib/parse/model/core/embed_managed.rb

@@ -642,6 +642,20 @@ module Parse
       # `Parse::Embeddings.validate_image_url!`, and call
       # `Provider#embed_image`. Digest tracking elides the provider
       # call when the source has not changed since last save.
+      # @!visibility private
+      # Run the shared binding validator when the declared provider is
+      # actually registered. Delegates to
+      # {Parse::Embeddings::BindingAudit} so the lazy path and the
+      # explicit `audit_all!` path apply identical rules.
+      def self.audit_binding!(klass, directive)
+        provider = begin
+          Parse::Embeddings.provider(directive.provider_name)
+        rescue Parse::Embeddings::ProviderNotRegistered
+          return
+        end
+        Parse::Embeddings::BindingAudit.verify!(klass, directive.into, provider)
+      end
+
       def self.recompute_embedding!(record, directive)
         input = build_source_input(record, directive)
         stored_digest = record.public_send(directive.digest_field)
@@ -658,6 +672,18 @@ module Parse
           return
         end
 
+        # Audit the binding BEFORE the digest early-return, not after.
+        # A drifted `model:` is a configuration fault, not a property of
+        # this particular record: if the check sat behind the digest
+        # gate, an unchanged record would skip it and the mismatch would
+        # stay invisible until some unrelated record happened to be
+        # edited. Saving any record with this directive should surface
+        # it. Skipped silently when the provider is not registered — the
+        # resolution below still raises for records that actually embed,
+        # so an unchanged save keeps its existing no-provider-needed
+        # behavior.
+        audit_binding!(record.class, directive)
+
         digest = digest_for(input)
         return if stored_digest == digest && target_present
 

data/lib/parse/model/core/properties.rb

@@ -159,6 +159,11 @@ module Parse
 
       # @return [Hash] per-property metadata for `:vector`-typed fields.
       # Maps property names (symbols) to a frozen options hash:
+      # Similarity functions Atlas vectorSearch accepts. Validated at
+      # declaration time so a typo surfaces when the class loads rather
+      # than as an Atlas index error much later.
+      VECTOR_SIMILARITIES = %w[euclidean cosine dotProduct].freeze
+
       # `{ dimensions: Integer, provider: Symbol, model: String, similarity: Symbol }`.
       # `dimensions:` is required; the rest are optional and only carry
       # meaning for the embedding provider plumbing layered above this
@@ -396,11 +401,42 @@ module Parse
                   "Property #{self}##{key} :vector dimensions #{dims} exceeds max " \
                   "#{Parse::Vector::MAX_DIMENSIONS}."
           end
+
+          # `searchable:` separates "can be stored" from "can be
+          # indexed". Parse::Vector tolerates up to 16384 dims, but
+          # Atlas caps a vectorSearch index at 8192 — so without this,
+          # a wider property is declarable, storable, and permanently
+          # unsearchable, with the failure surfacing only at query
+          # time. Default true: a :vector property exists to be
+          # searched, and opting out should be the deliberate act.
+          searchable = opts.key?(:searchable) ? opts[:searchable] : true
+          unless [true, false].include?(searchable)
+            raise ArgumentError,
+                  "Property #{self}##{key} :vector `searchable:` must be true or false " \
+                  "(got #{searchable.inspect})."
+          end
+          if searchable && defined?(Parse::VectorSearch) &&
+             dims > Parse::VectorSearch::MAX_DIMENSIONS
+            raise ArgumentError,
+                  "Property #{self}##{key} :vector dimensions #{dims} exceeds the Atlas " \
+                  "vectorSearch index cap (#{Parse::VectorSearch::MAX_DIMENSIONS}); such a " \
+                  "vector can be stored but never searched. Reduce dimensions, or declare " \
+                  "`searchable: false` to acknowledge it is storage-only."
+          end
+
+          similarity = opts[:similarity]
+          if similarity && !VECTOR_SIMILARITIES.include?(similarity.to_s)
+            raise ArgumentError,
+                  "Property #{self}##{key} :vector `similarity:` must be one of " \
+                  "#{VECTOR_SIMILARITIES.inspect} (got #{similarity.inspect})."
+          end
+
           vector_properties[key] = {
             dimensions: dims,
             provider: opts[:provider],
             model: opts[:model],
-            similarity: opts[:similarity],
+            similarity: similarity,
+            searchable: searchable,
           }.freeze
 
           validates_each key do |record, attribute, value|

data/lib/parse/model/core/vector_searchable.rb

@@ -149,6 +149,12 @@ module Parse
       #   (fields must be declared `type: "filter"` in the index).
       # @param index [String, nil] explicit vectorSearch index name.
       #   Skips auto-discovery when given.
+      # @param candidate_limit [Integer, nil] rows Atlas returns before
+      #   ACL / filter / pointer-field enforcement runs; the result is
+      #   trimmed to `k` afterwards. Defaults to a multiple of `k` for
+      #   any caller subject to enforcement, so heavy ACL attrition
+      #   does not silently return fewer than `k` rows. Raise it when
+      #   a principal can read only a small fraction of the class.
       # @param num_candidates [Integer, nil] HNSW search width.
       # @param max_time_ms [Integer, nil] server-side timeout.
       # @param raw [Boolean] when true return the raw Mongo documents
@@ -192,7 +198,8 @@ module Parse
       #   deadline on the embed step.
       def find_similar(vector: nil, text: nil, k: 10, field: nil, filter: nil,
                        vector_filter: nil, index: nil,
-                       num_candidates: nil, max_time_ms: nil, raw: false,
+                       num_candidates: nil, candidate_limit: nil,
+                       max_time_ms: nil, raw: false,
                        **scope_opts)
         if vector.nil? && text.nil?
           raise ArgumentError,
@@ -222,6 +229,7 @@ module Parse
           query_vector: query_vector,
           k: k,
           num_candidates: num_candidates,
+          candidate_limit: candidate_limit,
           filter: filter,
           vector_filter: vector_filter,
           index: index_name,
@@ -314,6 +322,7 @@ module Parse
             query_vector: qv, field: field_sym, index: vector_index,
             num_candidates: vec[:num_candidates], filter: vec[:filter],
             vector_filter: vec[:vector_filter],
+            candidate_limit: vec[:candidate_limit],
           },
           k: k,
           fusion: fusion,
@@ -327,9 +336,22 @@ module Parse
       private
 
       def resolve_vector_field!(field)
-        declared = vector_properties.keys
+        # A property declared `searchable: false` is storage-only. That
+        # may be because it exceeds the Atlas index cap, or simply
+        # because the author opted it out. Either way, including it in
+        # resolution would let a caller reach a field no index serves,
+        # so it is excluded from auto-resolution entirely and refused
+        # with an explanation when named outright.
+        all_declared = vector_properties.keys
+        declared = all_declared.reject { |f| vector_properties[f][:searchable] == false }
+
         if field
           sym = field.to_sym
+          if all_declared.include?(sym) && !declared.include?(sym)
+            raise NoVectorProperty,
+                  "#{self}.find_similar: field :#{sym} is declared `searchable: false` " \
+                  "(storage-only, not eligible for search) and cannot be searched."
+          end
           unless declared.include?(sym)
             raise NoVectorProperty,
                   "#{self}.find_similar: field :#{sym} is not a :vector property " \
@@ -342,7 +364,7 @@ module Parse
         end
         if declared.empty?
           raise NoVectorProperty,
-                "#{self}.find_similar: no :vector property declared on this class."
+                "#{self}.find_similar: no searchable :vector property declared on this class."
         end
         raise AmbiguousVectorField,
               "#{self}.find_similar: class declares multiple :vector properties " \
@@ -383,6 +405,11 @@ module Parse
                 "on the property, or pass an explicit `vector:`."
         end
         provider = Parse::Embeddings.provider(provider_name)
+        # A query embedded by a different model than the stored vectors
+        # returns plausible-looking nonsense rather than an error, so
+        # the binding is verified here too — before the spend cap is
+        # charged and before any provider round-trip.
+        Parse::Embeddings::BindingAudit.verify!(self, resolved_field, provider)
         # Spend cap: every query-embed path (find_similar(text:),
         # hybrid_search(text:), Retrieval.retrieve) funnels through this
         # method, so charging here closes the "direct callers bypass the

data/lib/parse/stack/version.rb

@@ -6,6 +6,6 @@ module Parse
   # The Parse Server SDK for Ruby
   module Stack
     # The current version.
-    VERSION = "5.5.6"
+    VERSION = "5.6.0"
   end
 end

data/lib/parse/vector_search/hybrid.rb

@@ -66,6 +66,22 @@ module Parse
       # uses a comparable internal oversample.
       DEFAULT_OVERSAMPLE_MULTIPLIER = 5
 
+      # Emitted once per {.search}, mirroring
+      # {Parse::VectorSearch::AS_NOTIFICATION_NAME} so hybrid attrition
+      # is observable through the same subscriber. Carries `method`
+      # (`:rrf_client` / `:rrf_native`), `branch_depth`,
+      # `candidate_window`, `post_filter_count`, `returned_count`, and
+      # `underfilled`.
+      #
+      # `branch_depth` is the rows each branch actually retained, and it
+      # legitimately DIFFERS between the two methods: the client path
+      # enforces ACL inside each branch, so it can retain the narrower
+      # fusion depth, while the native path enforces after
+      # `$rankFusion` and must therefore retain the full candidate
+      # window. Exact parity is not achievable while that ordering
+      # difference exists, so the number reported is the one that ran.
+      AS_NOTIFICATION_NAME = "parse.vector_search.hybrid"
+
       # Hard ceiling on the fused result count, matching
       # {Parse::VectorSearch::MAX_K}.
       MAX_K = Parse::VectorSearch::MAX_K
@@ -220,7 +236,14 @@ module Parse
           end
           k_constant = fusion[:k_constant] || DEFAULT_K_CONSTANT
           weights    = fusion[:weights]
-          oversample = [k_int * DEFAULT_OVERSAMPLE_MULTIPLIER, k_int].max
+          # Two distinct numbers, deliberately not one. `fusion_depth` is
+          # how many rows each branch RETAINS for RRF (bounded by
+          # VectorSearch::MAX_K, since it becomes a branch `k`);
+          # `candidate_window` is how many rows Atlas CONSIDERS before
+          # ACL enforcement (bounded by 10_000). Collapsing them lets
+          # the window be multiplied twice — once here and again inside
+          # VectorSearch.search — and lets a branch `k` exceed MAX_K.
+          fusion_depth, candidate_window = resolve_windows(k_int, vec[:candidate_limit])
 
           # NOTE (deviation from plan §8.3): the default fuses CLIENT-SIDE.
           # The native single-roundtrip `$rankFusion` path is OPT-IN
@@ -234,19 +257,94 @@ module Parse
           # native AND the cluster supports it. Native still falls back to
           # the client path on any execution error.
           if method == :rrf_native && rank_fusion_supported?(collection_name)
-            fused = run_native(collection_name, lex, vec, oversample,
+            # The native pipeline enforces ACL AFTER fusion, so its
+            # per-branch limit is the pre-ACL depth: the candidate
+            # window, not the fusion depth.
+            fused = run_native(collection_name, lex, vec, candidate_window,
                                k_constant: k_constant, weights: weights, scope_opts: scope_opts)
-            return fused.first(k_int) if fused
+            if fused
+              trimmed = fused.first(k_int)
+              # Native retains the full candidate window per branch,
+              # NOT the client path's fusion depth: its ACL `$match`
+              # runs after `$rankFusion`, so the branch limit and the
+              # pre-ACL window are necessarily the same number. Report
+              # what actually executed rather than the client figure.
+              emit_hybrid_stats(collection_name: collection_name, k: k_int,
+                                method: :rrf_native, branch_depth: candidate_window,
+                                candidate_window: candidate_window,
+                                post_filter_count: fused.length,
+                                returned_count: trimmed.length)
+              return trimmed
+            end
           end
 
-          lexical_rows = run_lexical(collection_name, lex, oversample, scope_opts)
-          vector_rows  = run_vector(collection_name, vec, oversample, scope_opts)
-          rrf({ lexical: lexical_rows, vector: vector_rows },
-              k_constant: k_constant, weights: weights).first(k_int)
+          lexical_rows = run_lexical(collection_name, lex, fusion_depth, scope_opts)
+          vector_rows  = run_vector(collection_name, vec, fusion_depth, candidate_window, scope_opts)
+          fused = rrf({ lexical: lexical_rows, vector: vector_rows },
+                      k_constant: k_constant, weights: weights)
+          trimmed = fused.first(k_int)
+          emit_hybrid_stats(collection_name: collection_name, k: k_int,
+                            method: :rrf_client, branch_depth: fusion_depth,
+                            candidate_window: candidate_window,
+                            post_filter_count: fused.length,
+                            returned_count: trimmed.length)
+          trimmed
         end
 
         private
 
+        # Resolve the two windows.
+        #
+        # * `candidate_window` — rows Atlas considers before ACL. Never
+        #   narrower than the plain search's window, so opting into
+        #   hybrid cannot make ACL underfill worse than a straight
+        #   vector search. Bounded by Atlas's 10_000 ceiling.
+        # * `fusion_depth` — rows each branch retains for RRF. This
+        #   becomes a branch `k`, so it is additionally bounded by
+        #   VectorSearch::MAX_K; without that bound a hybrid `k` above
+        #   100 produces a branch `k` the plain search refuses outright.
+        #
+        # An out-of-range explicit `candidate_limit` is REFUSED rather
+        # than clamped: silently shrinking a caller's stated window
+        # would hide the very underfill they were trying to avoid.
+        #
+        # @return [Array(Integer, Integer)] `[fusion_depth, candidate_window]`
+        def resolve_windows(k_int, candidate_limit)
+          window =
+            if candidate_limit
+              limit = Integer(candidate_limit)
+              if limit < k_int
+                raise ArgumentError,
+                      "hybrid search: vector[:candidate_limit] (#{limit}) must be >= k (#{k_int})."
+              end
+              if limit > Parse::VectorSearch::MAX_CANDIDATE_LIMIT
+                raise ArgumentError,
+                      "hybrid search: vector[:candidate_limit] (#{limit}) exceeds the Atlas " \
+                      "ceiling (#{Parse::VectorSearch::MAX_CANDIDATE_LIMIT})."
+              end
+              limit
+            else
+              multiplier = [DEFAULT_OVERSAMPLE_MULTIPLIER,
+                            Parse::VectorSearch::DEFAULT_CANDIDATE_MULTIPLIER].max
+              [[k_int * multiplier, k_int].max,
+               Parse::VectorSearch::MAX_CANDIDATE_LIMIT].min
+            end
+
+          depth = [[window, Parse::VectorSearch::MAX_K].min, k_int].max
+          [depth, window]
+        end
+
+        # Mirrors the plain search's attrition telemetry. `post_filter_count`
+        # is the fused row count after each branch has enforced ACL/CLP;
+        # `underfilled` means the caller received fewer than `k`.
+        def emit_hybrid_stats(**payload)
+          return unless defined?(ActiveSupport::Notifications)
+
+          payload[:underfilled] = payload[:returned_count] < payload[:k]
+          ActiveSupport::Notifications.instrument(AS_NOTIFICATION_NAME, payload)
+          nil
+        end
+
         # -- client-side branch execution --------------------------------
 
         def run_lexical(collection_name, lex, oversample, scope_opts)
@@ -263,12 +361,16 @@ module Parse
           )
         end
 
-        def run_vector(collection_name, vec, oversample, scope_opts)
+        def run_vector(collection_name, vec, fusion_depth, candidate_window, scope_opts)
           Parse::VectorSearch.search(
             collection_name,
             field: vec[:field],
             query_vector: vec[:query_vector],
-            k: oversample,
+            # `k` is what this branch keeps; `candidate_limit` is the
+            # pre-ACL window. Passing the window as `k` and letting the
+            # plain search derive its own would multiply it twice.
+            k: fusion_depth,
+            candidate_limit: candidate_window,
             num_candidates: vec[:num_candidates],
             filter: vec[:filter],
             vector_filter: vec[:vector_filter],
@@ -312,11 +414,17 @@ module Parse
           fusion = symbolize(fusion || {})
           lex = symbolize(lexical || {})
           vec = symbolize(vector || {})
-          oversample = [Integer(k) * DEFAULT_OVERSAMPLE_MULTIPLIER, Integer(k)].max
+          # Same resolution the live path uses, so the shape this
+          # returns is the shape that actually executes.
+          _depth, candidate_window = resolve_windows(Integer(k), vec[:candidate_limit])
           resolution = Parse::ACLScope.resolve!(scope_opts.dup, method_name: :"VectorSearch::Hybrid.search")
-          native_pipeline_for(lex, vec, oversample, resolution,
+          # `limit:` must match {#run_native}'s — the final `$limit` runs
+          # AFTER the ACL `$match`, so trimming to `k` there would
+          # reintroduce the underfill the window exists to prevent. The
+          # trim to `k` happens client-side once enforcement is done.
+          native_pipeline_for(lex, vec, candidate_window, resolution,
                               k_constant: fusion[:k_constant] || DEFAULT_K_CONSTANT,
-                              weights: fusion[:weights], limit: Integer(k))
+                              weights: fusion[:weights], limit: candidate_window)
         end
 
         def native_pipeline_for(lex, vec, oversample, resolution, k_constant:, weights:, limit:)

data/lib/parse/vector_search.rb

@@ -95,6 +95,44 @@ module Parse
     # one. Atlas's guidance: numCandidates ≥ 10 × limit, ≤ 10_000.
     DEFAULT_NUM_CANDIDATES_MULTIPLIER = 20
 
+    # How far past `k` the `$vectorSearch.limit` is raised when
+    # something downstream can drop rows (ACL enforcement, a
+    # caller-supplied `filter`, `protectedFields`, or pointer-field
+    # filtering). Atlas applies `limit` BEFORE any of those run, so
+    # without overfetching a caller who can read 2 of the top 10 asks
+    # for 10 and receives 2 — even when hundreds of readable matches
+    # exist further down the ranking.
+    #
+    # This is a mitigation, not a guarantee: a sufficiently selective
+    # ACL can still exhaust any finite candidate window. Deterministic
+    # fill would require `_rperm` declared as `type: "filter"` in the
+    # Atlas index (so the prefilter enforces visibility) or iterative
+    # candidate expansion. The instrumentation emitted by {.search}
+    # exists so an underfill is observable rather than silent.
+    DEFAULT_CANDIDATE_MULTIPLIER = 10
+
+    # Ceiling on the internally-raised candidate window. Atlas caps
+    # `numCandidates` at 10_000 and `numCandidates` must be >= `limit`,
+    # so the candidate window cannot usefully exceed that.
+    MAX_CANDIDATE_LIMIT = 10_000
+
+    # Emitted once per {.search}. The counts are deliberately named for
+    # where they are actually measured — there is no cheap way to learn
+    # how many rows `$vectorSearch` emitted before the server-side
+    # `$match` stages ran, so no field claims to be that number:
+    #
+    # * `candidate_limit` / `num_candidates` — the requested window.
+    # * `post_filter_count` — rows returned by the pipeline, i.e. after
+    #   the server-side ACL `$match` and any caller `filter`.
+    # * `post_pointer_count` — rows left after client-side redaction and
+    #   pointer-field filtering.
+    # * `returned_count` — rows handed back, after trimming to `k`.
+    # * `underfilled` — the caller received fewer than `k`.
+    #
+    # Obtaining a true pre-`$match` count would require a `$facet`, at
+    # the cost of a second pass over the candidate set.
+    AS_NOTIFICATION_NAME = "parse.vector_search.search"
+
     # Accepted {.index_drift_policy} values.
     INDEX_DRIFT_POLICIES = %i[warn raise ignore].freeze
 
@@ -165,8 +203,9 @@ module Parse
       #   `_vscore` rather than `_score` so hybrid pipelines with
       #   Atlas Search don't collide on the same key).
       def search(collection_name, field:, query_vector:, k: 10,
-                 num_candidates: nil, filter: nil, vector_filter: nil,
-                 index: nil, max_time_ms: nil, **scope_opts)
+                 num_candidates: nil, candidate_limit: nil, filter: nil,
+                 vector_filter: nil, index: nil, max_time_ms: nil, **scope_opts)
+        candidate_limit_override = candidate_limit
         require_available!
         index_name = (index || @default_index)
         if index_name.nil? || index_name.to_s.empty?
@@ -195,12 +234,62 @@ module Parse
           raise ArgumentError, "k must be in 1..#{MAX_K} (got #{k_int})."
         end
 
-        num_candidates_int = (num_candidates || (k_int * DEFAULT_NUM_CANDIDATES_MULTIPLIER)).to_i
+        # Anything that can drop rows AFTER Atlas has already applied
+        # `limit` forces an overfetch, otherwise the caller silently
+        # receives fewer than `k`. Master mode with no caller filter
+        # has no attrition, so it keeps the old one-for-one cost.
+        attrition_possible = !resolution.master? || !(filter.nil? || filter.empty?)
+        candidate_limit =
+          if candidate_limit_override
+            Integer(candidate_limit_override)
+          elsif attrition_possible
+            [k_int * DEFAULT_CANDIDATE_MULTIPLIER, MAX_CANDIDATE_LIMIT].min
+          else
+            k_int
+          end
+        if candidate_limit < k_int
+          raise ArgumentError,
+                "candidate_limit (#{candidate_limit}) must be >= k (#{k_int})."
+        end
+        if candidate_limit > MAX_CANDIDATE_LIMIT
+          raise ArgumentError,
+                "candidate_limit capped at #{MAX_CANDIDATE_LIMIT} by Atlas (got #{candidate_limit})."
+        end
+
+        # ANN width stays anchored to `k`, NOT to the raised window.
+        # Deriving it from `candidate_limit` would multiply twice and
+        # silently widen the HNSW search by the candidate multiplier —
+        # a scoped k=10 would jump from 200 to 2000 candidates. The
+        # window only needs numCandidates to be at least as large as
+        # `limit`, so take whichever is greater.
+        derived_num_candidates =
+          [k_int * DEFAULT_NUM_CANDIDATES_MULTIPLIER, candidate_limit].max
+        num_candidates_int = (num_candidates || derived_num_candidates).to_i
+        num_candidates_int = MAX_CANDIDATE_LIMIT if num_candidates_int > MAX_CANDIDATE_LIMIT &&
+                                                    num_candidates.nil?
+
+        # A caller-supplied num_candidates that predates the candidate
+        # window used to be valid whenever it was >= k. Clamping the
+        # implicit window down to it keeps those calls working rather
+        # than turning them into a new ArgumentError. An EXPLICIT
+        # candidate_limit still conflicts loudly — that combination can
+        # only be a mistake.
+        if num_candidates && num_candidates_int < candidate_limit
+          if candidate_limit_override
+            raise ArgumentError,
+                  "num_candidates (#{num_candidates_int}) must be >= candidate_limit " \
+                  "(#{candidate_limit})."
+          end
+          candidate_limit = [num_candidates_int, k_int].max
+        end
+
         if num_candidates_int < k_int
-          raise ArgumentError, "num_candidates (#{num_candidates_int}) must be >= k (#{k_int})."
+          raise ArgumentError,
+                "num_candidates (#{num_candidates_int}) must be >= k (#{k_int})."
         end
-        if num_candidates_int > 10_000
-          raise ArgumentError, "num_candidates capped at 10000 by Atlas (got #{num_candidates_int})."
+        if num_candidates_int > MAX_CANDIDATE_LIMIT
+          raise ArgumentError,
+                "num_candidates capped at #{MAX_CANDIDATE_LIMIT} by Atlas (got #{num_candidates_int})."
         end
 
         validated_vector = validate_query_vector!(query_vector)
@@ -223,7 +312,9 @@ module Parse
           "path"          => path,
           "queryVector"   => validated_vector,
           "numCandidates" => num_candidates_int,
-          "limit"         => k_int,
+          # Deliberately the raised candidate window, not `k` — the
+          # result set is trimmed to `k` after enforcement runs.
+          "limit"         => candidate_limit,
         }
         vs_stage["filter"] = vector_filter if vector_filter && !vector_filter.empty?
         pipeline = [{ "$vectorSearch" => vs_stage }]
@@ -248,6 +339,9 @@ module Parse
         pipeline << { "$match" => filter } if filter
 
         raw_results = run_pipeline!(collection_name, pipeline, max_time_ms: max_time_ms)
+        # Already past the server-side ACL `$match` and any caller
+        # `filter` — NOT the number $vectorSearch emitted.
+        post_filter_count = raw_results.length
 
         # Post-fetch enforcement: walk the rows the same way
         # Parse::MongoDB.aggregate would. Master mode skips every
@@ -266,9 +360,40 @@ module Parse
         # every mode, master included, so `_hashed_password` /
         # `_session_token` can never surface through this entry point.
         raw_results.map! { |doc| Parse::PipelineSecurity.strip_internal_fields(doc) }
+
+        # Trim only AFTER every enforcement layer has run — that
+        # ordering is the whole point of the raised candidate window.
+        post_pointer_count = raw_results.length
+        raw_results = raw_results.first(k_int) if post_pointer_count > k_int
+
+        emit_search_stats(
+          collection_name: collection_name, k: k_int,
+          candidate_limit: candidate_limit, num_candidates: num_candidates_int,
+          post_filter_count: post_filter_count, post_pointer_count: post_pointer_count,
+          returned_count: raw_results.length, master: resolution.master?,
+        )
+
         raw_results
       end
 
+      # @!visibility private
+      # Emit per-search counts so ACL/filter attrition is observable.
+      # `underfilled` means the caller received fewer rows than they
+      # asked for — the signal worth alerting on, since it means either
+      # the candidate window was too small for this principal or the
+      # collection simply ran out of matches. Distinguishing those two
+      # would need a pre-`$match` count, which this deliberately does
+      # not fabricate.
+      def emit_search_stats(**payload)
+        return unless defined?(ActiveSupport::Notifications)
+
+        payload[:pointer_attrition] =
+          payload[:post_filter_count] - payload[:post_pointer_count]
+        payload[:underfilled] = payload[:returned_count] < payload[:k]
+        ActiveSupport::Notifications.instrument(AS_NOTIFICATION_NAME, payload)
+        nil
+      end
+
       # Validate a query vector. Public so callers (and tests) can
       # invoke it independently of {.search}.
       #

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: parse-stack-next
 version: !ruby/object:Gem::Version
-  version: 5.5.6
+  version: 5.6.0
 platform: ruby
 authors:
 - Adrian Curtin
@@ -301,16 +301,20 @@ files:
 - lib/parse/console.rb
 - lib/parse/embeddings.rb
 - lib/parse/embeddings/batch_embedder.rb
+- lib/parse/embeddings/binding_audit.rb
 - lib/parse/embeddings/cache.rb
 - lib/parse/embeddings/cohere.rb
 - lib/parse/embeddings/fixture.rb
 - lib/parse/embeddings/image_fetch.rb
 - lib/parse/embeddings/jina.rb
 - lib/parse/embeddings/local_http.rb
+- lib/parse/embeddings/media_file.rb
 - lib/parse/embeddings/openai.rb
 - lib/parse/embeddings/provider.rb
 - lib/parse/embeddings/qwen.rb
 - lib/parse/embeddings/spend_cap.rb
+- lib/parse/embeddings/streaming_body.rb
+- lib/parse/embeddings/video_source.rb
 - lib/parse/embeddings/voyage.rb
 - lib/parse/graphql.rb
 - lib/parse/graphql/scalars.rb