parse-stack-next 5.4.1 → 5.5.1

data/lib/parse/model/core/vector_searchable.rb

@@ -55,6 +55,22 @@ module Parse
       # field and the caller didn't pass an explicit `index:` kwarg.
       class IndexNotResolved < ArgumentError; end
 
+      # Raised (under `Parse::VectorSearch.index_drift_policy = :raise`)
+      # when first-query verification finds the deployed vectorSearch
+      # index disagreeing with the model declaration — wrong
+      # `numDimensions`, wrong `similarity`, or a registered
+      # tenant-scope field missing from the index's `filter` paths.
+      # Under the default `:warn` policy the same findings emit a
+      # single `[Parse::VectorSearch:DRIFT]` warning instead.
+      class IndexDriftError < StandardError
+        # @return [Array<String>] human-readable drift findings.
+        attr_reader :findings
+        def initialize(message, findings: [])
+          @findings = findings
+          super(message)
+        end
+      end
+
       # Raised by the `find_similar(text:)` overload when the resolved
       # `:vector` property has no `provider:` (and therefore no way to
       # turn `text:` into a query vector). Distinct from
@@ -367,13 +383,23 @@ module Parse
                 "on the property, or pass an explicit `vector:`."
         end
         provider = Parse::Embeddings.provider(provider_name)
-        vectors = provider.embed_text([text], input_type: :search_query)
-        unless vectors.is_a?(Array) && vectors.length == 1 && vectors.first.is_a?(Array)
-          raise Parse::Embeddings::InvalidResponseError,
-                "#{self}.find_similar: provider #{provider_name.inspect} did not return " \
-                "a single vector for `text:` (got #{vectors.inspect[0, 80]})."
-        end
-        vectors.first
+        # Spend cap: every query-embed path (find_similar(text:),
+        # hybrid_search(text:), Retrieval.retrieve) funnels through this
+        # method, so charging here closes the "direct callers bypass the
+        # cap" gap. No-op when no limit is configured, or when an
+        # upstream caller (the semantic_search agent tool) has already
+        # charged with per-tenant identity (SpendCap.with_precharged).
+        #
+        # Deliberate: the charge runs BEFORE the cache lookup, so cache
+        # hits bill at full price. The cap bounds query *volume* (an
+        # abuse/probing control), not just provider spend — a caller
+        # replaying one cached query must not get unlimited throughput.
+        Parse::Embeddings::SpendCap.charge_query!(text)
+        # Query-embed cache: repeated identical queries skip the
+        # provider round-trip when Parse::Embeddings::Cache.enable! has
+        # been called; pass-through (with the provider's own response
+        # validation preserved) when disabled.
+        Parse::Embeddings::Cache.fetch_vector(provider, text, input_type: :search_query)
       end
 
       def coerce_query_vector(vector)
@@ -387,7 +413,10 @@ module Parse
       end
 
       def resolve_vector_index!(field, explicit_index)
-        return explicit_index if explicit_index && !explicit_index.to_s.empty?
+        if explicit_index && !explicit_index.to_s.empty?
+          verify_explicit_vector_index(field, explicit_index.to_s)
+          return explicit_index
+        end
         begin
           require_relative "../../atlas_search"
         rescue LoadError
@@ -402,9 +431,129 @@ module Parse
                 "#{parse_class}.#{field}; pass index: explicitly or create one " \
                 "via Parse::AtlasSearch::IndexCatalog.create_index."
         end
+        verify_vector_index!(field, idx)
         (idx["name"] || idx[:name]).to_s
       end
 
+      # Best-effort drift verification for an explicitly named `index:`.
+      # The auto-discovery path verifies the index it resolves; an
+      # explicit kwarg would otherwise skip verification entirely. Look
+      # the field's covering index up in the catalog and verify it when
+      # its name matches the explicit one. Lookup failures (catalog
+      # unavailable, index not discoverable, name targeting a different
+      # index) skip verification rather than failing the query — the
+      # explicit kwarg is an override, not a discovery request.
+      def verify_explicit_vector_index(field, index_name)
+        return if Parse::VectorSearch.index_drift_policy == :ignore
+        begin
+          require_relative "../../atlas_search"
+          idx = Parse::AtlasSearch::IndexCatalog.find_vector_index(parse_class, field: field)
+        rescue StandardError, LoadError
+          return
+        end
+        return if idx.nil?
+        return unless (idx["name"] || idx[:name]).to_s == index_name
+        verify_vector_index!(field, idx)
+      end
+
+      # First-query drift verification: compare the deployed index's
+      # `latestDefinition` against the model declaration. The drift
+      # findings are computed once per (field, index name) per class per
+      # process and cached; the policy check runs on EVERY query, so
+      # under `:raise` a drifted index keeps failing instead of failing
+      # once and then silently serving results. Under `:warn` the
+      # warning is emitted only on the first check to avoid log spam.
+      # Honors {Parse::VectorSearch.index_drift_policy} (`:warn` default
+      # / `:raise` / `:ignore`).
+      #
+      # Checks:
+      # 1. `numDimensions` on the covering `type: "vector"` entry vs the
+      #    property's declared `dimensions:`.
+      # 2. `similarity` vs the property's declared `similarity:` (only
+      #    when both sides declare one).
+      # 3. When the class registers an `agent_tenant_scope`, the scope
+      #    field must appear among the index's `type: "filter"` paths —
+      #    otherwise the tenant pre-filter that
+      #    {Parse::Retrieval.retrieve} folds into `$vectorSearch.filter`
+      #    fails Atlas-side at query time.
+      def verify_vector_index!(field, idx)
+        return if Parse::VectorSearch.index_drift_policy == :ignore
+        index_name = (idx["name"] || idx[:name]).to_s
+        @_verified_vector_indexes ||= {}
+        cache_key = "#{field}|#{index_name}"
+        findings = @_verified_vector_indexes[cache_key]
+        first_check = findings.nil?
+        if first_check
+          findings = vector_index_drift_findings(field, idx).freeze
+          @_verified_vector_indexes[cache_key] = findings
+        end
+        return if findings.empty?
+
+        message = "#{self} vectorSearch index #{index_name.inspect} drifts from the " \
+                  "model declaration for :#{field}: #{findings.join("; ")}"
+        if Parse::VectorSearch.index_drift_policy == :raise
+          # Raise on every query, not just the first: strict mode means a
+          # drifted index must never serve results.
+          raise IndexDriftError.new(message, findings: findings)
+        end
+        warn "[Parse::VectorSearch:DRIFT] #{message}" if first_check
+      end
+
+      # @!visibility private
+      # @return [Array<String>] drift findings (empty when in sync).
+      def vector_index_drift_findings(field, idx)
+        defn = idx["latestDefinition"] || idx[:latestDefinition] || {}
+        entries = defn["fields"] || defn[:fields] || []
+        field_str = field.to_s
+        vector_entry = entries.find do |f|
+          (f["type"] || f[:type]).to_s == "vector" && (f["path"] || f[:path]).to_s == field_str
+        end
+        findings = []
+        return findings if vector_entry.nil? # find_vector_index matched on it; defensive
+
+        declared_dims = vector_properties.dig(field.to_sym, :dimensions)
+        index_dims = vector_entry["numDimensions"] || vector_entry[:numDimensions]
+        if declared_dims && index_dims && Integer(index_dims) != Integer(declared_dims)
+          findings << "index numDimensions=#{index_dims} but property declares " \
+                      "dimensions: #{declared_dims} (every query will mismatch — " \
+                      "rebuild the index or run #{self}.reembed! after fixing the declaration)"
+        end
+
+        declared_sim = vector_properties.dig(field.to_sym, :similarity)
+        index_sim = vector_entry["similarity"] || vector_entry[:similarity]
+        if declared_sim && index_sim && index_sim.to_s != declared_sim.to_s
+          findings << "index similarity=#{index_sim.inspect} but property declares " \
+                      "similarity: #{declared_sim.inspect}"
+        end
+
+        scope_field = registered_tenant_scope_field
+        if scope_field
+          filter_paths = entries.select { |f| (f["type"] || f[:type]).to_s == "filter" }
+                                .map { |f| (f["path"] || f[:path]).to_s }
+          unless filter_paths.include?(scope_field)
+            findings << "agent_tenant_scope field #{scope_field.inspect} is not declared " \
+                        "as a type: \"filter\" path in the index — tenant-scoped " \
+                        "$vectorSearch.filter will fail Atlas-side"
+          end
+        end
+        findings
+      end
+
+      # @!visibility private
+      # Wire/storage name of the class's registered tenant-scope field,
+      # or nil. Mirrors the resolution Parse::Retrieval#wire_name uses
+      # when folding the scope into $vectorSearch.filter.
+      def registered_tenant_scope_field
+        return nil unless defined?(Parse::Agent::MetadataRegistry)
+        rule = Parse::Agent::MetadataRegistry.tenant_scope_rule(parse_class)
+        return nil unless rule
+        sym = rule[:field].to_sym
+        fmap = respond_to?(:field_map) ? field_map : {}
+        (fmap[sym] || sym.to_s.columnize).to_s
+      rescue StandardError
+        nil
+      end
+
       def build_vector_hits(raw_hits)
         return [] if raw_hits.nil? || raw_hits.empty?
         converted = Parse::MongoDB.convert_documents_to_parse(raw_hits, parse_class)

data/lib/parse/mongodb.rb

@@ -1651,6 +1651,18 @@ module Parse
             collection_name, perms_for_clp,
           )
           Parse::CLPScope.redact_protected_fields!(results, strip_set) if strip_set.any?
+
+          # Process-level floor: recursively strip Parse-internal credential
+          # columns (_hashed_password, _session_token, _auth_data_*, _rperm,
+          # ...) from every row AND every embedded sub-document. The
+          # protectedFields strip above is keyed on the OUTER class, and the
+          # ACL sub-doc walk only DROPS ACL-failing sub-docs — neither covers
+          # a foreign class (e.g. _User / _Session) pulled in via $lookup /
+          # $graphLookup / $unionWith under an arbitrary alias. Runs last, for
+          # scoped (non-master) callers only; master is unredacted by design.
+          results.each do |row|
+            Parse::PipelineSecurity.redact_internal_fields_deep!(row)
+          end
         end
 
         payload[:result_count] = results.size

data/lib/parse/pipeline_security.rb

@@ -105,6 +105,7 @@ module Parse
     DENIED_FIELD_REFS = %w[
       $_hashed_password $_password_history
       $_session_token $_sessionToken
+      $sessionToken $session_token
       $_email_verify_token $_perishable_token
       $_failed_login_count $_account_lockout_expires_at
       $_rperm $_wperm
@@ -161,6 +162,19 @@ module Parse
     # walk_for_denied! field-name screen.
     INTERNAL_FIELDS_PREFIX_DENYLIST = %w[_auth_data_].freeze
 
+    # The credential / sensitive subset of {INTERNAL_FIELDS_DENYLIST}. These
+    # columns must NEVER appear as a user-influenced `$match` field name —
+    # even on a pipeline that runs with `allow_internal_fields: true` (which
+    # exists to permit SDK-emitted `_rperm`/`_wperm` references from
+    # `readable_by_role` / `publicly_readable`). A `$match`/`$count` on a
+    # password hash, session/reset token, or auth-data column is a credential-
+    # exfiltration oracle (bisect the value char-by-char), and these columns
+    # have NO legitimate SDK query use — so the `allow_internal_fields` escape
+    # hatch must not relax them. Derived from {INTERNAL_FIELDS_DENYLIST} minus
+    # the ACL/bookkeeping columns (`_rperm`/`_wperm`/`_tombstone`) the ACL DSL
+    # legitimately emits, so the two lists never drift.
+    CREDENTIAL_FIELDS_DENYLIST = (INTERNAL_FIELDS_DENYLIST - %w[_rperm _wperm _tombstone]).freeze
+
     # Forensic string-introspection operators. When any of these
     # appears INSIDE `$expr` with a field-reference input string, the
     # query becomes a per-character oracle even though the operator
@@ -336,6 +350,48 @@ module Parse
       end
     end
 
+    # Depth bound for {redact_internal_fields_deep!}. `$lookup`/`$graphLookup`/
+    # `$unionWith` embed foreign documents at shallow alias depth, so this is
+    # generous; the bound exists only to fail safe on cyclic/pathological docs.
+    INTERNAL_REDACT_MAX_DEPTH = 32
+
+    # Recursively delete {INTERNAL_FIELDS_DENYLIST} / {INTERNAL_FIELDS_PREFIX_DENYLIST}
+    # keys from `node` AND every embedded sub-document/array element, in place.
+    #
+    # This is the process-level floor that stops Parse-Server-internal
+    # credential columns (`_hashed_password`, `_session_token`, `_auth_data_*`,
+    # `_rperm`/`_wperm`, ...) from reaching a scoped caller through ANY result
+    # shape — most importantly a foreign-class document pulled in via
+    # `$lookup`/`$graphLookup`/`$unionWith` under an arbitrary alias. Neither
+    # the per-class protectedFields strip (keyed on the OUTER class) nor the
+    # ACL sub-document walk (which only DROPS ACL-failing sub-docs, never
+    # strips field names) covers that alias. Unlike {strip_internal_fields}
+    # (one level, non-mutating), this walks the whole tree and mutates in
+    # place so it can run as the last step over a result set.
+    #
+    # Structural columns (`_id`, `_p_*`, `_created_at`, `_updated_at`, `_acl`)
+    # are intentionally NOT in the denylist, so object/ACL reconstruction is
+    # unaffected.
+    #
+    # @param node [Object] a result row (Hash), array, or scalar.
+    # @return [Object] the same node, mutated.
+    def redact_internal_fields_deep!(node, depth: INTERNAL_REDACT_MAX_DEPTH)
+      case node
+      when Hash
+        # Always clean the current level (even at the depth floor) so an
+        # embedded document sitting exactly at the bound is still scrubbed.
+        node.delete_if do |key, _value|
+          ks = key.to_s
+          INTERNAL_FIELDS_DENYLIST.include?(ks) ||
+            INTERNAL_FIELDS_PREFIX_DENYLIST.any? { |prefix| ks.start_with?(prefix) }
+        end
+        node.each_value { |v| redact_internal_fields_deep!(v, depth: depth - 1) } if depth > 0
+      when Array
+        node.each { |el| redact_internal_fields_deep!(el, depth: depth - 1) } if depth > 0
+      end
+      node
+    end
+
     # Wave-3 TRACK-CLP-4: refuse caller-supplied pipelines that
     # reference a protected field via `$<field>` on the RHS of a
     # `$project` / `$addFields` / `$set` / `$group` / `$bucket` /
@@ -510,21 +566,31 @@ module Parse
           # oracle as the where:-constraint path in ConstraintTranslator.
           # Operators ($-prefixed) are excluded because they are validated
           # separately by DENIED_OPERATORS.
-          if !allow_internal_fields &&
-             !key_str.start_with?("$") &&
-             (INTERNAL_FIELDS_DENYLIST.include?(key_str) ||
-              INTERNAL_FIELDS_PREFIX_DENYLIST.any? { |prefix| key_str.start_with?(prefix) })
-            raise Error.new(
-              "SECURITY: Pipeline references internal Parse Server field " \
-              "'#{key_str}' at nesting depth #{depth}" \
-              "#{stage_idx ? " inside stage #{stage_idx}" : ""}. " \
-              "This column (password hash, session token, auth data, or ACL " \
-              "pointer) must not appear in a user-influenced pipeline — " \
-              "it enables credential exfiltration via count/match oracles.",
-              stage: stage_idx,
-              operator: key_str,
-              reason: :denied_internal_field,
-            )
+          #
+          # CREDENTIAL columns (password hash, session/reset token, auth data)
+          # are refused UNCONDITIONALLY — `allow_internal_fields` (which exists
+          # so SDK-emitted `_rperm`/`_wperm` references survive on the mongo-
+          # direct path) must NOT relax them, or a `*_direct` terminal becomes
+          # a credential-bisection oracle. The remaining internal columns
+          # (`_rperm`/`_wperm`/`_tombstone`) stay gated by allow_internal_fields.
+          if !key_str.start_with?("$")
+            is_credential = CREDENTIAL_FIELDS_DENYLIST.include?(key_str) ||
+                            INTERNAL_FIELDS_PREFIX_DENYLIST.any? { |prefix| key_str.start_with?(prefix) }
+            is_internal = INTERNAL_FIELDS_DENYLIST.include?(key_str) ||
+                          INTERNAL_FIELDS_PREFIX_DENYLIST.any? { |prefix| key_str.start_with?(prefix) }
+            if is_credential || (is_internal && !allow_internal_fields)
+              raise Error.new(
+                "SECURITY: Pipeline references internal Parse Server field " \
+                "'#{key_str}' at nesting depth #{depth}" \
+                "#{stage_idx ? " inside stage #{stage_idx}" : ""}. " \
+                "This column (password hash, session token, auth data, or ACL " \
+                "pointer) must not appear in a user-influenced pipeline — " \
+                "it enables credential exfiltration via count/match oracles.",
+                stage: stage_idx,
+                operator: key_str,
+                reason: :denied_internal_field,
+              )
+            end
           end
           # Cap caller-supplied regex pattern length. Catches the two
           # shapes Mongo accepts: the find-form `{ field: { $regex: "..." } }`

data/lib/parse/query/constraint.rb

@@ -191,6 +191,28 @@ module Parse
       self.class.formatted_value(@value)
     end
 
+    # Supports the opt-in `{ value:, unicode: true }` form accepted by the
+    # regex-based constraints ({RegularExpressionConstraint},
+    # {StartsWithConstraint}, {ContainsConstraint}, {EndsWithConstraint}).
+    # When the `unicode` flag is set, the constraint adds the `u` flag to the
+    # compiled `$options`, asking the backend to treat the pattern and subject
+    # as UTF-8 for correct multibyte (e.g. accented or CJK) case-insensitive
+    # matching.
+    #
+    # The `u` flag is only honored by Parse Server 8.3.0+ over REST (older
+    # servers reject it) and by MongoDB 6.1+ on the mongo-direct path; it is
+    # therefore strictly opt-in and never emitted for the bare-value form.
+    #
+    # @param raw [Object] the raw constraint value (`@value`).
+    # @return [Array(Object, Boolean)] the unwrapped value and the unicode flag.
+    # @api private
+    def regex_unicode_option(raw)
+      return [raw, false] unless raw.is_a?(Hash)
+
+      opts = raw.symbolize_keys
+      [opts[:value], opts[:unicode] ? true : false]
+    end
+
     # Registers the default constraint of equality
     register :eq, Constraint
     precedence 100