parse-stack-next 5.7.6 → 5.8.0

data/lib/parse/clp_scope.rb

@@ -21,6 +21,10 @@ module Parse
     EMPTY_SET = Set.new.freeze
     private_constant :EMPTY_SET
 
+    # Parse Server's default server-config `protectedFields` option. See
+    # {.default_protected_fields}.
+    DEFAULT_PROTECTED_FIELDS = { "_User" => { "*" => ["email"].freeze }.freeze }.freeze
+
     # Cache-entry shape. `kind:` is the disposition of the most recent
     # schema-fetch attempt:
     #
@@ -288,31 +292,185 @@ module Parse
         # `permits?` already refused the query, so this branch is only
         # reached when callers ask for the protected-fields set directly
         # (e.g. for documentation or audit tooling).
-        return EMPTY_SET if entry.kind == :no_clp || entry.kind == :unresolvable
-        protected_map = entry.clp["protectedFields"] || entry.clp[:protectedFields]
+        return EMPTY_SET if entry.kind == :unresolvable
+        stored_map = if entry.kind == :no_clp
+            nil
+          else
+            entry.clp["protectedFields"] || entry.clp[:protectedFields]
+          end
+        protected_map = merge_default_protected_fields(class_name, stored_map)
         return EMPTY_SET if protected_map.nil? || protected_map.empty?
 
-        strip = Set.new(Array(protected_map["*"] || protected_map[:"*"]).map(&:to_s))
-
         claim_set = permission_strings.is_a?(Set) ? permission_strings : permission_strings.to_set
-        claim_set.each do |claim|
-          next if claim == "*"
-          override = protected_map[claim.to_s] || protected_map[claim.to_sym]
-          next if override.nil?
-          override_set = Set.new(Array(override).map(&:to_s))
-          strip &= override_set
-        end
+        protected_sets_for(protected_map, claim_set).reduce { |acc, set| acc & set }&.freeze || EMPTY_SET
+      end
 
-        strip.freeze
+      # Parse Server's server-config `protectedFields` default, merged into
+      # every class's CLP `protectedFields` the way Parse Server's
+      # `SchemaData` merges them (union per group key).
+      #
+      # Parse Server ships `{ _User: { "*": ["email"] } }` as the default
+      # and does NOT expose the server-config value through
+      # `GET /schemas/<Class>`, so the SDK cannot discover it. Without this
+      # default a scoped mongo-direct read returned every user's email,
+      # and an `email` filter matched, where REST hides the field and
+      # refuses the filter. Set this to the same value as the server's
+      # `protectedFields` option when it is customized; `{}` disables the
+      # merge.
+      #
+      # @return [Hash{String => Hash{String => Array<String>}}]
+      def default_protected_fields
+        @default_protected_fields ||= DEFAULT_PROTECTED_FIELDS
+      end
+
+      # @param value [Hash, nil] class name => protectedFields map; nil
+      #   restores Parse Server's default.
+      def default_protected_fields=(value)
+        @default_protected_fields = value.nil? ? DEFAULT_PROTECTED_FIELDS : value
       end
 
-      def redact_protected_fields!(documents, strip_set)
+      # Strip protected fields from result rows, top-level keys only.
+      #
+      # Parse Server deletes `object[key]` for each protected key on the
+      # returned object and does not descend into nested values, so an
+      # `:object` column holding `{ "secret" => ... }` keeps that key even
+      # when a top-level `secret` column is protected. Included objects are
+      # separate rows of another class; strip them with that class's own set
+      # (see {Parse::MongoDB.aggregate}).
+      #
+      # Parse Server also skips the strip for a `_User` row that IS the
+      # requesting user, so a user always sees their own `email`. Pass
+      # `class_name:` and `user_id:` to apply that exemption.
+      #
+      # The exemption reads the row's `_id` (or `objectId`) as it comes out
+      # of the pipeline. A caller stage can rewrite that value, so pass
+      # `user_id:` only when ownership was settled before any caller stage
+      # ran: either no caller stage can change the identity (see
+      # {Parse::PipelineSecurity.identity_preserving?}), or the rows went
+      # through {.protected_strip_stage} at the head of their pipeline.
+      #
+      # @param documents [Array<Hash>] rows (Mongo storage or Parse form).
+      # @param strip_set [Set<String>] protected field names.
+      # @param class_name [String, nil] the rows' class.
+      # @param user_id [String, nil] the requesting user's objectId.
+      # @return [Array<Hash>] the same array, modified in place.
+      def redact_protected_fields!(documents, strip_set, class_name: nil, user_id: nil)
         return documents if documents.nil? || documents.empty?
         return documents if strip_set.nil? || strip_set.empty?
-        documents.each { |doc| walk_and_delete!(doc, strip_set) }
+        self_exempt = class_name.to_s == Parse::Model::CLASS_USER && !user_id.to_s.empty?
+        documents.each do |doc|
+          next unless doc.is_a?(Hash)
+          if self_exempt
+            row_id = doc["_id"] || doc[:_id] || doc["objectId"] || doc[:objectId]
+            next if row_id.to_s == user_id.to_s
+          end
+          strip_top_level!(doc, strip_set)
+        end
         documents
       end
 
+      # A pipeline stage that removes protected fields at the database,
+      # placed at the head of a (sub-)pipeline before any caller stage runs.
+      #
+      # Ownership of a `_User` row is decided here, on the stored `_id`, so
+      # the self exemption cannot be claimed by a later stage that rewrites
+      # `_id`. For a `_User` read by a known user the fields are kept only on
+      # that user's own row; everywhere else they are unset. The `_p_<field>`
+      # storage column of a protected pointer is removed with the field.
+      #
+      # @param strip_set [Set<String>] protected field names for the scope.
+      # @param class_name [String, nil] the class the stage reads.
+      # @param user_id [String, nil] the requesting user's objectId.
+      # @return [Hash, nil] a `$unset` or `$set` stage, or nil when nothing
+      #   is protected.
+      def protected_strip_stage(strip_set, class_name: nil, user_id: nil)
+        return nil if strip_set.nil? || strip_set.empty?
+        columns = strip_set.flat_map { |f| [f.to_s, "_p_#{f}"] }.uniq
+        if class_name.to_s == Parse::Model::CLASS_USER && !user_id.to_s.empty?
+          own_row = { "$eq" => ["$_id", { "$literal" => user_id.to_s }] }
+          { "$set" => columns.to_h { |c| [c, { "$cond" => [own_row, "$#{c}", "$$REMOVE"] }] } }
+        else
+          { "$unset" => columns }
+        end
+      end
+
+      # Evaluate the `op` CLP for a resolved mongo-direct scope with the same
+      # mutually exclusive branches Parse Server uses, and return the row
+      # constraint the caller must apply.
+      #
+      # A public, user, or role grant permits every row, even when the CLP
+      # also lists `pointerFields` / `readUserFields` (Parse Server only
+      # consults those when no other branch grants). When the only grant is
+      # a pointer branch, the caller must keep just the rows whose named
+      # pointer fields reference the requesting user. `readUserFields`
+      # counts as a pointer branch here; the older {permits?} plus
+      # {pointer_fields_for} pair ignored it, so a `readUserFields` class
+      # let every authenticated user read every row on the direct path.
+      #
+      # @param class_name [String]
+      # @param op [Symbol] one of {OPERATIONS}.
+      # @param resolution [Parse::ACLScope::Resolution, nil]
+      # @param client [Parse::Client, nil] application whose schema owns the CLP.
+      # @param label [String, nil] scope label used in the error message.
+      # @return [Array<String>, nil] pointer field names that must reference
+      #   the requesting user, or nil when every row is permitted. Always nil
+      #   for master and nil resolutions.
+      # @raise [Denied] when the scope cannot perform `op` at all, including
+      #   when the CLP is unresolvable (fail closed).
+      def row_constraint_for!(class_name, op, resolution, client: nil, label: nil)
+        return nil if resolution.nil?
+        return nil if resolution.respond_to?(:master?) && resolution.master?
+        perms = resolution.respond_to?(:permission_strings) ? resolution.permission_strings : nil
+        return nil if perms.nil?
+
+        user_id = resolution.respond_to?(:user_id) ? resolution.user_id.to_s : ""
+        evaluation = evaluate_access(
+          class_name, op,
+          claims: perms,
+          authenticated: !user_id.empty?,
+          user_id: user_id.empty? ? nil : user_id,
+          client: client,
+        )
+        if evaluation.allowed?
+          fields = Array(evaluation.pointer_fields)
+          return fields.empty? ? nil : fields
+        end
+
+        warn_unresolvable_once!(class_name) if evaluation.reason == :clp_unresolvable
+        scope = label ? "the current #{label} scope" : "the current scope"
+        message = if %i[concrete_user_required authentication_required].include?(evaluation.reason) &&
+                     !pointer_fields_from_entry(class_name, op, client).empty?
+            "CLP requires user identity (pointerFields=#{pointer_fields_from_entry(class_name, op, client).inspect}) " \
+            "but #{scope} has no user_id."
+          else
+            "CLP refuses #{op} on '#{class_name}' for #{scope}."
+          end
+        raise Denied.new(class_name, op, message)
+      end
+
+      # A `$match` predicate keeping only rows whose pointer fields reference
+      # `user_id`, in Mongo storage form. A single pointer is stored as
+      # `_p_<field>: "_User$<id>"`; an array of pointers is stored under the
+      # bare field name as `{ __type, className, objectId }` elements. Both
+      # forms are matched, ORed across fields, as Parse Server's
+      # `addPointerPermissions` does.
+      #
+      # @param pointer_fields [Array<String>]
+      # @param user_id [String]
+      # @return [Hash] a `$match` predicate.
+      def pointer_fields_predicate(pointer_fields, user_id)
+        uid = user_id.to_s
+        storage = "#{Parse::Model::CLASS_USER}$#{uid}"
+        clauses = Array(pointer_fields).flat_map do |field|
+          f = field.to_s
+          [
+            { "_p_#{f}" => storage },
+            { f => { "$elemMatch" => { "className" => Parse::Model::CLASS_USER, "objectId" => uid } } },
+          ]
+        end
+        { "$or" => clauses }
+      end
+
       def filter_by_pointer_fields(documents, pointer_fields, user_id)
         return documents if pointer_fields.nil? || pointer_fields.empty?
         return [] if user_id.nil? || user_id.to_s.empty?
@@ -373,6 +531,54 @@ module Parse
 
       private
 
+      # Collect one field set per protectedFields group that applies to the
+      # caller, mirroring Parse Server's `DatabaseController.addProtectedFields`.
+      # Only groups PRESENT in the map participate. A missing `"*"` key does
+      # not mean "nothing protected": a map of `{ "role:Restricted" => ["secret"] }`
+      # still strips `secret` for a member of `Restricted`. The final strip set
+      # is the intersection of the applicable sets, so membership in a group
+      # with a narrower (or empty) list relaxes protection.
+      #
+      # Groups:
+      # - `"*"` applies to everyone.
+      # - `"authenticated"` applies when the claim set carries a concrete user id.
+      # - `"role:<name>"` applies for each role claim.
+      # - `"<userId>"` applies for the caller's own user id.
+      # - `"userField:<field>"` entries are per-object pointer rules that
+      #   Parse Server evaluates against each row. They are skipped here,
+      #   which can only over-protect, never under-protect.
+      #
+      # Role entries apply even when the claim set has no user id (an
+      # `acl_role:` scope). Parse Server has no REST equivalent of a
+      # role-only caller, and the SDK treats a granted role claim as held.
+      #
+      # @return [Array<Set<String>>]
+      def protected_sets_for(protected_map, claim_set)
+        lookup = lambda do |key|
+          value = protected_map[key]
+          value = protected_map[key.to_sym] if value.nil?
+          value.nil? ? nil : Set.new(Array(value).map(&:to_s))
+        end
+
+        sets = []
+        public_set = lookup.call("*")
+        sets << public_set if public_set
+
+        user_claims = claim_set.map(&:to_s).reject { |c| c == "*" || c.start_with?("role:") }
+        if user_claims.any?
+          auth_set = lookup.call("authenticated")
+          sets << auth_set if auth_set
+        end
+
+        claim_set.each do |claim|
+          key = claim.to_s
+          next if key == "*" || key == "authenticated" || key.start_with?("userField:")
+          set = lookup.call(key)
+          sets << set if set
+        end
+        sets
+      end
+
       # Always returns a {CacheEntry}. On schema-fetch failure (network
       # error, unsuccessful response, raised exception, missing client)
       # the entry has `kind: :unresolvable` and is held for
@@ -400,7 +606,7 @@ module Parse
             unresolvable_entry
           else
             begin
-              response = resolved_client.schema(class_key)
+              response = fetch_schema_response(resolved_client, class_key)
               if response&.success?
                 schema = response.result || {}
                 clp = schema["classLevelPermissions"] || {}
@@ -418,6 +624,37 @@ module Parse
         entry
       end
 
+      # Fetch a class schema with the master key, explicitly.
+      #
+      # `GET /schemas/<Class>` is master-key-only on Parse Server. A plain
+      # `client.schema(name)` call inherits the request layer's auth
+      # resolution, so inside `Parse.with_session(token)` (or on a client in
+      # `client_mode`) it sends the user's session token instead of the master
+      # key. Parse Server refuses that, the entry is recorded as
+      # `:unresolvable`, and every scoped mongo-direct read in the block is
+      # denied by CLP (and the denial is negatively cached). Passing
+      # `use_master_key: true` makes the request layer skip both the ambient
+      # and the client-bound session token. A client that holds no master key
+      # still cannot read schemas, which stays fail-closed.
+      #
+      # Objects that only implement `#schema` (test doubles, custom schema
+      # sources installed via {.schema_client}), and clients whose `#schema`
+      # was overridden on the instance, keep the old call.
+      def fetch_schema_response(resolved_client, class_key)
+        stock_schema = begin
+            resolved_client.method(:schema).owner == Parse::API::Schema
+          rescue NameError
+            false
+          end
+        if resolved_client.is_a?(Parse::Client) && stock_schema
+          safe = Parse::API::PathSegment.identifier!(class_key, kind: "class name")
+          resolved_client.request(:get, "schemas/#{safe}",
+                                  opts: { cache: false, use_master_key: true })
+        else
+          resolved_client.schema(class_key)
+        end
+      end
+
       def unresolvable_entry
         CacheEntry.new(kind: :unresolvable, clp: nil, fetched_at: monotonic_now)
       end
@@ -514,15 +751,36 @@ module Parse
         s != "*" && !s.start_with?("role:")
       end
 
-      def walk_and_delete!(node, strip_set)
-        case node
-        when Hash
-          strip_set.each { |k| node.delete(k) }
-          node.each_value { |v| walk_and_delete!(v, strip_set) }
-        when Array
-          node.each { |v| walk_and_delete!(v, strip_set) }
+      # Delete protected keys from one row. Mongo stores a pointer column as
+      # `_p_<field>`, so the storage form of a protected pointer is removed
+      # along with the Parse-form key.
+      def strip_top_level!(doc, strip_set)
+        strip_set.each do |k|
+          key = k.to_s
+          doc.delete(key)
+          doc.delete(key.to_sym)
+          doc.delete("_p_#{key}")
+        end
+        doc
+      end
+
+      def merge_default_protected_fields(class_name, stored_map)
+        defaults = default_protected_fields
+        extra = defaults.is_a?(Hash) ? (defaults[class_name.to_s] || defaults[class_name.to_s.to_sym]) : nil
+        return stored_map if extra.nil? || extra.empty?
+        merged = {}
+        (stored_map || {}).each { |k, v| merged[k.to_s] = Array(v).map(&:to_s) }
+        extra.each do |k, v|
+          key = k.to_s
+          merged[key] = (Array(merged[key]) + Array(v).map(&:to_s)).uniq
         end
-        node
+        merged
+      end
+
+      def pointer_fields_from_entry(class_name, op, client)
+        entry = fetch(class_name, client: client)
+        return [] unless entry.kind == :cached_clp
+        pointer_fields_from(entry.clp, op)
       end
 
       def any_pointer_matches?(doc, pointer_fields, user_id)

data/lib/parse/console.rb

@@ -46,8 +46,8 @@ module Parse
     #     puts "[#{event}] #{obj.title} (#{obj.id})"
     #   end
     #
-    # @example Admin-style with master key (no ambient session)
-    #   Parse.with_session(nil) { Parse.watch(JobRun) }
+    # @example Outside any session block (master key on a server client)
+    #   Parse.watch(JobRun)
     #
     # @param klass [Class] a Parse::Object subclass.
     # @param where [Hash] optional query constraints.

data/lib/parse/embeddings/voyage.rb

@@ -116,7 +116,64 @@ module Parse
     #   {Parse::Middleware::BodyBuilder::REDACTED_HEADERS}.
     class Voyage < Provider
       class AuthenticationError < Error; end
-      class BadRequestError < Error; end
+      # A 4xx the provider refused outright. Carries the HTTP status and
+      # the provider's own error text (bounded and terminal-sanitized) so
+      # callers can tell a request that was merely too large from one that
+      # is malformed.
+      class BadRequestError < Error
+        # Phrases that positively identify a request rejected for its
+        # SIZE (too many inputs or too many tokens summed across the
+        # batch). Matched case-insensitively against the provider's error
+        # text. Kept deliberately narrow: an unrelated 400 (bad parameter,
+        # invalid JSON) must never be mistaken for a size error, because
+        # the response to a size error is to split and resend.
+        #
+        # Voyage documents these 400 causes as "Batch size is too large"
+        # and "Total number of tokens in the batch exceeds the limit"; its
+        # messages read like "The max allowed tokens per submitted batch
+        # is 120000" and "The batch size limit is 128".
+        REQUEST_TOO_LARGE_PATTERNS = [
+          /batch size is too large/i,
+          /batch size limit/i,
+          /total number of tokens in the batch exceeds/i,
+          /max(?:imum)? allowed tokens per (?:submitted )?batch/i,
+        ].freeze
+
+        # Phrases identifying ONE input that is longer than the model's
+        # context window ("Number of tokens in an example exceeds the
+        # context length"). Splitting the batch cannot fix this.
+        INPUT_TOO_LONG_PATTERNS = [
+          /tokens in an example exceeds the context length/i,
+        ].freeze
+
+        # @return [Integer, nil] the HTTP status the provider returned.
+        attr_reader :status
+        # @return [String, nil] the provider's error text, truncated and
+        #   sanitized for terminals and logs.
+        attr_reader :detail
+
+        def initialize(message = nil, status: nil, detail: nil)
+          super(message)
+          @status = status
+          @detail = detail
+        end
+
+        # @return [Boolean] true when the provider rejected the request
+        #   because the batch (input count or summed tokens) is too large,
+        #   so resending it in smaller pieces can succeed.
+        def request_too_large?
+          return true if @status == 413
+          return false if @detail.nil? || input_too_long?
+          REQUEST_TOO_LARGE_PATTERNS.any? { |re| re.match?(@detail) }
+        end
+
+        # @return [Boolean] true when a single input exceeds the model's
+        #   context length.
+        def input_too_long?
+          return false if @detail.nil?
+          INPUT_TOO_LONG_PATTERNS.any? { |re| re.match?(@detail) }
+        end
+      end
       class RateLimitError < Error; end
       class TransientError < Error; end
 
@@ -266,10 +323,31 @@ module Parse
       # at most this many documents, and this many chunks summed
       # across them.
       # The endpoint also caps a request at 120k tokens summed across
-      # every document, which the SDK cannot check without a tokenizer.
+      # every document ({CONTEXT_MAX_REQUEST_TOKENS}), which the SDK can
+      # only estimate: it has no tokenizer.
       MAX_CONTEXT_DOCUMENTS = 1_000
       MAX_CONTEXT_CHUNKS = 16_000
 
+      # Voyage's cap on input tokens summed across every document in one
+      # contextualized request.
+      CONTEXT_MAX_REQUEST_TOKENS = 120_000
+
+      # Bytes per token assumed when ESTIMATING a request's input tokens
+      # for packing. Real tokenizers average closer to four bytes per token
+      # for English text, so dividing by three over-estimates and packs
+      # conservatively. This is an estimate, not a token count: the SDK has
+      # no tokenizer. When the estimate still lets an over-limit request
+      # through, the provider's size rejection is handled by splitting the
+      # request (see {CONTEXT_MAX_SPLIT_DEPTH}).
+      CONTEXT_ESTIMATED_BYTES_PER_TOKEN = 3
+
+      # How many times one contextualized request may be halved after the
+      # provider rejects it as too large. Ten halvings take the largest
+      # allowed request ({MAX_CONTEXT_DOCUMENTS} documents) down to single
+      # documents, so this bounds the retries without stopping short of a
+      # one-document request.
+      CONTEXT_MAX_SPLIT_DEPTH = 10
+
       # Default `embed_batch_size` for {CONTEXTUALIZED_MODELS}. Each
       # string is a whole document there, so 128 paragraph-sized inputs
       # would overrun the 120k-token request cap; 32 leaves room for
@@ -704,25 +782,83 @@ module Parse
       # Embed documents through the contextualized endpoint and return one
       # Array of chunk vectors per document, in input order.
       #
-      # Voyage accepts up to {MAX_CONTEXT_CHUNKS} chunks per request, but
-      # that many vectors serialize to far more than {MAX_RESPONSE_BYTES}.
-      # Whole documents are therefore grouped so each response's estimated
-      # size stays within the cap, and the groups are sent in turn. A
-      # single document too large for the cap on its own is sent alone
-      # with a response allowance sized to its chunk count.
+      # This is robust adaptation, not exact token counting. Whole
+      # documents are packed into requests that stay within every limit
+      # the SDK can check or estimate:
+      #
+      # * at most {MAX_CONTEXT_DOCUMENTS} documents;
+      # * a chunk count whose response stays within {MAX_RESPONSE_BYTES}
+      #   (sized by the vectors coming back);
+      # * an ESTIMATED input-token total within
+      #   {CONTEXT_MAX_REQUEST_TOKENS} (see
+      #   {CONTEXT_ESTIMATED_BYTES_PER_TOKEN}).
+      #
+      # A document is never split across requests. A document that alone
+      # exceeds a budget is sent on its own (its response allowance is
+      # sized to its chunk count). If the provider still rejects a request
+      # as too large, that request is halved by document and each half
+      # resent (see {#embed_contextualized_group}), so a mis-estimate costs
+      # extra requests rather than failing the batch.
       def embed_contextualized(documents, input_type, wire_input_type)
-        per_vector = response_bytes_per_vector
-        chunks_per_request = [MAX_RESPONSE_BYTES / per_vector, 1].max
+        chunks_per_request = [MAX_RESPONSE_BYTES / response_bytes_per_vector, 1].max
         groups = []
-        documents.each do |doc|
+        documents.each_with_index do |doc, i|
           last = groups.last
-          if last && last.sum(&:length) + doc.length <= chunks_per_request
-            last << doc
+          tokens = estimate_input_tokens(doc)
+          if last &&
+             last[:docs].length < MAX_CONTEXT_DOCUMENTS &&
+             last[:chunks] + doc.length <= chunks_per_request &&
+             last[:tokens] + tokens <= CONTEXT_MAX_REQUEST_TOKENS
+            last[:docs] << doc
+            last[:chunks] += doc.length
+            last[:tokens] += tokens
           else
-            groups << [doc]
+            groups << { docs: [doc], offset: i, chunks: doc.length, tokens: tokens }
           end
         end
-        groups.flat_map { |group| embed_contextualized_request(group, input_type, wire_input_type) }
+        groups.flat_map do |group|
+          embed_contextualized_group(group[:docs], group[:offset], input_type, wire_input_type, 0)
+        end
+      end
+
+      # Send one packed group, halving it by document and resending each
+      # half when the provider rejects it as too large. Only an error that
+      # {BadRequestError#request_too_large?} positively identifies is split;
+      # any other error propagates unchanged. Results are concatenated in
+      # input order, so alignment with the caller's documents is preserved
+      # however the group was divided.
+      #
+      # @param offset [Integer] index of `documents.first` in the caller's
+      #   original input, for error messages.
+      # @param depth [Integer] halvings so far, bounded by
+      #   {CONTEXT_MAX_SPLIT_DEPTH}.
+      def embed_contextualized_group(documents, offset, input_type, wire_input_type, depth)
+        embed_contextualized_request(documents, input_type, wire_input_type)
+      rescue BadRequestError => e
+        raise unless e.request_too_large?
+
+        if documents.length == 1
+          raise BadRequestError.new(
+            "Parse::Embeddings::Voyage: document #{offset} (#{documents.first.length} chunk(s), " \
+            "about #{estimate_input_tokens(documents.first)} estimated tokens) is too large for a " \
+            "single #{@model} request even on its own. Split that document into fewer or shorter " \
+            "chunks, or into several documents. Provider said: #{e.detail || e.message}",
+            status: e.status, detail: e.detail,
+          )
+        end
+        raise if depth >= CONTEXT_MAX_SPLIT_DEPTH
+
+        mid = documents.length / 2
+        embed_contextualized_group(documents[0...mid], offset, input_type, wire_input_type, depth + 1) +
+          embed_contextualized_group(documents[mid..], offset + mid, input_type, wire_input_type, depth + 1)
+      end
+
+      # Conservative ESTIMATE of a document's input tokens: its chunks'
+      # bytes divided by {CONTEXT_ESTIMATED_BYTES_PER_TOKEN}, rounded up.
+      # Used only for packing, never to refuse a document.
+      def estimate_input_tokens(chunks)
+        bytes = chunks.sum(&:bytesize)
+        (bytes + CONTEXT_ESTIMATED_BYTES_PER_TOKEN - 1) / CONTEXT_ESTIMATED_BYTES_PER_TOKEN
       end
 
       # @return [Integer] the planning estimate for one vector's JSON size.
@@ -1037,11 +1173,39 @@ module Parse
             sleep(backoff_seconds(attempts))
             next
           end
-          raise BadRequestError,
-                "Parse::Embeddings::Voyage: #{status} from POST /#{path}."
+          detail = provider_error_detail(response.body)
+          message = +"Parse::Embeddings::Voyage: #{status} from POST /#{path}."
+          message << " #{detail}" if detail
+          raise BadRequestError.new(message, status: status, detail: detail)
         end
       end
 
+      # Longest provider error text kept on a {BadRequestError}.
+      MAX_ERROR_DETAIL_CHARS = 500
+
+      # Extract the provider's error text from a non-2xx body, bounded and
+      # made safe to print. Voyage returns `{ "detail": "..." }`; a
+      # `message`, a string `error`, or a nested `error.message` is accepted
+      # too. Returns nil for an empty, oversized, or unparseable body rather
+      # than raising, since the status code alone still describes the
+      # failure.
+      def provider_error_detail(body)
+        s = body.to_s
+        return nil if s.empty? || s.bytesize > 64 * 1024
+        parsed = JSON.parse(s, max_nesting: 8)
+        return nil unless parsed.is_a?(Hash)
+        text = parsed["detail"] || parsed["message"]
+        err = parsed["error"]
+        text ||= err["message"] if err.is_a?(Hash)
+        text ||= err if err.is_a?(String)
+        text = text.to_json unless text.nil? || text.is_a?(String)
+        return nil if text.nil? || text.strip.empty?
+        text = text[0, MAX_ERROR_DETAIL_CHARS]
+        defined?(Parse::TerminalSafe) ? Parse::TerminalSafe.sanitize_line(text) : text
+      rescue JSON::ParserError
+        nil
+      end
+
       def parse_json_body!(body, max_bytes = MAX_RESPONSE_BYTES)
         s = body.to_s
         if s.bytesize > max_bytes

data/lib/parse/graphql/type_generator.rb

@@ -29,6 +29,9 @@ module Parse
         string: ::GraphQL::Types::String,
         integer: ::GraphQL::Types::Int,
         float: ::GraphQL::Types::Float,
+        # A :number property may hold fractional values, which GraphQL's
+        # 32-bit Int cannot represent, so it is exposed as Float.
+        number: ::GraphQL::Types::Float,
         boolean: ::GraphQL::Types::Boolean,
         date: ::GraphQL::Types::ISO8601DateTime,
         timezone: ::GraphQL::Types::String,