parse-stack-next 5.1.1 → 5.2.0

data/lib/parse/retrieval/agent_tool.rb

@@ -0,0 +1,369 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+
+require_relative "../retrieval"
+
+module Parse
+  module Retrieval
+    # The `semantic_search` agent tool: the agent-aware wrapper around
+    # {Parse::Retrieval.retrieve}. It applies the agent security
+    # envelope that {Parse::Retrieval.retrieve} (a model-layer method) is
+    # deliberately kept free of:
+    #
+    # * Class allowlist via {Parse::Agent::MetadataRegistry.resolve_searchable!}
+    #   (`agent_searchable` opt-in, hidden-class refusal, tenant-scope gate).
+    # * Recursive underscore-key refusal + filter-field allowlist on
+    #   caller-supplied `filter:` / `vector_filter:`.
+    # * Tenant scope merged into the Atlas pre-filter AND re-asserted on
+    #   every returned source record (NEW-TOOLS-3 guard).
+    # * `field_allowlist` projection of each source record on the way out.
+    # * Score quantization in non-admin contexts.
+    #
+    # ACL is enforced mongo-direct inside `find_similar` via the agent's
+    # `acl_scope_kwargs` (`session_token:` / `acl_user:` / `acl_role:` /
+    # `master:`), which is why the tool is `client_safe: true`: a
+    # session-token client routes through the one path with first-class
+    # SDK-side `_rperm` enforcement.
+    module AgentTool
+      module_function
+
+      # Upper bound on `k` (mirrors the registered parameter schema).
+      MAX_K = 20
+      # Default neighbour count for the agent tool. Intentionally lower than
+      # Parse::Retrieval.retrieve's library default of 10: an LLM tool result
+      # is paid for in context tokens, so the agent surface defaults
+      # conservatively. Callers/LLMs can raise it up to MAX_K per call.
+      DEFAULT_K = 5
+
+      # Default ceiling on total returned chunk-content tokens (estimated as
+      # chars/4). The retrieve count caps (k * max_chunks_per_document) bound
+      # the NUMBER of chunks but not their total size, so a few long documents
+      # could silently blow the context window. This budget trims the
+      # (score-ordered) chunk list and reports `budget_truncated` so the
+      # truncation is never silent. Pass `max_total_tokens: 0` to disable.
+      DEFAULT_MAX_TOTAL_TOKENS = 20_000
+
+      # @param agent [Parse::Agent]
+      # @param text_field [String, Symbol, nil] which embedded text source to
+      #   chunk and return as `content`. Required only for models with more
+      #   than one `embed` text source (otherwise inferred). Must name one of
+      #   the class's declared embed sources — an arbitrary field is refused so
+      #   the chunk `content` can't disclose a non-embedded field.
+      # @param max_chunks_per_document [Integer, nil] cap on chunks emitted per
+      #   matched document (forwarded to the chunker).
+      # @param max_total_tokens [Integer, nil] ceiling on total returned
+      #   chunk-content tokens (estimated chars/4). nil uses
+      #   {DEFAULT_MAX_TOTAL_TOKENS}; 0 disables the budget.
+      # @return [Hash] `{ chunks: Array<Hash>, documents: Hash, count: Integer }`
+      #   — each chunk's parent record is hoisted once into `documents` (keyed
+      #   by objectId) instead of being duplicated on every chunk. When the
+      #   token budget trims the result, `budget_truncated: true` and
+      #   `budget_dropped: <n>` are added.
+      def semantic_search(agent, class_name: nil, query: nil, k: DEFAULT_K,
+                          filter: nil, vector_filter: nil, text_field: nil,
+                          chunk_size: nil, chunk_overlap: nil, chunk_by: nil,
+                          max_chunks_per_document: nil, max_total_tokens: nil,
+                          # Back-compat / ergonomic aliases for direct callers:
+                          # `klass:`/`class:` for class_name, and the chunker's
+                          # own `size:`/`overlap:`/`by:` names.
+                          klass: nil, size: nil, overlap: nil, by: nil,
+                          **rest)
+        class_name    ||= klass || rest.delete(:class)
+        chunk_size    ||= size
+        chunk_overlap ||= overlap
+        chunk_by      ||= by
+
+        klass = Parse::Agent::MetadataRegistry.resolve_searchable!(class_name)
+        cname = klass.parse_class
+
+        unless query.is_a?(String) && !query.strip.empty?
+          raise Parse::Agent::ValidationError, "semantic_search: `query` must be a non-empty String."
+        end
+
+        resolved_text_field = normalize_text_field!(text_field, klass)
+
+        # Reject reserved underscore keys at any depth, then enforce the
+        # per-class filter-field allowlist on top-level keys.
+        Parse::Retrieval.assert_no_underscore_keys!(filter) unless filter.nil?
+        Parse::Retrieval.assert_no_underscore_keys!(vector_filter) unless vector_filter.nil?
+        allowed = Parse::Agent::MetadataRegistry.searchable_filter_fields(cname).map(&:to_s)
+        assert_filter_fields_allowed!(filter, allowed)
+        assert_filter_fields_allowed!(vector_filter, allowed)
+
+        # Tenant scope (nil for unscoped classes / bypassed admins; raises
+        # AccessDenied for an un-bound agent on a scoped class).
+        scope = Parse::Agent::Tools.resolve_tenant_scope!(agent, cname)
+
+        # Non-admin agents get quantized scores (membership-inference
+        # defense); admin agents get full precision. Keyed on the
+        # permission tier, not master-key posture.
+        score_quantize = (agent.permissions != :admin)
+        vector_field = Parse::Agent::MetadataRegistry.searchable_field(cname)
+
+        chunks = Parse::Retrieval.retrieve(
+          query: query,
+          klass: klass,
+          field: vector_field,
+          text_field: resolved_text_field,
+          k: clamp_k(k),
+          filter: filter,
+          vector_filter: vector_filter,
+          chunker: build_chunker(chunk_size, chunk_overlap, chunk_by, max_chunks_per_document),
+          tenant_scope: scope,
+          score_quantize: score_quantize,
+          source_transform: source_projector(agent, cname, scope),
+          **agent.acl_scope_kwargs,
+        )
+
+        # Token budget (B4): trim the score-ordered chunk list before
+        # building the envelope so `documents` only carries parents whose
+        # chunks survived.
+        kept, dropped = apply_token_budget(chunks, resolve_token_budget(max_total_tokens))
+
+        # Source dedup (A3): a document's (projected) source record is
+        # identical across all its chunks. Hoist it into a `documents` map
+        # keyed by objectId and drop the inline `source` from each chunk —
+        # ~46 tok/chunk saved for every chunk past the first of a document.
+        documents = {}
+        chunk_hashes = kept.map do |chunk|
+          h = chunk.to_h
+          oid = h.dig(:metadata, :object_id)
+          if oid && !oid.to_s.empty?
+            documents[oid] ||= h[:source]
+            h = h.reject { |key, _| key == :source }
+          end
+          h
+        end
+        stamp_chunk_provenance!(chunk_hashes, cname) if Parse::Agent.include_source_provenance?
+
+        envelope = { chunks: chunk_hashes, documents: documents, count: chunk_hashes.length }
+        if dropped > 0
+          envelope[:budget_truncated] = true
+          envelope[:budget_dropped] = dropped
+        end
+        envelope
+      end
+
+      # @!visibility private
+      # nil -> DEFAULT_MAX_TOTAL_TOKENS; <=0 -> nil (unlimited); else the int.
+      def resolve_token_budget(max_total_tokens)
+        return DEFAULT_MAX_TOTAL_TOKENS if max_total_tokens.nil?
+        n = max_total_tokens.to_i
+        n <= 0 ? nil : n
+      end
+
+      # @!visibility private
+      # Greedily keep score-ordered chunks until the cumulative content
+      # token estimate (chars/4) would exceed `budget`. Always keeps at
+      # least the first chunk so a single oversize chunk still returns
+      # something (flagged truncated).
+      # @return [Array(Array<Chunk>, Integer)] [kept, dropped_count]
+      def apply_token_budget(chunks, budget)
+        return [chunks, 0] if budget.nil? || chunks.empty?
+        total = 0
+        kept = []
+        chunks.each do |chunk|
+          est = (chunk.content.to_s.length / 4.0).ceil
+          break unless kept.empty? || total + est <= budget
+          kept << chunk
+          total += est
+        end
+        [kept, chunks.length - kept.length]
+      end
+
+      # @!visibility private
+      # Per-chunk `_source` provenance. The chunk already carries a
+      # `source` key (the projected parent record), so provenance uses the
+      # distinct `_source` key. object_id comes from the chunk metadata
+      # (or the projected source record).
+      def stamp_chunk_provenance!(chunk_hashes, cname)
+        chunk_hashes.each do |c|
+          next unless c.is_a?(Hash)
+          next if c.key?(:_source)
+          oid = c.dig(:metadata, :object_id)
+          oid ||= (c[:source]["objectId"] || c[:source][:objectId]) if c[:source].is_a?(Hash)
+          c[:_source] = { "class" => cname.to_s, "tool" => "semantic_search", "object_id" => oid }
+        end
+      end
+
+      # @!visibility private
+      # Build the per-record OUTPUT transform: convert the raw storage-
+      # form Mongo hit to Parse/wire form, re-assert tenant scope (raises
+      # AccessDenied — fail closed for the whole call), redact hidden
+      # nested classes, then project through `field_allowlist`.
+      def source_projector(agent, cname, scope)
+        lambda do |raw_doc|
+          converted = convert_to_parse_form(raw_doc, cname)
+          Parse::Agent::Tools.assert_record_in_tenant_scope!(converted, scope, cname) if scope
+          projected = Parse::Agent::Tools.project_object_to_allowlist(cname, converted)
+          redacted = Parse::Agent::Tools.redact_hidden_classes!(projected, agent: agent)
+          # Normalize to the same LLM-friendly, ACL-stripped form the other
+          # read tools emit so the `documents` map is consistent (and ACL-
+          # free) even for a searchable class with no agent_fields allowlist,
+          # where project_object_to_allowlist is a pass-through.
+          Parse::Agent::ResultFormatter.simplify_object(redacted)
+        end
+      end
+
+      # @!visibility private
+      def convert_to_parse_form(raw_doc, cname)
+        Parse::MongoDB.convert_documents_to_parse([raw_doc], cname).first || raw_doc
+      rescue StandardError
+        # Conversion failed for this hit. Do NOT surface the raw storage-form
+        # Mongo document: it carries internal metadata (_acl, _rperm/_wperm,
+        # storage-form _p_* pointers, _id, _created_at/_updated_at) that the
+        # success path strips. For a searchable class with NO agent_fields
+        # allowlist, project_object_to_allowlist downstream is a pass-through, so
+        # this fallback is the only thing standing between those keys and the
+        # LLM. Drop every storage-internal (underscore-prefixed) key. NOTE:
+        # reusing Parse::PipelineSecurity.strip_internal_fields is NOT enough —
+        # its denylist EXCLUDES _acl, which is exactly the field that discloses
+        # other principals' object ids and roles. The chunk's object_id is read
+        # from the raw doc before this transform runs, so dropping _id is
+        # harmless.
+        raw_doc.is_a?(Hash) ? raw_doc.reject { |k, _| k.to_s.start_with?("_") } : {}
+      end
+
+      # @!visibility private
+      def clamp_k(k)
+        n = k.to_i
+        n = DEFAULT_K if n <= 0
+        [n, MAX_K].min
+      end
+
+      # @!visibility private
+      def build_chunker(size, overlap, by, max_chunks_per_document = nil)
+        return nil if size.nil? && overlap.nil? && by.nil? && max_chunks_per_document.nil?
+        opts = {
+          size: (size || 800).to_i,
+          overlap: (overlap || 100).to_i,
+          by: (by || :chars).to_sym,
+        }
+        # Only override the chunker's own default (200) when the caller asked,
+        # so an unset cap keeps the library default rather than forcing it here.
+        opts[:max_chunks_per_document] = max_chunks_per_document.to_i unless max_chunks_per_document.nil?
+        Parse::Retrieval::Chunker::FixedSizeOverlap.new(**opts)
+      rescue ArgumentError => e
+        raise Parse::Agent::ValidationError, "semantic_search: invalid chunker options — #{e.message}"
+      end
+
+      # @!visibility private
+      # The class's declared embed TEXT sources — the only fields an agent may
+      # name as `text_field:`. Chunk `content` is the text_field's value, so
+      # restricting it to embedded sources stops the tool from surfacing a
+      # field the model never opted into embedding.
+      def searchable_text_fields(klass)
+        return [] unless klass.respond_to?(:embed_directives)
+        klass.embed_directives.values
+             .reject { |d| d.respond_to?(:image?) && d.image? }
+             .flat_map(&:sources).map(&:to_s).uniq
+      end
+
+      # @!visibility private
+      # Validate a caller-supplied text_field against the embedded-source
+      # allowlist. nil/blank → nil (retrieve infers; works for single-source
+      # models, raises AmbiguousTextField for multi-source so the agent knows
+      # to pass one).
+      def normalize_text_field!(text_field, klass)
+        return nil if text_field.nil? || text_field.to_s.strip.empty?
+        allowed = searchable_text_fields(klass)
+        unless allowed.include?(text_field.to_s)
+          raise Parse::Agent::ValidationError,
+                "semantic_search: text_field #{text_field.to_s.inspect} is not an embedded " \
+                "text source for this class (allowed: #{allowed.inspect})."
+        end
+        text_field.to_sym
+      end
+
+      # @!visibility private
+      # Refuse any top-level filter key not in the class's declared
+      # `filter_fields` allowlist (compound operators included — the
+      # allowlist is the complete set of keys the agent may use).
+      def assert_filter_fields_allowed!(filter, allowed)
+        return if filter.nil? || (filter.respond_to?(:empty?) && filter.empty?)
+        unless filter.is_a?(Hash)
+          raise Parse::Agent::ValidationError, "semantic_search: filter must be an object."
+        end
+        offending = filter.keys.map(&:to_s).reject { |key| allowed.include?(key) }
+        unless offending.empty?
+          raise Parse::Agent::ValidationError,
+                "semantic_search: filter field(s) #{offending.inspect} are not in the " \
+                "agent_searchable filter_fields allowlist (#{allowed.inspect})."
+        end
+      end
+
+      # JSON Schema for the registered tool's parameters.
+      PARAMETERS = {
+        "type" => "object",
+        "properties" => {
+          "class_name"    => { "type" => "string", "description" => "Parse class name (must be agent_searchable)." },
+          "query"         => { "type" => "string", "description" => "Natural-language query." },
+          "k"             => { "type" => "integer", "default" => DEFAULT_K, "minimum" => 1, "maximum" => MAX_K },
+          "filter"        => { "type" => "object", "description" => "Post-search field filter (allowlisted fields only)." },
+          "vector_filter" => { "type" => "object", "description" => "Atlas pre-search filter (allowlisted fields only)." },
+          "text_field"    => { "type" => "string", "description" => "Which embedded text source to chunk and return as content. Required only when the class embeds more than one text field; must name one of those sources." },
+          "chunk_size"    => { "type" => "integer", "description" => "Override chunk window size." },
+          "chunk_overlap" => { "type" => "integer", "description" => "Override chunk overlap." },
+          "chunk_by"      => { "type" => "string", "enum" => %w[chars tokens], "description" => "Chunk unit." },
+          "max_chunks_per_document" => { "type" => "integer", "minimum" => 1, "description" => "Cap on chunks emitted per matched document." },
+          "max_total_tokens" => { "type" => "integer", "minimum" => 0, "description" => "Ceiling on total returned chunk-content tokens (approx chars/4). Trims lowest-ranked chunks first and sets budget_truncated. 0 disables." },
+        },
+        "required" => %w[class_name query],
+      }.freeze
+
+      # MCP outputSchema → mirrored as structuredContent on results.
+      # The parent record of each chunk is hoisted into `documents` (keyed
+      # by objectId) rather than duplicated inline on every chunk; map a
+      # chunk to its source via `metadata.object_id`.
+      OUTPUT_SCHEMA = {
+        "type" => "object",
+        "properties" => {
+          "chunks" => {
+            "type" => "array",
+            "items" => {
+              "type" => "object",
+              "properties" => {
+                "id"      => { "type" => "string" },
+                "score"   => { "type" => %w[number null] },
+                "content" => { "type" => "string" },
+                "metadata" => { "type" => "object" },
+              },
+            },
+          },
+          "documents" => {
+            "type" => "object",
+            "description" => "objectId => projected source record (sent once per matched document).",
+          },
+          "count" => { "type" => "integer" },
+          "budget_truncated" => { "type" => "boolean", "description" => "Present when the token budget dropped lowest-ranked chunks." },
+          "budget_dropped" => { "type" => "integer", "description" => "Number of chunks dropped by the token budget." },
+        },
+      }.freeze
+
+      # Register the tool. Idempotent-ish: re-requiring is a no-op because
+      # require caches; an explicit re-register after reset_registry! is
+      # supported via {.register!}.
+      def register!
+        Parse::Agent::Tools.register(
+          name: :semantic_search,
+          description: "Find documents semantically similar to a natural-language query and " \
+                       "return scored text chunks. Use when keyword matching is unlikely to " \
+                       "work or the question needs synthesizing across documents. The target " \
+                       "class must be declared `agent_searchable`.",
+          parameters: PARAMETERS,
+          permission: :readonly,
+          timeout: 30,
+          output_schema: OUTPUT_SCHEMA,
+          client_safe: true,
+          handler: ->(agent, **args) { Parse::Retrieval::AgentTool.semantic_search(agent, **args) },
+        )
+      end
+    end
+  end
+end
+
+# Register at load. Requires Parse::Agent::Tools (TOOL_DEFINITIONS for the
+# collision check), Parse::Retrieval (loaded with the model layer), and
+# Parse::Object + MetadataDSL — all present by the time agent.rb requires
+# this file at its tail.
+Parse::Retrieval::AgentTool.register!

data/lib/parse/retrieval/chunk.rb

@@ -0,0 +1,74 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+
+module Parse
+  module Retrieval
+    # A single retrieved passage: one chunk of one source document,
+    # carrying the document's vector-search score and (optionally
+    # projected) source record.
+    #
+    # Produced by {Parse::Retrieval.retrieve}. Because embedding is
+    # one-vector-per-record (see {Parse::Core::EmbedManaged}), every
+    # chunk split from a document shares that document's single score —
+    # the chunking is presentation-only, applied after retrieval.
+    #
+    # @!attribute [r] id
+    #   @return [String] stable synthetic chunk id, `"<objectId>#<index>"`.
+    # @!attribute [r] score
+    #   @return [Float, nil] the parent document's Atlas vectorSearchScore,
+    #     already quantized when the caller requested it.
+    # @!attribute [r] content
+    #   @return [String] the chunk text.
+    # @!attribute [r] source
+    #   @return [Hash] the parent document record. When the producer
+    #     supplied a `source_transform:` (the agent tool does, projecting
+    #     through `field_allowlist`), this is the projected/redacted form.
+    # @!attribute [r] metadata
+    #   @return [Hash] presentation metadata: `:chunk_index`,
+    #     `:chunk_count`, `:chunks_truncated`, and any producer-supplied
+    #     signals (e.g. `:token_chunking_degraded`).
+    class Chunk
+      attr_reader :id, :score, :content, :source, :metadata
+
+      # @param id [String]
+      # @param score [Float, nil]
+      # @param content [String]
+      # @param source [Hash]
+      # @param metadata [Hash]
+      def initialize(id:, content:, source:, score: nil, metadata: {})
+        @id = id.to_s
+        @score = score
+        @content = content
+        @source = source
+        @metadata = metadata
+        freeze
+      end
+
+      # @return [Hash] plain-Hash form for tool output / JSON.
+      def to_h
+        {
+          id: @id,
+          score: @score,
+          content: @content,
+          source: @source,
+          metadata: @metadata,
+        }
+      end
+
+      # Value equality on the identifying triple — convenient for tests
+      # and de-duplication. `source`/`metadata` are intentionally not
+      # part of identity.
+      def ==(other)
+        other.is_a?(Chunk) &&
+          other.id == @id &&
+          other.score == @score &&
+          other.content == @content
+      end
+      alias eql? ==
+
+      def hash
+        [@id, @score, @content].hash
+      end
+    end
+  end
+end

data/lib/parse/retrieval/chunker.rb

@@ -0,0 +1,208 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+
+module Parse
+  module Retrieval
+    # Pluggable text-chunking strategies for the retrieval layer.
+    #
+    # A chunker splits a source document's text into smaller, overlapping
+    # windows for presentation. {Parse::Retrieval.retrieve} fetches the
+    # top-k whole records via Atlas `$vectorSearch`, then runs each
+    # record's text field through a chunker so callers get focused,
+    # citable passages rather than whole documents.
+    #
+    # == Presentation chunking, not embedding chunking
+    #
+    # Embedding remains one-vector-per-record (see
+    # {Parse::Core::EmbedManaged}). Chunking here is purely a
+    # *presentation* step applied after retrieval: every chunk produced
+    # from a document inherits that document's single vector-search
+    # score. The chunker never calls an embedding provider.
+    #
+    # == Extending
+    #
+    # {FixedSizeOverlap} is the default and the only strategy shipped.
+    # Subclass {Base} for semantic, sentence-aware, or true
+    # token-aware chunking:
+    #
+    #   class SentenceChunker < Parse::Retrieval::Chunker::Base
+    #     def chunk(text)
+    #       normalize(text).split(/(?<=[.!?])\s+/)
+    #     end
+    #   end
+    #
+    #   Parse::Retrieval.retrieve(
+    #     query: "onboarding steps",
+    #     klass: KnowledgeArticle,
+    #     chunker: SentenceChunker.new,
+    #   )
+    module Chunker
+      # Abstract base. Subclasses MUST implement {#chunk}.
+      #
+      # Subclasses get one free behavior from {Base}: {#chunk_with_meta},
+      # which wraps {#chunk} and reports whether the result was capped.
+      # {Parse::Retrieval.retrieve} calls {#chunk_with_meta} so it can
+      # stamp a truncation signal onto each emitted chunk's metadata.
+      class Base
+        # @param text [String, nil] source document text.
+        # @return [Array<String>] zero or more chunks. MUST return `[]`
+        #   for blank/`nil` input.
+        # @raise [NotImplementedError] unless overridden.
+        def chunk(text)
+          raise NotImplementedError, "#{self.class}#chunk must return Array<String>."
+        end
+
+        # Wrap {#chunk} with truncation metadata. The default
+        # implementation here does NOT cap — it reports the chunk list as
+        # produced. {FixedSizeOverlap} overrides this to enforce its
+        # `max_chunks_per_document` cap and report the pre-cap count.
+        #
+        # @param text [String, nil]
+        # @return [Hash] `{ chunks: Array<String>, truncated: Boolean,
+        #   total_before_truncation: Integer }`.
+        def chunk_with_meta(text)
+          chunks = Array(chunk(text))
+          { chunks: chunks, truncated: false, total_before_truncation: chunks.length }
+        end
+
+        # @!visibility private
+        # Shared input normalization. Returns `nil` for `nil`,
+        # non-String, empty, or whitespace-only input — every concrete
+        # `#chunk` treats a `nil` return as "no chunks". A non-String is
+        # treated as blank rather than raised so a document with an
+        # unexpected non-text value in the chunked field is skipped, not
+        # fatal, during retrieval.
+        def normalize(text)
+          return nil if text.nil?
+          return nil unless text.is_a?(String)
+          stripped = text.strip
+          stripped.empty? ? nil : text
+        end
+      end
+
+      # Fixed-size sliding-window chunker with overlap.
+      #
+      # Splits text into windows of `size` units, advancing by
+      # `size - overlap` each step so consecutive chunks share `overlap`
+      # units of context. `by: :chars` (default) counts characters;
+      # `by: :tokens` counts whitespace-delimited tokens (a cheap
+      # approximation — there is no model tokenizer here; see the
+      # `:tokens` note below).
+      #
+      #   c = Parse::Retrieval::Chunker::FixedSizeOverlap.new(size: 800, overlap: 100)
+      #   c.chunk(long_text) #=> ["…800 chars…", "…overlap+800…", …]
+      #
+      # == Amplification cap
+      #
+      # `max_chunks_per_document` (default 200) bounds how many chunks a
+      # single document can yield. Beyond the cap the chunker
+      # *truncates* — it returns the first `max_chunks_per_document`
+      # chunks rather than raising — and {#chunk_with_meta} reports
+      # `truncated: true`. This is the DoS guard: a 10 MB field at
+      # 800-char windows would otherwise yield ~12,500 chunks.
+      #
+      # == `:tokens`
+      #
+      # `by: :tokens` treats `size`/`overlap` as literal whitespace-token
+      # counts supplied by the caller. The chunker does NOT consult an
+      # embedding provider's `max_input_tokens`; that hint is the
+      # caller's concern (see {Parse::Retrieval.retrieve}). The chunker
+      # always does exactly what it was constructed with and never
+      # silently switches modes.
+      class FixedSizeOverlap < Base
+        # @return [Integer] window width in `by:` units.
+        attr_reader :size
+        # @return [Integer] units shared between consecutive windows.
+        attr_reader :overlap
+        # @return [Symbol] `:chars` or `:tokens`.
+        attr_reader :by
+        # @return [Integer] hard cap on chunks emitted per document.
+        attr_reader :max_chunks_per_document
+
+        # @param size [Integer] window width (> 0).
+        # @param overlap [Integer] shared units between windows
+        #   (`0 <= overlap < size`).
+        # @param by [Symbol] `:chars` (default) or `:tokens`.
+        # @param max_chunks_per_document [Integer] cap (> 0, default 200).
+        # @raise [ArgumentError] on any out-of-range argument. In
+        #   particular `overlap >= size` is refused: a non-shrinking
+        #   stride would never advance and would loop forever.
+        def initialize(size: 800, overlap: 100, by: :chars, max_chunks_per_document: 200)
+          unless size.is_a?(Integer) && size > 0
+            raise ArgumentError, "size must be a positive Integer (got #{size.inspect})."
+          end
+          unless overlap.is_a?(Integer) && overlap >= 0
+            raise ArgumentError, "overlap must be a non-negative Integer (got #{overlap.inspect})."
+          end
+          if overlap >= size
+            raise ArgumentError,
+                  "overlap (#{overlap}) must be strictly less than size (#{size}); " \
+                  "a stride of size - overlap <= 0 would never advance."
+          end
+          unless %i[chars tokens].include?(by)
+            raise ArgumentError, "by must be :chars or :tokens (got #{by.inspect})."
+          end
+          unless max_chunks_per_document.is_a?(Integer) && max_chunks_per_document > 0
+            raise ArgumentError,
+                  "max_chunks_per_document must be a positive Integer " \
+                  "(got #{max_chunks_per_document.inspect})."
+          end
+          @size = size
+          @overlap = overlap
+          @by = by
+          @max_chunks_per_document = max_chunks_per_document
+          @stride = size - overlap
+        end
+
+        # @param text [String, nil]
+        # @return [Array<String>] chunks (capped at
+        #   {#max_chunks_per_document}). `[]` for blank input.
+        def chunk(text)
+          chunk_with_meta(text)[:chunks]
+        end
+
+        # (see Base#chunk_with_meta)
+        def chunk_with_meta(text)
+          source = normalize(text)
+          return { chunks: [], truncated: false, total_before_truncation: 0 } if source.nil?
+
+          all = (@by == :tokens) ? window_tokens(source) : window_chars(source)
+          total = all.length
+          if total > @max_chunks_per_document
+            { chunks: all.first(@max_chunks_per_document),
+              truncated: true,
+              total_before_truncation: total }
+          else
+            { chunks: all, truncated: false, total_before_truncation: total }
+          end
+        end
+
+        private
+
+        def window_chars(text)
+          len = text.length
+          out = []
+          start = 0
+          while start < len
+            out << text[start, @size]
+            start += @stride
+          end
+          out
+        end
+
+        def window_tokens(text)
+          tokens = text.split(/\s+/).reject(&:empty?)
+          return [] if tokens.empty?
+          out = []
+          start = 0
+          n = tokens.length
+          while start < n
+            out << tokens[start, @size].join(" ")
+            start += @stride
+          end
+          out
+        end
+      end
+    end
+  end
+end