agentfootprint 8.9.0 → 8.10.0

package/dist/esm/rag/splitters/byParagraph.d.ts

@@ -0,0 +1,24 @@
+/**
+ * byParagraph — cut on blank lines, pack up to `maxChars`.
+ *
+ * Pattern: Strategy (one of `Splitter`).
+ * Role:    rag/ layer. The right default for prose that carries no headings.
+ * Emits:   N/A.
+ *
+ * A blank line is the weakest structural signal a document reliably has, and
+ * unlike a character count it at least falls where the author paused. Adjacent
+ * paragraphs are packed together until the next one would exceed `maxChars`,
+ * so a document of short paragraphs yields whole chunks rather than one chunk
+ * per line.
+ *
+ * A single paragraph longer than `maxChars` is hard-cut — see the note in
+ * `shared.ts` on why packing and cutting are separate decisions.
+ */
+import type { Splitter } from '../types.js';
+export interface ByParagraphOptions {
+    /** Target chunk size in characters. Default 1000 — see the family docstring. */
+    readonly maxChars?: number;
+    /** Backward overlap between adjacent chunks. Default 150. */
+    readonly overlapChars?: number;
+}
+export declare function byParagraph(options?: ByParagraphOptions): Splitter;

package/dist/esm/rag/splitters/byParagraph.js

@@ -0,0 +1,16 @@
+import { DEFAULT_MAX_CHARS, DEFAULT_OVERLAP_CHARS } from './constants.js';
+import { applyOverlap, foldRunts, hardCut, packSpans, paragraphSpans, toPieces } from './shared.js';
+export function byParagraph(options = {}) {
+    const maxChars = options.maxChars ?? DEFAULT_MAX_CHARS;
+    const overlapChars = options.overlapChars ?? DEFAULT_OVERLAP_CHARS;
+    return {
+        name: 'byParagraph',
+        split(doc) {
+            const paragraphs = paragraphSpans(doc.text);
+            const packed = packSpans(paragraphs, maxChars);
+            const cut = packed.flatMap((span) => span.end - span.start > maxChars ? hardCut(span, maxChars, overlapChars) : [span]);
+            return toPieces(doc, applyOverlap(foldRunts(cut), overlapChars));
+        },
+    };
+}
+//# sourceMappingURL=byParagraph.js.map

package/dist/esm/rag/splitters/byParagraph.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"byParagraph.js","sourceRoot":"","sources":["../../../../src/rag/splitters/byParagraph.ts"],"names":[],"mappings":"AAiBA,OAAO,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAC1E,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AASpG,MAAM,UAAU,WAAW,CAAC,UAA8B,EAAE;IAC1D,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,iBAAiB,CAAC;IACvD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,qBAAqB,CAAC;IAEnE,OAAO;QACL,IAAI,EAAE,aAAa;QACnB,KAAK,CAAC,GAAmB;YACvB,MAAM,UAAU,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC5C,MAAM,MAAM,GAAG,SAAS,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;YAC/C,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAClC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAClF,CAAC;YACF,OAAO,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,YAAY,CAAC,CAAC,CAAC;QACnE,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/esm/rag/splitters/constants.d.ts

@@ -0,0 +1,19 @@
+/**
+ * The two numbers every splitter defaults to, in one place so they cannot
+ * drift apart between strategies.
+ *
+ * See `splitters/index.ts` for the measurement they come from: `localEmbedder`
+ * truncates at 512 wordpiece tokens (~1,800–2,000 characters), silently, and
+ * 1,000 characters is ~250 tokens — comfortably inside it.
+ */
+/** Target chunk size in characters. ~250 wordpiece tokens of English. */
+export declare const DEFAULT_MAX_CHARS = 1000;
+/** Overlap between adjacent chunks, so a boundary-spanning sentence is whole somewhere. */
+export declare const DEFAULT_OVERLAP_CHARS = 150;
+/**
+ * Below this, a trailing fragment is folded back into the previous chunk
+ * rather than shipped on its own. A 12-character chunk consisting of one
+ * heading and no body embeds to noise and takes a top-K slot from a real
+ * passage.
+ */
+export declare const MIN_CHUNK_CHARS = 50;

package/dist/esm/rag/splitters/constants.js

@@ -0,0 +1,20 @@
+/**
+ * The two numbers every splitter defaults to, in one place so they cannot
+ * drift apart between strategies.
+ *
+ * See `splitters/index.ts` for the measurement they come from: `localEmbedder`
+ * truncates at 512 wordpiece tokens (~1,800–2,000 characters), silently, and
+ * 1,000 characters is ~250 tokens — comfortably inside it.
+ */
+/** Target chunk size in characters. ~250 wordpiece tokens of English. */
+export const DEFAULT_MAX_CHARS = 1000;
+/** Overlap between adjacent chunks, so a boundary-spanning sentence is whole somewhere. */
+export const DEFAULT_OVERLAP_CHARS = 150;
+/**
+ * Below this, a trailing fragment is folded back into the previous chunk
+ * rather than shipped on its own. A 12-character chunk consisting of one
+ * heading and no body embeds to noise and takes a top-K slot from a real
+ * passage.
+ */
+export const MIN_CHUNK_CHARS = 50;
+//# sourceMappingURL=constants.js.map

package/dist/esm/rag/splitters/constants.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../../../src/rag/splitters/constants.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,yEAAyE;AACzE,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,CAAC;AAEtC,2FAA2F;AAC3F,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAEzC;;;;;GAKG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,EAAE,CAAC"}

package/dist/esm/rag/splitters/fixedWithOverlap.d.ts

@@ -0,0 +1,31 @@
+/**
+ * fixedWithOverlap — cut every `chars` characters, overlapping by `overlapChars`.
+ *
+ * Pattern: Strategy (one of `Splitter`).
+ * Role:    rag/ layer. The fallback for text with no structure to read:
+ *          transcripts, OCR output, scraped bodies with the markup gone.
+ * Emits:   N/A.
+ *
+ * It is the least informed strategy and it is honest about that: it cuts where
+ * the counter says, which will sometimes be mid-sentence. The overlap is what
+ * makes that survivable — a sentence broken at a boundary is whole in the next
+ * chunk. Prefer `byHeading` or `byParagraph` whenever the document gives you
+ * anything to work with.
+ *
+ * Cuts are nudged to the nearest word boundary within a small window, so the
+ * common case does not split a word in half. The offsets move with the nudge;
+ * they always index the source.
+ */
+import type { Splitter } from '../types.js';
+export interface FixedWithOverlapOptions {
+    /** Characters per chunk. Default 1000. */
+    readonly chars?: number;
+    /** Backward overlap. Default 150. */
+    readonly overlapChars?: number;
+    /**
+     * How far to look for a word boundary before accepting a mid-word cut.
+     * Default 40 characters. Set 0 to cut exactly on the count.
+     */
+    readonly wordWindow?: number;
+}
+export declare function fixedWithOverlap(options?: FixedWithOverlapOptions): Splitter;

package/dist/esm/rag/splitters/fixedWithOverlap.js

@@ -0,0 +1,34 @@
+import { DEFAULT_MAX_CHARS, DEFAULT_OVERLAP_CHARS } from './constants.js';
+import { applyOverlap, foldRunts, toPieces } from './shared.js';
+export function fixedWithOverlap(options = {}) {
+    const chars = options.chars ?? DEFAULT_MAX_CHARS;
+    const overlapChars = options.overlapChars ?? DEFAULT_OVERLAP_CHARS;
+    const wordWindow = options.wordWindow ?? 40;
+    return {
+        name: 'fixedWithOverlap',
+        split(doc) {
+            const text = doc.text;
+            const spans = [];
+            let start = 0;
+            while (start < text.length) {
+                let end = Math.min(text.length, start + chars);
+                // Nudge back to whitespace, but never past the window and never so far
+                // that the chunk collapses.
+                if (end < text.length && wordWindow > 0) {
+                    const floor = Math.max(start + 1, end - wordWindow);
+                    let candidate = end;
+                    while (candidate > floor && !/\s/.test(text[candidate] ?? ''))
+                        candidate -= 1;
+                    if (candidate > floor)
+                        end = candidate;
+                }
+                spans.push({ start, end });
+                if (end >= text.length)
+                    break;
+                start = end;
+            }
+            return toPieces(doc, applyOverlap(foldRunts(spans), overlapChars));
+        },
+    };
+}
+//# sourceMappingURL=fixedWithOverlap.js.map

package/dist/esm/rag/splitters/fixedWithOverlap.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fixedWithOverlap.js","sourceRoot":"","sources":["../../../../src/rag/splitters/fixedWithOverlap.ts"],"names":[],"mappings":"AAmBA,OAAO,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAC1E,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,QAAQ,EAAa,MAAM,aAAa,CAAC;AAc3E,MAAM,UAAU,gBAAgB,CAAC,UAAmC,EAAE;IACpE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,iBAAiB,CAAC;IACjD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,qBAAqB,CAAC;IACnE,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC;IAE5C,OAAO;QACL,IAAI,EAAE,kBAAkB;QACxB,KAAK,CAAC,GAAmB;YACvB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;YACtB,MAAM,KAAK,GAAW,EAAE,CAAC;YACzB,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;gBAC3B,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,GAAG,KAAK,CAAC,CAAC;gBAC/C,uEAAuE;gBACvE,4BAA4B;gBAC5B,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;oBACxC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,EAAE,GAAG,GAAG,UAAU,CAAC,CAAC;oBACpD,IAAI,SAAS,GAAG,GAAG,CAAC;oBACpB,OAAO,SAAS,GAAG,KAAK,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;wBAAE,SAAS,IAAI,CAAC,CAAC;oBAC9E,IAAI,SAAS,GAAG,KAAK;wBAAE,GAAG,GAAG,SAAS,CAAC;gBACzC,CAAC;gBACD,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;gBAC3B,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM;oBAAE,MAAM;gBAC9B,KAAK,GAAG,GAAG,CAAC;YACd,CAAC;YACD,OAAO,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC,CAAC,CAAC;QACrE,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/esm/rag/splitters/index.d.ts

@@ -0,0 +1,50 @@
+/**
+ * rag/splitters — where a document gets cut, and why there.
+ *
+ * Pattern: Strategy family (factory functions), the same shape as the window
+ *          and retrieval families. Not a `{ kind }` union: a splitter you write
+ *          yourself is the same shape as these, unused ones stay out of a
+ *          bundle, and adding one needs no switch edited anywhere.
+ * Role:    rag/ layer.
+ * Emits:   N/A.
+ *
+ * ── The defaults, and the measurement behind them ───────────────────────────
+ *
+ * `maxChars: 1000`, `overlapChars: 150`. Those numbers are chosen by
+ * CONSTRAINT, not by fashion, and the constraint is measurable:
+ *
+ * **`localEmbedder`'s default model silently truncates at 512 wordpiece
+ * tokens.** Measured directly — embed a long text, embed the same text with an
+ * unrelated tail appended, and compare. At 508 base tokens the tail still
+ * moves the vector (cosine 0.9965). At 596 it does not (cosine 0.999999): the
+ * tail was discarded and nothing said so. 512 wordpiece tokens is roughly
+ * 1,800–2,000 characters of English.
+ *
+ * A chunk of 1,000 characters is about 250 tokens. That sits comfortably
+ * inside that cliff, inside the 256-token length the sentence-transformer
+ * family was actually trained at, and trivially inside `openaiEmbedder`'s
+ * 8,191. Raise it and you approach a limit whose failure mode is silence —
+ * half your chunk embedded, full score reported.
+ *
+ * The 150-character overlap (15%) exists so a sentence spanning a boundary is
+ * whole in one of the two neighbours. It is the smallest overlap that reliably
+ * does that for English prose; there is nothing deeper claimed for it.
+ *
+ * ── Which one to use ────────────────────────────────────────────────────────
+ *
+ * | splitter | cuts on | use when |
+ * |---|---|---|
+ * | `byHeading()` | Markdown `#` lines | the document declares its own sections — **not a heuristic** |
+ * | `byParagraph()` | blank lines | prose with no headings |
+ * | `fixedWithOverlap()` | character count | text with no structure at all (transcripts, OCR) |
+ * | `wholeDocument()` | nothing | short documents that are already one idea |
+ *
+ * `byHeading` is the only one that is not guessing. The author already marked
+ * where the sections are; the others are inferring boundaries from typography.
+ * That is why it is the default for Markdown.
+ */
+export { byParagraph, type ByParagraphOptions } from './byParagraph.js';
+export { byHeading, type ByHeadingOptions } from './byHeading.js';
+export { fixedWithOverlap, type FixedWithOverlapOptions } from './fixedWithOverlap.js';
+export { wholeDocument } from './wholeDocument.js';
+export { DEFAULT_MAX_CHARS, DEFAULT_OVERLAP_CHARS } from './constants.js';

package/dist/esm/rag/splitters/index.js

@@ -0,0 +1,51 @@
+/**
+ * rag/splitters — where a document gets cut, and why there.
+ *
+ * Pattern: Strategy family (factory functions), the same shape as the window
+ *          and retrieval families. Not a `{ kind }` union: a splitter you write
+ *          yourself is the same shape as these, unused ones stay out of a
+ *          bundle, and adding one needs no switch edited anywhere.
+ * Role:    rag/ layer.
+ * Emits:   N/A.
+ *
+ * ── The defaults, and the measurement behind them ───────────────────────────
+ *
+ * `maxChars: 1000`, `overlapChars: 150`. Those numbers are chosen by
+ * CONSTRAINT, not by fashion, and the constraint is measurable:
+ *
+ * **`localEmbedder`'s default model silently truncates at 512 wordpiece
+ * tokens.** Measured directly — embed a long text, embed the same text with an
+ * unrelated tail appended, and compare. At 508 base tokens the tail still
+ * moves the vector (cosine 0.9965). At 596 it does not (cosine 0.999999): the
+ * tail was discarded and nothing said so. 512 wordpiece tokens is roughly
+ * 1,800–2,000 characters of English.
+ *
+ * A chunk of 1,000 characters is about 250 tokens. That sits comfortably
+ * inside that cliff, inside the 256-token length the sentence-transformer
+ * family was actually trained at, and trivially inside `openaiEmbedder`'s
+ * 8,191. Raise it and you approach a limit whose failure mode is silence —
+ * half your chunk embedded, full score reported.
+ *
+ * The 150-character overlap (15%) exists so a sentence spanning a boundary is
+ * whole in one of the two neighbours. It is the smallest overlap that reliably
+ * does that for English prose; there is nothing deeper claimed for it.
+ *
+ * ── Which one to use ────────────────────────────────────────────────────────
+ *
+ * | splitter | cuts on | use when |
+ * |---|---|---|
+ * | `byHeading()` | Markdown `#` lines | the document declares its own sections — **not a heuristic** |
+ * | `byParagraph()` | blank lines | prose with no headings |
+ * | `fixedWithOverlap()` | character count | text with no structure at all (transcripts, OCR) |
+ * | `wholeDocument()` | nothing | short documents that are already one idea |
+ *
+ * `byHeading` is the only one that is not guessing. The author already marked
+ * where the sections are; the others are inferring boundaries from typography.
+ * That is why it is the default for Markdown.
+ */
+export { byParagraph } from './byParagraph.js';
+export { byHeading } from './byHeading.js';
+export { fixedWithOverlap } from './fixedWithOverlap.js';
+export { wholeDocument } from './wholeDocument.js';
+export { DEFAULT_MAX_CHARS, DEFAULT_OVERLAP_CHARS } from './constants.js';
+//# sourceMappingURL=index.js.map

package/dist/esm/rag/splitters/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/rag/splitters/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AACH,OAAO,EAAE,WAAW,EAA2B,MAAM,kBAAkB,CAAC;AACxE,OAAO,EAAE,SAAS,EAAyB,MAAM,gBAAgB,CAAC;AAClE,OAAO,EAAE,gBAAgB,EAAgC,MAAM,uBAAuB,CAAC;AACvF,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC"}

package/dist/esm/rag/splitters/shared.d.ts

@@ -0,0 +1,87 @@
+/**
+ * Shared machinery for the splitter family.
+ *
+ * Pattern: internal helpers, not exported from the door.
+ * Role:    rag/ layer. The offset arithmetic lives here ONCE, because it is the
+ *          part every splitter must get exactly right and the part that is
+ *          easy to get subtly wrong per-strategy.
+ * Emits:   N/A.
+ *
+ * **The invariant, stated once:** for every piece produced,
+ * `doc.text.slice(charStart, charEnd) === piece.text`. Every helper here
+ * derives its text BY SLICING rather than by concatenating what it read, which
+ * is what makes the invariant true by construction rather than by care.
+ */
+import type { LoadedDocument, SplitPiece } from '../types.js';
+/** A half-open span of the source text. */
+export interface Span {
+    readonly start: number;
+    readonly end: number;
+    readonly heading?: string;
+    /**
+     * Where this span's OWN content begins, before overlap was prepended.
+     *
+     * Overlap deliberately reaches backward into the previous chunk, which means
+     * `start` can sit on the previous PAGE. Attributing the chunk to that page
+     * would cite a page whose text is only the borrowed run-up — so page lookup
+     * uses this instead, and it is set only when the two differ.
+     */
+    readonly contentStart?: number;
+}
+/**
+ * Which page an offset falls on, 1-based, or `undefined` when the document is
+ * not paginated.
+ *
+ * Pages were joined with `'\n\n'` by the loader, so the boundaries are
+ * reconstructible exactly. A chunk that straddles a page break is attributed
+ * to the page it STARTS on — one number, the one you would turn to first.
+ */
+export declare function pageLocator(doc: LoadedDocument): (offset: number) => number | undefined;
+/**
+ * Trim a span inward past leading/trailing whitespace, keeping the offsets
+ * honest.
+ *
+ * The naive version — `slice().trim()` — produces text that no longer matches
+ * its own offsets. This moves the offsets instead, so the invariant holds.
+ * Returns `undefined` when the span is entirely whitespace.
+ */
+export declare function trimSpan(text: string, span: Span): Span | undefined;
+/**
+ * Pack consecutive units (paragraphs, lines) into spans of at most `maxChars`,
+ * then apply `overlapChars` of backward overlap.
+ *
+ * Units are never split by this function — a single unit longer than `maxChars`
+ * becomes its own oversized span, and the caller decides whether to hard-cut
+ * it. That separation matters: a 4,000-character paragraph is a real thing in
+ * real documents, and silently slicing it mid-word is a different decision from
+ * packing.
+ */
+export declare function packSpans(units: readonly Span[], maxChars: number): Span[];
+/**
+ * Extend each span backward by `overlapChars`, without crossing the previous
+ * span's start.
+ *
+ * Backward rather than forward so a chunk always ENDS on the boundary its
+ * strategy chose — a heading section that ends mid-sentence because of overlap
+ * would be a section that lies about where it ends.
+ */
+export declare function applyOverlap(spans: readonly Span[], overlapChars: number): Span[];
+/**
+ * Fold a too-small span back into its predecessor.
+ *
+ * A fragment of a few characters — a stray heading, a signature line — embeds
+ * to noise and then competes for a top-K slot with real passages.
+ *
+ * **It never folds across a heading boundary.** A merged chunk carries the
+ * FIRST span's heading, so folding a short section into the one before it
+ * would produce a chunk labelled `heading="Alpha"` whose text is Beta's — a
+ * citation that names the wrong section, which is worse than a small chunk.
+ * Two spans merge only when they are already the same section.
+ */
+export declare function foldRunts(spans: readonly Span[], minChars?: number): Span[];
+/** Turn spans into pieces, slicing the text so the invariant holds by construction. */
+export declare function toPieces(doc: LoadedDocument, spans: readonly Span[]): SplitPiece[];
+/** Hard-cut a span that no packing could fit, at `maxChars` with overlap. */
+export declare function hardCut(span: Span, maxChars: number, overlapChars: number): Span[];
+/** Split into paragraph spans on blank lines, keeping offsets. */
+export declare function paragraphSpans(text: string): Span[];

package/dist/esm/rag/splitters/shared.js

@@ -0,0 +1,188 @@
+import { MIN_CHUNK_CHARS } from './constants.js';
+/**
+ * Which page an offset falls on, 1-based, or `undefined` when the document is
+ * not paginated.
+ *
+ * Pages were joined with `'\n\n'` by the loader, so the boundaries are
+ * reconstructible exactly. A chunk that straddles a page break is attributed
+ * to the page it STARTS on — one number, the one you would turn to first.
+ */
+export function pageLocator(doc) {
+    const pages = doc.pages;
+    if (!pages || pages.length === 0)
+        return () => undefined;
+    const bounds = [];
+    let cursor = 0;
+    for (const page of pages) {
+        bounds.push(cursor);
+        cursor += page.length + 2; // the '\n\n' the loader joined with
+    }
+    return (offset) => {
+        let page = 1;
+        for (let i = 0; i < bounds.length; i++) {
+            if (offset >= (bounds[i] ?? 0))
+                page = i + 1;
+            else
+                break;
+        }
+        return page;
+    };
+}
+/**
+ * Trim a span inward past leading/trailing whitespace, keeping the offsets
+ * honest.
+ *
+ * The naive version — `slice().trim()` — produces text that no longer matches
+ * its own offsets. This moves the offsets instead, so the invariant holds.
+ * Returns `undefined` when the span is entirely whitespace.
+ */
+export function trimSpan(text, span) {
+    let start = span.start;
+    let end = span.end;
+    while (start < end && /\s/.test(text[start] ?? ''))
+        start += 1;
+    while (end > start && /\s/.test(text[end - 1] ?? ''))
+        end -= 1;
+    if (start >= end)
+        return undefined;
+    return {
+        start,
+        end,
+        ...(span.heading !== undefined && { heading: span.heading }),
+        ...(span.contentStart !== undefined && { contentStart: span.contentStart }),
+    };
+}
+/**
+ * Pack consecutive units (paragraphs, lines) into spans of at most `maxChars`,
+ * then apply `overlapChars` of backward overlap.
+ *
+ * Units are never split by this function — a single unit longer than `maxChars`
+ * becomes its own oversized span, and the caller decides whether to hard-cut
+ * it. That separation matters: a 4,000-character paragraph is a real thing in
+ * real documents, and silently slicing it mid-word is a different decision from
+ * packing.
+ */
+export function packSpans(units, maxChars) {
+    const out = [];
+    let current;
+    for (const unit of units) {
+        if (current === undefined) {
+            current = unit;
+            continue;
+        }
+        // Same heading, and the merged span still fits.
+        const fits = unit.end - current.start <= maxChars;
+        const sameSection = current.heading === unit.heading;
+        if (fits && sameSection) {
+            current = {
+                start: current.start,
+                end: unit.end,
+                ...(current.heading !== undefined && { heading: current.heading }),
+            };
+            continue;
+        }
+        out.push(current);
+        current = unit;
+    }
+    if (current !== undefined)
+        out.push(current);
+    return out;
+}
+/**
+ * Extend each span backward by `overlapChars`, without crossing the previous
+ * span's start.
+ *
+ * Backward rather than forward so a chunk always ENDS on the boundary its
+ * strategy chose — a heading section that ends mid-sentence because of overlap
+ * would be a section that lies about where it ends.
+ */
+export function applyOverlap(spans, overlapChars) {
+    if (overlapChars <= 0)
+        return [...spans];
+    return spans.map((span, i) => {
+        if (i === 0)
+            return span;
+        const floor = spans[i - 1]?.start ?? 0;
+        const start = Math.max(floor, span.start - overlapChars);
+        // Remember where the chunk's own content starts, so the page it is
+        // attributed to is the page it is ABOUT, not the page its run-up borrowed
+        // from.
+        return start === span.start ? span : { ...span, start, contentStart: span.start };
+    });
+}
+/**
+ * Fold a too-small span back into its predecessor.
+ *
+ * A fragment of a few characters — a stray heading, a signature line — embeds
+ * to noise and then competes for a top-K slot with real passages.
+ *
+ * **It never folds across a heading boundary.** A merged chunk carries the
+ * FIRST span's heading, so folding a short section into the one before it
+ * would produce a chunk labelled `heading="Alpha"` whose text is Beta's — a
+ * citation that names the wrong section, which is worse than a small chunk.
+ * Two spans merge only when they are already the same section.
+ */
+export function foldRunts(spans, minChars = MIN_CHUNK_CHARS) {
+    const out = [];
+    for (const span of spans) {
+        const previous = out[out.length - 1];
+        const sameSection = previous !== undefined && previous.heading === span.heading;
+        if (previous !== undefined && sameSection && span.end - span.start < minChars) {
+            out[out.length - 1] = {
+                start: previous.start,
+                end: span.end,
+                ...(previous.heading !== undefined && { heading: previous.heading }),
+            };
+            continue;
+        }
+        out.push(span);
+    }
+    return out;
+}
+/** Turn spans into pieces, slicing the text so the invariant holds by construction. */
+export function toPieces(doc, spans) {
+    const pageAt = pageLocator(doc);
+    const pieces = [];
+    for (const span of spans) {
+        const trimmed = trimSpan(doc.text, span);
+        if (trimmed === undefined)
+            continue;
+        // See `Span.contentStart`: the page is the one this chunk is about.
+        const page = pageAt(span.contentStart ?? trimmed.start);
+        pieces.push({
+            // Sliced, never concatenated. This is the invariant.
+            text: doc.text.slice(trimmed.start, trimmed.end),
+            charStart: trimmed.start,
+            charEnd: trimmed.end,
+            ...(trimmed.heading !== undefined && { heading: trimmed.heading }),
+            ...(page !== undefined && { page }),
+        });
+    }
+    return pieces;
+}
+/** Hard-cut a span that no packing could fit, at `maxChars` with overlap. */
+export function hardCut(span, maxChars, overlapChars) {
+    const out = [];
+    const stride = Math.max(1, maxChars - overlapChars);
+    for (let start = span.start; start < span.end; start += stride) {
+        const end = Math.min(span.end, start + maxChars);
+        out.push({ start, end, ...(span.heading !== undefined && { heading: span.heading }) });
+        if (end >= span.end)
+            break;
+    }
+    return out;
+}
+/** Split into paragraph spans on blank lines, keeping offsets. */
+export function paragraphSpans(text) {
+    const spans = [];
+    const separator = /\n[ \t]*\n/g;
+    let cursor = 0;
+    let match;
+    while ((match = separator.exec(text)) !== null) {
+        spans.push({ start: cursor, end: match.index });
+        cursor = match.index + match[0].length;
+    }
+    spans.push({ start: cursor, end: text.length });
+    return spans.filter((s) => s.end > s.start);
+}
+//# sourceMappingURL=shared.js.map

package/dist/esm/rag/splitters/shared.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../../../src/rag/splitters/shared.ts"],"names":[],"mappings":"AAeA,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAkBjD;;;;;;;GAOG;AACH,MAAM,UAAU,WAAW,CAAC,GAAmB;IAC7C,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;IACxB,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,GAAG,EAAE,CAAC,SAAS,CAAC;IACzD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACpB,MAAM,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,oCAAoC;IACjE,CAAC;IACD,OAAO,CAAC,MAAc,EAAsB,EAAE;QAC5C,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBAAE,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;;gBACxC,MAAM;QACb,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,IAAU;IAC/C,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACvB,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;IACnB,OAAO,KAAK,GAAG,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAAE,KAAK,IAAI,CAAC,CAAC;IAC/D,OAAO,GAAG,GAAG,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAAE,GAAG,IAAI,CAAC,CAAC;IAC/D,IAAI,KAAK,IAAI,GAAG;QAAE,OAAO,SAAS,CAAC;IACnC,OAAO;QACL,KAAK;QACL,GAAG;QACH,GAAG,CAAC,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC;QAC5D,GAAG,CAAC,IAAI,CAAC,YAAY,KAAK,SAAS,IAAI,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,CAAC;KAC5E,CAAC;AACJ,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,SAAS,CAAC,KAAsB,EAAE,QAAgB;IAChE,MAAM,GAAG,GAAW,EAAE,CAAC;IACvB,IAAI,OAAyB,CAAC;IAE9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,OAAO,GAAG,IAAI,CAAC;YACf,SAAS;QACX,CAAC;QACD,gDAAgD;QAChD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,KAAK,IAAI,QAAQ,CAAC;QAClD,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO,CAAC;QACrD,IAAI,IAAI,IAAI,WAAW,EAAE,CAAC;YACxB,OAAO,GAAG;gBACR,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,GAAG,CAAC,OAAO,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;aACnE,CAAC;YACF,SAAS;QACX,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAClB,OAAO,GAAG,IAAI,CAAC;IACjB,CAAC;IACD,IAAI,OAAO,KAAK,SAAS;QAAE,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC7C,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,YAAY,CAAC,KAAsB,EAAE,YAAoB;IACvE,IAAI,YAAY,IAAI,CAAC;QAAE,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC;IACzC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;QAC3B,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QACzB,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;QACvC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,GAAG,YAAY,CAAC,CAAC;QACzD,mEAAmE;QACnE,0EAA0E;QAC1E,QAAQ;QACR,OAAO,KAAK,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;IACpF,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,SAAS,CAAC,KAAsB,EAAE,WAAmB,eAAe;IAClF,MAAM,GAAG,GAAW,EAAE,CAAC;IACvB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACrC,MAAM,WAAW,GAAG,QAAQ,KAAK,SAAS,IAAI,QAAQ,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO,CAAC;QAChF,IAAI,QAAQ,KAAK,SAAS,IAAI,WAAW,IAAI,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,GAAG,QAAQ,EAAE,CAAC;YAC9E,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG;gBACpB,KAAK,EAAE,QAAQ,CAAC,KAAK;gBACrB,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,GAAG,CAAC,QAAQ,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC;aACrE,CAAC;YACF,SAAS;QACX,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,uFAAuF;AACvF,MAAM,UAAU,QAAQ,CAAC,GAAmB,EAAE,KAAsB;IAClE,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;IAChC,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACzC,IAAI,OAAO,KAAK,SAAS;YAAE,SAAS;QACpC,oEAAoE;QACpE,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,YAAY,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC;QACxD,MAAM,CAAC,IAAI,CAAC;YACV,qDAAqD;YACrD,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC;YAChD,SAAS,EAAE,OAAO,CAAC,KAAK;YACxB,OAAO,EAAE,OAAO,CAAC,GAAG;YACpB,GAAG,CAAC,OAAO,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;YAClE,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,CAAC;SACpC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,OAAO,CAAC,IAAU,EAAE,QAAgB,EAAE,YAAoB;IACxE,MAAM,GAAG,GAAW,EAAE,CAAC;IACvB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,YAAY,CAAC,CAAC;IACpD,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,IAAI,MAAM,EAAE,CAAC;QAC/D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,GAAG,QAAQ,CAAC,CAAC;QACjD,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;QACvF,IAAI,GAAG,IAAI,IAAI,CAAC,GAAG;YAAE,MAAM;IAC7B,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,KAAK,GAAW,EAAE,CAAC;IACzB,MAAM,SAAS,GAAG,aAAa,CAAC;IAChC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,KAA6B,CAAC;IAClC,OAAO,CAAC,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC/C,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;QAChD,MAAM,GAAG,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACzC,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAChD,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;AAC9C,CAAC"}

package/dist/esm/rag/splitters/wholeDocument.d.ts

@@ -0,0 +1,16 @@
+/**
+ * wholeDocument — one chunk per document, no cutting at all.
+ *
+ * Pattern: Strategy (one of `Splitter`).
+ * Role:    rag/ layer. For corpora whose documents are already one idea each —
+ *          FAQ entries, product records, support macros, changelog entries.
+ * Emits:   N/A.
+ *
+ * The honest caveat: an embedder truncates, so a document past the model's
+ * input ceiling is embedded from its opening only, and the rest is invisible
+ * to retrieval while still being injected if it is retrieved. `indexCorpus`
+ * records exactly that as `truncated[]` rather than letting it pass — but the
+ * fix is a real splitter, not a bigger number.
+ */
+import type { Splitter } from '../types.js';
+export declare function wholeDocument(): Splitter;

package/dist/esm/rag/splitters/wholeDocument.js

@@ -0,0 +1,10 @@
+import { toPieces } from './shared.js';
+export function wholeDocument() {
+    return {
+        name: 'wholeDocument',
+        split(doc) {
+            return toPieces(doc, [{ start: 0, end: doc.text.length }]);
+        },
+    };
+}
+//# sourceMappingURL=wholeDocument.js.map

package/dist/esm/rag/splitters/wholeDocument.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"wholeDocument.js","sourceRoot":"","sources":["../../../../src/rag/splitters/wholeDocument.ts"],"names":[],"mappings":"AAeA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,MAAM,UAAU,aAAa;IAC3B,OAAO;QACL,IAAI,EAAE,eAAe;QACrB,KAAK,CAAC,GAAmB;YACvB,OAAO,QAAQ,CAAC,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC7D,CAAC;KACF,CAAC;AACJ,CAAC"}