@aid-framework/runtime 0.0.0-stage → 0.1.39

package/dist/retrieval/chunking.js

@@ -0,0 +1,188 @@
+/**
+ * Turning documents into deterministic, size-bounded chunks.
+ *
+ * The IR declares a strategy, a size and an overlap; it does not declare the split
+ * algorithm. This module pins one, and the pin is the spec validator's own reading
+ * of the intent: *"spec v0.1 chunks by size (fixed, recursive, code)"*. `semantic`
+ * is refused because it needs an embedding pass over the whole source at ingest --
+ * a different shape of job, not a different boundary rule.
+ *
+ * So all three implemented strategies are size-bounded splitters and the only thing
+ * that distinguishes them is **where a boundary is allowed to fall**:
+ *
+ * `fixed`
+ *     Anywhere. The window always fills to `size`, cutting mid-word when that is
+ *     where the budget ends. This is the honest baseline other strategies improve on.
+ * `recursive`
+ *     Prefer the last paragraph break, then the last line break, then the last
+ *     sentence end, then the last word break inside the window.
+ * `code`
+ *     Prefer the last blank line, then the last line break, then the last statement
+ *     end. Code is line-structured, so a blank line is the closest thing it has to a
+ *     paragraph, and cutting mid-statement is the failure worth avoiding.
+ *
+ * Chunks are pure functions of the text, so the same input always produces the same
+ * chunks with the same ids in the same order -- which is what makes re-ingesting a
+ * corpus a no-op instead of a duplicate.
+ */
+import { ConfigurationError, FeatureDeferredError } from '../errors.js';
+import { repr } from '../internal/json.js';
+import { Chunk, chunkIdFor } from './types.js';
+/**
+ * A boundary found in the first half of a window is ignored and the window is
+ * filled to `size` instead. Without this, one early paragraph break near the end of
+ * a split would produce a stream of very short chunks, and a chunker that emits
+ * 5-character chunks is worse than one that cuts a sentence in half.
+ */
+const MIN_FILL_NUMERATOR = 1;
+const MIN_FILL_DENOMINATOR = 2;
+/** Shared windowing. Subclasses only choose where a boundary may fall. */
+class SizeChunker {
+    _size;
+    _overlap;
+    _keep;
+    constructor(init) {
+        if (init.size <= 0) {
+            throw new ConfigurationError('chunking size must be positive');
+        }
+        if (init.overlap < 0) {
+            throw new ConfigurationError('chunking overlap must not be negative');
+        }
+        if (init.overlap >= init.size) {
+            throw new ConfigurationError(`chunking overlap ${init.overlap} must be smaller than size ${init.size}; otherwise a ` +
+                'window would not advance and chunking would never terminate');
+        }
+        this._size = init.size;
+        this._overlap = init.overlap;
+        this._keep = [...(init.keepMetadata ?? [])];
+    }
+    chunk(document) {
+        const text = document.text;
+        const chunks = [];
+        let start = 0;
+        while (start < text.length) {
+            const limit = Math.min(start + this._size, text.length);
+            let end = limit >= text.length ? limit : this.cut(text, start, limit);
+            // A boundary at or before `start` would stall the loop, so progress is
+            // guaranteed here rather than trusted from the strategy hook.
+            end = Math.max(end, start + 1);
+            const window = text.slice(start, end);
+            if (window.trim().length > 0) {
+                chunks.push(this.build(document, chunks.length, start, end, window));
+            }
+            if (end >= text.length) {
+                break;
+            }
+            start = Math.max(end - this._overlap, start + 1);
+        }
+        return chunks;
+    }
+    build(document, index, start, end, text) {
+        const metadata = {};
+        for (const key of this._keep) {
+            const value = document.metadata[key];
+            if (value !== undefined)
+                metadata[key] = value;
+        }
+        return new Chunk({
+            chunkId: chunkIdFor(document.docId, index, text),
+            docId: document.docId,
+            uri: document.uri,
+            index,
+            start,
+            end,
+            text,
+            metadata,
+        });
+    }
+    /**
+     * The end offset of the last `marker` in `text.slice(start, limit)`, if usable.
+     *
+     * `null` means "no boundary worth taking", and the caller falls back to `limit`.
+     * The minimum-fill rule is applied here rather than by each strategy so no
+     * strategy can forget it.
+     */
+    boundary(text, start, limit, markers) {
+        const floor = start + Math.floor((this._size * MIN_FILL_NUMERATOR) / MIN_FILL_DENOMINATOR);
+        const window = text.slice(start, limit);
+        for (const marker of markers) {
+            const position = window.lastIndexOf(marker);
+            if (position === -1)
+                continue;
+            const end = start + position + marker.length;
+            if (end >= Math.max(floor, start + 1)) {
+                return end;
+            }
+        }
+        return null;
+    }
+    /** The first usable boundary from `markerGroups`, in preference order. */
+    fill(text, start, limit, markerGroups) {
+        for (const markers of markerGroups) {
+            const found = this.boundary(text, start, limit, markers);
+            if (found !== null)
+                return found;
+        }
+        return limit;
+    }
+}
+/** Cuts on size alone. */
+export class FixedChunker extends SizeChunker {
+    get strategy() {
+        return 'fixed';
+    }
+    cut(_text, _start, limit) {
+        return limit;
+    }
+}
+/** Prefers prose boundaries: paragraph, then line, then sentence, then word. */
+export class RecursiveChunker extends SizeChunker {
+    get strategy() {
+        return 'recursive';
+    }
+    cut(text, start, limit) {
+        return this.fill(text, start, limit, [['\n\n'], ['\n'], ['. ', '! ', '? '], [' ']]);
+    }
+}
+/** Prefers code boundaries: blank line, then line, then statement, then token. */
+export class CodeChunker extends SizeChunker {
+    get strategy() {
+        return 'code';
+    }
+    cut(text, start, limit) {
+        return this.fill(text, start, limit, [['\n\n'], ['\n'], [';\n', ';\r\n', '; '], [' ']]);
+    }
+}
+/**
+ * Resolve a chunking block to an implementation.
+ *
+ * `semantic` is refused here as well as in the spec validator, because the runtime
+ * is reachable from generated code the validator never saw.
+ */
+export function chunkerFor(spec) {
+    const options = {
+        size: spec.size,
+        overlap: spec.overlap,
+        keepMetadata: spec.keepMetadata,
+    };
+    switch (spec.strategy) {
+        case 'fixed':
+            return new FixedChunker(options);
+        case 'recursive':
+            return new RecursiveChunker(options);
+        case 'code':
+            return new CodeChunker(options);
+        case 'semantic':
+            throw new FeatureDeferredError(`chunking strategy ${repr(spec.strategy)} is declared but not implemented: spec ` +
+                'v0.1 chunks by size (fixed, recursive, code), and semantic chunking needs an ' +
+                'embedding pass over the whole source at ingest');
+        /* istanbul ignore next -- unreachable while the union above is exhaustive */
+        default:
+            throw new FeatureDeferredError(`chunking strategy ${repr(String(spec.strategy))} is declared but not implemented`);
+    }
+}
+/** Chunk one document with the strategy the spec names. */
+export function chunkDocument(document, spec) {
+    return chunkerFor(spec).chunk(document);
+}
+//# sourceMappingURL=chunking.js.map

package/dist/retrieval/chunking.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunking.js","sourceRoot":"","sources":["../../src/retrieval/chunking.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AACxE,OAAO,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAE3C,OAAO,EAAE,KAAK,EAA4C,UAAU,EAAE,MAAM,YAAY,CAAC;AAEzF;;;;;GAKG;AACH,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAC7B,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAW/B,0EAA0E;AAC1E,MAAe,WAAW;IACP,KAAK,CAAS;IACd,QAAQ,CAAS;IACjB,KAAK,CAAoB;IAE1C,YAAY,IAIX;QACC,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC;YACnB,MAAM,IAAI,kBAAkB,CAAC,gCAAgC,CAAC,CAAC;QACjE,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;YACrB,MAAM,IAAI,kBAAkB,CAAC,uCAAuC,CAAC,CAAC;QACxE,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAC9B,MAAM,IAAI,kBAAkB,CAC1B,oBAAoB,IAAI,CAAC,OAAO,8BAA8B,IAAI,CAAC,IAAI,gBAAgB;gBACrF,6DAA6D,CAChE,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,IAAI,CAAC,KAAK,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,CAAC;IAC9C,CAAC;IAOD,KAAK,CAAC,QAAwB;QAC5B,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;QAC3B,MAAM,MAAM,GAAY,EAAE,CAAC;QAC3B,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACxD,IAAI,GAAG,GAAG,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;YACtE,uEAAuE;YACvE,8DAA8D;YAC9D,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;YAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YACtC,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;YACvE,CAAC;YACD,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACvB,MAAM;YACR,CAAC;YACD,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,QAAQ,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;QACnD,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CACX,QAAwB,EACxB,KAAa,EACb,KAAa,EACb,GAAW,EACX,IAAY;QAEZ,MAAM,QAAQ,GAA2B,EAAE,CAAC;QAC5C,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YACrC,IAAI,KAAK,KAAK,SAAS;gBAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACjD,CAAC;QACD,OAAO,IAAI,KAAK,CAAC;YACf,OAAO,EAAE,UAAU,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC;YAChD,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,GAAG,EAAE,QAAQ,CAAC,GAAG;YACjB,KAAK;YACL,KAAK;YACL,GAAG;YACH,IAAI;YACJ,QAAQ;SACT,CAAC,CAAC;IACL,CAAC;IAED;;;;;;OAMG;IACK,QAAQ,CACd,IAAY,EACZ,KAAa,EACb,KAAa,EACb,OAA0B;QAE1B,MAAM,KAAK,GAAG,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,kBAAkB,CAAC,GAAG,oBAAoB,CAAC,CAAC;QAC3F,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACxC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YAC5C,IAAI,QAAQ,KAAK,CAAC,CAAC;gBAAE,SAAS;YAC9B,MAAM,GAAG,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;YAC7C,IAAI,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC;gBACtC,OAAO,GAAG,CAAC;YACb,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,0EAA0E;IAChE,IAAI,CAAC,IAAY,EAAE,KAAa,EAAE,KAAa,EAAE,YAA0B;QACnF,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;YACzD,IAAI,KAAK,KAAK,IAAI;gBAAE,OAAO,KAAK,CAAC;QACnC,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;CACF;AAED,0BAA0B;AAC1B,MAAM,OAAO,YAAa,SAAQ,WAAW;IAC3C,IAAa,QAAQ;QACnB,OAAO,OAAO,CAAC;IACjB,CAAC;IAEkB,GAAG,CAAC,KAAa,EAAE,MAAc,EAAE,KAAa;QACjE,OAAO,KAAK,CAAC;IACf,CAAC;CACF;AAED,gFAAgF;AAChF,MAAM,OAAO,gBAAiB,SAAQ,WAAW;IAC/C,IAAa,QAAQ;QACnB,OAAO,WAAW,CAAC;IACrB,CAAC;IAEkB,GAAG,CAAC,IAAY,EAAE,KAAa,EAAE,KAAa;QAC/D,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACtF,CAAC;CACF;AAED,kFAAkF;AAClF,MAAM,OAAO,WAAY,SAAQ,WAAW;IAC1C,IAAa,QAAQ;QACnB,OAAO,MAAM,CAAC;IAChB,CAAC;IAEkB,GAAG,CAAC,IAAY,EAAE,KAAa,EAAE,KAAa;QAC/D,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC1F,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,UAAU,UAAU,CAAC,IAAkB;IAC3C,MAAM,OAAO,GAAG;QACd,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,YAAY,EAAE,IAAI,CAAC,YAAY;KAChC,CAAC;IACF,QAAQ,IAAI,CAAC,QAAQ,EAAE,CAAC;QACtB,KAAK,OAAO;YACV,OAAO,IAAI,YAAY,CAAC,OAAO,CAAC,CAAC;QACnC,KAAK,WAAW;YACd,OAAO,IAAI,gBAAgB,CAAC,OAAO,CAAC,CAAC;QACvC,KAAK,MAAM;YACT,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC;QAClC,KAAK,UAAU;YACb,MAAM,IAAI,oBAAoB,CAC5B,qBAAqB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,yCAAyC;gBAC/E,+EAA+E;gBAC/E,gDAAgD,CACnD,CAAC;QACJ,6EAA6E;QAC7E;YACE,MAAM,IAAI,oBAAoB,CAC5B,qBAAqB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,kCAAkC,CACnF,CAAC;IACN,CAAC;AACH,CAAC;AAED,2DAA2D;AAC3D,MAAM,UAAU,aAAa,CAAC,QAAwB,EAAE,IAAkB;IACxE,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;AAC1C,CAAC"}

package/dist/retrieval/citations.d.ts

@@ -0,0 +1,98 @@
+/**
+ * Retrieved passages, their provenance, and the context they assemble into.
+ *
+ * A retrieved set is only useful to a model if the passages arrive with provenance
+ * the model can repeat back, so ranking and citation are one type here rather than a
+ * search result plus a separate lookup table. {@link RetrievalResult} is the
+ * runtime's answer type: it is what a `retrieve` step produces and what every
+ * retrieval metric scores.
+ *
+ * The assembled context is numbered by **rank**, not by position in the array. The
+ * two are the same in practice because ranks are assigned contiguously from 1, but
+ * naming the rank is what makes a model's `[2]` resolvable by the caller without
+ * re-running the search.
+ */
+import type { Chunk } from './types.js';
+/**
+ * Where one retrieved passage came from.
+ *
+ * `docId` is the document identity the IR's retrieval metric already uses
+ * (`expectDocs: [getting-started.md]`), so a citation and a gate expectation are the
+ * same vocabulary rather than two spellings of the same thing.
+ */
+export declare class Citation {
+    readonly docId: string;
+    readonly chunkId: string;
+    readonly uri: string;
+    readonly index: number;
+    constructor(init: {
+        readonly docId: string;
+        readonly chunkId: string;
+        readonly uri?: string;
+        readonly index: number;
+    });
+    toString(): string;
+}
+/** One ranked passage: the chunk, its score, and its provenance. */
+export declare class RetrievedItem {
+    readonly chunk: Chunk;
+    readonly score: number;
+    readonly rank: number;
+    readonly citation: Citation;
+    constructor(init: {
+        readonly chunk: Chunk;
+        readonly score: number;
+        readonly rank: number;
+        readonly citation: Citation;
+    });
+}
+/**
+ * What a `retrieve` step returns.
+ *
+ * `items` is rank-ordered and may be empty -- "the corpus has nothing for this
+ * query" is a result, not an error, and a caller that needs a non-empty answer says
+ * so with a metric rather than by crashing.
+ */
+export declare class RetrievalResult {
+    readonly retriever: string;
+    readonly query: string;
+    readonly items: readonly RetrievedItem[];
+    constructor(init: {
+        readonly retriever: string;
+        readonly query: string;
+        readonly items?: readonly RetrievedItem[];
+    });
+    get length(): number;
+    /** The first `k` items, or all of them when the result is shorter. */
+    top(k: number): readonly RetrievedItem[];
+    /** The numbered excerpts a prompt binds. */
+    get context(): string;
+    /** One citation per item, in rank order. */
+    get citations(): readonly Citation[];
+    /**
+     * Rank-ordered, de-duplicated document ids.
+     *
+     * De-duplicated because recall is about *documents found*, and a document that
+     * contributed three passages was found once. First occurrence wins, so the order
+     * still reflects relevance.
+     */
+    get docIds(): readonly string[];
+}
+/**
+ * The citation for a chunk.
+ *
+ * Derived from the chunk rather than passed alongside it, so a citation cannot drift
+ * from the passage it names.
+ */
+export declare function cite(chunk: Chunk): Citation;
+/**
+ * Render ranked items as numbered excerpts, each tagged with its citation.
+ *
+ * An empty retrieval renders as the empty string rather than as a header with nothing
+ * under it: a header alone reads to a model as "sources were found", and a prompt
+ * that invites a model to invent a source is worse than an empty prompt.
+ */
+export declare function assembleContext(items: readonly RetrievedItem[], options?: {
+    readonly separator?: string;
+}): string;
+//# sourceMappingURL=citations.d.ts.map

package/dist/retrieval/citations.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"citations.d.ts","sourceRoot":"","sources":["../../src/retrieval/citations.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAGH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAExC;;;;;;GAMG;AACH,qBAAa,QAAQ;IACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAEvB,YAAY,IAAI,EAAE;QAChB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;KACxB,EAYA;IAED,QAAQ,IAAI,MAAM,CAEjB;CACF;AAED,oEAAoE;AACpE,qBAAa,aAAa;IACxB,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAE5B,YAAY,IAAI,EAAE;QAChB,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC;QACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;KAC7B,EAQA;CACF;AAED;;;;;;GAMG;AACH,qBAAa,eAAe;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,SAAS,aAAa,EAAE,CAAC;IAEzC,YAAY,IAAI,EAAE;QAChB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;KAC3C,EAOA;IAED,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED,sEAAsE;IACtE,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,SAAS,aAAa,EAAE,CAEvC;IAED,4CAA4C;IAC5C,IAAI,OAAO,IAAI,MAAM,CAEpB;IAED,4CAA4C;IAC5C,IAAI,SAAS,IAAI,SAAS,QAAQ,EAAE,CAEnC;IAED;;;;;;OAMG;IACH,IAAI,MAAM,IAAI,SAAS,MAAM,EAAE,CAE9B;CACF;AAED;;;;;GAKG;AACH,wBAAgB,IAAI,CAAC,KAAK,EAAE,KAAK,GAAG,QAAQ,CAO3C;AAED;;;;;;GAMG;AACH,wBAAgB,eAAe,CAC7B,KAAK,EAAE,SAAS,aAAa,EAAE,EAC/B,OAAO,GAAE;IAAE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAO,GAC5C,MAAM,CAKR"}

package/dist/retrieval/citations.js

@@ -0,0 +1,134 @@
+/**
+ * Retrieved passages, their provenance, and the context they assemble into.
+ *
+ * A retrieved set is only useful to a model if the passages arrive with provenance
+ * the model can repeat back, so ranking and citation are one type here rather than a
+ * search result plus a separate lookup table. {@link RetrievalResult} is the
+ * runtime's answer type: it is what a `retrieve` step produces and what every
+ * retrieval metric scores.
+ *
+ * The assembled context is numbered by **rank**, not by position in the array. The
+ * two are the same in practice because ranks are assigned contiguously from 1, but
+ * naming the rank is what makes a model's `[2]` resolvable by the caller without
+ * re-running the search.
+ */
+import { ConfigurationError } from '../errors.js';
+/**
+ * Where one retrieved passage came from.
+ *
+ * `docId` is the document identity the IR's retrieval metric already uses
+ * (`expectDocs: [getting-started.md]`), so a citation and a gate expectation are the
+ * same vocabulary rather than two spellings of the same thing.
+ */
+export class Citation {
+    docId;
+    chunkId;
+    uri;
+    index;
+    constructor(init) {
+        if (init.docId.length === 0)
+            throw new ConfigurationError('citation docId must not be empty');
+        if (init.chunkId.length === 0) {
+            throw new ConfigurationError('citation chunkId must not be empty');
+        }
+        if (!Number.isInteger(init.index) || init.index < 0) {
+            throw new ConfigurationError('citation index must be a non-negative integer');
+        }
+        this.docId = init.docId;
+        this.chunkId = init.chunkId;
+        this.uri = init.uri ?? '';
+        this.index = init.index;
+    }
+    toString() {
+        return `${this.docId}#${this.index}`;
+    }
+}
+/** One ranked passage: the chunk, its score, and its provenance. */
+export class RetrievedItem {
+    chunk;
+    score;
+    rank;
+    citation;
+    constructor(init) {
+        if (!Number.isInteger(init.rank) || init.rank < 1) {
+            throw new ConfigurationError('retrieved item rank must be at least 1');
+        }
+        this.chunk = init.chunk;
+        this.score = init.score;
+        this.rank = init.rank;
+        this.citation = init.citation;
+    }
+}
+/**
+ * What a `retrieve` step returns.
+ *
+ * `items` is rank-ordered and may be empty -- "the corpus has nothing for this
+ * query" is a result, not an error, and a caller that needs a non-empty answer says
+ * so with a metric rather than by crashing.
+ */
+export class RetrievalResult {
+    retriever;
+    query;
+    items;
+    constructor(init) {
+        if (init.retriever.length === 0) {
+            throw new ConfigurationError('retriever must not be empty');
+        }
+        this.retriever = init.retriever;
+        this.query = init.query;
+        this.items = [...(init.items ?? [])];
+    }
+    get length() {
+        return this.items.length;
+    }
+    /** The first `k` items, or all of them when the result is shorter. */
+    top(k) {
+        return this.items.slice(0, k);
+    }
+    /** The numbered excerpts a prompt binds. */
+    get context() {
+        return assembleContext(this.items);
+    }
+    /** One citation per item, in rank order. */
+    get citations() {
+        return this.items.map((item) => item.citation);
+    }
+    /**
+     * Rank-ordered, de-duplicated document ids.
+     *
+     * De-duplicated because recall is about *documents found*, and a document that
+     * contributed three passages was found once. First occurrence wins, so the order
+     * still reflects relevance.
+     */
+    get docIds() {
+        return [...new Set(this.items.map((item) => item.citation.docId))];
+    }
+}
+/**
+ * The citation for a chunk.
+ *
+ * Derived from the chunk rather than passed alongside it, so a citation cannot drift
+ * from the passage it names.
+ */
+export function cite(chunk) {
+    return new Citation({
+        docId: chunk.docId,
+        chunkId: chunk.chunkId,
+        uri: chunk.uri,
+        index: chunk.index,
+    });
+}
+/**
+ * Render ranked items as numbered excerpts, each tagged with its citation.
+ *
+ * An empty retrieval renders as the empty string rather than as a header with nothing
+ * under it: a header alone reads to a model as "sources were found", and a prompt
+ * that invites a model to invent a source is worse than an empty prompt.
+ */
+export function assembleContext(items, options = {}) {
+    const separator = options.separator ?? '\n\n';
+    return items
+        .map((item) => `[${item.rank}] ${item.citation.toString()} :: ${item.chunk.text}`)
+        .join(separator);
+}
+//# sourceMappingURL=citations.js.map

package/dist/retrieval/citations.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"citations.js","sourceRoot":"","sources":["../../src/retrieval/citations.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAGlD;;;;;;GAMG;AACH,MAAM,OAAO,QAAQ;IACV,KAAK,CAAS;IACd,OAAO,CAAS;IAChB,GAAG,CAAS;IACZ,KAAK,CAAS;IAEvB,YAAY,IAKX;QACC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,kBAAkB,CAAC,kCAAkC,CAAC,CAAC;QAC9F,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9B,MAAM,IAAI,kBAAkB,CAAC,oCAAoC,CAAC,CAAC;QACrE,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;YACpD,MAAM,IAAI,kBAAkB,CAAC,+CAA+C,CAAC,CAAC;QAChF,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC;QAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IAC1B,CAAC;IAED,QAAQ;QACN,OAAO,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;IACvC,CAAC;CACF;AAED,oEAAoE;AACpE,MAAM,OAAO,aAAa;IACf,KAAK,CAAQ;IACb,KAAK,CAAS;IACd,IAAI,CAAS;IACb,QAAQ,CAAW;IAE5B,YAAY,IAKX;QACC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAClD,MAAM,IAAI,kBAAkB,CAAC,wCAAwC,CAAC,CAAC;QACzE,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACtB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;IAChC,CAAC;CACF;AAED;;;;;;GAMG;AACH,MAAM,OAAO,eAAe;IACjB,SAAS,CAAS;IAClB,KAAK,CAAS;IACd,KAAK,CAA2B;IAEzC,YAAY,IAIX;QACC,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,MAAM,IAAI,kBAAkB,CAAC,6BAA6B,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,KAAK,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;IAED,sEAAsE;IACtE,GAAG,CAAC,CAAS;QACX,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAChC,CAAC;IAED,4CAA4C;IAC5C,IAAI,OAAO;QACT,OAAO,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;IAED,4CAA4C;IAC5C,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACjD,CAAC;IAED;;;;;;OAMG;IACH,IAAI,MAAM;QACR,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACrE,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,UAAU,IAAI,CAAC,KAAY;IAC/B,OAAO,IAAI,QAAQ,CAAC;QAClB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,KAAK,EAAE,KAAK,CAAC,KAAK;KACnB,CAAC,CAAC;AACL,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,eAAe,CAC7B,KAA+B,EAC/B,OAAO,GAAoC,EAAE;IAE7C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC;IAC9C,OAAO,KAAK;SACT,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;SACjF,IAAI,CAAC,SAAS,CAAC,CAAC;AACrB,CAAC"}

package/dist/retrieval/connectors.d.ts

@@ -0,0 +1,74 @@
+/**
+ * Reading a `file` source.
+ *
+ * The IR names five source kinds and Phase 1 implements one. That is the spec
+ * validator's own position ("spec v0.1 indexes a local `file` source; the remote
+ * loaders belong to the phase that ships them"), so the other four raise
+ * {@link FeatureDeferredError} here rather than indexing an empty corpus and
+ * looking healthy.
+ *
+ * **Document identity is the POSIX path relative to the source root.** That is
+ * the vocabulary the IR's retrieval metric already uses -- its `expectDocs` hold
+ * `getting-started.md`, not `./docs/getting-started.md` -- and it is
+ * machine-independent, so the same corpus indexed on Windows and on Linux yields
+ * the same ids and the same eval outcome. Absolute paths would make a baseline
+ * non-portable.
+ *
+ * The protocol is `async` even though the only implementation reads local files
+ * synchronously, because the kinds that follow (`s3`, `http`, `db`, `mcp`) are
+ * genuinely concurrent. Making it sync now would be a breaking change to the seam
+ * exactly when the first real loader arrives.
+ */
+import type { SourceKind, SourceSpec } from './types.js';
+/** One document read from a source, before chunking. */
+export declare class SourceDocument {
+    /** POSIX path relative to the source root; the document's identity everywhere. */
+    readonly docId: string;
+    /** The absolute resolved location, for a human reading a citation. Never matched on. */
+    readonly uri: string;
+    readonly text: string;
+    readonly metadata: Readonly<Record<string, string>>;
+    constructor(init: {
+        readonly docId: string;
+        readonly uri: string;
+        readonly text: string;
+        readonly metadata?: Readonly<Record<string, string>>;
+    });
+}
+/** A source that yields documents in a deterministic order. */
+export interface Connector {
+    readonly kind: SourceKind;
+    documents(): Promise<readonly SourceDocument[]>;
+    close(): Promise<void>;
+}
+/**
+ * Reads `*.md`-style files under a local directory.
+ *
+ * Files are read and returned in sorted `docId` order, never in directory order:
+ * directory order is a filesystem accident, and a corpus whose chunk ids depend on
+ * it would reindex itself differently on a different machine.
+ */
+export declare class FileConnector implements Connector {
+    private readonly _root;
+    private readonly _glob;
+    constructor(root: string, options?: {
+        readonly glob?: string | null;
+    });
+    get kind(): SourceKind;
+    documents(): Promise<readonly SourceDocument[]>;
+    close(): Promise<void>;
+    private paths;
+    private docId;
+    private read;
+}
+/**
+ * Open the connector for a source entry.
+ *
+ * `baseDir` resolves a relative source `uri`; it is explicit because the runtime
+ * has no opinion about a working directory, and a corpus must not depend on where a
+ * process happened to be started.
+ */
+export declare function openConnector(spec: SourceSpec, options?: {
+    readonly baseDir?: string | null;
+}): Connector;
+//# sourceMappingURL=connectors.d.ts.map

package/dist/retrieval/connectors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"connectors.d.ts","sourceRoot":"","sources":["../../src/retrieval/connectors.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAOH,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAqGzD,wDAAwD;AACxD,qBAAa,cAAc;IACzB,kFAAkF;IAClF,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,wFAAwF;IACxF,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAEpD,YAAY,IAAI,EAAE;QAChB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;KACtD,EAWA;CACF;AAED,+DAA+D;AAC/D,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,SAAS,IAAI,OAAO,CAAC,SAAS,cAAc,EAAE,CAAC,CAAC;IAChD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED;;;;;;GAMG;AACH,qBAAa,aAAc,YAAW,SAAS;IAC7C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAgB;IAEtC,YAAY,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE;QAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;KAAO,EASxE;IAED,IAAI,IAAI,IAAI,UAAU,CAErB;IAEK,SAAS,IAAI,OAAO,CAAC,SAAS,cAAc,EAAE,CAAC,CAEpD;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAE3B;IAED,OAAO,CAAC,KAAK;IA0Bb,OAAO,CAAC,KAAK;IAIb,OAAO,CAAC,IAAI;CAUb;AAED;;;;;;GAMG;AACH,wBAAgB,aAAa,CAC3B,IAAI,EAAE,UAAU,EAChB,OAAO,GAAE;IAAE,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAAO,GACjD,SAAS,CAaX"}