@keel_flow/kb-pipeline 0.2.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 jglasskatz
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/dist/chunking/ast.d.ts

@@ -0,0 +1,8 @@
+export interface AstChunkOpts {
+    language?: "typescript" | "javascript" | "python" | "auto";
+    maxSize?: number;
+    preserveImports?: boolean;
+}
+export declare function astImportPrefixLength(text: string, language?: "typescript" | "javascript" | "python" | "auto"): number;
+export declare function astChunk(text: string, opts?: AstChunkOpts): string[];
+//# sourceMappingURL=ast.d.ts.map

package/dist/chunking/ast.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ast.d.ts","sourceRoot":"","sources":["../../src/chunking/ast.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,YAAY,GAAG,YAAY,GAAG,QAAQ,GAAG,MAAM,CAAC;IAC3D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAOD,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,YAAY,GAAG,YAAY,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,CAGtH;AAED,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,GAAE,YAAiB,GAAG,MAAM,EAAE,CAoCxE"}

package/dist/chunking/ast.js

@@ -0,0 +1,86 @@
+import { recursiveChunk } from "./recursive.js";
+// Regex-based heuristic boundary chunker for TypeScript/JavaScript and Python.
+// tree-sitter would give exact AST boundaries, but its native bindings often
+// fail to install in restricted environments; we deliberately fall back to a
+// regex pass that handles `function`/`class`/`export`/`def` boundaries with
+// import preservation. Behaviour matches the documented fallback in the spec.
+export function astImportPrefixLength(text, language) {
+    const lang = language === "auto" || language === undefined ? detectLanguage(text) : language;
+    return extractImports(text, lang).length;
+}
+export function astChunk(text, opts = {}) {
+    const lang = opts.language === "auto" || opts.language === undefined
+        ? detectLanguage(text)
+        : opts.language;
+    const maxSize = opts.maxSize ?? 1200;
+    const preserveImports = opts.preserveImports ?? true;
+    if (text.length === 0)
+        return [];
+    const imports = preserveImports ? extractImports(text, lang) : "";
+    const body = preserveImports ? text.slice(imports.length) : text;
+    const boundaries = findBoundaries(body, lang);
+    if (boundaries.length <= 1) {
+        const single = imports + body;
+        if (single.length <= maxSize)
+            return [single];
+        return recursiveChunk(single, { maxSize });
+    }
+    const segments = [];
+    for (let i = 0; i < boundaries.length; i++) {
+        const start = boundaries[i] ?? 0;
+        const end = boundaries[i + 1] ?? body.length;
+        segments.push(body.slice(start, end));
+    }
+    const chunks = [];
+    for (const seg of segments) {
+        const candidate = imports + seg;
+        if (candidate.length <= maxSize) {
+            chunks.push(candidate);
+        }
+        else {
+            for (const sub of recursiveChunk(candidate, { maxSize }))
+                chunks.push(sub);
+        }
+    }
+    return chunks;
+}
+function detectLanguage(text) {
+    if (/\bdef\s+\w+\s*\(|^\s*class\s+\w+\s*\(/m.test(text))
+        return "python";
+    if (/\binterface\s+\w+|:\s*\w+\s*[=,)]/.test(text))
+        return "typescript";
+    return "javascript";
+}
+function extractImports(text, lang) {
+    const lines = text.split("\n");
+    const out = [];
+    const importRe = lang === "python"
+        ? /^\s*(import\s+|from\s+\S+\s+import\s+)/
+        : /^\s*(import\s+|export\s+\*\s+from|export\s+\{[^}]*\}\s+from)/;
+    let i = 0;
+    for (; i < lines.length; i++) {
+        const line = lines[i] ?? "";
+        if (importRe.test(line) || line.trim() === "" || line.trim().startsWith("//") || line.trim().startsWith("#")) {
+            out.push(line);
+        }
+        else {
+            break;
+        }
+    }
+    if (out.length === 0)
+        return "";
+    return out.join("\n") + "\n";
+}
+function findBoundaries(body, lang) {
+    const re = lang === "python"
+        ? /^(?:def|class)\s+\w+/gm
+        : /^(?:export\s+(?:default\s+)?(?:async\s+)?(?:function|class)|(?:async\s+)?function|class)\s+\w+/gm;
+    const positions = [];
+    let m;
+    while ((m = re.exec(body)) !== null)
+        positions.push(m.index);
+    if (positions[0] !== 0)
+        positions.unshift(0);
+    return positions;
+}
+//# sourceMappingURL=ast.js.map

package/dist/chunking/ast.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ast.js","sourceRoot":"","sources":["../../src/chunking/ast.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAQhD,+EAA+E;AAC/E,6EAA6E;AAC7E,6EAA6E;AAC7E,4EAA4E;AAC5E,8EAA8E;AAC9E,MAAM,UAAU,qBAAqB,CAAC,IAAY,EAAE,QAA0D;IAC5G,MAAM,IAAI,GAAG,QAAQ,KAAK,MAAM,IAAI,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC7F,OAAO,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,OAAqB,EAAE;IAC5D,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,KAAK,MAAM,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS;QAClE,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC;QACtB,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC;IAClB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC;IACrC,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC;IAErD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEjC,MAAM,OAAO,GAAG,eAAe,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAClE,MAAM,IAAI,GAAG,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEjE,MAAM,UAAU,GAAG,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC9C,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,OAAO,GAAG,IAAI,CAAC;QAC9B,IAAI,MAAM,CAAC,MAAM,IAAI,OAAO;YAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAC9C,OAAO,cAAc,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,GAAG,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC;QAC7C,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,MAAM,SAAS,GAAG,OAAO,GAAG,GAAG,CAAC;QAChC,IAAI,SAAS,CAAC,MAAM,IAAI,OAAO,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACzB,CAAC;aAAM,CAAC;YACN,KAAK,MAAM,GAAG,IAAI,cAAc,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,CAAC;gBAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,cAAc,CAAC,IAAY;IAClC,IAAI,wCAAwC,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,QAAQ,CAAC;IACzE,IAAI,mCAAmC,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,YAAY,CAAC;IACxE,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,SAAS,cAAc,CAAC,IAAY,EAAE,IAA4C;IAChF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,QAAQ,GACZ,IAAI,KAAK,QAAQ;QACf,CAAC,CAAC,wCAAwC;QAC1C,CAAC,CAAC,8DAA8D,CAAC;IACrE,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7G,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjB,CAAC;aAAM,CAAC;YACN,MAAM;QACR,CAAC;IACH,CAAC;IACD,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAChC,OAAO,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AAC/B,CAAC;AAED,SAAS,cAAc,CAAC,IAAY,EAAE,IAA4C;IAChF,MAAM,EAAE,GACN,IAAI,KAAK,QAAQ;QACf,CAAC,CAAC,wBAAwB;QAC1B,CAAC,CAAC,kGAAkG,CAAC;IACzG,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,IAAI,CAAyB,CAAC;IAC9B,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI;QAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC7D,IAAI,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC;QAAE,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC7C,OAAO,SAAS,CAAC;AACnB,CAAC"}

package/dist/chunking/contextual-fallback.d.ts

@@ -0,0 +1,15 @@
+export interface ContextProvider {
+    generate(args: {
+        systemPrompt: string;
+        userPrompt: string;
+    }): Promise<string>;
+}
+export interface AddChunkContextArgs {
+    chunk: string;
+    document: string;
+    provider: ContextProvider;
+    documentBudgetChars?: number;
+}
+export declare function addChunkContext(args: AddChunkContextArgs): Promise<string>;
+export declare function prependContext(chunk: string, contextParagraph: string): string;
+//# sourceMappingURL=contextual-fallback.d.ts.map

package/dist/chunking/contextual-fallback.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"contextual-fallback.d.ts","sourceRoot":"","sources":["../../src/chunking/contextual-fallback.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,eAAe;IAI9B,QAAQ,CAAC,IAAI,EAAE;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC/E;AAED,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,eAAe,CAAC;IAG1B,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAKD,wBAAsB,eAAe,CAAC,IAAI,EAAE,mBAAmB,GAAG,OAAO,CAAC,MAAM,CAAC,CAqBhF;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,GAAG,MAAM,CAG9E"}

package/dist/chunking/contextual-fallback.js

@@ -0,0 +1,33 @@
+// Per-chunk LLM context generator (Anthropic Contextual Retrieval recipe). Only
+// invoked when the embedder is NOT context-aware (so we get the same lift on
+// OpenAI / MiniLM that voyage-context-3 gets for free). The prompt asks for a
+// short paragraph explaining where the chunk sits in the document.
+const SYSTEM_PROMPT = "You write a single ~50–100 token context paragraph that situates a chunk inside its source document. The paragraph is prepended to the chunk before embedding, so retrieval can use both the chunk's content and its surrounding context. Output the paragraph and nothing else — no preamble, no markdown.";
+export async function addChunkContext(args) {
+    const docBudget = args.documentBudgetChars ?? 16_000;
+    const doc = args.document.length > docBudget
+        ? args.document.slice(0, docBudget) + "\n[... truncated ...]"
+        : args.document;
+    const userPrompt = [
+        "<document>",
+        doc,
+        "</document>",
+        "",
+        "<chunk>",
+        args.chunk,
+        "</chunk>",
+        "",
+        "Write the ~50–100 token context paragraph for this chunk.",
+    ].join("\n");
+    const out = await args.provider.generate({
+        systemPrompt: SYSTEM_PROMPT,
+        userPrompt,
+    });
+    return out.trim();
+}
+export function prependContext(chunk, contextParagraph) {
+    if (contextParagraph.length === 0)
+        return chunk;
+    return `${contextParagraph}\n\n${chunk}`;
+}
+//# sourceMappingURL=contextual-fallback.js.map

package/dist/chunking/contextual-fallback.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"contextual-fallback.js","sourceRoot":"","sources":["../../src/chunking/contextual-fallback.ts"],"names":[],"mappings":"AAAA,gFAAgF;AAChF,6EAA6E;AAC7E,8EAA8E;AAC9E,mEAAmE;AAkBnE,MAAM,aAAa,GACjB,6SAA6S,CAAC;AAEhT,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAyB;IAC7D,MAAM,SAAS,GAAG,IAAI,CAAC,mBAAmB,IAAI,MAAM,CAAC;IACrD,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,SAAS;QAC1C,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG,uBAAuB;QAC7D,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC;IAClB,MAAM,UAAU,GAAG;QACjB,YAAY;QACZ,GAAG;QACH,aAAa;QACb,EAAE;QACF,SAAS;QACT,IAAI,CAAC,KAAK;QACV,UAAU;QACV,EAAE;QACF,2DAA2D;KAC5D,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACb,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACvC,YAAY,EAAE,aAAa;QAC3B,UAAU;KACX,CAAC,CAAC;IACH,OAAO,GAAG,CAAC,IAAI,EAAE,CAAC;AACpB,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAAa,EAAE,gBAAwB;IACpE,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAChD,OAAO,GAAG,gBAAgB,OAAO,KAAK,EAAE,CAAC;AAC3C,CAAC"}

package/dist/chunking/fixed.d.ts

@@ -0,0 +1,6 @@
+export interface FixedChunkOpts {
+    windowSize?: number;
+    overlap?: number;
+}
+export declare function fixedChunk(text: string, opts?: FixedChunkOpts): string[];
+//# sourceMappingURL=fixed.d.ts.map

package/dist/chunking/fixed.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fixed.d.ts","sourceRoot":"","sources":["../../src/chunking/fixed.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,cAAc;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,GAAE,cAAmB,GAAG,MAAM,EAAE,CAmB5E"}

package/dist/chunking/fixed.js

@@ -0,0 +1,24 @@
+export function fixedChunk(text, opts = {}) {
+    const windowSize = opts.windowSize ?? 800;
+    const overlap = opts.overlap ?? 100;
+    if (windowSize <= 0)
+        throw new Error("windowSize must be > 0");
+    if (text.length === 0)
+        return [];
+    if (text.length <= windowSize)
+        return [text];
+    if (overlap < 0 || overlap >= windowSize) {
+        throw new Error("overlap must be in [0, windowSize)");
+    }
+    const chunks = [];
+    let start = 0;
+    while (start < text.length) {
+        const end = Math.min(start + windowSize, text.length);
+        chunks.push(text.slice(start, end));
+        if (end === text.length)
+            break;
+        start += windowSize - overlap;
+    }
+    return chunks;
+}
+//# sourceMappingURL=fixed.js.map

package/dist/chunking/fixed.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fixed.js","sourceRoot":"","sources":["../../src/chunking/fixed.ts"],"names":[],"mappings":"AAKA,MAAM,UAAU,UAAU,CAAC,IAAY,EAAE,OAAuB,EAAE;IAChE,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,GAAG,CAAC;IAC1C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,GAAG,CAAC;IACpC,IAAI,UAAU,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;IAC/D,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACjC,IAAI,IAAI,CAAC,MAAM,IAAI,UAAU;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7C,IAAI,OAAO,GAAG,CAAC,IAAI,OAAO,IAAI,UAAU,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;IACxD,CAAC;IAED,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACtD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;QACpC,IAAI,GAAG,KAAK,IAAI,CAAC,MAAM;YAAE,MAAM;QAC/B,KAAK,IAAI,UAAU,GAAG,OAAO,CAAC;IAChC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/chunking/index.d.ts

@@ -0,0 +1,24 @@
+export type ChunkMode = "fixed" | "recursive" | "ast" | "late";
+export interface ChunkDocumentOpts {
+    content: string;
+    mode?: ChunkMode;
+    mimeType?: string;
+    maxSize?: number;
+    overlap?: number;
+}
+export interface Chunk {
+    content: string;
+    ordinal: number;
+    charStart: number;
+    charEnd: number;
+    mode: ChunkMode;
+}
+export declare function chunkDocument(opts: ChunkDocumentOpts): Chunk[];
+export { fixedChunk } from "./fixed.js";
+export { recursiveChunk } from "./recursive.js";
+export { astChunk } from "./ast.js";
+export { meanPoolByRange } from "./late.js";
+export type { LateChunkRange, LateChunkTokenPosition } from "./late.js";
+export { addChunkContext, prependContext } from "./contextual-fallback.js";
+export type { ContextProvider, AddChunkContextArgs } from "./contextual-fallback.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/chunking/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/chunking/index.ts"],"names":[],"mappings":"AAIA,MAAM,MAAM,SAAS,GAAG,OAAO,GAAG,WAAW,GAAG,KAAK,GAAG,MAAM,CAAC;AAE/D,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,KAAK;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,SAAS,CAAC;CACjB;AAWD,wBAAgB,aAAa,CAAC,IAAI,EAAE,iBAAiB,GAAG,KAAK,EAAE,CAiD9D;AAiBD,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACpC,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5C,YAAY,EAAE,cAAc,EAAE,sBAAsB,EAAE,MAAM,WAAW,CAAC;AACxE,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC3E,YAAY,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC"}

package/dist/chunking/index.js

@@ -0,0 +1,86 @@
+import { fixedChunk } from "./fixed.js";
+import { recursiveChunk } from "./recursive.js";
+import { astChunk, astImportPrefixLength } from "./ast.js";
+const CODE_MIME_PREFIXES = [
+    "text/x-typescript",
+    "text/typescript",
+    "application/typescript",
+    "text/javascript",
+    "application/javascript",
+    "text/x-python",
+];
+export function chunkDocument(opts) {
+    const mode = opts.mode ?? autoMode(opts.mimeType);
+    const maxSize = opts.maxSize ?? 800;
+    const overlap = opts.overlap ?? 100;
+    let raw;
+    if (mode === "ast") {
+        raw = astChunk(opts.content, { maxSize, language: pickLanguage(opts.mimeType) });
+    }
+    else if (mode === "recursive") {
+        raw = recursiveChunk(opts.content, { maxSize, overlap });
+    }
+    else if (mode === "late") {
+        raw = recursiveChunk(opts.content, { maxSize, overlap });
+    }
+    else {
+        raw = fixedChunk(opts.content, { windowSize: maxSize, overlap });
+    }
+    const importPrefixLen = mode === "ast" ? astImportPrefixLength(opts.content, pickLanguage(opts.mimeType)) : 0;
+    let cursor = 0;
+    return raw.map((content, i) => {
+        const searchText = content.slice(importPrefixLen);
+        const searchFrom = i === 0 ? 0 : cursor;
+        const found = searchText.length > 0
+            ? opts.content.indexOf(searchText, searchFrom)
+            : opts.content.indexOf(content, searchFrom);
+        let charStart;
+        let charEnd;
+        if (found >= 0) {
+            charStart = found;
+            charEnd = found + searchText.length;
+            cursor = charStart + 1;
+        }
+        else {
+            const directFound = opts.content.indexOf(content, searchFrom);
+            if (directFound >= 0) {
+                charStart = directFound;
+                charEnd = directFound + content.length;
+                cursor = charStart + 1;
+            }
+            else {
+                charStart = cursor;
+                charEnd = cursor + content.length;
+            }
+        }
+        return {
+            content,
+            ordinal: i,
+            charStart,
+            charEnd,
+            mode,
+        };
+    });
+}
+function autoMode(mimeType) {
+    if (mimeType && CODE_MIME_PREFIXES.some((p) => mimeType.startsWith(p)))
+        return "ast";
+    return "recursive";
+}
+function pickLanguage(mimeType) {
+    if (!mimeType)
+        return "auto";
+    if (mimeType.includes("typescript"))
+        return "typescript";
+    if (mimeType.includes("javascript"))
+        return "javascript";
+    if (mimeType.includes("python"))
+        return "python";
+    return "auto";
+}
+export { fixedChunk } from "./fixed.js";
+export { recursiveChunk } from "./recursive.js";
+export { astChunk } from "./ast.js";
+export { meanPoolByRange } from "./late.js";
+export { addChunkContext, prependContext } from "./contextual-fallback.js";
+//# sourceMappingURL=index.js.map

package/dist/chunking/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/chunking/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,QAAQ,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAoB3D,MAAM,kBAAkB,GAAG;IACzB,mBAAmB;IACnB,iBAAiB;IACjB,wBAAwB;IACxB,iBAAiB;IACjB,wBAAwB;IACxB,eAAe;CAChB,CAAC;AAEF,MAAM,UAAU,aAAa,CAAC,IAAuB;IACnD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,GAAG,CAAC;IACpC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,GAAG,CAAC;IAEpC,IAAI,GAAa,CAAC;IAClB,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;QACnB,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IACnF,CAAC;SAAM,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;QAChC,GAAG,GAAG,cAAc,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IAC3D,CAAC;SAAM,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;QAC3B,GAAG,GAAG,cAAc,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IAC3D,CAAC;SAAM,CAAC;QACN,GAAG,GAAG,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IACnE,CAAC;IAED,MAAM,eAAe,GAAG,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,qBAAqB,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9G,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE;QAC5B,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;QAClD,MAAM,UAAU,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QACxC,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC;YACjC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,UAAU,CAAC;YAC9C,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QAC9C,IAAI,SAAiB,CAAC;QACtB,IAAI,OAAe,CAAC;QACpB,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YACf,SAAS,GAAG,KAAK,CAAC;YAClB,OAAO,GAAG,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC;YACpC,MAAM,GAAG,SAAS,GAAG,CAAC,CAAC;QACzB,CAAC;aAAM,CAAC;YACN,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;YAC9D,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;gBACrB,SAAS,GAAG,WAAW,CAAC;gBACxB,OAAO,GAAG,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;gBACvC,MAAM,GAAG,SAAS,GAAG,CAAC,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACN,SAAS,GAAG,MAAM,CAAC;gBACnB,OAAO,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;YACpC,CAAC;QACH,CAAC;QACD,OAAO;YACL,OAAO;YACP,OAAO,EAAE,CAAC;YACV,SAAS;YACT,OAAO;YACP,IAAI;SACL,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,QAAQ,CAAC,QAAiB;IACjC,IAAI,QAAQ,IAAI,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAAE,OAAO,KAAK,CAAC;IACrF,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,SAAS,YAAY,CACnB,QAAiB;IAEjB,IAAI,CAAC,QAAQ;QAAE,OAAO,MAAM,CAAC;IAC7B,IAAI,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC;QAAE,OAAO,YAAY,CAAC;IACzD,IAAI,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC;QAAE,OAAO,YAAY,CAAC;IACzD,IAAI,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAAE,OAAO,QAAQ,CAAC;IACjD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACpC,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAE5C,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC"}

package/dist/chunking/late.d.ts

@@ -0,0 +1,11 @@
+export interface LateChunkRange {
+    start: number;
+    end: number;
+}
+export interface LateChunkTokenPosition {
+    charStart: number;
+    charEnd: number;
+    vector: number[];
+}
+export declare function meanPoolByRange(tokens: LateChunkTokenPosition[], ranges: LateChunkRange[]): number[][];
+//# sourceMappingURL=late.d.ts.map

package/dist/chunking/late.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"late.d.ts","sourceRoot":"","sources":["../../src/chunking/late.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,sBAAsB;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAMD,wBAAgB,eAAe,CAC7B,MAAM,EAAE,sBAAsB,EAAE,EAChC,MAAM,EAAE,cAAc,EAAE,GACvB,MAAM,EAAE,EAAE,CAEZ"}

package/dist/chunking/late.js

@@ -0,0 +1,27 @@
+// Late chunking: given token-level embeddings over the entire document and a
+// list of (charStart, charEnd) chunk ranges, returns one mean-pooled vector
+// per chunk. The contract: the token positions are sorted; we accumulate any
+// token whose midpoint falls inside the chunk's char range and average.
+export function meanPoolByRange(tokens, ranges) {
+    return ranges.map((r) => meanPoolOne(tokens, r));
+}
+function meanPoolOne(tokens, range) {
+    const matching = tokens.filter((t) => {
+        const mid = (t.charStart + t.charEnd) / 2;
+        return mid >= range.start && mid < range.end;
+    });
+    if (matching.length === 0) {
+        const dim = tokens[0]?.vector.length ?? 0;
+        return new Array(dim).fill(0);
+    }
+    const dim = matching[0]?.vector.length ?? 0;
+    const out = new Array(dim).fill(0);
+    for (const tok of matching) {
+        for (let i = 0; i < dim; i++)
+            out[i] = (out[i] ?? 0) + (tok.vector[i] ?? 0);
+    }
+    for (let i = 0; i < dim; i++)
+        out[i] = (out[i] ?? 0) / matching.length;
+    return out;
+}
+//# sourceMappingURL=late.js.map

package/dist/chunking/late.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"late.js","sourceRoot":"","sources":["../../src/chunking/late.ts"],"names":[],"mappings":"AAWA,6EAA6E;AAC7E,4EAA4E;AAC5E,6EAA6E;AAC7E,wEAAwE;AACxE,MAAM,UAAU,eAAe,CAC7B,MAAgC,EAChC,MAAwB;IAExB,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;AACnD,CAAC;AAED,SAAS,WAAW,CAAC,MAAgC,EAAE,KAAqB;IAC1E,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACnC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC1C,OAAO,GAAG,IAAI,KAAK,CAAC,KAAK,IAAI,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;IAC/C,CAAC,CAAC,CAAC;IACH,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC;QAC1C,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAa,CAAC;IAC5C,CAAC;IACD,MAAM,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC;IAC5C,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAa,CAAC;IAC/C,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE;YAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC9E,CAAC;IACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE;QAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;IACvE,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/chunking/recursive.d.ts

@@ -0,0 +1,7 @@
+export interface RecursiveChunkOpts {
+    maxSize?: number;
+    separators?: string[];
+    overlap?: number;
+}
+export declare function recursiveChunk(text: string, opts?: RecursiveChunkOpts): string[];
+//# sourceMappingURL=recursive.d.ts.map

package/dist/chunking/recursive.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"recursive.d.ts","sourceRoot":"","sources":["../../src/chunking/recursive.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,kBAAkB;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAKD,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,GAAE,kBAAuB,GAAG,MAAM,EAAE,CAUpF"}

package/dist/chunking/recursive.js

@@ -0,0 +1,87 @@
+// Recursive separator-priority splitter — for each separator in order, splits the
+// text and merges adjacent pieces while staying under maxSize. Falls back to a
+// hard slice when a single piece exceeds maxSize and no smaller separator helps.
+export function recursiveChunk(text, opts = {}) {
+    const maxSize = opts.maxSize ?? 800;
+    const overlap = opts.overlap ?? 100;
+    const separators = opts.separators ?? ["\n\n", "\n", " ", ""];
+    if (text.length === 0)
+        return [];
+    if (text.length <= maxSize)
+        return [text];
+    const split = splitWithSeparator(text, separators, maxSize);
+    return mergeAdjacent(split, maxSize, overlap);
+}
+function splitWithSeparator(text, separators, maxSize) {
+    if (text.length <= maxSize)
+        return [text];
+    const sep = separators[0];
+    if (sep === undefined)
+        return [text];
+    let parts;
+    if (sep === "") {
+        parts = sliceFixed(text, maxSize);
+    }
+    else {
+        parts = splitKeepDelimiter(text, sep);
+    }
+    const result = [];
+    for (const part of parts) {
+        if (part.length <= maxSize) {
+            result.push(part);
+        }
+        else {
+            const rest = separators.slice(1);
+            for (const sub of splitWithSeparator(part, rest.length > 0 ? rest : [""], maxSize)) {
+                result.push(sub);
+            }
+        }
+    }
+    return result;
+}
+function splitKeepDelimiter(text, sep) {
+    const out = [];
+    let i = 0;
+    while (i < text.length) {
+        const idx = text.indexOf(sep, i);
+        if (idx === -1) {
+            out.push(text.slice(i));
+            break;
+        }
+        out.push(text.slice(i, idx + sep.length));
+        i = idx + sep.length;
+    }
+    return out.filter((s) => s.length > 0);
+}
+function sliceFixed(text, size) {
+    const parts = [];
+    for (let i = 0; i < text.length; i += size)
+        parts.push(text.slice(i, i + size));
+    return parts;
+}
+function mergeAdjacent(parts, maxSize, overlap) {
+    const merged = [];
+    let current = "";
+    for (const p of parts) {
+        if (current.length === 0) {
+            current = p;
+            continue;
+        }
+        if (current.length + p.length <= maxSize) {
+            current += p;
+        }
+        else {
+            merged.push(current);
+            const overlapText = overlap > 0 ? current.slice(-overlap) : "";
+            current = overlapText + p;
+            if (current.length > maxSize) {
+                merged.push(current.slice(0, maxSize));
+                current = current.slice(maxSize - overlap < 0 ? 0 : maxSize - overlap);
+            }
+        }
+    }
+    if (current.length > 0)
+        merged.push(current);
+    return merged;
+}
+//# sourceMappingURL=recursive.js.map

package/dist/chunking/recursive.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"recursive.js","sourceRoot":"","sources":["../../src/chunking/recursive.ts"],"names":[],"mappings":"AAMA,kFAAkF;AAClF,+EAA+E;AAC/E,iFAAiF;AACjF,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,OAA2B,EAAE;IACxE,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,GAAG,CAAC;IACpC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,GAAG,CAAC;IACpC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;IAE9D,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACjC,IAAI,IAAI,CAAC,MAAM,IAAI,OAAO;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAE1C,MAAM,KAAK,GAAG,kBAAkB,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IAC5D,OAAO,aAAa,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY,EAAE,UAAoB,EAAE,OAAe;IAC7E,IAAI,IAAI,CAAC,MAAM,IAAI,OAAO;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;IAC1B,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAErC,IAAI,KAAe,CAAC;IACpB,IAAI,GAAG,KAAK,EAAE,EAAE,CAAC;QACf,KAAK,GAAG,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACpC,CAAC;SAAM,CAAC;QACN,KAAK,GAAG,kBAAkB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,MAAM,IAAI,OAAO,EAAE,CAAC;YAC3B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACjC,KAAK,MAAM,GAAG,IAAI,kBAAkB,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,OAAO,CAAC,EAAE,CAAC;gBACnF,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY,EAAE,GAAW;IACnD,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACjC,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;YACf,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACxB,MAAM;QACR,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QAC1C,CAAC,GAAG,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC;IACvB,CAAC;IACD,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACzC,CAAC;AAED,SAAS,UAAU,CAAC,IAAY,EAAE,IAAY;IAC5C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI;QAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;IAChF,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,aAAa,CAAC,KAAe,EAAE,OAAe,EAAE,OAAe;IACtE,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,GAAG,CAAC,CAAC;YACZ,SAAS;QACX,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,IAAI,OAAO,EAAE,CAAC;YACzC,OAAO,IAAI,CAAC,CAAC;QACf,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,MAAM,WAAW,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/D,OAAO,GAAG,WAAW,GAAG,CAAC,CAAC;YAC1B,IAAI,OAAO,CAAC,MAAM,GAAG,OAAO,EAAE,CAAC;gBAC7B,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;gBACvC,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,OAAO,CAAC,CAAC;YACzE,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC7C,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/embedding/index.d.ts

@@ -0,0 +1,14 @@
+import type { Embedder } from "./types.js";
+import { createVoyageContext3Embedder } from "./voyage-context-3.js";
+import { createOpenAIEmbedder } from "./openai-text-3-small.js";
+import { createLocalMiniLMEmbedder } from "./local-minilm.js";
+export type EmbedderModel = "voyage-context-3" | "text-embedding-3-small" | "all-MiniLM-L6-v2" | "auto";
+export interface CreateEmbedderOpts {
+    model?: EmbedderModel;
+    dimensions?: number;
+    env?: NodeJS.ProcessEnv;
+}
+export declare function createEmbedder(opts?: CreateEmbedderOpts): Embedder;
+export { createVoyageContext3Embedder, createOpenAIEmbedder, createLocalMiniLMEmbedder, };
+export type { Embedder, EmbedKind } from "./types.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/embedding/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/embedding/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,4BAA4B,EAAE,MAAM,uBAAuB,CAAC;AACrE,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAAE,yBAAyB,EAAE,MAAM,mBAAmB,CAAC;AAE9D,MAAM,MAAM,aAAa,GACrB,kBAAkB,GAClB,wBAAwB,GACxB,kBAAkB,GAClB,MAAM,CAAC;AAEX,MAAM,WAAW,kBAAkB;IACjC,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;CACzB;AAID,wBAAgB,cAAc,CAAC,IAAI,GAAE,kBAAuB,GAAG,QAAQ,CAmBtE;AAQD,OAAO,EACL,4BAA4B,EAC5B,oBAAoB,EACpB,yBAAyB,GAC1B,CAAC;AACF,YAAY,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC"}

package/dist/embedding/index.js

@@ -0,0 +1,33 @@
+import { createVoyageContext3Embedder } from "./voyage-context-3.js";
+import { createOpenAIEmbedder } from "./openai-text-3-small.js";
+import { createLocalMiniLMEmbedder } from "./local-minilm.js";
+// Auto-select an embedder based on which provider keys are present.
+// Precedence: voyage > openai > local. Callers can force a specific model.
+export function createEmbedder(opts = {}) {
+    const env = opts.env ?? process.env;
+    const model = opts.model ?? autoModel(env);
+    if (model === "voyage-context-3") {
+        return createVoyageContext3Embedder({
+            ...(env["VOYAGE_API_KEY"] ? { apiKey: env["VOYAGE_API_KEY"] } : {}),
+            ...(opts.dimensions !== undefined
+                ? { outputDimension: opts.dimensions }
+                : {}),
+        });
+    }
+    if (model === "text-embedding-3-small") {
+        return createOpenAIEmbedder({
+            ...(env["OPENAI_API_KEY"] ? { apiKey: env["OPENAI_API_KEY"] } : {}),
+            ...(opts.dimensions !== undefined ? { dimensions: opts.dimensions } : {}),
+        });
+    }
+    return createLocalMiniLMEmbedder();
+}
+function autoModel(env) {
+    if (env["VOYAGE_API_KEY"])
+        return "voyage-context-3";
+    if (env["OPENAI_API_KEY"])
+        return "text-embedding-3-small";
+    return "all-MiniLM-L6-v2";
+}
+export { createVoyageContext3Embedder, createOpenAIEmbedder, createLocalMiniLMEmbedder, };
+//# sourceMappingURL=index.js.map

package/dist/embedding/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/embedding/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,4BAA4B,EAAE,MAAM,uBAAuB,CAAC;AACrE,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAAE,yBAAyB,EAAE,MAAM,mBAAmB,CAAC;AAc9D,oEAAoE;AACpE,2EAA2E;AAC3E,MAAM,UAAU,cAAc,CAAC,OAA2B,EAAE;IAC1D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;IACpC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,SAAS,CAAC,GAAG,CAAC,CAAC;IAE3C,IAAI,KAAK,KAAK,kBAAkB,EAAE,CAAC;QACjC,OAAO,4BAA4B,CAAC;YAClC,GAAG,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnE,GAAG,CAAC,IAAI,CAAC,UAAU,KAAK,SAAS;gBAC/B,CAAC,CAAC,EAAE,eAAe,EAAE,IAAI,CAAC,UAAqC,EAAE;gBACjE,CAAC,CAAC,EAAE,CAAC;SACR,CAAC,CAAC;IACL,CAAC;IACD,IAAI,KAAK,KAAK,wBAAwB,EAAE,CAAC;QACvC,OAAO,oBAAoB,CAAC;YAC1B,GAAG,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnE,GAAG,CAAC,IAAI,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC1E,CAAC,CAAC;IACL,CAAC;IACD,OAAO,yBAAyB,EAAE,CAAC;AACrC,CAAC;AAED,SAAS,SAAS,CAAC,GAAsB;IACvC,IAAI,GAAG,CAAC,gBAAgB,CAAC;QAAE,OAAO,kBAAkB,CAAC;IACrD,IAAI,GAAG,CAAC,gBAAgB,CAAC;QAAE,OAAO,wBAAwB,CAAC;IAC3D,OAAO,kBAAkB,CAAC;AAC5B,CAAC;AAED,OAAO,EACL,4BAA4B,EAC5B,oBAAoB,EACpB,yBAAyB,GAC1B,CAAC"}

package/dist/embedding/local-minilm.d.ts

@@ -0,0 +1,8 @@
+import type { Embedder } from "./types.js";
+export interface LocalMiniLMOpts {
+    pipelineFactory?: () => Promise<(text: string, opts: object) => Promise<{
+        data: Float32Array;
+    }>>;
+}
+export declare function createLocalMiniLMEmbedder(opts?: LocalMiniLMOpts): Embedder;
+//# sourceMappingURL=local-minilm.d.ts.map

package/dist/embedding/local-minilm.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"local-minilm.d.ts","sourceRoot":"","sources":["../../src/embedding/local-minilm.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,QAAQ,EAAa,MAAM,YAAY,CAAC;AAuBtD,MAAM,WAAW,eAAe;IAG9B,eAAe,CAAC,EAAE,MAAM,OAAO,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC;QAAE,IAAI,EAAE,YAAY,CAAA;KAAE,CAAC,CAAC,CAAC;CAClG;AAED,wBAAgB,yBAAyB,CAAC,IAAI,GAAE,eAAoB,GAAG,QAAQ,CAyC9E"}

package/dist/embedding/local-minilm.js

@@ -0,0 +1,51 @@
+const MODEL = "Xenova/all-MiniLM-L6-v2";
+const DIM = 384;
+// Loader factored into a named function so consumers (apps/api in particular)
+// can mock it via vi.mock("@keel_flow/kb-pipeline/dist/embedding/loader.js") when the
+// upstream @huggingface/transformers package isn't installable in their test
+// env. Default behaviour is an unconditional dynamic import.
+async function loadHuggingface() {
+    try {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any -- dynamic import of the optional @huggingface/transformers peer; module shape is validated at the call site
+        return await import("@huggingface/transformers");
+    }
+    catch (cause) {
+        throw new Error("The local-minilm embedder requires @huggingface/transformers, which is an optional peer dependency of @keel_flow/kb-pipeline. " +
+            "Install it with: pnpm add @huggingface/transformers — " +
+            "or set OPENAI_API_KEY to use the OpenAI text-embedding-3-small path instead.", { cause });
+    }
+}
+export function createLocalMiniLMEmbedder(opts = {}) {
+    let pipelinePromise = null;
+    async function getPipeline() {
+        if (pipelinePromise)
+            return pipelinePromise;
+        pipelinePromise = (async () => {
+            if (opts.pipelineFactory)
+                return opts.pipelineFactory();
+            const mod = (await loadHuggingface());
+            return mod.pipeline("feature-extraction", MODEL, { revision: "main" });
+        })();
+        pipelinePromise.catch(() => {
+            pipelinePromise = null;
+        });
+        return pipelinePromise;
+    }
+    return {
+        modelId: MODEL,
+        dim: DIM,
+        isContextAware: false,
+        async embed(texts, _kind) {
+            if (texts.length === 0)
+                return [];
+            const pipeline = await getPipeline();
+            const out = [];
+            for (const text of texts) {
+                const result = await pipeline(text, { pooling: "mean", normalize: true });
+                out.push(Array.from(result.data));
+            }
+            return out;
+        },
+    };
+}
+//# sourceMappingURL=local-minilm.js.map