@speakeasy-api/docs-mcp-core 0.1.0

package/dist/embedding-cache.js

@@ -0,0 +1,230 @@
+import { mkdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { connect } from "@lancedb/lancedb";
+import { sha256hex, toEmbeddingInput } from "./embedding.js";
+/**
+ * Bumped only if toEmbeddingInput() changes in a way that isn't captured
+ * by the input text itself (e.g., normalization, encoding).
+ */
+export const EMBEDDING_FORMAT_VERSION = "1";
+const CACHE_VERSION = "1";
+const CACHE_DIR_NAME = ".embedding-cache";
+const CACHE_TMP_DIR_NAME = ".embedding-cache.tmp";
+const CACHE_OLD_DIR_NAME = ".embedding-cache.old";
+const CACHE_META_FILE = "cache-meta.json";
+const CACHE_TABLE_NAME = "embeddings";
+/**
+ * Compute a fingerprint for a chunk that captures both the embedding model config
+ * and the exact text that would be sent to the model.
+ */
+export function computeFingerprint(chunk, config) {
+    const input = [
+        EMBEDDING_FORMAT_VERSION,
+        config.configFingerprint,
+        toEmbeddingInput(chunk),
+    ].join("\0");
+    return sha256hex(input);
+}
+function cacheDirPath(baseDir) {
+    return path.join(baseDir, CACHE_DIR_NAME);
+}
+function cacheTmpDirPath(baseDir) {
+    return path.join(baseDir, CACHE_TMP_DIR_NAME);
+}
+function cacheOldDirPath(baseDir) {
+    return path.join(baseDir, CACHE_OLD_DIR_NAME);
+}
+/**
+ * Remove leftover .tmp and .old directories from interrupted builds.
+ */
+async function cleanupStaleDirectories(baseDir) {
+    await rm(cacheTmpDirPath(baseDir), { recursive: true, force: true });
+    await rm(cacheOldDirPath(baseDir), { recursive: true, force: true });
+}
+/**
+ * Load the embedding cache from disk.
+ * Returns null (with a warning to stderr) if the cache is missing, corrupt, or incompatible.
+ */
+export async function loadCache(baseDir, currentConfig) {
+    await cleanupStaleDirectories(baseDir);
+    const cacheDir = cacheDirPath(baseDir);
+    const metaPath = path.join(cacheDir, CACHE_META_FILE);
+    let metaRaw;
+    try {
+        metaRaw = await readFile(metaPath, "utf8");
+    }
+    catch {
+        // cache-meta.json missing — check if the cache dir itself exists (orphaned LanceDB files)
+        try {
+            await stat(cacheDir);
+            console.warn("warn: embedding cache invalidated: cache-meta.json missing but cache directory exists");
+            await rm(cacheDir, { recursive: true, force: true });
+        }
+        catch {
+            // No cache dir at all — normal first-build case, no warning needed
+        }
+        return null;
+    }
+    let meta;
+    try {
+        meta = JSON.parse(metaRaw);
+    }
+    catch {
+        console.warn("warn: embedding cache invalidated: corrupt cache-meta.json");
+        await rm(cacheDir, { recursive: true, force: true });
+        return null;
+    }
+    if (typeof meta.cache_version !== "string" || meta.cache_version.split(".")[0] !== CACHE_VERSION.split(".")[0]) {
+        console.warn(`warn: embedding cache invalidated: cache_version mismatch (got ${meta.cache_version}, expected ${CACHE_VERSION})`);
+        await rm(cacheDir, { recursive: true, force: true });
+        return null;
+    }
+    if (meta.format_version !== EMBEDDING_FORMAT_VERSION) {
+        console.warn(`warn: embedding cache invalidated: format_version mismatch (got ${meta.format_version}, expected ${EMBEDDING_FORMAT_VERSION})`);
+        await rm(cacheDir, { recursive: true, force: true });
+        return null;
+    }
+    if (meta.config_fingerprint !== currentConfig.configFingerprint) {
+        console.warn("warn: embedding cache invalidated: config_fingerprint mismatch (provider config changed)");
+        await rm(cacheDir, { recursive: true, force: true });
+        return null;
+    }
+    try {
+        const db = await connect(cacheDir);
+        try {
+            const tableNames = await db.tableNames();
+            if (!tableNames.includes(CACHE_TABLE_NAME)) {
+                // Valid meta but no table — could be a zero-chunk build. Return empty cache.
+                return { entries: new Map() };
+            }
+            const table = await db.openTable(CACHE_TABLE_NAME);
+            const rows = await table.query().toArray();
+            const entries = new Map();
+            // NOTE: This loads all vectors into JS memory as number[] (Float64).
+            // For a 50K-chunk corpus at 3072 dims this is ~600MB+ of heap. Acceptable
+            // for v1; a streaming/query approach could reduce peak memory later.
+            for (const row of rows) {
+                const fingerprint = row.fingerprint;
+                const chunk_id = row.chunk_id;
+                const vector = Array.from(row.vector);
+                entries.set(fingerprint, { fingerprint, chunk_id, vector });
+            }
+            table.close();
+            return { entries };
+        }
+        finally {
+            db.close();
+        }
+    }
+    catch {
+        console.warn("warn: embedding cache invalidated: failed to read cache table");
+        await rm(cacheDir, { recursive: true, force: true });
+        return null;
+    }
+}
+/**
+ * Save the embedding cache to disk using the rotate-and-rename pattern
+ * for crash safety.
+ */
+export async function saveCache(baseDir, cache, config) {
+    const tmpDir = cacheTmpDirPath(baseDir);
+    const liveDir = cacheDirPath(baseDir);
+    const oldDir = cacheOldDirPath(baseDir);
+    // Clean up any leftover temp/old dirs
+    await rm(tmpDir, { recursive: true, force: true });
+    await rm(oldDir, { recursive: true, force: true });
+    // Write cache-meta.json
+    await mkdir(tmpDir, { recursive: true });
+    const meta = {
+        cache_version: CACHE_VERSION,
+        format_version: EMBEDDING_FORMAT_VERSION,
+        config_fingerprint: config.configFingerprint,
+    };
+    await writeFile(path.join(tmpDir, CACHE_META_FILE), JSON.stringify(meta, null, 2));
+    // Write LanceDB cache table
+    const rows = Array.from(cache.entries.values()).map((entry) => ({
+        fingerprint: entry.fingerprint,
+        chunk_id: entry.chunk_id,
+        vector: entry.vector,
+    }));
+    if (rows.length > 0) {
+        const db = await connect(tmpDir);
+        try {
+            const table = await db.createTable(CACHE_TABLE_NAME, rows, { mode: "overwrite" });
+            table.close();
+        }
+        finally {
+            db.close();
+        }
+    }
+    // Atomic swap: live → old, tmp → live, delete old
+    try {
+        await rename(liveDir, oldDir);
+    }
+    catch {
+        // No prior cache — that's fine
+    }
+    await rename(tmpDir, liveDir);
+    await rm(oldDir, { recursive: true, force: true }).catch(() => { });
+}
+/**
+ * Embed chunks incrementally: reuse cached vectors for unchanged chunks,
+ * only call the provider for misses.
+ */
+export async function embedChunksIncremental(chunks, provider, cache) {
+    const config = {
+        provider: provider.provider,
+        model: provider.model,
+        dimensions: provider.dimensions,
+        configFingerprint: provider.configFingerprint,
+    };
+    // Partition chunks into hits and misses
+    const fingerprints = chunks.map((chunk) => computeFingerprint(chunk, config));
+    const hitIndices = [];
+    const missIndices = [];
+    for (let i = 0; i < chunks.length; i++) {
+        const fp = fingerprints[i];
+        if (cache?.entries.has(fp)) {
+            hitIndices.push(i);
+        }
+        else {
+            missIndices.push(i);
+        }
+    }
+    // Embed only misses
+    const missTexts = missIndices.map((i) => toEmbeddingInput(chunks[i]));
+    const missVectors = missTexts.length > 0 ? await provider.embed(missTexts) : [];
+    if (missVectors.length !== missTexts.length) {
+        throw new Error(`Embedding provider returned ${missVectors.length} vectors for ${missTexts.length} chunks`);
+    }
+    // Merge results
+    const vectorsByChunkId = new Map();
+    const newEntries = new Map();
+    // Cache hits
+    for (const i of hitIndices) {
+        const fp = fingerprints[i];
+        const chunk = chunks[i];
+        const entry = cache.entries.get(fp);
+        vectorsByChunkId.set(chunk.chunk_id, entry.vector);
+        newEntries.set(fp, entry);
+    }
+    // Cache misses (freshly embedded)
+    for (let j = 0; j < missIndices.length; j++) {
+        const i = missIndices[j];
+        const fp = fingerprints[i];
+        const chunk = chunks[i];
+        const vector = missVectors[j];
+        vectorsByChunkId.set(chunk.chunk_id, vector);
+        newEntries.set(fp, { fingerprint: fp, chunk_id: chunk.chunk_id, vector });
+    }
+    return {
+        vectorsByChunkId,
+        updatedCache: { entries: newEntries },
+        stats: {
+            total: chunks.length,
+            hits: hitIndices.length,
+            misses: missIndices.length,
+        },
+    };
+}
+//# sourceMappingURL=embedding-cache.js.map

package/dist/embedding-cache.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedding-cache.js","sourceRoot":"","sources":["../src/embedding-cache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAChF,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAG7D;;;GAGG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,GAAG,CAAC;AAE5C,MAAM,aAAa,GAAG,GAAG,CAAC;AAC1B,MAAM,cAAc,GAAG,kBAAkB,CAAC;AAC1C,MAAM,kBAAkB,GAAG,sBAAsB,CAAC;AAClD,MAAM,kBAAkB,GAAG,sBAAsB,CAAC;AAClD,MAAM,eAAe,GAAG,iBAAiB,CAAC;AAC1C,MAAM,gBAAgB,GAAG,YAAY,CAAC;AAwBtC;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAAY,EAAE,MAAuB;IACtE,MAAM,KAAK,GAAG;QACZ,wBAAwB;QACxB,MAAM,CAAC,iBAAiB;QACxB,gBAAgB,CAAC,KAAK,CAAC;KACxB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACb,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,YAAY,CAAC,OAAe;IACnC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;AAC5C,CAAC;AACD,SAAS,eAAe,CAAC,OAAe;IACtC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;AAChD,CAAC;AACD,SAAS,eAAe,CAAC,OAAe;IACtC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;AAChD,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,uBAAuB,CAAC,OAAe;IACpD,MAAM,EAAE,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACrE,MAAM,EAAE,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACvE,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,OAAe,EACf,aAA8B;IAE9B,MAAM,uBAAuB,CAAC,OAAO,CAAC,CAAC;IAEvC,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;IAEtD,IAAI,OAAe,CAAC;IACpB,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC;IAAC,MAAM,CAAC;QACP,0FAA0F;QAC1F,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC,uFAAuF,CAAC,CAAC;YACtG,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACvD,CAAC;QAAC,MAAM,CAAC;YACP,mEAAmE;QACrE,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,IAAe,CAAC;IACpB,IAAI,CAAC;QACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAc,CAAC;IAC1C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,IAAI,CAAC,4DAA4D,CAAC,CAAC;QAC3E,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,OAAO,IAAI,CAAC,aAAa,KAAK,QAAQ,IAAI,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/G,OAAO,CAAC,IAAI,CAAC,kEAAkE,IAAI,CAAC,aAAa,cAAc,aAAa,GAAG,CAAC,CAAC;QACjI,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,IAAI,CAAC,cAAc,KAAK,wBAAwB,EAAE,CAAC;QACrD,OAAO,CAAC,IAAI,CAAC,mEAAmE,IAAI,CAAC,cAAc,cAAc,wBAAwB,GAAG,CAAC,CAAC;QAC9I,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,IAAI,CAAC,kBAAkB,KAAK,aAAa,CAAC,iBAAiB,EAAE,CAAC;QAChE,OAAO,CAAC,IAAI,CAAC,0FAA0F,CAAC,CAAC;QACzG,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,UAAU,EAAE,CAAC;YACzC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;gBAC3C,6EAA6E;gBAC7E,OAAO,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;YAChC,CAAC;YAED,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;YACnD,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,OAAO,EAAE,CAAC;YAC3C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAsB,CAAC;YAE9C,qEAAqE;YACrE,0EAA0E;YAC1E,qEAAqE;YACrE,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,MAAM,WAAW,GAAG,GAAG,CAAC,WAAqB,CAAC;gBAC9C,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAkB,CAAC;gBACxC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAiC,CAAC,CAAC;gBACjE,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;YAC9D,CAAC;YAED,KAAK,CAAC,KAAK,EAAE,CAAC;YACd,OAAO,EAAE,OAAO,EAAE,CAAC;QACrB,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,KAAK,EAAE,CAAC;QACb,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;QAC9E,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,OAAe,EACf,KAAqB,EACrB,MAAuB;IAEvB,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IACxC,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAExC,sCAAsC;IACtC,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACnD,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAEnD,wBAAwB;IACxB,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,IAAI,GAAc;QACtB,aAAa,EAAE,aAAa;QAC5B,cAAc,EAAE,wBAAwB;QACxC,kBAAkB,EAAE,MAAM,CAAC,iBAAiB;KAC7C,CAAC;IACF,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAEnF,4BAA4B;IAC5B,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC9D,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,MAAM,EAAE,KAAK,CAAC,MAAM;KACrB,CAAC,CAAC,CAAC;IAEJ,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpB,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,WAAW,CAAC,gBAAgB,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;YAClF,KAAK,CAAC,KAAK,EAAE,CAAC;QAChB,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,KAAK,EAAE,CAAC;QACb,CAAC;IACH,CAAC;IAED,kDAAkD;IAClD,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,+BAA+B;IACjC,CAAC;IACD,MAAM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9B,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;AACrE,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,MAAe,EACf,QAA2E,EAC3E,KAA4B;IAM5B,MAAM,MAAM,GAAoB;QAC9B,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,UAAU,EAAE,QAAQ,CAAC,UAAU;QAC/B,iBAAiB,EAAE,QAAQ,CAAC,iBAAiB;KAC9C,CAAC;IAEF,wCAAwC;IACxC,MAAM,YAAY,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,kBAAkB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;IAC9E,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,MAAM,WAAW,GAAa,EAAE,CAAC;IAEjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,EAAE,GAAG,YAAY,CAAC,CAAC,CAAE,CAAC;QAC5B,IAAI,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;YAC3B,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrB,CAAC;aAAM,CAAC;YACN,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAED,oBAAoB;IACpB,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC;IACvE,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAChF,IAAI,WAAW,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CACb,+BAA+B,WAAW,CAAC,MAAM,gBAAgB,SAAS,CAAC,MAAM,SAAS,CAC3F,CAAC;IACJ,CAAC;IAED,gBAAgB;IAChB,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAoB,CAAC;IACrD,MAAM,UAAU,GAAG,IAAI,GAAG,EAAsB,CAAC;IAEjD,aAAa;IACb,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,MAAM,EAAE,GAAG,YAAY,CAAC,CAAC,CAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;QACzB,MAAM,KAAK,GAAG,KAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;QACtC,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QACnD,UAAU,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;IAC5B,CAAC;IAED,kCAAkC;IAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5C,MAAM,CAAC,GAAG,WAAW,CAAC,CAAC,CAAE,CAAC;QAC1B,MAAM,EAAE,GAAG,YAAY,CAAC,CAAC,CAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;QACzB,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAE,CAAC;QAC/B,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC7C,UAAU,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5E,CAAC;IAED,OAAO;QACL,gBAAgB;QAChB,YAAY,EAAE,EAAE,OAAO,EAAE,UAAU,EAAE;QACrC,KAAK,EAAE;YACL,KAAK,EAAE,MAAM,CAAC,MAAM;YACpB,IAAI,EAAE,UAAU,CAAC,MAAM;YACvB,MAAM,EAAE,WAAW,CAAC,MAAM;SAC3B;KACF,CAAC;AACJ,CAAC"}

package/dist/embedding.d.ts

@@ -0,0 +1,66 @@
+import type { Chunk, EmbeddingProvider } from "./types.js";
+/**
+ * The single source of truth for constructing the text payload sent to embedding models.
+ * Both `provider.embed()` and `computeFingerprint()` use this function.
+ */
+export declare function toEmbeddingInput(chunk: Chunk): string;
+export declare function sha256hex(input: string): string;
+export interface OpenAIEmbeddingProviderOptions {
+    apiKey: string;
+    model?: string;
+    dimensions?: number;
+    batchSize?: number;
+    baseUrl?: string;
+    concurrency?: number;
+    maxRetries?: number;
+    retryBaseDelayMs?: number;
+    retryMaxDelayMs?: number;
+}
+export declare class NoopEmbeddingProvider implements EmbeddingProvider {
+    readonly name = "none";
+    readonly model = "none";
+    readonly dimensions = 0;
+    readonly configFingerprint: string;
+    constructor();
+    embed(texts: string[]): Promise<number[][]>;
+}
+export declare class HashEmbeddingProvider implements EmbeddingProvider {
+    readonly name = "hash";
+    readonly model: string;
+    readonly dimensions: number;
+    readonly configFingerprint: string;
+    constructor(options?: {
+        dimensions?: number;
+        model?: string;
+    });
+    embed(texts: string[]): Promise<number[][]>;
+}
+export declare class OpenAIEmbeddingProvider implements EmbeddingProvider {
+    readonly name = "openai";
+    readonly model: string;
+    readonly dimensions: number;
+    readonly configFingerprint: string;
+    private readonly apiKey;
+    private readonly batchSize;
+    private readonly baseUrl;
+    private readonly concurrency;
+    private readonly maxRetries;
+    private readonly retryBaseDelayMs;
+    private readonly retryMaxDelayMs;
+    constructor(options: OpenAIEmbeddingProviderOptions);
+    embed(texts: string[]): Promise<number[][]>;
+    private embedBatchWithRetry;
+}
+export declare function createEmbeddingProvider(input: {
+    provider: "none" | "hash" | "openai";
+    dimensions?: number;
+    apiKey?: string;
+    model?: string;
+    baseUrl?: string;
+    batchSize?: number;
+    concurrency?: number;
+    maxRetries?: number;
+    retryBaseDelayMs?: number;
+    retryMaxDelayMs?: number;
+}): EmbeddingProvider;
+//# sourceMappingURL=embedding.d.ts.map

package/dist/embedding.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedding.d.ts","sourceRoot":"","sources":["../src/embedding.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAE3D;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,CAGrD;AAED,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAE/C;AASD,MAAM,WAAW,8BAA8B;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,qBAAa,qBAAsB,YAAW,iBAAiB;IAC7D,QAAQ,CAAC,IAAI,UAAU;IACvB,QAAQ,CAAC,KAAK,UAAU;IACxB,QAAQ,CAAC,UAAU,KAAK;IACxB,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;;IAU7B,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CAGlD;AAED,qBAAa,qBAAsB,YAAW,iBAAiB;IAC7D,QAAQ,CAAC,IAAI,UAAU;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;gBAEvB,OAAO,GAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAO;IAU3D,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CAGlD;AAED,qBAAa,uBAAwB,YAAW,iBAAiB;IAC/D,QAAQ,CAAC,IAAI,YAAY;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IAEnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;gBAE7B,OAAO,EAAE,8BAA8B;IAsB7C,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YA4BnC,mBAAmB;CAwDlC;AAED,wBAAgB,uBAAuB,CAAC,KAAK,EAAE;IAC7C,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAC;IACrC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,GAAG,iBAAiB,CA4DpB"}

package/dist/embedding.js

@@ -0,0 +1,264 @@
+import { createHash } from "node:crypto";
+/**
+ * The single source of truth for constructing the text payload sent to embedding models.
+ * Both `provider.embed()` and `computeFingerprint()` use this function.
+ */
+export function toEmbeddingInput(chunk) {
+    const context = chunk.breadcrumb || chunk.filepath;
+    return `Context: ${context}\n\nContent:\n${chunk.content_text}`;
+}
+export function sha256hex(input) {
+    return createHash("sha256").update(input).digest("hex");
+}
+function computeConfigFingerprint(fields) {
+    const parts = Object.entries(fields)
+        .sort(([a], [b]) => a.localeCompare(b))
+        .map(([k, v]) => `${k}=${v}`);
+    return sha256hex(parts.join("\0"));
+}
+export class NoopEmbeddingProvider {
+    name = "none";
+    model = "none";
+    dimensions = 0;
+    configFingerprint;
+    constructor() {
+        this.configFingerprint = computeConfigFingerprint({
+            provider: "none",
+            model: "none",
+            dimensions: 0,
+        });
+    }
+    async embed(texts) {
+        return texts.map(() => []);
+    }
+}
+export class HashEmbeddingProvider {
+    name = "hash";
+    model;
+    dimensions;
+    configFingerprint;
+    constructor(options = {}) {
+        this.dimensions = options.dimensions ?? 256;
+        this.model = options.model ?? "hash-v1";
+        this.configFingerprint = computeConfigFingerprint({
+            provider: "hash",
+            model: this.model,
+            dimensions: this.dimensions,
+        });
+    }
+    async embed(texts) {
+        return texts.map((text) => hashToUnitVector(text, this.dimensions));
+    }
+}
+export class OpenAIEmbeddingProvider {
+    name = "openai";
+    model;
+    dimensions;
+    configFingerprint;
+    apiKey;
+    batchSize;
+    baseUrl;
+    concurrency;
+    maxRetries;
+    retryBaseDelayMs;
+    retryMaxDelayMs;
+    constructor(options) {
+        if (!options.apiKey.trim()) {
+            throw new Error("OpenAIEmbeddingProvider requires a non-empty apiKey");
+        }
+        this.apiKey = options.apiKey;
+        this.model = options.model ?? "text-embedding-3-large";
+        this.dimensions = options.dimensions ?? 3072;
+        this.batchSize = options.batchSize ?? 128;
+        this.baseUrl = (options.baseUrl ?? "https://api.openai.com/v1").replace(/\/$/, "");
+        this.concurrency = normalizeInt(options.concurrency, 4, 1, 32);
+        this.maxRetries = normalizeInt(options.maxRetries, 3, 0, 10);
+        this.retryBaseDelayMs = normalizeInt(options.retryBaseDelayMs, 500, 50, 60_000);
+        this.retryMaxDelayMs = normalizeInt(options.retryMaxDelayMs, 10_000, 100, 120_000);
+        this.configFingerprint = computeConfigFingerprint({
+            provider: "openai",
+            model: this.model,
+            dimensions: this.dimensions,
+            baseUrl: this.baseUrl,
+        });
+    }
+    async embed(texts) {
+        if (texts.length === 0) {
+            return [];
+        }
+        const vectors = new Array(texts.length);
+        const batches = [];
+        for (let i = 0; i < texts.length; i += this.batchSize) {
+            batches.push({
+                offset: i,
+                values: texts.slice(i, i + this.batchSize)
+            });
+        }
+        await runWithConcurrency(batches, this.concurrency, async (batch) => {
+            const batchVectors = await this.embedBatchWithRetry(batch.values);
+            for (let i = 0; i < batchVectors.length; i += 1) {
+                vectors[batch.offset + i] = batchVectors[i];
+            }
+        });
+        if (vectors.some((vector) => vector === undefined)) {
+            throw new Error("OpenAI embeddings response was missing one or more vectors");
+        }
+        return vectors;
+    }
+    async embedBatchWithRetry(batch) {
+        let attempt = 0;
+        while (true) {
+            const response = await fetch(`${this.baseUrl}/embeddings`, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    Authorization: `Bearer ${this.apiKey}`
+                },
+                body: JSON.stringify({
+                    model: this.model,
+                    input: batch,
+                    dimensions: this.dimensions
+                })
+            });
+            if (response.ok) {
+                const payload = (await response.json());
+                if (!Array.isArray(payload.data)) {
+                    throw new Error("OpenAI embeddings response missing data array");
+                }
+                const ordered = [...payload.data].sort((a, b) => a.index - b.index);
+                if (ordered.length !== batch.length) {
+                    throw new Error(`OpenAI embeddings response size mismatch: expected ${batch.length}, got ${ordered.length}`);
+                }
+                return ordered.map((item) => item.embedding);
+            }
+            const body = await response.text();
+            if (!isRetryableStatus(response.status) || attempt >= this.maxRetries) {
+                throw new Error(`OpenAI embeddings request failed (${response.status}): ${body.slice(0, 500)}`);
+            }
+            const retryAfterMs = parseRetryAfterMs(response.headers.get("retry-after"));
+            const backoffMs = Math.min(this.retryMaxDelayMs, Math.max(retryAfterMs, this.retryBaseDelayMs * Math.pow(2, attempt) + Math.floor(Math.random() * 250)));
+            await sleep(backoffMs);
+            attempt += 1;
+        }
+    }
+}
+export function createEmbeddingProvider(input) {
+    if (input.provider !== "none" &&
+        input.provider !== "hash" &&
+        input.provider !== "openai") {
+        throw new Error(`unsupported embedding provider '${String(input.provider)}'. Expected one of: none, hash, openai`);
+    }
+    if (input.provider === "none") {
+        return new NoopEmbeddingProvider();
+    }
+    if (input.provider === "hash") {
+        const options = {};
+        if (input.dimensions !== undefined) {
+            options.dimensions = input.dimensions;
+        }
+        if (input.model !== undefined) {
+            options.model = input.model;
+        }
+        return new HashEmbeddingProvider(options);
+    }
+    const apiKey = input.apiKey?.trim();
+    if (!apiKey) {
+        throw new Error(`${input.provider} embedding provider requires --embedding-api-key or OPENAI_API_KEY`);
+    }
+    const options = { apiKey };
+    if (input.model !== undefined) {
+        options.model = input.model;
+    }
+    if (input.dimensions !== undefined) {
+        options.dimensions = input.dimensions;
+    }
+    if (input.baseUrl !== undefined) {
+        options.baseUrl = input.baseUrl;
+    }
+    if (input.batchSize !== undefined) {
+        options.batchSize = input.batchSize;
+    }
+    if (input.concurrency !== undefined) {
+        options.concurrency = input.concurrency;
+    }
+    if (input.maxRetries !== undefined) {
+        options.maxRetries = input.maxRetries;
+    }
+    if (input.retryBaseDelayMs !== undefined) {
+        options.retryBaseDelayMs = input.retryBaseDelayMs;
+    }
+    if (input.retryMaxDelayMs !== undefined) {
+        options.retryMaxDelayMs = input.retryMaxDelayMs;
+    }
+    return new OpenAIEmbeddingProvider(options);
+}
+function hashToUnitVector(text, dimensions) {
+    const vector = new Array(dimensions).fill(0);
+    const bytes = new TextEncoder().encode(text);
+    for (let i = 0; i < bytes.length; i += 1) {
+        const bucket = i % dimensions;
+        const byte = bytes[i] ?? 0;
+        const signed = i % 2 === 0 ? byte : -byte;
+        const current = vector[bucket] ?? 0;
+        vector[bucket] = current + signed;
+    }
+    let norm = 0;
+    for (const value of vector) {
+        norm += value * value;
+    }
+    norm = Math.sqrt(norm);
+    if (!norm) {
+        return vector;
+    }
+    return vector.map((value) => value / norm);
+}
+function normalizeInt(value, fallback, min, max) {
+    if (value === undefined || !Number.isFinite(value)) {
+        return fallback;
+    }
+    return Math.max(min, Math.min(max, Math.floor(value)));
+}
+function isRetryableStatus(status) {
+    return status === 429 || (status >= 500 && status <= 599);
+}
+function parseRetryAfterMs(value) {
+    if (!value) {
+        return 0;
+    }
+    const asSeconds = Number.parseFloat(value);
+    if (Number.isFinite(asSeconds) && asSeconds > 0) {
+        return Math.round(asSeconds * 1000);
+    }
+    const asDate = Date.parse(value);
+    if (Number.isFinite(asDate)) {
+        return Math.max(0, asDate - Date.now());
+    }
+    return 0;
+}
+function sleep(ms) {
+    return new Promise((resolve) => {
+        setTimeout(resolve, ms);
+    });
+}
+async function runWithConcurrency(items, concurrency, worker) {
+    if (items.length === 0) {
+        return;
+    }
+    const limit = Math.max(1, Math.min(items.length, Math.floor(concurrency)));
+    let cursor = 0;
+    await Promise.all(Array.from({ length: limit }, async () => {
+        while (true) {
+            const index = cursor;
+            cursor += 1;
+            if (index >= items.length) {
+                break;
+            }
+            const item = items[index];
+            if (!item) {
+                continue;
+            }
+            await worker(item);
+        }
+    }));
+}
+//# sourceMappingURL=embedding.js.map

package/dist/embedding.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedding.js","sourceRoot":"","sources":["../src/embedding.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGzC;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAY;IAC3C,MAAM,OAAO,GAAG,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,QAAQ,CAAC;IACnD,OAAO,YAAY,OAAO,iBAAiB,KAAK,CAAC,YAAY,EAAE,CAAC;AAClE,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAAa;IACrC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC1D,CAAC;AAED,SAAS,wBAAwB,CAAC,MAAuC;IACvE,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;SACjC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;SACtC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAChC,OAAO,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AACrC,CAAC;AAcD,MAAM,OAAO,qBAAqB;IACvB,IAAI,GAAG,MAAM,CAAC;IACd,KAAK,GAAG,MAAM,CAAC;IACf,UAAU,GAAG,CAAC,CAAC;IACf,iBAAiB,CAAS;IAEnC;QACE,IAAI,CAAC,iBAAiB,GAAG,wBAAwB,CAAC;YAChD,QAAQ,EAAE,MAAM;YAChB,KAAK,EAAE,MAAM;YACb,UAAU,EAAE,CAAC;SACd,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAAe;QACzB,OAAO,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IAC7B,CAAC;CACF;AAED,MAAM,OAAO,qBAAqB;IACvB,IAAI,GAAG,MAAM,CAAC;IACd,KAAK,CAAS;IACd,UAAU,CAAS;IACnB,iBAAiB,CAAS;IAEnC,YAAY,UAAmD,EAAE;QAC/D,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,GAAG,CAAC;QAC5C,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,SAAS,CAAC;QACxC,IAAI,CAAC,iBAAiB,GAAG,wBAAwB,CAAC;YAChD,QAAQ,EAAE,MAAM;YAChB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,UAAU,EAAE,IAAI,CAAC,UAAU;SAC5B,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAAe;QACzB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;IACtE,CAAC;CACF;AAED,MAAM,OAAO,uBAAuB;IACzB,IAAI,GAAG,QAAQ,CAAC;IAChB,KAAK,CAAS;IACd,UAAU,CAAS;IACnB,iBAAiB,CAAS;IAElB,MAAM,CAAS;IACf,SAAS,CAAS;IAClB,OAAO,CAAS;IAChB,WAAW,CAAS;IACpB,UAAU,CAAS;IACnB,gBAAgB,CAAS;IACzB,eAAe,CAAS;IAEzC,YAAY,OAAuC;QACjD,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;QACzE,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,wBAAwB,CAAC;QACvD,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC;QAC7C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,GAAG,CAAC;QAC1C,IAAI,CAAC,OAAO,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,2BAA2B,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACnF,IAAI,CAAC,WAAW,GAAG,YAAY,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAC/D,IAAI,CAAC,UAAU,GAAG,YAAY,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAC7D,IAAI,CAAC,gBAAgB,GAAG,YAAY,CAAC,OAAO,CAAC,gBAAgB,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC;QAChF,IAAI,CAAC,eAAe,GAAG,YAAY,CAAC,OAAO,CAAC,eAAe,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;QACnF,IAAI,CAAC,iBAAiB,GAAG,wBAAwB,CAAC;YAChD,QAAQ,EAAE,QAAQ;YAClB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAAe;QACzB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,OAAO,GAAgC,IAAI,KAAK,CAAuB,KAAK,CAAC,MAAM,CAAC,CAAC;QAC3F,MAAM,OAAO,GAAgD,EAAE,CAAC;QAChE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACtD,OAAO,CAAC,IAAI,CAAC;gBACX,MAAM,EAAE,CAAC;gBACT,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC;aAC3C,CAAC,CAAC;QACL,CAAC;QAED,MAAM,kBAAkB,CAAC,OAAO,EAAE,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;YAClE,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAClE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;gBAChD,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,KAAK,SAAS,CAAC,EAAE,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;QAChF,CAAC;QAED,OAAO,OAAqB,CAAC;IAC/B,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,KAAe;QAC/C,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,aAAa,EAAE;gBACzD,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;iBACvC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,KAAK,EAAE,KAAK;oBACZ,UAAU,EAAE,IAAI,CAAC,UAAU;iBAC5B,CAAC;aACH,CAAC,CAAC;YAEH,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;gBAChB,MAAM,OAAO,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAErC,CAAC;gBAEF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;oBACjC,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;gBACnE,CAAC;gBAED,MAAM,OAAO,GAAG,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;gBACpE,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;oBACpC,MAAM,IAAI,KAAK,CACb,sDAAsD,KAAK,CAAC,MAAM,SAAS,OAAO,CAAC,MAAM,EAAE,CAC5F,CAAC;gBACJ,CAAC;gBAED,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC/C,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,OAAO,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBACtE,MAAM,IAAI,KAAK,CACb,qCAAqC,QAAQ,CAAC,MAAM,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAC/E,CAAC;YACJ,CAAC;YAED,MAAM,YAAY,GAAG,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC;YAC5E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACxB,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,GAAG,CACN,YAAY,EACZ,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAC/E,CACF,CAAC;YAEF,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC;YACvB,OAAO,IAAI,CAAC,CAAC;QACf,CAAC;IACH,CAAC;CACF;AAED,MAAM,UAAU,uBAAuB,CAAC,KAWvC;IACC,IACE,KAAK,CAAC,QAAQ,KAAK,MAAM;QACzB,KAAK,CAAC,QAAQ,KAAK,MAAM;QACzB,KAAK,CAAC,QAAQ,KAAK,QAAQ,EAC3B,CAAC;QACD,MAAM,IAAI,KAAK,CACb,mCAAmC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,wCAAwC,CAClG,CAAC;IACJ,CAAC;IAED,IAAI,KAAK,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAC;QAC9B,OAAO,IAAI,qBAAqB,EAAE,CAAC;IACrC,CAAC;IAED,IAAI,KAAK,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAC;QAC9B,MAAM,OAAO,GAA4C,EAAE,CAAC;QAC5D,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YACnC,OAAO,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC;QACxC,CAAC;QACD,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC9B,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;QAC9B,CAAC;QACD,OAAO,IAAI,qBAAqB,CAAC,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC;IACpC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CACb,GAAG,KAAK,CAAC,QAAQ,oEAAoE,CACtF,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAmC,EAAE,MAAM,EAAE,CAAC;IAC3D,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC9B,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;IAC9B,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QACnC,OAAO,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC;IACxC,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAChC,OAAO,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;IAClC,CAAC;IACD,IAAI,KAAK,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QAClC,OAAO,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC;IACtC,CAAC;IACD,IAAI,KAAK,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;QACpC,OAAO,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW,CAAC;IAC1C,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QACnC,OAAO,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC;IACxC,CAAC;IACD,IAAI,KAAK,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;QACzC,OAAO,CAAC,gBAAgB,GAAG,KAAK,CAAC,gBAAgB,CAAC;IACpD,CAAC;IACD,IAAI,KAAK,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;QACxC,OAAO,CAAC,eAAe,GAAG,KAAK,CAAC,eAAe,CAAC;IAClD,CAAC;IAED,OAAO,IAAI,uBAAuB,CAAC,OAAO,CAAC,CAAC;AAC9C,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAY,EAAE,UAAkB;IACxD,MAAM,MAAM,GAAG,IAAI,KAAK,CAAS,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAErD,MAAM,KAAK,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACzC,MAAM,MAAM,GAAG,CAAC,GAAG,UAAU,CAAC;QAC9B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC3B,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC1C,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACpC,MAAM,CAAC,MAAM,CAAC,GAAG,OAAO,GAAG,MAAM,CAAC;IACpC,CAAC;IAED,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,IAAI,KAAK,GAAG,KAAK,CAAC;IACxB,CAAC;IACD,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEvB,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;AAC7C,CAAC;AAED,SAAS,YAAY,CACnB,KAAyB,EACzB,QAAgB,EAChB,GAAW,EACX,GAAW;IAEX,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACnD,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,iBAAiB,CAAC,MAAc;IACvC,OAAO,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,MAAM,IAAI,GAAG,CAAC,CAAC;AAC5D,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAoB;IAC7C,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;IAC3C,IAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QAChD,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;IACtC,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACjC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAC1B,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,kBAAkB,CAC/B,KAAU,EACV,WAAmB,EACnB,MAAkC;IAElC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO;IACT,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;IAC3E,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,KAAK,IAAI,EAAE;QACvC,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,KAAK,GAAG,MAAM,CAAC;YACrB,MAAM,IAAI,CAAC,CAAC;YACZ,IAAI,KAAK,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;gBAC1B,MAAM;YACR,CAAC;YACD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAC1B,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,SAAS;YACX,CAAC;YACD,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;IACH,CAAC,CAAC,CACH,CAAC;AACJ,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,11 @@
+export * from "./types.js";
+export * from "./chunking.js";
+export * from "./parser.js";
+export * from "./cursor.js";
+export * from "./manifest.js";
+export * from "./metadata.js";
+export * from "./search.js";
+export * from "./embedding.js";
+export * from "./embedding-cache.js";
+export * from "./lancedb.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,aAAa,CAAC;AAC5B,cAAc,aAAa,CAAC;AAC5B,cAAc,eAAe,CAAC;AAC9B,cAAc,eAAe,CAAC;AAC9B,cAAc,aAAa,CAAC;AAC5B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,sBAAsB,CAAC;AACrC,cAAc,cAAc,CAAC"}

package/dist/index.js

@@ -0,0 +1,11 @@
+export * from "./types.js";
+export * from "./chunking.js";
+export * from "./parser.js";
+export * from "./cursor.js";
+export * from "./manifest.js";
+export * from "./metadata.js";
+export * from "./search.js";
+export * from "./embedding.js";
+export * from "./embedding-cache.js";
+export * from "./lancedb.js";
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,aAAa,CAAC;AAC5B,cAAc,aAAa,CAAC;AAC5B,cAAc,eAAe,CAAC;AAC9B,cAAc,eAAe,CAAC;AAC9B,cAAc,aAAa,CAAC;AAC5B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,sBAAsB,CAAC;AACrC,cAAc,cAAc,CAAC"}