@reposkein/mcp 0.8.0 → 0.9.0

package/binary-digests.json

@@ -1,6 +1,6 @@
 {
-  "reposkein-indexer-darwin-arm64": "a9d796f0e331979c68d30087c2aece4fa042c324008736e8f08346747066a638",
-  "reposkein-indexer-linux-arm64": "b7f57cd831a0b252a225bb535d5ae3b57b0571e43d5593e600e2baf0d7ac477e",
-  "reposkein-indexer-linux-x64": "68c4ae3371ac35fb24198ea871f13dd8a0ccfed449eec166a4b1e488953eb5b0",
-  "reposkein-indexer-win32-x64.exe": "b761e7bb308aea16c9e08d4c71cf130c353fe143d4d2c6c71e29e3c13e191119"
+  "reposkein-indexer-darwin-arm64": "5b6804c12a676c8028b4cbafd28782eda6d2d277b21da143dea8ea358a700f92",
+  "reposkein-indexer-linux-arm64": "96ff841ca329ae49548a009dee316640be81eb8c25caed019414db5717228b91",
+  "reposkein-indexer-linux-x64": "de1dc2f52747332fa5da81b2dd75baaa2dac5428dc4d42643e742369f5b2f813",
+  "reposkein-indexer-win32-x64.exe": "e4fe5a208ae29cf2ef4101a017883412fa6cb00745e325a146381c92c6d0cb48"
 }

package/dist/embed/batch.d.ts

@@ -0,0 +1,47 @@
+/**
+ * The batching seam for embeddings.
+ *
+ * An EmbeddingProvider can only express ONE request (embedBatch). Everything
+ * that decides how much work a request may carry lives here, so no adapter can
+ * opt out of the bound — the local HTTP adapter used to post an entire cold
+ * corpus in a single request (~5 GB at 50k symbols), because chunking was
+ * adapter-local and only the cloud adapter implemented it.
+ *
+ * Batches are handed to the caller as they complete, never accumulated, so peak
+ * memory on the embed path is proportional to one batch rather than the corpus.
+ *
+ * Config env vars (both optional; may only LOWER a provider's declared limits):
+ *   REPOSKEIN_EMBED_MAX_BATCH_ITEMS   max texts per request
+ *   REPOSKEIN_EMBED_MAX_BATCH_TOKENS  max estimated tokens per request
+ */
+import type { BatchLimits, EmbeddingProvider, EmbedKind } from "./provider.js";
+/**
+ * Deterministic token estimate: ~4 characters per token, plus a small
+ * per-text overhead for the delimiters and special tokens a server adds.
+ *
+ * This is a memory budget, not a billing figure — it only has to be
+ * monotonic in length and never wildly under-count.
+ */
+export declare function estimateTokens(text: string): number;
+/**
+ * The limits actually applied to a provider: its own, optionally lowered by env.
+ *
+ * The env can only tighten. A provider's declared limit is a hard fact about
+ * its backend (Voyage rejects >1000 inputs); letting an env var exceed it would
+ * turn a memory knob into a source of 4xx errors.
+ */
+export declare function resolveBatchLimits(provider: EmbeddingProvider, env?: NodeJS.ProcessEnv): BatchLimits;
+/**
+ * Embed `texts` in bounded requests, handing each batch's vectors to `onBatch`
+ * as it arrives together with the offset into `texts` it starts at.
+ *
+ * Nothing is accumulated here: the caller decides what to keep. That is what
+ * lets embedCorpus persist progress per batch, so an interrupted cold run
+ * resumes instead of restarting.
+ *
+ * A text that exceeds maxTokens on its own is sent alone rather than dropped or
+ * split — the server truncates it if it must, and the loop always advances.
+ *
+ * Throws on the first provider failure; callers fall back to lexical.
+ */
+export declare function embedInBatches(provider: EmbeddingProvider, texts: string[], kind: EmbedKind, onBatch: (offset: number, vectors: number[][]) => void | Promise<void>, env?: NodeJS.ProcessEnv): Promise<void>;

package/dist/embed/batch.js

@@ -0,0 +1,92 @@
+/**
+ * The batching seam for embeddings.
+ *
+ * An EmbeddingProvider can only express ONE request (embedBatch). Everything
+ * that decides how much work a request may carry lives here, so no adapter can
+ * opt out of the bound — the local HTTP adapter used to post an entire cold
+ * corpus in a single request (~5 GB at 50k symbols), because chunking was
+ * adapter-local and only the cloud adapter implemented it.
+ *
+ * Batches are handed to the caller as they complete, never accumulated, so peak
+ * memory on the embed path is proportional to one batch rather than the corpus.
+ *
+ * Config env vars (both optional; may only LOWER a provider's declared limits):
+ *   REPOSKEIN_EMBED_MAX_BATCH_ITEMS   max texts per request
+ *   REPOSKEIN_EMBED_MAX_BATCH_TOKENS  max estimated tokens per request
+ */
+/**
+ * Deterministic token estimate: ~4 characters per token, plus a small
+ * per-text overhead for the delimiters and special tokens a server adds.
+ *
+ * This is a memory budget, not a billing figure — it only has to be
+ * monotonic in length and never wildly under-count.
+ */
+export function estimateTokens(text) {
+    return Math.ceil(text.length / 4) + 2;
+}
+/** Read a positive-integer env override, or undefined when absent/malformed. */
+function positiveInt(raw) {
+    if (raw === undefined)
+        return undefined;
+    const n = Number(raw);
+    if (!Number.isInteger(n) || n <= 0)
+        return undefined;
+    return n;
+}
+/**
+ * The limits actually applied to a provider: its own, optionally lowered by env.
+ *
+ * The env can only tighten. A provider's declared limit is a hard fact about
+ * its backend (Voyage rejects >1000 inputs); letting an env var exceed it would
+ * turn a memory knob into a source of 4xx errors.
+ */
+export function resolveBatchLimits(provider, env = process.env) {
+    const declared = provider.limits();
+    const items = positiveInt(env["REPOSKEIN_EMBED_MAX_BATCH_ITEMS"]);
+    const tokens = positiveInt(env["REPOSKEIN_EMBED_MAX_BATCH_TOKENS"]);
+    return {
+        maxItems: items === undefined ? declared.maxItems : Math.min(items, declared.maxItems),
+        maxTokens: tokens === undefined ? declared.maxTokens : Math.min(tokens, declared.maxTokens),
+    };
+}
+/**
+ * Embed `texts` in bounded requests, handing each batch's vectors to `onBatch`
+ * as it arrives together with the offset into `texts` it starts at.
+ *
+ * Nothing is accumulated here: the caller decides what to keep. That is what
+ * lets embedCorpus persist progress per batch, so an interrupted cold run
+ * resumes instead of restarting.
+ *
+ * A text that exceeds maxTokens on its own is sent alone rather than dropped or
+ * split — the server truncates it if it must, and the loop always advances.
+ *
+ * Throws on the first provider failure; callers fall back to lexical.
+ */
+export async function embedInBatches(provider, texts, kind, onBatch, env = process.env) {
+    if (texts.length === 0)
+        return;
+    const { maxItems, maxTokens } = resolveBatchLimits(provider, env);
+    let start = 0;
+    while (start < texts.length) {
+        let end = start;
+        let tokens = 0;
+        while (end < texts.length) {
+            const next = estimateTokens(texts[end]);
+            // Always take at least one text, however oversized, so the loop advances.
+            if (end > start && tokens + next > maxTokens)
+                break;
+            tokens += next;
+            end++;
+            if (end - start >= maxItems)
+                break;
+        }
+        const batch = texts.slice(start, end);
+        const vectors = await provider.embedBatch(batch, kind);
+        if (vectors.length !== batch.length) {
+            throw new Error(`Embedding provider returned ${vectors.length} vectors for a batch of ${batch.length} texts — count mismatch`);
+        }
+        await onBatch(start, vectors);
+        start = end;
+    }
+}
+//# sourceMappingURL=batch.js.map

package/dist/embed/batch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"batch.js","sourceRoot":"","sources":["../../src/embed/batch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAIH;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;AACxC,CAAC;AAED,gFAAgF;AAChF,SAAS,WAAW,CAAC,GAAuB;IAC1C,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IACxC,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IACtB,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACrD,OAAO,CAAC,CAAC;AACX,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAChC,QAA2B,EAC3B,MAAyB,OAAO,CAAC,GAAG;IAEpC,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;IACnC,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC,CAAC;IAClE,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC,CAAC;IACpE,OAAO;QACL,QAAQ,EAAE,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,QAAQ,CAAC;QACtF,SAAS,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,SAAS,CAAC;KAC5F,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,QAA2B,EAC3B,KAAe,EACf,IAAe,EACf,OAAsE,EACtE,MAAyB,OAAO,CAAC,GAAG;IAEpC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAC/B,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,kBAAkB,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAElE,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QAC5B,IAAI,GAAG,GAAG,KAAK,CAAC;QAChB,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,OAAO,GAAG,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,cAAc,CAAC,KAAK,CAAC,GAAG,CAAE,CAAC,CAAC;YACzC,0EAA0E;YAC1E,IAAI,GAAG,GAAG,KAAK,IAAI,MAAM,GAAG,IAAI,GAAG,SAAS;gBAAE,MAAM;YACpD,MAAM,IAAI,IAAI,CAAC;YACf,GAAG,EAAE,CAAC;YACN,IAAI,GAAG,GAAG,KAAK,IAAI,QAAQ;gBAAE,MAAM;QACrC,CAAC;QAED,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QACvD,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CACb,+BAA+B,OAAO,CAAC,MAAM,2BAA2B,KAAK,CAAC,MAAM,yBAAyB,CAC9G,CAAC;QACJ,CAAC;QACD,MAAM,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC9B,KAAK,GAAG,GAAG,CAAC;IACd,CAAC;AACH,CAAC"}

package/dist/embed/cache.d.ts

@@ -1,35 +1,34 @@
 /**
- * Derived embedding cache for semantic_find.
+ * How a corpus node becomes a document string, and how that string is keyed.
  *
- * Vectors are stored in `.reposkein/local/embeddings/<providerId>__<modelId>__d<dims>.jsonl`
- * — gitignored, never committed, never required.
+ * Storage itself lives in vectorStore.ts — fixed-stride float32 beside a JSONL
+ * index — and the orchestration in corpusVectors.ts. This file is only the
+ * text: what we embed, and the hash that decides when to re-embed it.
  *
- * Cache key / invalidation:
- *   1. Filename encodes provider + model + dims (switching any → different file → miss).
- *   2. Per-row `doc_hash` must match hash of the freshly-built document string
- *      (changes to qualified_name, signature, semantic_summary, or file_path → re-embed).
- *   Note: CorpusNode does not expose the committed content_hash, so doc_hash is the
- *   sole per-row invalidation key. It covers all code/summary changes since the doc
- *   string is built from the same fields (qualified_name + signature + summary + file_path).
- *
- * Mirrors the atomic-write + best-effort pattern from mcp/src/store/sidecar.ts.
- * Any I/O or provider failure in embedCorpus must NOT propagate — callers catch and
- * fall back to the lexical result.
+ * Invalidation: `doc_hash` must match the hash of a freshly-built document
+ * string, so a change to qualified_name, signature, semantic_summary or
+ * file_path re-embeds the node. CorpusNode does not expose the committed
+ * content_hash, so this is the sole per-row key — which is sufficient, since
+ * the document is built from exactly those fields.
  */
-import type { EmbeddingProvider } from "./provider.js";
 import type { CorpusNode } from "../store/GraphStore.js";
-/** One cached record per embedded node. */
-export interface EmbedRecord {
-    id: string;
-    /** doc_hash: SHA-256 of the embedded document string (invalidation key). */
-    doc_hash: string;
-    v: number[];
-}
 /** Build the document string for a corpus node (deterministic, same for all calls).
  *  voyage-code-3 is code-specialized, so including code-ish context plays to its strength.
  *  Document = qualified_name + optional signature + optional summary + file_path.
  */
-export declare function buildDocString(node: CorpusNode): string;
+export declare function buildDocString(node: CorpusNode, budget?: number): string;
+/**
+ * Longest document string we will hand a provider.
+ *
+ * This is a correctness bound, not a preference. The bundled embedding server
+ * rejects an over-long input with 413 (EMBED_MAX_INPUT_CHARS, default 8000),
+ * and one rejected document fails the whole embed call — so `semantic_find`
+ * would fall back to lexical, and keep falling back, because the offending
+ * document is never embedded and every later attempt reissues it. Keep this at
+ * or below the server's cap.
+ */
+export declare const DEFAULT_DOC_CHAR_BUDGET = 8000;
+export declare function docCharBudget(env?: NodeJS.ProcessEnv): number;
 /** SHA-256 of a string, hex-encoded. */
 export declare function sha256(s: string): string;
 /**
@@ -39,28 +38,3 @@ export declare function sha256(s: string): string;
  * the FS. Replace any run of unsafe characters with "_".
  */
 export declare function sanitizeModelId(modelId: string): string;
-/** Derive the cache file path for a given repo root + provider. */
-export declare function cachePath(repoPath: string, provider: EmbeddingProvider): string;
-/**
- * Load cache into a Map keyed by node id. Missing file → empty map. Best-effort.
- *
- * Row validation: any row whose `v` is not an array of finite numbers of the
- * correct length (matching provider.dims) is dropped — treated as a miss so
- * it will be re-embedded.  Pass `expectedDims` to enable dim-length validation;
- * omit (undefined) to skip the length check (e.g. when the provider is not known
- * at load time — but the embedCorpus function always passes it).
- */
-export declare function loadCache(path: string, expectedDims?: number): Map<string, EmbedRecord>;
-/** Rewrite the cache file (sorted by id). Atomic: write to .tmp then rename. Best-effort. */
-export declare function saveCache(path: string, records: Map<string, EmbedRecord>): void;
-/**
- * Embed the corpus nodes using the provider, leveraging the cache for nodes
- * whose doc_hash hasn't changed.
- *
- * Returns a Map<id, vector> for all nodes in corpus.
- * Only embeds nodes whose cache entry is missing or stale.
- * Persists updated cache to disk.
- *
- * Throws on embedding failure — callers must catch and fall back to lexical.
- */
-export declare function embedCorpus(provider: EmbeddingProvider, repoPath: string, corpus: CorpusNode[]): Promise<Map<string, number[]>>;

package/dist/embed/cache.js

@@ -1,36 +1,57 @@
 /**
- * Derived embedding cache for semantic_find.
+ * How a corpus node becomes a document string, and how that string is keyed.
  *
- * Vectors are stored in `.reposkein/local/embeddings/<providerId>__<modelId>__d<dims>.jsonl`
- * — gitignored, never committed, never required.
+ * Storage itself lives in vectorStore.ts — fixed-stride float32 beside a JSONL
+ * index — and the orchestration in corpusVectors.ts. This file is only the
+ * text: what we embed, and the hash that decides when to re-embed it.
  *
- * Cache key / invalidation:
- *   1. Filename encodes provider + model + dims (switching any → different file → miss).
- *   2. Per-row `doc_hash` must match hash of the freshly-built document string
- *      (changes to qualified_name, signature, semantic_summary, or file_path → re-embed).
- *   Note: CorpusNode does not expose the committed content_hash, so doc_hash is the
- *   sole per-row invalidation key. It covers all code/summary changes since the doc
- *   string is built from the same fields (qualified_name + signature + summary + file_path).
- *
- * Mirrors the atomic-write + best-effort pattern from mcp/src/store/sidecar.ts.
- * Any I/O or provider failure in embedCorpus must NOT propagate — callers catch and
- * fall back to the lexical result.
+ * Invalidation: `doc_hash` must match the hash of a freshly-built document
+ * string, so a change to qualified_name, signature, semantic_summary or
+ * file_path re-embeds the node. CorpusNode does not expose the committed
+ * content_hash, so this is the sole per-row key — which is sufficient, since
+ * the document is built from exactly those fields.
  */
-import { existsSync, readFileSync, writeFileSync, renameSync, unlinkSync, mkdirSync } from "node:fs";
-import { join, dirname } from "node:path";
 import { createHash } from "node:crypto";
 /** Build the document string for a corpus node (deterministic, same for all calls).
  *  voyage-code-3 is code-specialized, so including code-ish context plays to its strength.
  *  Document = qualified_name + optional signature + optional summary + file_path.
  */
-export function buildDocString(node) {
-    const parts = [node.qualified_name];
+export function buildDocString(node, budget = docCharBudget()) {
+    const head = [node.qualified_name];
     if (node.signature)
-        parts.push(node.signature);
-    if (node.summary)
-        parts.push(node.summary);
-    parts.push(node.file_path);
-    return parts.join("\n");
+        head.push(node.signature);
+    const tail = [node.file_path];
+    // Trim the SUMMARY to fit, not the tail. It is the one unbounded field —
+    // agents author it — and the identifiers are what a search matches on.
+    const overhead = [...head, ...tail].join("\n").length + (node.summary ? 1 : 0);
+    let summary = node.summary ?? "";
+    if (overhead + summary.length > budget) {
+        summary = summary.slice(0, Math.max(0, budget - overhead));
+    }
+    const parts = [...head];
+    if (summary)
+        parts.push(summary);
+    parts.push(...tail);
+    // Last resort, for a node whose own name and path already exceed the budget.
+    return parts.join("\n").slice(0, budget);
+}
+/**
+ * Longest document string we will hand a provider.
+ *
+ * This is a correctness bound, not a preference. The bundled embedding server
+ * rejects an over-long input with 413 (EMBED_MAX_INPUT_CHARS, default 8000),
+ * and one rejected document fails the whole embed call — so `semantic_find`
+ * would fall back to lexical, and keep falling back, because the offending
+ * document is never embedded and every later attempt reissues it. Keep this at
+ * or below the server's cap.
+ */
+export const DEFAULT_DOC_CHAR_BUDGET = 8000;
+export function docCharBudget(env = process.env) {
+    const raw = env["REPOSKEIN_EMBED_MAX_INPUT_CHARS"];
+    if (raw === undefined)
+        return DEFAULT_DOC_CHAR_BUDGET;
+    const n = Number(raw);
+    return Number.isInteger(n) && n > 0 ? n : DEFAULT_DOC_CHAR_BUDGET;
 }
 /** SHA-256 of a string, hex-encoded. */
 export function sha256(s) {
@@ -47,145 +68,4 @@ export function sanitizeModelId(modelId) {
     // Also collapse multiple consecutive underscores to avoid "___"-confusing names.
     return modelId.replace(/[/\\:*?"<>|\x00-\x1f]+/g, "_");
 }
-/** Derive the cache file path for a given repo root + provider. */
-export function cachePath(repoPath, provider) {
-    const safeModel = sanitizeModelId(provider.modelId());
-    const name = `${provider.id()}__${safeModel}__d${provider.dims()}`;
-    return join(repoPath, ".reposkein", "local", "embeddings", `${name}.jsonl`);
-}
-/**
- * Load cache into a Map keyed by node id. Missing file → empty map. Best-effort.
- *
- * Row validation: any row whose `v` is not an array of finite numbers of the
- * correct length (matching provider.dims) is dropped — treated as a miss so
- * it will be re-embedded.  Pass `expectedDims` to enable dim-length validation;
- * omit (undefined) to skip the length check (e.g. when the provider is not known
- * at load time — but the embedCorpus function always passes it).
- */
-export function loadCache(path, expectedDims) {
-    const map = new Map();
-    if (!existsSync(path))
-        return map;
-    let text;
-    try {
-        text = readFileSync(path, "utf8");
-    }
-    catch {
-        return map;
-    }
-    for (const line of text.split("\n")) {
-        if (line.trim() === "")
-            continue;
-        try {
-            const o = JSON.parse(line);
-            if (typeof o["id"] === "string" &&
-                typeof o["doc_hash"] === "string" &&
-                Array.isArray(o["v"])) {
-                const v = o["v"];
-                // Validate: every element must be a finite number
-                const allFinite = v.every((x) => typeof x === "number" && isFinite(x));
-                if (!allFinite)
-                    continue; // corrupt row — drop it (will be re-embedded)
-                // Validate: length must match the expected dims (when known)
-                if (expectedDims !== undefined && v.length !== expectedDims)
-                    continue;
-                map.set(o["id"], {
-                    id: o["id"],
-                    doc_hash: o["doc_hash"],
-                    v: v,
-                });
-            }
-        }
-        catch {
-            // skip malformed line
-        }
-    }
-    return map;
-}
-/** Rewrite the cache file (sorted by id). Atomic: write to .tmp then rename. Best-effort. */
-export function saveCache(path, records) {
-    const lines = [...records.keys()].sort().map((id) => {
-        const r = records.get(id);
-        return JSON.stringify({ id: r.id, doc_hash: r.doc_hash, v: r.v });
-    });
-    const tmp = `${path}.tmp`;
-    try {
-        mkdirSync(dirname(path), { recursive: true });
-        writeFileSync(tmp, lines.length ? lines.join("\n") + "\n" : "");
-        renameSync(tmp, path);
-    }
-    catch {
-        // best-effort; write failure must not break the tool call
-        try {
-            if (existsSync(tmp))
-                unlinkSync(tmp);
-        }
-        catch { /* ignore */ }
-    }
-}
-/**
- * Embed the corpus nodes using the provider, leveraging the cache for nodes
- * whose doc_hash hasn't changed.
- *
- * Returns a Map<id, vector> for all nodes in corpus.
- * Only embeds nodes whose cache entry is missing or stale.
- * Persists updated cache to disk.
- *
- * Throws on embedding failure — callers must catch and fall back to lexical.
- */
-export async function embedCorpus(provider, repoPath, corpus) {
-    const path = cachePath(repoPath, provider);
-    const cache = loadCache(path, provider.dims());
-    // Compute document strings + hashes for all corpus nodes
-    const docStrings = new Map();
-    const docHashes = new Map();
-    for (const node of corpus) {
-        const doc = buildDocString(node);
-        docStrings.set(node.id, doc);
-        docHashes.set(node.id, sha256(doc));
-    }
-    // Identify which nodes need embedding (cache miss or stale doc_hash)
-    const toEmbed = [];
-    for (const node of corpus) {
-        const cached = cache.get(node.id);
-        const isHit = cached !== undefined && cached.doc_hash === docHashes.get(node.id);
-        if (!isHit) {
-            toEmbed.push(node);
-        }
-    }
-    // Embed only misses
-    if (toEmbed.length > 0) {
-        const texts = toEmbed.map((n) => docStrings.get(n.id));
-        const vectors = await provider.embed(texts, "document");
-        // Guard: provider must return exactly as many vectors as we sent.
-        // A count mismatch means the provider is broken or mis-aligned — throw so
-        // the caller's try/catch falls back to lexical.  Never write corrupt data.
-        if (vectors.length !== toEmbed.length) {
-            throw new Error(`Embedding provider returned ${vectors.length} vectors for ${toEmbed.length} texts — count mismatch; refusing to cache`);
-        }
-        const expectedDims = provider.dims();
-        for (let i = 0; i < toEmbed.length; i++) {
-            const vec = vectors[i];
-            if (!Array.isArray(vec) || vec.length !== expectedDims) {
-                throw new Error(`Embedding provider returned a vector with ${Array.isArray(vec) ? vec.length : "undefined"} dims at index ${i}; expected ${expectedDims} — refusing to cache`);
-            }
-            const node = toEmbed[i];
-            cache.set(node.id, {
-                id: node.id,
-                doc_hash: docHashes.get(node.id),
-                v: vec,
-            });
-        }
-        saveCache(path, cache);
-    }
-    // Build result map for all corpus nodes
-    const result = new Map();
-    for (const node of corpus) {
-        const rec = cache.get(node.id);
-        if (rec) {
-            result.set(node.id, rec.v);
-        }
-    }
-    return result;
-}
 //# sourceMappingURL=cache.js.map

package/dist/embed/cache.js.map

@@ -1 +1 @@
-{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../src/embed/cache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACrG,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAYzC;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,IAAgB;IAC7C,MAAM,KAAK,GAAa,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC9C,IAAI,IAAI,CAAC,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC/C,IAAI,IAAI,CAAC,OAAO;QAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC3B,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,wCAAwC;AACxC,MAAM,UAAU,MAAM,CAAC,CAAS;IAC9B,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC9D,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,eAAe,CAAC,OAAe;IAC7C,oEAAoE;IACpE,iFAAiF;IACjF,OAAO,OAAO,CAAC,OAAO,CAAC,yBAAyB,EAAE,GAAG,CAAC,CAAC;AACzD,CAAC;AAED,mEAAmE;AACnE,MAAM,UAAU,SAAS,CAAC,QAAgB,EAAE,QAA2B;IACrE,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC;IACtD,MAAM,IAAI,GAAG,GAAG,QAAQ,CAAC,EAAE,EAAE,KAAK,SAAS,MAAM,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;IACnE,OAAO,IAAI,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,IAAI,QAAQ,CAAC,CAAC;AAC9E,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,YAAqB;IAC3D,MAAM,GAAG,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC3C,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,GAAG,CAAC;IAClC,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACH,IAAI,GAAG,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC;IACb,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,SAAS;QACjC,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAA4B,CAAC;YACtD,IACE,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,QAAQ;gBAC3B,OAAO,CAAC,CAAC,UAAU,CAAC,KAAK,QAAQ;gBACjC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,EACrB,CAAC;gBACD,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,CAAc,CAAC;gBAC9B,kDAAkD;gBAClD,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;gBACvE,IAAI,CAAC,SAAS;oBAAE,SAAS,CAAC,8CAA8C;gBACxE,6DAA6D;gBAC7D,IAAI,YAAY,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,KAAK,YAAY;oBAAE,SAAS;gBACtE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAW,EAAE;oBACzB,EAAE,EAAE,CAAC,CAAC,IAAI,CAAW;oBACrB,QAAQ,EAAE,CAAC,CAAC,UAAU,CAAW;oBACjC,CAAC,EAAE,CAAa;iBACjB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,sBAAsB;QACxB,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,6FAA6F;AAC7F,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,OAAiC;IACvE,MAAM,KAAK,GAAG,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QAClD,MAAM,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;QAC3B,OAAO,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACpE,CAAC,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,GAAG,IAAI,MAAM,CAAC;IAC1B,IAAI,CAAC;QACH,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,aAAa,CAAC,GAAG,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAChE,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,0DAA0D;QAC1D,IAAI,CAAC;YAAC,IAAI,UAAU,CAAC,GAAG,CAAC;gBAAE,UAAU,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IACtE,CAAC;AACH,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,QAA2B,EAC3B,QAAgB,EAChB,MAAoB;IAEpB,MAAM,IAAI,GAAG,SAAS,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAE/C,yDAAyD;IACzD,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC7C,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC5C,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACjC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QAC7B,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;IAED,qEAAqE;IACrE,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAClC,MAAM,KAAK,GAAG,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,KAAK,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjF,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,oBAAoB;IACpB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAE,CAAC,CAAC;QACxD,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAExD,kEAAkE;QAClE,0EAA0E;QAC1E,2EAA2E;QAC3E,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CACb,+BAA+B,OAAO,CAAC,MAAM,gBAAgB,OAAO,CAAC,MAAM,4CAA4C,CACxH,CAAC;QACJ,CAAC;QAED,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;QACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YACvB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;gBACvD,MAAM,IAAI,KAAK,CACb,6CAA6C,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,kBAAkB,CAAC,cAAc,YAAY,sBAAsB,CAC9J,CAAC;YACJ,CAAC;YACD,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC;YACzB,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE;gBACjB,EAAE,EAAE,IAAI,CAAC,EAAE;gBACX,QAAQ,EAAE,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAE;gBACjC,CAAC,EAAE,GAAG;aACP,CAAC,CAAC;QACL,CAAC;QAED,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACzB,CAAC;IAED,wCAAwC;IACxC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC3C,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/B,IAAI,GAAG,EAAE,CAAC;YACR,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
+{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../src/embed/cache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGzC;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,IAAgB,EAAE,MAAM,GAAG,aAAa,EAAE;IACvE,MAAM,IAAI,GAAa,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC7C,IAAI,IAAI,CAAC,SAAS;QAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC9C,MAAM,IAAI,GAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAExC,yEAAyE;IACzE,uEAAuE;IACvE,MAAM,QAAQ,GAAG,CAAC,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/E,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;IACjC,IAAI,QAAQ,GAAG,OAAO,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC;QACvC,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC;IAC7D,CAAC;IAED,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;IACxB,IAAI,OAAO;QAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACjC,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;IACpB,6EAA6E;IAC7E,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;AAC3C,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,IAAI,CAAC;AAE5C,MAAM,UAAU,aAAa,CAAC,MAAyB,OAAO,CAAC,GAAG;IAChE,MAAM,GAAG,GAAG,GAAG,CAAC,iCAAiC,CAAC,CAAC;IACnD,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,uBAAuB,CAAC;IACtD,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IACtB,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,uBAAuB,CAAC;AACpE,CAAC;AAED,wCAAwC;AACxC,MAAM,UAAU,MAAM,CAAC,CAAS;IAC9B,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC9D,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,eAAe,CAAC,OAAe;IAC7C,oEAAoE;IACpE,iFAAiF;IACjF,OAAO,OAAO,CAAC,OAAO,CAAC,yBAAyB,EAAE,GAAG,CAAC,CAAC;AACzD,CAAC"}

package/dist/embed/corpusVectors.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Corpus vectors for `semantic_find`, behind the vector store.
+ *
+ * Replaces the JSON cache's "hand me every vector as number[]" shape. Callers
+ * get a store they can ask for a ranked top-k; they never hold the corpus.
+ */
+import type { EmbeddingProvider } from "./provider.js";
+import { type VectorStore } from "./vectorStore.js";
+import type { CorpusNode } from "../store/GraphStore.js";
+/** Base path (no extension) for a provider's vectors in this repo. */
+export declare function vectorBasePath(repoPath: string, provider: EmbeddingProvider): string;
+/**
+ * Ensure every corpus node has a current vector, and return the store.
+ *
+ * Only misses are embedded, in bounded batches, each flushed as it lands so an
+ * interrupted cold run resumes rather than restarting.
+ *
+ * Throws on embedding failure — callers catch and fall back to lexical.
+ */
+export declare function ensureCorpusVectors(provider: EmbeddingProvider, repoPath: string, corpus: CorpusNode[]): Promise<VectorStore>;

package/dist/embed/corpusVectors.js

@@ -0,0 +1,60 @@
+/**
+ * Corpus vectors for `semantic_find`, behind the vector store.
+ *
+ * Replaces the JSON cache's "hand me every vector as number[]" shape. Callers
+ * get a store they can ask for a ranked top-k; they never hold the corpus.
+ */
+import { join } from "node:path";
+import { embedInBatches } from "./batch.js";
+import { buildDocString, sanitizeModelId, sha256 } from "./cache.js";
+import { openVectorStore, removeLegacyJsonCache } from "./vectorStore.js";
+/** Base path (no extension) for a provider's vectors in this repo. */
+export function vectorBasePath(repoPath, provider) {
+    const name = `${provider.id()}__${sanitizeModelId(provider.modelId())}__d${provider.dims()}`;
+    return join(repoPath, ".reposkein", "local", "embeddings", name);
+}
+/**
+ * Ensure every corpus node has a current vector, and return the store.
+ *
+ * Only misses are embedded, in bounded batches, each flushed as it lands so an
+ * interrupted cold run resumes rather than restarting.
+ *
+ * Throws on embedding failure — callers catch and fall back to lexical.
+ */
+export async function ensureCorpusVectors(provider, repoPath, corpus) {
+    const base = vectorBasePath(repoPath, provider);
+    // The pre-binary cache is superseded, and at 200k nodes it was over 4 GB of
+    // dead JSON. Drop it the first time we write the store beside it.
+    removeLegacyJsonCache(base);
+    const store = openVectorStore(base, provider.dims());
+    const docs = new Map();
+    const hashes = new Map();
+    const misses = [];
+    for (const node of corpus) {
+        const doc = buildDocString(node);
+        docs.set(node.id, doc);
+        const hash = sha256(doc);
+        hashes.set(node.id, hash);
+        if (store.docHash(node.id) !== hash)
+            misses.push(node);
+    }
+    if (misses.length === 0)
+        return store;
+    const texts = misses.map((n) => docs.get(n.id));
+    const expectedDims = provider.dims();
+    await embedInBatches(provider, texts, "document", (offset, vectors) => {
+        const batch = [];
+        for (let i = 0; i < vectors.length; i++) {
+            const vec = vectors[i];
+            if (!Array.isArray(vec) || vec.length !== expectedDims) {
+                throw new Error(`Embedding provider returned a vector with ${Array.isArray(vec) ? vec.length : "undefined"} dims at index ${offset + i}; expected ${expectedDims} — refusing to store`);
+            }
+            const node = misses[offset + i];
+            batch.push({ id: node.id, docHash: hashes.get(node.id), vector: vec });
+        }
+        store.upsertMany(batch);
+        store.flush();
+    });
+    return store;
+}
+//# sourceMappingURL=corpusVectors.js.map

package/dist/embed/corpusVectors.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"corpusVectors.js","sourceRoot":"","sources":["../../src/embed/corpusVectors.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AACrE,OAAO,EAAE,eAAe,EAAE,qBAAqB,EAAoB,MAAM,kBAAkB,CAAC;AAG5F,sEAAsE;AACtE,MAAM,UAAU,cAAc,CAAC,QAAgB,EAAE,QAA2B;IAC1E,MAAM,IAAI,GAAG,GAAG,QAAQ,CAAC,EAAE,EAAE,KAAK,eAAe,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;IAC7F,OAAO,IAAI,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;AACnE,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,QAA2B,EAC3B,QAAgB,EAChB,MAAoB;IAEpB,MAAM,IAAI,GAAG,cAAc,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAChD,4EAA4E;IAC5E,kEAAkE;IAClE,qBAAqB,CAAC,IAAI,CAAC,CAAC;IAE5B,MAAM,KAAK,GAAG,eAAe,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAErD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;IACvC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QACvB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QACzB,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAC1B,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,IAAI;YAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACzD,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAEtC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAE,CAAC,CAAC;IACjD,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;IAErC,MAAM,cAAc,CAAC,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE;QACpE,MAAM,KAAK,GAAG,EAAE,CAAC;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YACvB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;gBACvD,MAAM,IAAI,KAAK,CACb,6CAA6C,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,kBAAkB,MAAM,GAAG,CAAC,cAAc,YAAY,sBAAsB,CACvK,CAAC;YACJ,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC;YACjC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACxB,KAAK,CAAC,KAAK,EAAE,CAAC;IAChB,CAAC,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/embed/provider.d.ts

@@ -12,6 +12,19 @@
  *   REPOSKEIN_EMBED_URL       base URL for http provider
  */
 export type EmbedKind = "document" | "query";
+/**
+ * What a single request to this provider may carry.
+ *
+ * The shared batcher (embed/batch.ts) enforces these; an adapter never chunks
+ * for itself. `maxTokens` is measured with estimateTokens, a deterministic
+ * length heuristic — it is a memory budget, not a billing figure.
+ */
+export interface BatchLimits {
+    /** Max texts in one request. */
+    maxItems: number;
+    /** Max estimated tokens across one request's texts. */
+    maxTokens: number;
+}
 export interface EmbeddingProvider {
     /** Stable provider id, e.g. "voyage". Forms part of the cache key. */
     id(): string;
@@ -19,13 +32,19 @@ export interface EmbeddingProvider {
     modelId(): string;
     /** Output vector dimensionality, e.g. 1024. Part of the cache key. */
     dims(): number;
+    /** Per-request limits. Callers go through embedInBatches, which enforces them. */
+    limits(): BatchLimits;
     /**
-     * Embed a batch of texts.
+     * Embed ONE request's worth of texts. Implementations must NOT batch
+     * internally: the caller has already sized this call against limits().
+     * Splitting here would put the memory bound back inside each adapter, which
+     * is exactly how the local HTTP adapter came to send whole corpora at once.
+     *
      * kind maps to Voyage's input_type ("document" vs "query") — Voyage uses
      * asymmetric embeddings so this materially changes the vectors.
      * Returns one number[] per input, in order.
      */
-    embed(texts: string[], kind: EmbedKind): Promise<number[][]>;
+    embedBatch(texts: string[], kind: EmbedKind): Promise<number[][]>;
 }
 /**
  * Build a provider from env vars, or null if embeddings are disabled (default).

package/dist/embed/provider.js.map

@@ -1 +1 @@
-{"version":3,"file":"provider.js","sourceRoot":"","sources":["../../src/embed/provider.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAoBH;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAyB,OAAO,CAAC,GAAG;IACxE,MAAM,YAAY,GAAG,CAAC,GAAG,CAAC,0BAA0B,CAAC,IAAI,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;IAC/E,IAAI,YAAY,KAAK,MAAM,IAAI,YAAY,KAAK,EAAE;QAAE,OAAO,IAAI,CAAC;IAEhE,IAAI,YAAY,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,EAAE,uBAAuB,EAAE,GAAG,MAAM,MAAM,CAAC,uBAAuB,CAAC,CAAC;QAC1E,OAAO,IAAI,uBAAuB,CAAC,GAAG,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,YAAY,KAAK,MAAM,EAAE,CAAC;QAC5B,MAAM,EAAE,qBAAqB,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAC;QACtE,OAAO,IAAI,qBAAqB,CAAC,GAAG,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,IAAI,KAAK,CACb,sCAAsC,YAAY,kCAAkC,CACrF,CAAC;AACJ,CAAC"}
+{"version":3,"file":"provider.js","sourceRoot":"","sources":["../../src/embed/provider.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAwCH;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAyB,OAAO,CAAC,GAAG;IACxE,MAAM,YAAY,GAAG,CAAC,GAAG,CAAC,0BAA0B,CAAC,IAAI,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;IAC/E,IAAI,YAAY,KAAK,MAAM,IAAI,YAAY,KAAK,EAAE;QAAE,OAAO,IAAI,CAAC;IAEhE,IAAI,YAAY,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,EAAE,uBAAuB,EAAE,GAAG,MAAM,MAAM,CAAC,uBAAuB,CAAC,CAAC;QAC1E,OAAO,IAAI,uBAAuB,CAAC,GAAG,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,YAAY,KAAK,MAAM,EAAE,CAAC;QAC5B,MAAM,EAAE,qBAAqB,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAC;QACtE,OAAO,IAAI,qBAAqB,CAAC,GAAG,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,IAAI,KAAK,CACb,sCAAsC,YAAY,kCAAkC,CACrF,CAAC;AACJ,CAAC"}