codesift-mcp 0.15.3 → 0.17.0

package/dist/search/bm25-store.js

@@ -0,0 +1,164 @@
+import { createReadStream, createWriteStream } from "node:fs";
+import { rename, unlink, stat } from "node:fs/promises";
+import { createInterface } from "node:readline";
+/**
+ * Persist a BM25 index next to the code index it was built from.
+ *
+ * Rebuilding costs 10.04 s on the largest repo here (352,166 symbols, 12.9M tokens) and is paid
+ * once per repo per process — so every daemon restart, and every eviction under the cache budget,
+ * charges it again. Measured alternative on the same index: reconstructing the maps from flat
+ * arrays is **0.70 s** and reading 400 MB off this disk is 0.06 s. 13x, and the reason is that the
+ * expensive half of a build is TOKENISING every symbol, not assembling the maps.
+ *
+ * Line-delimited, not one JSON document: `JSON.stringify` on the whole structure exceeds V8's
+ * maximum string length and throws outright. The same reason `embeddings.ndjson` is streamed.
+ *
+ * `symbols` is deliberately NOT written. Every one of those objects is already in the code index
+ * that gets loaded first, so persisting them would double the bytes on a disk that is, on this
+ * machine, the actual bottleneck. They are reattached on load from that index.
+ */
+/** Bump on any format change: a mismatch rebuilds rather than misreads. */
+const FORMAT_VERSION = 1;
+const FIELDS = ["name", "signature", "docstring", "body", "comments"];
+function headerFor(index, code) {
+    return {
+        v: FORMAT_VERSION,
+        docCount: index.docCount,
+        avg: index.avgFieldLengths,
+        tot: index.totalFieldLengths,
+        symbolCount: code.symbols.length,
+        fileCount: code.files.length,
+        indexUpdatedAt: code.updated_at ?? code.created_at ?? 0,
+    };
+}
+/**
+ * A cache that does not describe the current index is worse than no cache: it returns confident,
+ * wrong search results, which is the one failure mode a search tool must never have. So the check
+ * is deliberately cheap AND strict — any disagreement rebuilds, and nothing here tries to repair a
+ * partial match.
+ */
+function isStale(header, code) {
+    if (header.v !== FORMAT_VERSION)
+        return true;
+    if (header.symbolCount !== code.symbols.length)
+        return true;
+    if (header.fileCount !== code.files.length)
+        return true;
+    return header.indexUpdatedAt !== (code.updated_at ?? code.created_at ?? 0);
+}
+export function bm25PathFor(indexPath) {
+    return indexPath.replace(/\.index\.json$/, "").replace(/\.index\.db$/, "") + ".bm25.ndjson";
+}
+export async function saveBM25Index(indexPath, index, code) {
+    const target = bm25PathFor(indexPath);
+    // Temp + rename, like every other artifact here: a process killed mid-write must not leave a
+    // truncated file that the next start would read as a complete index.
+    const temp = `${target}.tmp.${process.pid}`;
+    const out = createWriteStream(temp, { encoding: "utf-8" });
+    const write = (line) => out.write(line) ? Promise.resolve() : new Promise((r) => out.once("drain", () => r()));
+    try {
+        await write(`${JSON.stringify(headerFor(index, code))}\n`);
+        for (const field of FIELDS) {
+            for (const [token, postings] of index.fields[field]) {
+                // Flat [id, tf, id, tf, …]: half the JSON of an array of pairs, and it rebuilds with one
+                // loop rather than a destructuring per entry.
+                const flat = [];
+                for (const [id, tf] of postings) {
+                    flat.push(id);
+                    flat.push(tf);
+                }
+                await write(`${JSON.stringify(["p", field, token, flat])}\n`);
+            }
+        }
+        for (const [id, lengths] of index.fieldLengths) {
+            await write(`${JSON.stringify(["l", id, lengths.name, lengths.signature, lengths.docstring, lengths.body, lengths.comments])}\n`);
+        }
+        for (const [file, score] of index.centrality) {
+            await write(`${JSON.stringify(["c", file, score])}\n`);
+        }
+        await new Promise((resolve, reject) => {
+            out.end(() => resolve());
+            out.on("error", reject);
+        });
+        await rename(temp, target);
+    }
+    catch {
+        await unlink(temp).catch(() => { });
+        // Never fail a build over its cache. The caller has a working index in memory either way.
+    }
+}
+export async function loadBM25Index(indexPath, code) {
+    const target = bm25PathFor(indexPath);
+    try {
+        if (!(await stat(target)).isFile())
+            return null;
+    }
+    catch {
+        return null;
+    }
+    const fields = {
+        name: new Map(), signature: new Map(), docstring: new Map(),
+        body: new Map(), comments: new Map(),
+    };
+    const fieldLengths = new Map();
+    const centrality = new Map();
+    let header = null;
+    try {
+        const rl = createInterface({ input: createReadStream(target, { encoding: "utf-8" }), crlfDelay: Infinity });
+        for await (const line of rl) {
+            if (!line)
+                continue;
+            if (header === null) {
+                header = JSON.parse(line);
+                // Checked BEFORE reading 8.9M posting entries, not after: a stale cache should cost one
+                // line, not a full parse followed by a discard.
+                if (isStale(header, code)) {
+                    rl.close();
+                    return null;
+                }
+                continue;
+            }
+            const row = JSON.parse(line);
+            const kind = row[0];
+            if (kind === "p") {
+                const field = row[1];
+                const flat = row[3];
+                const postings = new Map();
+                for (let i = 0; i < flat.length; i += 2)
+                    postings.set(flat[i], flat[i + 1]);
+                fields[field].set(row[2], postings);
+            }
+            else if (kind === "l") {
+                fieldLengths.set(row[1], {
+                    name: row[2], signature: row[3], docstring: row[4],
+                    body: row[5], comments: row[6],
+                });
+            }
+            else if (kind === "c") {
+                centrality.set(row[1], row[2]);
+            }
+        }
+    }
+    catch {
+        // Truncated, corrupt, or written by a version that disagrees. Rebuilding is always correct;
+        // reading half an index is not.
+        return null;
+    }
+    if (header === null)
+        return null;
+    // Reattached rather than persisted — see the file header. The map must be built the same way the
+    // builder does it (last write wins), because symbol ids are documented as non-unique.
+    const symbols = new Map();
+    for (const symbol of code.symbols)
+        symbols.set(symbol.id, symbol);
+    return {
+        fields,
+        avgFieldLengths: header.avg,
+        docCount: header.docCount,
+        symbols,
+        centrality,
+        fieldLengths,
+        totalFieldLengths: header.tot,
+    };
+}
+//# sourceMappingURL=bm25-store.js.map

package/dist/search/bm25-store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bm25-store.js","sourceRoot":"","sources":["../../src/search/bm25-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAIhD;;;;;;;;;;;;;;;GAeG;AAEH,2EAA2E;AAC3E,MAAM,cAAc,GAAG,CAAC,CAAC;AAGzB,MAAM,MAAM,GAAgB,CAAC,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;AAanF,SAAS,SAAS,CAAC,KAAgB,EAAE,IAAe;IAClD,OAAO;QACL,CAAC,EAAE,cAAc;QACjB,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,GAAG,EAAE,KAAK,CAAC,eAAe;QAC1B,GAAG,EAAE,KAAK,CAAC,iBAAiB;QAC5B,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;QAChC,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;QAC5B,cAAc,EAAE,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC;KACxD,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,SAAS,OAAO,CAAC,MAAc,EAAE,IAAe;IAC9C,IAAI,MAAM,CAAC,CAAC,KAAK,cAAc;QAAE,OAAO,IAAI,CAAC;IAC7C,IAAI,MAAM,CAAC,WAAW,KAAK,IAAI,CAAC,OAAO,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAC5D,IAAI,MAAM,CAAC,SAAS,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IACxD,OAAO,MAAM,CAAC,cAAc,KAAK,CAAC,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC;AAC7E,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,SAAiB;IAC3C,OAAO,SAAS,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,GAAG,cAAc,CAAC;AAC9F,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,SAAiB,EACjB,KAAgB,EAChB,IAAe;IAEf,MAAM,MAAM,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;IACtC,6FAA6F;IAC7F,qEAAqE;IACrE,MAAM,IAAI,GAAG,GAAG,MAAM,QAAQ,OAAO,CAAC,GAAG,EAAE,CAAC;IAC5C,MAAM,GAAG,GAAG,iBAAiB,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;IAE3D,MAAM,KAAK,GAAG,CAAC,IAAY,EAAiB,EAAE,CAC5C,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAEzF,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;QAC3D,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;gBACpD,yFAAyF;gBACzF,8CAA8C;gBAC9C,MAAM,IAAI,GAAwB,EAAE,CAAC;gBACrC,KAAK,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,QAAQ,EAAE,CAAC;oBAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;oBAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAAC,CAAC;gBAClE,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;QACD,KAAK,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;YAC/C,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC;QACpI,CAAC;QACD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YAC7C,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;QACzD,CAAC;QACD,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1C,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;YACzB,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QACH,MAAM,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACnC,0FAA0F;IAC5F,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,SAAiB,EACjB,IAAe;IAEf,MAAM,MAAM,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;IACtC,IAAI,CAAC;QACH,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE;YAAE,OAAO,IAAI,CAAC;IAClD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,MAAM,GAAwD;QAClE,IAAI,EAAE,IAAI,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,GAAG,EAAE;QAC3D,IAAI,EAAE,IAAI,GAAG,EAAE,EAAE,QAAQ,EAAE,IAAI,GAAG,EAAE;KACrC,CAAC;IACF,MAAM,YAAY,GAAG,IAAI,GAAG,EAAqC,CAAC;IAClE,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC7C,IAAI,MAAM,GAAkB,IAAI,CAAC;IAEjC,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,gBAAgB,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC5G,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;YAC5B,IAAI,CAAC,IAAI;gBAAE,SAAS;YACpB,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;gBACpB,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAW,CAAC;gBACpC,wFAAwF;gBACxF,gDAAgD;gBAChD,IAAI,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;oBAAC,EAAE,CAAC,KAAK,EAAE,CAAC;oBAAC,OAAO,IAAI,CAAC;gBAAC,CAAC;gBACvD,SAAS;YACX,CAAC;YACD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAc,CAAC;YAC1C,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;YACpB,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;gBACjB,MAAM,KAAK,GAAG,GAAG,CAAC,CAAC,CAAc,CAAC;gBAClC,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAwB,CAAC;gBAC3C,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;gBAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC;oBAAE,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAW,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAW,CAAC,CAAC;gBAChG,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAW,EAAE,QAAQ,CAAC,CAAC;YAChD,CAAC;iBAAM,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;gBACxB,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAW,EAAE;oBACjC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAW,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAW,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAW;oBAChF,IAAI,EAAE,GAAG,CAAC,CAAC,CAAW,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAW;iBACnD,CAAC,CAAC;YACL,CAAC;iBAAM,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;gBACxB,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAW,EAAE,GAAG,CAAC,CAAC,CAAW,CAAC,CAAC;YACrD,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,4FAA4F;QAC5F,gCAAgC;QAChC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAEjC,iGAAiG;IACjG,sFAAsF;IACtF,MAAM,OAAO,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC9C,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO;QAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAElE,OAAO;QACL,MAAM;QACN,eAAe,EAAE,MAAM,CAAC,GAAG;QAC3B,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,OAAO;QACP,UAAU;QACV,YAAY;QACZ,iBAAiB,EAAE,MAAM,CAAC,GAAG;KAC9B,CAAC;AACJ,CAAC"}

package/dist/search/bm25.d.ts

@@ -13,6 +13,12 @@ export interface BM25Index {
     centrality: Map<string, number>;
     /** Pre-computed per-document field lengths (avoids O(n*m) recomputation per search) */
     fieldLengths: Map<string, Record<FieldName, number>>;
+    /**
+     * Running per-field token totals. `avgFieldLengths` is derived from these, and keeping the
+     * numerator lets one file's symbols be swapped without rescanning every document. Deriving it
+     * back as `avg * docCount` would work on paper and accumulate float error in practice.
+     */
+    totalFieldLengths: Record<FieldName, number>;
 }
 /**
  * General-purpose tokenizer for signature, docstring, and body text.
@@ -20,7 +26,31 @@ export interface BM25Index {
  * lowercases, and filters tokens shorter than 2 chars.
  */
 export declare function tokenizeText(text: string): string[];
+/**
+ * Swap one file's symbols in place, instead of throwing the whole index away.
+ *
+ * Editing a single file used to delete the repository's entire BM25 index, and the next search
+ * rebuilt it from scratch — measured 6.8 s on a 372k-symbol repository, against 952 `index_file`
+ * calls in a week. The agent loop is edit-then-search, so that rebuild was being paid constantly.
+ *
+ * `centrality` is deliberately NOT recomputed. It is an O(imports x files) scan over every symbol
+ * in the repository — the thing this function exists to avoid — and it is a ranking bonus derived
+ * from a substring heuristic, not a correctness input. One file's imports moving leaves it
+ * marginally stale until the next full build; a 6.8 s pause would not.
+ */
+export declare function updateBM25ForFile(index: BM25Index, file: string, symbols: CodeSymbol[]): void;
+/**
+ * Synchronous build. Correct, and fine for small inputs — the tool-ranker index is ~150 entries.
+ * Do NOT use it on a repository index inside the daemon: see buildBM25IndexYielding.
+ */
 export declare function buildBM25Index(symbols: CodeSymbol[]): BM25Index;
+/**
+ * Same index, built without monopolising the event loop.
+ *
+ * Identical work and identical output — it simply hands the loop a turn every SYMBOLS_PER_TURN
+ * symbols, so other clients keep getting answers while a large repository is indexed.
+ */
+export declare function buildBM25IndexYielding(symbols: CodeSymbol[]): Promise<BM25Index>;
 export declare function searchBM25(index: BM25Index, query: string, topK: number, fieldWeights: Record<FieldName, number>): SearchResult[];
 export declare function applyCutoff(results: SearchResult[]): SearchResult[];
 export {};

package/dist/search/bm25.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"bm25.d.ts","sourceRoot":"","sources":["../../src/search/bm25.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAe5D,KAAK,SAAS,GAAG,MAAM,GAAG,WAAW,GAAG,WAAW,GAAG,MAAM,GAAG,UAAU,CAAC;AAE1E,MAAM,WAAW,SAAS;IACxB,sEAAsE;IACtE,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IAC5D,oDAAoD;IACpD,eAAe,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC3C,wCAAwC;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,0BAA0B;IAC1B,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IACjC,sFAAsF;IACtF,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,uFAAuF;IACvF,YAAY,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;CACtD;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAqBnD;AA8CD,wBAAgB,cAAc,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,SAAS,CAoF/D;AAED,wBAAgB,UAAU,CACxB,KAAK,EAAE,SAAS,EAChB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,GACtC,YAAY,EAAE,CAwFhB;AAKD,wBAAgB,WAAW,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,YAAY,EAAE,CAWnE"}
+{"version":3,"file":"bm25.d.ts","sourceRoot":"","sources":["../../src/search/bm25.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAe5D,KAAK,SAAS,GAAG,MAAM,GAAG,WAAW,GAAG,WAAW,GAAG,MAAM,GAAG,UAAU,CAAC;AAE1E,MAAM,WAAW,SAAS;IACxB,sEAAsE;IACtE,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IAC5D,oDAAoD;IACpD,eAAe,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC3C,wCAAwC;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,0BAA0B;IAC1B,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IACjC,sFAAsF;IACtF,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,uFAAuF;IACvF,YAAY,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;IACrD;;;;OAIG;IACH,iBAAiB,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;CAC9C;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAqBnD;AA8MD;;;;;;;;;;;GAWG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,IAAI,CAsB7F;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,SAAS,CAI/D;AAED;;;;;GAKG;AACH,wBAAsB,sBAAsB,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,SAAS,CAAC,CAQtF;AAED,wBAAgB,UAAU,CACxB,KAAK,EAAE,SAAS,EAChB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,GACtC,YAAY,EAAE,CAwFhB;AAKD,wBAAgB,WAAW,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,YAAY,EAAE,CAWnE"}

package/dist/search/bm25.js

@@ -73,44 +73,67 @@ function countTermFrequencies(tokens) {
     }
     return tf;
 }
-export function buildBM25Index(symbols) {
-    const fieldNames = ["name", "signature", "docstring", "body", "comments"];
-    const fields = {
-        name: new Map(),
-        signature: new Map(),
-        docstring: new Map(),
-        body: new Map(),
-        comments: new Map(),
+/**
+ * Symbols ingested per turn before the event loop gets one.
+ *
+ * Building this index is the single longest synchronous burst in the process. Measured on the
+ * largest real index here (372,949 symbols, 20,132 files): **19.5 seconds during which a 20 ms
+ * timer fired ZERO times**. In the shared daemon that is 19.5 seconds where nothing is answered —
+ * not another client's search, not `/health` — and it is the largest single contributor to what
+ * users reported as "CodeSift is down".
+ *
+ * The split matters for where to yield: 18.3 s of that is the tokenise-and-map loop below, and only
+ * 1.2 s the import-centrality pass after it. So the ingest loop is what yields.
+ */
+/** Module scope now that ingestion is extracted — it was a local of the old single function. */
+const fieldNames = ["name", "signature", "docstring", "body", "comments"];
+const SYMBOLS_PER_TURN = 2000;
+function yieldToEventLoop() {
+    return new Promise((resolve) => setImmediate(resolve));
+}
+function newAccumulator() {
+    return {
+        fields: {
+            name: new Map(), signature: new Map(), docstring: new Map(),
+            body: new Map(), comments: new Map(),
+        },
+        totalFieldLengths: { name: 0, signature: 0, docstring: 0, body: 0, comments: 0 },
+        symbolMap: new Map(),
+        fieldLengths: new Map(),
     };
-    const totalFieldLengths = {
-        name: 0,
-        signature: 0,
-        docstring: 0,
-        body: 0,
-        comments: 0,
+}
+/** One symbol into the inverted index. Extracted so both builders run byte-identical work. */
+function ingestSymbol(acc, symbol) {
+    acc.symbolMap.set(symbol.id, symbol);
+    const fieldTokens = getFieldTokens(symbol);
+    const lengths = {
+        name: 0, signature: 0, docstring: 0, body: 0, comments: 0,
     };
-    const symbolMap = new Map();
-    const fieldLengths = new Map();
-    for (const symbol of symbols) {
-        symbolMap.set(symbol.id, symbol);
-        const fieldTokens = getFieldTokens(symbol);
-        const lengths = { name: 0, signature: 0, docstring: 0, body: 0, comments: 0 };
-        for (const field of fieldNames) {
-            const tokens = fieldTokens[field];
-            totalFieldLengths[field] += tokens.length;
-            lengths[field] = tokens.length;
-            const tf = countTermFrequencies(tokens);
-            for (const [token, freq] of tf) {
-                let postings = fields[field].get(token);
-                if (!postings) {
-                    postings = new Map();
-                    fields[field].set(token, postings);
-                }
-                postings.set(symbol.id, freq);
+    for (const field of fieldNames) {
+        const tokens = fieldTokens[field];
+        acc.totalFieldLengths[field] += tokens.length;
+        lengths[field] = tokens.length;
+        const tf = countTermFrequencies(tokens);
+        for (const [token, freq] of tf) {
+            let postings = acc.fields[field].get(token);
+            if (!postings) {
+                postings = new Map();
+                acc.fields[field].set(token, postings);
             }
+            postings.set(symbol.id, freq);
         }
-        fieldLengths.set(symbol.id, lengths);
     }
+    acc.fieldLengths.set(symbol.id, lengths);
+}
+/**
+ * Averages plus import centrality — 1.2 s of the 19.5 on the largest index, so it stays synchronous.
+ *
+ * The inner `for (const file of allFiles)` is a linear scan per import match, i.e. O(imports x files).
+ * At 6% of the build it was not worth changing while fixing the blocking, but it is the obvious next
+ * thing if this pass ever grows.
+ */
+function finishBuild(acc, symbols) {
+    const { fields, totalFieldLengths, symbolMap, fieldLengths } = acc;
     const docCount = symbols.length;
     const avgFieldLengths = {
         name: docCount > 0 ? totalFieldLengths.name / docCount : 0,
@@ -147,7 +170,109 @@ export function buildBM25Index(symbols) {
     for (const [file, count] of importCount) {
         centrality.set(file, Math.log2(1 + count));
     }
-    return { fields, avgFieldLengths, docCount, symbols: symbolMap, centrality, fieldLengths };
+    return { fields, avgFieldLengths, docCount, symbols: symbolMap, centrality, fieldLengths, totalFieldLengths };
+}
+function recomputeAverages(index) {
+    const n = index.docCount;
+    for (const field of fieldNames) {
+        index.avgFieldLengths[field] = n > 0 ? index.totalFieldLengths[field] / n : 0;
+    }
+}
+/**
+ * Undo one symbol's contribution to the inverted index.
+ *
+ * No reverse token map is needed: the tokens a symbol contributed are a pure function of the
+ * symbol, and the symbol itself is still in `index.symbols`. Re-deriving them is exact and costs
+ * one `getFieldTokens` call, against a reverse map that would have to be kept correct forever.
+ *
+ * Field lengths come from the STORED record rather than the re-derived tokens, so the running
+ * totals stay symmetric with what ingest actually added even if tokenisation ever changes under a
+ * long-lived index.
+ */
+function removeSymbolFromIndex(index, symbol) {
+    const fieldTokens = getFieldTokens(symbol);
+    const stored = index.fieldLengths.get(symbol.id);
+    for (const field of fieldNames) {
+        const tokens = fieldTokens[field];
+        index.totalFieldLengths[field] -= stored ? stored[field] : tokens.length;
+        const postings = index.fields[field];
+        for (const token of new Set(tokens)) {
+            const forToken = postings.get(token);
+            if (!forToken)
+                continue;
+            forToken.delete(symbol.id);
+            // A token nobody carries any more must go, or the vocabulary grows without bound across a
+            // long-lived daemon and every idf denominator drifts.
+            if (forToken.size === 0)
+                postings.delete(token);
+        }
+    }
+    index.symbols.delete(symbol.id);
+    index.fieldLengths.delete(symbol.id);
+    index.docCount--;
+}
+/**
+ * Swap one file's symbols in place, instead of throwing the whole index away.
+ *
+ * Editing a single file used to delete the repository's entire BM25 index, and the next search
+ * rebuilt it from scratch — measured 6.8 s on a 372k-symbol repository, against 952 `index_file`
+ * calls in a week. The agent loop is edit-then-search, so that rebuild was being paid constantly.
+ *
+ * `centrality` is deliberately NOT recomputed. It is an O(imports x files) scan over every symbol
+ * in the repository — the thing this function exists to avoid — and it is a ranking bonus derived
+ * from a substring heuristic, not a correctness input. One file's imports moving leaves it
+ * marginally stale until the next full build; a 6.8 s pause would not.
+ */
+export function updateBM25ForFile(index, file, symbols) {
+    // Select by the STORED symbol's file, never by parsing the incoming ids: ids are
+    // `repo:file:name:line` and are documented as non-unique, so an incoming id can collide with a
+    // symbol that lives in a different file. Matching on the stored record cannot touch it.
+    const stale = [];
+    for (const existing of index.symbols.values()) {
+        if (existing.file === file)
+            stale.push(existing);
+    }
+    for (const symbol of stale)
+        removeSymbolFromIndex(index, symbol);
+    const acc = {
+        fields: index.fields,
+        totalFieldLengths: index.totalFieldLengths,
+        symbolMap: index.symbols,
+        fieldLengths: index.fieldLengths,
+    };
+    for (const symbol of symbols) {
+        ingestSymbol(acc, symbol);
+        index.docCount++;
+    }
+    recomputeAverages(index);
+}
+/**
+ * Synchronous build. Correct, and fine for small inputs — the tool-ranker index is ~150 entries.
+ * Do NOT use it on a repository index inside the daemon: see buildBM25IndexYielding.
+ */
+export function buildBM25Index(symbols) {
+    const acc = newAccumulator();
+    for (const symbol of symbols)
+        ingestSymbol(acc, symbol);
+    return finishBuild(acc, symbols);
+}
+/**
+ * Same index, built without monopolising the event loop.
+ *
+ * Identical work and identical output — it simply hands the loop a turn every SYMBOLS_PER_TURN
+ * symbols, so other clients keep getting answers while a large repository is indexed.
+ */
+export async function buildBM25IndexYielding(symbols) {
+    const acc = newAccumulator();
+    let sinceYield = 0;
+    for (const symbol of symbols) {
+        ingestSymbol(acc, symbol);
+        if (++sinceYield >= SYMBOLS_PER_TURN) {
+            sinceYield = 0;
+            await yieldToEventLoop();
+        }
+    }
+    return finishBuild(acc, symbols);
 }
 export function searchBM25(index, query, topK, fieldWeights) {
     if (index.docCount === 0 || !query.trim()) {

package/dist/search/bm25.js.map

@@ -1 +1 @@
-{"version":3,"file":"bm25.js","sourceRoot":"","sources":["../../src/search/bm25.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAC/D,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAGnD,kBAAkB;AAClB,MAAM,EAAE,GAAG,GAAG,CAAC;AACf,MAAM,CAAC,GAAG,IAAI,CAAC;AAEf,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B;;;;GAIG;AACH,MAAM,0BAA0B,GAAG,GAAG,CAAC;AAmBvC;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,uCAAuC;IACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAE7D,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,kEAAkE;QAClE,MAAM,QAAQ,GAAG,IAAI;aAClB,OAAO,CAAC,oBAAoB,EAAE,QAAQ,CAAC;aACvC,OAAO,CAAC,uBAAuB,EAAE,QAAQ,CAAC;aAC1C,KAAK,CAAC,IAAI,CAAC,CAAC;QAEf,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;YAChC,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACtB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,cAAc,CAAC,MAAkB;IACxC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,IAAI,EAAE,CAAC;IAC9D,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAExD,OAAO;QACL,IAAI,EAAE,kBAAkB,CAAC,MAAM,CAAC,IAAI,CAAC;QACrC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE;QACjE,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE;QACjE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE;QACtC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE;KACjD,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,oBAAoB,CAAC,MAAc;IAC1C,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,yCAAyC;IACzC,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE;QACnD,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrB,OAAO,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAAC,EAAE,EAAE;QACpC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrB,OAAO,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;AAC9D,CAAC;AAED,SAAS,oBAAoB,CAAC,MAAgB;IAC5C,MAAM,EAAE,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,OAAqB;IAClD,MAAM,UAAU,GAAgB,CAAC,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;IAEvF,MAAM,MAAM,GAAwD;QAClE,IAAI,EAAE,IAAI,GAAG,EAAE;QACf,SAAS,EAAE,IAAI,GAAG,EAAE;QACpB,SAAS,EAAE,IAAI,GAAG,EAAE;QACpB,IAAI,EAAE,IAAI,GAAG,EAAE;QACf,QAAQ,EAAE,IAAI,GAAG,EAAE;KACpB,CAAC;IAEF,MAAM,iBAAiB,GAA8B;QACnD,IAAI,EAAE,CAAC;QACP,SAAS,EAAE,CAAC;QACZ,SAAS,EAAE,CAAC;QACZ,IAAI,EAAE,CAAC;QACP,QAAQ,EAAE,CAAC;KACZ,CAAC;IAEF,MAAM,SAAS,GAAG,IAAI,GAAG,EAAsB,CAAC;IAChD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAqC,CAAC;IAElE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACjC,MAAM,WAAW,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;QAC3C,MAAM,OAAO,GAA8B,EAAE,IAAI,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;QAEzG,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YAC/B,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;YAClC,iBAAiB,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC;YAC1C,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;YAE/B,MAAM,EAAE,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;YACxC,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC/B,IAAI,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBACxC,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAC;oBACrB,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;gBACrC,CAAC;gBACD,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;QACD,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;IACvC,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IAChC,MAAM,eAAe,GAA8B;QACjD,IAAI,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC1D,SAAS,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACpE,SAAS,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACpE,IAAI,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC1D,QAAQ,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,QAAQ,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;KACnE,CAAC;IAEF,mEAAmE;IACnE,2FAA2F;IAC3F,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC9C,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,KAAK,MAAM,GAAG,IAAI,OAAO;QAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAElD,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,IAAI,CAAC,GAAG,CAAC,MAAM;YAAE,SAAS;QAC1B,yDAAyD;QACzD,MAAM,QAAQ,GAAG,mCAAmC,CAAC;QACrD,IAAI,KAA6B,CAAC;QAClC,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACpD,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;YAC3B,mCAAmC;YACnC,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;gBAC5B,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC5B,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;oBACxD,MAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,gFAAgF;IAChF,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC7C,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,WAAW,EAAE,CAAC;QACxC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;IAC7C,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,YAAY,EAAE,CAAC;AAC7F,CAAC;AAED,MAAM,UAAU,UAAU,CACxB,KAAgB,EAChB,KAAa,EACb,IAAY,EACZ,YAAuC;IAEvC,IAAI,KAAK,CAAC,QAAQ,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QAC1C,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,WAAW,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IACxC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,UAAU,GAAgB,CAAC,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;IAEvF,iCAAiC;IACjC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,gDAAgD;IAChD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAuB,CAAC;IAErD,sEAAsE;IACtE,MAAM,EAAE,YAAY,EAAE,GAAG,KAAK,CAAC;IAE/B,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE,CAAC;QACjC,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YAC/B,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACjD,IAAI,CAAC,QAAQ;gBAAE,SAAS;YAExB,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC;YACzB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,QAAQ,GAAG,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACnE,MAAM,KAAK,GAAG,KAAK,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAC3C,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YAEnC,KAAK,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,QAAQ,EAAE,CAAC;gBACtC,MAAM,EAAE,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACpD,MAAM,IAAI,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBACxC,MAAM,OAAO,GAAG,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;gBACjE,MAAM,UAAU,GAAG,GAAG,GAAG,OAAO,GAAG,MAAM,CAAC;gBAE1C,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC;gBAE/D,IAAI,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBAC3C,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAC;oBACrB,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;gBACxC,CAAC;gBACD,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;IACH,CAAC;IAED,0EAA0E;IAC1E,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;IAChE,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM;YAAE,SAAS;QAEtB,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,2DAA2D;QAC3D,MAAM,cAAc,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9D,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;YACvB,QAAQ,IAAI,KAAK,GAAG,GAAG,GAAG,CAAC,cAAc,GAAG,aAAa,CAAC,CAAC;QAC7D,CAAC;QAED,uEAAuE;QACvE,IAAI,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,QAAQ,IAAI,0BAA0B,CAAC;QACzC,CAAC;QAED,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACjC,CAAC;IAED,uCAAuC;IACvC,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;SACjC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SAC3B,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAElB,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM;YAAE,SAAS;QAEtB,OAAO,CAAC,IAAI,CAAC;YACX,MAAM;YACN,KAAK;YACL,OAAO,EAAE,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;SAClD,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAC9B,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAE7B,MAAM,UAAU,WAAW,CAAC,OAAuB;IACjD,IAAI,OAAO,CAAC,MAAM,IAAI,kBAAkB;QAAE,OAAO,OAAO,CAAC;IACzD,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;IACxC,IAAI,QAAQ,IAAI,CAAC;QAAE,OAAO,OAAO,CAAC;IAClC,MAAM,SAAS,GAAG,QAAQ,GAAG,gBAAgB,CAAC;IAC9C,KAAK,IAAI,CAAC,GAAG,kBAAkB,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzD,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,SAAS,EAAE,CAAC;YACzC,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}
+{"version":3,"file":"bm25.js","sourceRoot":"","sources":["../../src/search/bm25.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAC/D,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAGnD,kBAAkB;AAClB,MAAM,EAAE,GAAG,GAAG,CAAC;AACf,MAAM,CAAC,GAAG,IAAI,CAAC;AAEf,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B;;;;GAIG;AACH,MAAM,0BAA0B,GAAG,GAAG,CAAC;AAyBvC;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,uCAAuC;IACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAE7D,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,kEAAkE;QAClE,MAAM,QAAQ,GAAG,IAAI;aAClB,OAAO,CAAC,oBAAoB,EAAE,QAAQ,CAAC;aACvC,OAAO,CAAC,uBAAuB,EAAE,QAAQ,CAAC;aAC1C,KAAK,CAAC,IAAI,CAAC,CAAC;QAEf,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;YAChC,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACtB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,cAAc,CAAC,MAAkB;IACxC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,IAAI,EAAE,CAAC;IAC9D,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAExD,OAAO;QACL,IAAI,EAAE,kBAAkB,CAAC,MAAM,CAAC,IAAI,CAAC;QACrC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE;QACjE,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE;QACjE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE;QACtC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE;KACjD,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,oBAAoB,CAAC,MAAc;IAC1C,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,yCAAyC;IACzC,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE;QACnD,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrB,OAAO,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAAC,EAAE,EAAE;QACpC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrB,OAAO,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;AAC9D,CAAC;AAED,SAAS,oBAAoB,CAAC,MAAgB;IAC5C,MAAM,EAAE,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;;;;;;;;;;GAWG;AACH,gGAAgG;AAChG,MAAM,UAAU,GAAgB,CAAC,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;AAEvF,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAE9B,SAAS,gBAAgB;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;AACzD,CAAC;AAUD,SAAS,cAAc;IACrB,OAAO;QACL,MAAM,EAAE;YACN,IAAI,EAAE,IAAI,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,GAAG,EAAE;YAC3D,IAAI,EAAE,IAAI,GAAG,EAAE,EAAE,QAAQ,EAAE,IAAI,GAAG,EAAE;SACrC;QACD,iBAAiB,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE;QAChF,SAAS,EAAE,IAAI,GAAG,EAAE;QACpB,YAAY,EAAE,IAAI,GAAG,EAAE;KACxB,CAAC;AACJ,CAAC;AAED,8FAA8F;AAC9F,SAAS,YAAY,CAAC,GAAoB,EAAE,MAAkB;IAC5D,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IACrC,MAAM,WAAW,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,OAAO,GAA8B;QACzC,IAAI,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC;KAC1D,CAAC;IAEF,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAClC,GAAG,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC;QAC9C,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QAE/B,MAAM,EAAE,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;QACxC,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YAC/B,IAAI,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC5C,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAC;gBACrB,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YACzC,CAAC;YACD,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IACD,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;AAC3C,CAAC;AAED;;;;;;GAMG;AACH,SAAS,WAAW,CAAC,GAAoB,EAAE,OAAqB;IAE9D,MAAM,EAAE,MAAM,EAAE,iBAAiB,EAAE,SAAS,EAAE,YAAY,EAAE,GAAG,GAAG,CAAC;IACnE,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IAChC,MAAM,eAAe,GAA8B;QACjD,IAAI,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC1D,SAAS,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACpE,SAAS,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACpE,IAAI,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC1D,QAAQ,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,QAAQ,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;KACnE,CAAC;IAEF,mEAAmE;IACnE,2FAA2F;IAC3F,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC9C,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,KAAK,MAAM,GAAG,IAAI,OAAO;QAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAElD,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,IAAI,CAAC,GAAG,CAAC,MAAM;YAAE,SAAS;QAC1B,yDAAyD;QACzD,MAAM,QAAQ,GAAG,mCAAmC,CAAC;QACrD,IAAI,KAA6B,CAAC;QAClC,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACpD,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;YAC3B,mCAAmC;YACnC,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;gBAC5B,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC5B,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;oBACxD,MAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,gFAAgF;IAChF,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC7C,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,WAAW,EAAE,CAAC;QACxC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;IAC7C,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,YAAY,EAAE,iBAAiB,EAAE,CAAC;AAChH,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAgB;IACzC,MAAM,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC;IACzB,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;QAC/B,KAAK,CAAC,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED;;;;;;;;;;GAUG;AACH,SAAS,qBAAqB,CAAC,KAAgB,EAAE,MAAkB;IACjE,MAAM,WAAW,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAEjD,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAClC,KAAK,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;QAEzE,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACrC,KAAK,MAAM,KAAK,IAAI,IAAI,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YACpC,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACrC,IAAI,CAAC,QAAQ;gBAAE,SAAS;YACxB,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAC3B,0FAA0F;YAC1F,sDAAsD;YACtD,IAAI,QAAQ,CAAC,IAAI,KAAK,CAAC;gBAAE,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAChC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACrC,KAAK,CAAC,QAAQ,EAAE,CAAC;AACnB,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAgB,EAAE,IAAY,EAAE,OAAqB;IACrF,iFAAiF;IACjF,+FAA+F;IAC/F,wFAAwF;IACxF,MAAM,KAAK,GAAiB,EAAE,CAAC;IAC/B,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QAC9C,IAAI,QAAQ,CAAC,IAAI,KAAK,IAAI;YAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACnD,CAAC;IACD,KAAK,MAAM,MAAM,IAAI,KAAK;QAAE,qBAAqB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAEjE,MAAM,GAAG,GAAoB;QAC3B,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,iBAAiB,EAAE,KAAK,CAAC,iBAAiB;QAC1C,SAAS,EAAE,KAAK,CAAC,OAAO;QACxB,YAAY,EAAE,KAAK,CAAC,YAAY;KACjC,CAAC;IACF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,YAAY,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAC1B,KAAK,CAAC,QAAQ,EAAE,CAAC;IACnB,CAAC;IAED,iBAAiB,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,OAAqB;IAClD,MAAM,GAAG,GAAG,cAAc,EAAE,CAAC;IAC7B,KAAK,MAAM,MAAM,IAAI,OAAO;QAAE,YAAY,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IACxD,OAAO,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;AACnC,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,OAAqB;IAChE,MAAM,GAAG,GAAG,cAAc,EAAE,CAAC;IAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,YAAY,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAC1B,IAAI,EAAE,UAAU,IAAI,gBAAgB,EAAE,CAAC;YAAC,UAAU,GAAG,CAAC,CAAC;YAAC,MAAM,gBAAgB,EAAE,CAAC;QAAC,CAAC;IACrF,CAAC;IACD,OAAO,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,UAAU,CACxB,KAAgB,EAChB,KAAa,EACb,IAAY,EACZ,YAAuC;IAEvC,IAAI,KAAK,CAAC,QAAQ,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QAC1C,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,WAAW,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IACxC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,UAAU,GAAgB,CAAC,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;IAEvF,iCAAiC;IACjC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,gDAAgD;IAChD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAuB,CAAC;IAErD,sEAAsE;IACtE,MAAM,EAAE,YAAY,EAAE,GAAG,KAAK,CAAC;IAE/B,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE,CAAC;QACjC,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YAC/B,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACjD,IAAI,CAAC,QAAQ;gBAAE,SAAS;YAExB,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC;YACzB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,QAAQ,GAAG,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACnE,MAAM,KAAK,GAAG,KAAK,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAC3C,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YAEnC,KAAK,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,QAAQ,EAAE,CAAC;gBACtC,MAAM,EAAE,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACpD,MAAM,IAAI,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBACxC,MAAM,OAAO,GAAG,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;gBACjE,MAAM,UAAU,GAAG,GAAG,GAAG,OAAO,GAAG,MAAM,CAAC;gBAE1C,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC;gBAE/D,IAAI,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBAC3C,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAC;oBACrB,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;gBACxC,CAAC;gBACD,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;IACH,CAAC;IAED,0EAA0E;IAC1E,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;IAChE,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM;YAAE,SAAS;QAEtB,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,2DAA2D;QAC3D,MAAM,cAAc,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9D,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;YACvB,QAAQ,IAAI,KAAK,GAAG,GAAG,GAAG,CAAC,cAAc,GAAG,aAAa,CAAC,CAAC;QAC7D,CAAC;QAED,uEAAuE;QACvE,IAAI,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,QAAQ,IAAI,0BAA0B,CAAC;QACzC,CAAC;QAED,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACjC,CAAC;IAED,uCAAuC;IACvC,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;SACjC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SAC3B,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAElB,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM;YAAE,SAAS;QAEtB,OAAO,CAAC,IAAI,CAAC;YACX,MAAM;YACN,KAAK;YACL,OAAO,EAAE,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;SAClD,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAC9B,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAE7B,MAAM,UAAU,WAAW,CAAC,OAAuB;IACjD,IAAI,OAAO,CAAC,MAAM,IAAI,kBAAkB;QAAE,OAAO,OAAO,CAAC;IACzD,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;IACxC,IAAI,QAAQ,IAAI,CAAC;QAAE,OAAO,OAAO,CAAC;IAClC,MAAM,SAAS,GAAG,QAAQ,GAAG,gBAAgB,CAAC;IAC9C,KAAK,IAAI,CAAC,GAAG,kBAAkB,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzD,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,SAAS,EAAE,CAAC;YACzC,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/search/embed-gate.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Run `fn` with at most `limit` others.
+ *
+ * FIFO, so a batch that has waited longest goes next — a stack would starve the first arrival under
+ * sustained load, which is exactly the caller whose timeout is closest to firing.
+ */
+export declare function withEmbedSlot<T>(fn: () => Promise<T>, options?: {
+    env?: NodeJS.ProcessEnv;
+}): Promise<T>;
+/** Tests need to observe the gate rather than infer it from timing. */
+export declare function embedGateStateForTesting(): {
+    active: number;
+    waiting: number;
+};
+/** Tests need a clean slate between cases. */
+export declare function resetEmbedGateForTesting(): void;
+//# sourceMappingURL=embed-gate.d.ts.map

package/dist/search/embed-gate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embed-gate.d.ts","sourceRoot":"","sources":["../../src/search/embed-gate.ts"],"names":[],"mappings":"AA8BA;;;;;GAKG;AACH,wBAAsB,aAAa,CAAC,CAAC,EACnC,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EACpB,OAAO,CAAC,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAA;CAAE,GACpC,OAAO,CAAC,CAAC,CAAC,CAiBZ;AAED,uEAAuE;AACvE,wBAAgB,wBAAwB,IAAI;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAE9E;AAED,8CAA8C;AAC9C,wBAAgB,wBAAwB,IAAI,IAAI,CAG/C"}

package/dist/search/embed-gate.js

@@ -0,0 +1,64 @@
+/**
+ * Cap on how many embedding requests are in flight at once.
+ *
+ * Nothing limited them, and the remote does: `OLLAMA_NUM_PARALLEL=4` on the tailnet host this
+ * machine embeds against. So the daemon fired unbounded concurrent batches at a server that serves
+ * four, everything past the fourth queued until it exceeded the client's timeout, and
+ * `embedBatchWithStallRetry` answered each timeout by splitting the batch in two — producing MORE
+ * requests against the same queue. That is a congestion collapse we cause ourselves, and it is what
+ * filled `daemon.err.log` with hundreds of `embed batch of N stalled … retrying as N/2+N/2` lines
+ * and left the daemon unable to answer /health for fifteen minutes after a restart.
+ *
+ * A gate does not make embedding slower: the work was already serialised by the remote. It makes
+ * the QUEUE visible to us instead of hidden behind timeouts, so a batch waits rather than failing
+ * and multiplying.
+ *
+ * Default 4 to match the common Ollama setting. `CODESIFT_EMBED_CONCURRENCY` overrides it — a
+ * bigger GPU host or a hosted provider can take more, and a value of 0 or less disables the gate
+ * for callers who know their provider is unmetered.
+ */
+const DEFAULT_CONCURRENCY = 4;
+function configuredLimit(env) {
+    const raw = Number(env["CODESIFT_EMBED_CONCURRENCY"]);
+    if (Number.isFinite(raw))
+        return Math.floor(raw);
+    return DEFAULT_CONCURRENCY;
+}
+let active = 0;
+const waiting = [];
+/**
+ * Run `fn` with at most `limit` others.
+ *
+ * FIFO, so a batch that has waited longest goes next — a stack would starve the first arrival under
+ * sustained load, which is exactly the caller whose timeout is closest to firing.
+ */
+export async function withEmbedSlot(fn, options) {
+    const limit = configuredLimit(options?.env ?? process.env);
+    if (limit <= 0)
+        return fn();
+    if (active >= limit) {
+        await new Promise((resolve) => waiting.push(resolve));
+    }
+    active++;
+    try {
+        return await fn();
+    }
+    finally {
+        active--;
+        // Release in arrival order. `shift()` on an array this short costs nothing, and the ordering is
+        // the property that matters.
+        const next = waiting.shift();
+        if (next)
+            next();
+    }
+}
+/** Tests need to observe the gate rather than infer it from timing. */
+export function embedGateStateForTesting() {
+    return { active, waiting: waiting.length };
+}
+/** Tests need a clean slate between cases. */
+export function resetEmbedGateForTesting() {
+    active = 0;
+    waiting.length = 0;
+}
+//# sourceMappingURL=embed-gate.js.map

package/dist/search/embed-gate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embed-gate.js","sourceRoot":"","sources":["../../src/search/embed-gate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAE9B,SAAS,eAAe,CAAC,GAAsB;IAC7C,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC,CAAC;IACtD,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACjD,OAAO,mBAAmB,CAAC;AAC7B,CAAC;AAED,IAAI,MAAM,GAAG,CAAC,CAAC;AACf,MAAM,OAAO,GAAsB,EAAE,CAAC;AAEtC;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,EAAoB,EACpB,OAAqC;IAErC,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,EAAE,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC;IAC3D,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,EAAE,EAAE,CAAC;IAE5B,IAAI,MAAM,IAAI,KAAK,EAAE,CAAC;QACpB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IAC9D,CAAC;IACD,MAAM,EAAE,CAAC;IACT,IAAI,CAAC;QACH,OAAO,MAAM,EAAE,EAAE,CAAC;IACpB,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC;QACT,gGAAgG;QAChG,6BAA6B;QAC7B,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;QAC7B,IAAI,IAAI;YAAE,IAAI,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED,uEAAuE;AACvE,MAAM,UAAU,wBAAwB;IACtC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;AAC7C,CAAC;AAED,8CAA8C;AAC9C,MAAM,UAAU,wBAAwB;IACtC,MAAM,GAAG,CAAC,CAAC;IACX,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;AACrB,CAAC"}

package/dist/search/semantic.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"semantic.d.ts","sourceRoot":"","sources":["../../src/search/semantic.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAkC5D,oFAAoF;AACpF,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAMnE;AAMD;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG,UAAU,GAAG,OAAO,CAAC;AAEjD,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAClE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAiB1D;AAMD,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,YAAY,GAAG,MAAM,CAiBzE;AAED;;;GAGG;AACH;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,uBAAuB,CACrC,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,GACpC;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAM9B;AAED,kDAAkD;AAClD,wBAAgB,wBAAwB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CASpF;AAED,wBAAgB,cAAc,CAC5B,cAAc,EAAE,YAAY,EAC5B,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,EACrC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,EAChC,IAAI,EAAE,MAAM,GACX,YAAY,EAAE,CAoBhB;AA6DD,qBAAa,cAAe,YAAW,iBAAiB;IACtD,QAAQ,CAAC,KAAK,mBAAmB;IACjC,QAAQ,CAAC,UAAU,QAAQ;IAC3B,OAAO,CAAC,MAAM,CAAS;IAEvB,YAAY,MAAM,EAAE,MAAM,EAEzB;IAEK,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,IAAI,GAAE,aAA0B,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAOlF;CACF;AAMD,qBAAa,cAAe,YAAW,iBAAiB;IACtD,QAAQ,CAAC,KAAK,4BAA4B;IAC1C,QAAQ,CAAC,UAAU,QAAQ;IAC3B,OAAO,CAAC,MAAM,CAAS;IAEvB,YAAY,MAAM,EAAE,MAAM,EAEzB;IAEK,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,KAAK,GAAE,aAA0B,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAOnF;CACF;AAMD,qBAAa,cAAe,YAAW,iBAAiB;IACtD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,OAAO,CAAC,OAAO,CAAS;IAMxB,YAAY,OAAO,EAAE,MAAM,EAAE,KAAK,SAAqB,EAAE,UAAU,SAAM,EAIxE;IAEK,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,KAAK,GAAE,aAA0B,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAuCnF;CACF;AAeD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,MAAM,EAAE,EACf,MAAM,SAAqB,EAC3B,SAAS,SAAmB,GAC3B,KAAK,CAAC;IAAE,KAAK,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAuB5B;AAED;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CACpC,QAAQ,EAAE,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,EAClD,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,EAC1B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAC1B,MAAM,CAOR;AAmBD,+EAA+E;AAC/E,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,GAAG,MAAM,CAKpE;AAiCD;;;;;;;;;;;GAWG;AACH,wBAAsB,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC,CAK3D;AA2DD,qBAAa,aAAc,YAAW,iBAAiB;IACrD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAE5B,YAAY,KAAK,GAAE,MAA4B,EAAE,UAAU,CAAC,EAAE,MAAM,EAGnE;IAEK,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,IAAI,GAAE,aAA0B,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CA0BlF;CACF;AAED,sDAAsD;AACtD,wBAAgB,mBAAmB,IAAI,IAAI,CAK1C;AAMD,wBAAgB,uBAAuB,CACrC,QAAQ,EAAE,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,EAClD,MAAM,EAAE;IACN,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B,GACA,iBAAiB,CA0BnB"}
+{"version":3,"file":"semantic.d.ts","sourceRoot":"","sources":["../../src/search/semantic.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAmC5D,oFAAoF;AACpF,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAMnE;AAMD;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG,UAAU,GAAG,OAAO,CAAC;AAEjD,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAClE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAiB1D;AAMD,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,YAAY,GAAG,MAAM,CAiBzE;AAED;;;GAGG;AACH;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,uBAAuB,CACrC,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,GACpC;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAM9B;AAED,kDAAkD;AAClD,wBAAgB,wBAAwB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CASpF;AAED,wBAAgB,cAAc,CAC5B,cAAc,EAAE,YAAY,EAC5B,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,EACrC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,EAChC,IAAI,EAAE,MAAM,GACX,YAAY,EAAE,CAoBhB;AA6DD,qBAAa,cAAe,YAAW,iBAAiB;IACtD,QAAQ,CAAC,KAAK,mBAAmB;IACjC,QAAQ,CAAC,UAAU,QAAQ;IAC3B,OAAO,CAAC,MAAM,CAAS;IAEvB,YAAY,MAAM,EAAE,MAAM,EAEzB;IAEK,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,IAAI,GAAE,aAA0B,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAOlF;CACF;AAMD,qBAAa,cAAe,YAAW,iBAAiB;IACtD,QAAQ,CAAC,KAAK,4BAA4B;IAC1C,QAAQ,CAAC,UAAU,QAAQ;IAC3B,OAAO,CAAC,MAAM,CAAS;IAEvB,YAAY,MAAM,EAAE,MAAM,EAEzB;IAEK,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,KAAK,GAAE,aAA0B,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAOnF;CACF;AAMD,qBAAa,cAAe,YAAW,iBAAiB;IACtD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,OAAO,CAAC,OAAO,CAAS;IAMxB,YAAY,OAAO,EAAE,MAAM,EAAE,KAAK,SAAqB,EAAE,UAAU,SAAM,EAIxE;IAEK,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,KAAK,GAAE,aAA0B,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAyCnF;CACF;AAeD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,MAAM,EAAE,EACf,MAAM,SAAqB,EAC3B,SAAS,SAAmB,GAC3B,KAAK,CAAC;IAAE,KAAK,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAuB5B;AAED;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CACpC,QAAQ,EAAE,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,EAClD,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,EAC1B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAC1B,MAAM,CAOR;AAmBD,+EAA+E;AAC/E,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,GAAG,MAAM,CAKpE;AAiCD;;;;;;;;;;;GAWG;AACH,wBAAsB,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC,CAK3D;AA2DD,qBAAa,aAAc,YAAW,iBAAiB;IACrD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAE5B,YAAY,KAAK,GAAE,MAA4B,EAAE,UAAU,CAAC,EAAE,MAAM,EAGnE;IAEK,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,IAAI,GAAE,aAA0B,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CA0BlF;CACF;AAED,sDAAsD;AACtD,wBAAgB,mBAAmB,IAAI,IAAI,CAK1C;AAMD,wBAAgB,uBAAuB,CACrC,QAAQ,EAAE,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,EAClD,MAAM,EAAE;IACN,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B,GACA,iBAAiB,CA0BnB"}

package/dist/search/semantic.js

@@ -1,3 +1,4 @@
+import { withEmbedSlot } from "./embed-gate.js";
 import { importTransformers, localEmbeddingRemedy } from "./optional-transformers.js";
 import { StaticEmbeddingProvider } from "./static-embedding-provider.js";
 const MAX_SYMBOL_SOURCE_CHARS = 200;
@@ -232,12 +233,14 @@ export class OllamaProvider {
         // /api/embeddings once per text — 90K sequential HTTP round-trips for a big
         // repo, which throws away most of the GPU speedup Ollama gives on Apple
         // Silicon. /api/embed embeds the whole array in one request.
-        const response = await fetch(`${this.baseUrl}/api/embed`, {
+        // Through the gate: the remote serves a fixed number at a time, and firing more than that at it
+        // turns queueing into timeouts, and timeouts into retries that queue again. See embed-gate.ts.
+        const response = await withEmbedSlot(() => fetch(`${this.baseUrl}/api/embed`, {
             method: "POST",
             headers: { "Content-Type": "application/json" },
             body: JSON.stringify({ model: this.model, input: capped }),
             signal: AbortSignal.timeout(embeddingTimeoutMs(capped)),
-        });
+        }));
         if (!response.ok) {
             // SEC-004: log raw body to stderr only — do not forward to the MCP client.
             const body = await response.text();