@reposkein/mcp 0.7.0 → 0.9.0

package/dist/search/bm25f.d.ts

@@ -27,5 +27,21 @@ export declare function tokenize(s: string): string[];
 /**
  * Rank corpus nodes by BM25F for the given query, returning up to limit results.
  * Deterministic: identical corpus + query → identical ranking.
+ *
+ * Takes an ITERABLE, and iterates it twice — BM25F needs corpus-wide statistics
+ * (average field lengths, document frequencies) before any document can be
+ * scored, so the corpus cannot be consumed in a single pass. What it does NOT
+ * do any more is RETAIN the corpus: the old form built a tokenized copy of all
+ * five fields of every node and held it through scoring, then collected every
+ * positive-scoring document before sorting. Retention is now bounded by
+ * `limit`; tokens are discarded as soon as a document has been scored.
+ *
+ * The two passes cost tokenization twice. That is the trade: CPU proportional
+ * to the corpus, memory proportional to the limit — which is the right way
+ * round for a query path that runs inside an agent's tool call.
+ *
+ * The iterable must be re-iterable (an array, or a generator function's result
+ * re-invoked). Results are identical to the single-pass form: same maths, and
+ * the same total ordering (score desc, id asc) applied to a bounded heap.
  */
-export declare function rankCorpus(corpus: CorpusNode[], query: string, limit: number): Scored[];
+export declare function rankCorpus(corpus: Iterable<CorpusNode>, query: string, limit: number): Scored[];

package/dist/search/bm25f.js

@@ -57,58 +57,93 @@ export function tokenize(s) {
 function fieldTokens(value) {
     return tokenize(value);
 }
+/** Count occurrences without allocating. The old form was
+ *  `toks.filter((t) => t === qt).length`, which built a throwaway array for
+ *  every (document x field x query token) — millions of them on a large
+ *  corpus, for a number. */
+function countOccurrences(toks, needle) {
+    let n = 0;
+    for (const t of toks)
+        if (t === needle)
+            n++;
+    return n;
+}
+/** Tokenize one node's fields. Callers must NOT retain the result. */
+function tokenizeFields(node) {
+    const fields = {};
+    const lengths = {};
+    for (const f of FIELDS) {
+        const toks = fieldTokens(node[f] ?? "");
+        fields[f] = toks;
+        lengths[f] = toks.length;
+    }
+    return { fields, lengths };
+}
 /**
  * Rank corpus nodes by BM25F for the given query, returning up to limit results.
  * Deterministic: identical corpus + query → identical ranking.
+ *
+ * Takes an ITERABLE, and iterates it twice — BM25F needs corpus-wide statistics
+ * (average field lengths, document frequencies) before any document can be
+ * scored, so the corpus cannot be consumed in a single pass. What it does NOT
+ * do any more is RETAIN the corpus: the old form built a tokenized copy of all
+ * five fields of every node and held it through scoring, then collected every
+ * positive-scoring document before sorting. Retention is now bounded by
+ * `limit`; tokens are discarded as soon as a document has been scored.
+ *
+ * The two passes cost tokenization twice. That is the trade: CPU proportional
+ * to the corpus, memory proportional to the limit — which is the right way
+ * round for a query path that runs inside an agent's tool call.
+ *
+ * The iterable must be re-iterable (an array, or a generator function's result
+ * re-invoked). Results are identical to the single-pass form: same maths, and
+ * the same total ordering (score desc, id asc) applied to a bounded heap.
  */
 export function rankCorpus(corpus, query, limit) {
-    if (corpus.length === 0 || !query.trim())
+    if (limit <= 0 || !query.trim())
         return [];
-    // 1. Tokenize query
     const queryTokens = tokenize(query);
     if (queryTokens.length === 0)
         return [];
     const uniqueQueryTokens = [...new Set(queryTokens)];
-    // 2. Build per-node documents in id-sorted order (corpus is already sorted by store)
-    const docs = corpus.map((node) => {
-        const fields = {};
-        const fieldLengths = {};
-        for (const f of FIELDS) {
-            const value = node[f] ?? "";
-            const toks = fieldTokens(value);
-            fields[f] = toks;
-            fieldLengths[f] = toks.length;
-        }
-        return { node, fields, fieldLengths };
-    });
-    // 3. Compute average field lengths across corpus
-    const avgFieldLen = {};
-    for (const f of FIELDS) {
-        const total = docs.reduce((sum, d) => sum + d.fieldLengths[f], 0);
-        avgFieldLen[f] = docs.length > 0 ? total / docs.length : 1;
-    }
-    // 4. Build inverted index: token → set of doc indices (for IDF)
+    // Pass 1 — corpus statistics only. Field-length totals and document
+    // frequencies; every token is dropped as soon as it has been counted.
+    const totalFieldLen = {};
+    for (const f of FIELDS)
+        totalFieldLen[f] = 0;
     const dfMap = new Map();
-    for (const qt of uniqueQueryTokens) {
-        let df = 0;
-        for (const doc of docs) {
-            // Check if token appears in any field of this doc
-            let found = false;
+    for (const qt of uniqueQueryTokens)
+        dfMap.set(qt, 0);
+    let N = 0;
+    for (const node of corpus) {
+        N++;
+        const { fields, lengths } = tokenizeFields(node);
+        for (const f of FIELDS)
+            totalFieldLen[f] += lengths[f];
+        for (const qt of uniqueQueryTokens) {
             for (const f of FIELDS) {
-                if (doc.fields[f].includes(qt)) {
-                    found = true;
+                if (fields[f].includes(qt)) {
+                    dfMap.set(qt, dfMap.get(qt) + 1);
                     break;
                 }
             }
-            if (found)
-                df++;
         }
-        dfMap.set(qt, df);
     }
-    const N = docs.length;
-    // 5. Score each document
-    const scored = [];
-    for (const doc of docs) {
+    if (N === 0)
+        return [];
+    const avgFieldLen = {};
+    for (const f of FIELDS)
+        avgFieldLen[f] = totalFieldLen[f] / N;
+    // Pass 2 — score, keeping only the best `limit`.
+    const best = [];
+    const worseThanWorst = (score, id) => {
+        const w = best[best.length - 1];
+        if (score !== w.score)
+            return score < w.score;
+        return id > w.node.id; // ties: lower id wins, matching the sort below
+    };
+    for (const node of corpus) {
+        const { fields, lengths } = tokenizeFields(node);
         let totalScore = 0;
         const matchedTokens = new Set();
         for (const qt of uniqueQueryTokens) {
@@ -117,35 +152,36 @@ export function rankCorpus(corpus, query, limit) {
                 continue;
             // IDF: ln(1 + (N - df + 0.5) / (df + 0.5))  — always positive for df < N
             const idf = Math.log(1 + (N - df + 0.5) / (df + 0.5));
-            // Weighted TF across fields (BM25F-style)
             let weightedTf = 0;
             for (const f of FIELDS) {
-                const toks = doc.fields[f];
-                const tf = toks.filter((t) => t === qt).length;
+                const tf = countOccurrences(fields[f], qt);
                 if (tf === 0)
                     continue;
                 matchedTokens.add(qt);
-                const fieldLen = doc.fieldLengths[f];
+                const fieldLen = lengths[f];
                 const avgLen = avgFieldLen[f] > 0 ? avgFieldLen[f] : 1;
-                // BM25 saturation with length normalization
                 const saturated = (tf * (K1 + 1)) / (tf + K1 * (1 - B + B * (fieldLen / avgLen)));
                 weightedTf += WEIGHTS[f] * saturated;
             }
             totalScore += idf * weightedTf;
         }
-        if (totalScore > 0) {
-            // Round to fixed precision to prevent cross-platform FP jitter from reordering ties
-            const roundedScore = Math.round(totalScore * SCORE_PRECISION) / SCORE_PRECISION;
-            scored.push({ node: doc.node, score: roundedScore, matched: [...matchedTokens].sort() });
-        }
+        if (totalScore <= 0)
+            continue;
+        // Round to fixed precision to prevent cross-platform FP jitter from
+        // reordering ties.
+        const score = Math.round(totalScore * SCORE_PRECISION) / SCORE_PRECISION;
+        if (best.length >= limit && worseThanWorst(score, node.id))
+            continue;
+        best.push({ node, score, matched: [...matchedTokens].sort() });
+        best.sort((a, b) => {
+            const diff = b.score - a.score;
+            if (diff !== 0)
+                return diff;
+            return a.node.id < b.node.id ? -1 : a.node.id > b.node.id ? 1 : 0;
+        });
+        if (best.length > limit)
+            best.pop();
     }
-    // 6. Sort: descending score, tie-break ascending node_id
-    scored.sort((a, b) => {
-        const diff = b.score - a.score;
-        if (diff !== 0)
-            return diff;
-        return a.node.id < b.node.id ? -1 : a.node.id > b.node.id ? 1 : 0;
-    });
-    return scored.slice(0, limit);
+    return best;
 }
 //# sourceMappingURL=bm25f.js.map

package/dist/search/bm25f.js.map

@@ -1 +1 @@
-{"version":3,"file":"bm25f.js","sourceRoot":"","sources":["../../src/search/bm25f.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAUH,8BAA8B;AAC9B,MAAM,OAAO,GAA2B;IACtC,cAAc,EAAE,EAAE;IAClB,IAAI,EAAE,CAAC;IACP,OAAO,EAAE,CAAC;IACV,SAAS,EAAE,CAAC;IACZ,SAAS,EAAE,CAAC;CACb,CAAC;AAEF,2EAA2E;AAC3E,MAAM,MAAM,GAAG,CAAC,gBAAgB,EAAE,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,WAAW,CAAU,CAAC;AAGxF,MAAM,EAAE,GAAG,GAAG,CAAC;AACf,MAAM,CAAC,GAAG,IAAI,CAAC;AACf,MAAM,eAAe,GAAG,GAAG,CAAC,CAAC,4BAA4B;AAEzD;;;;;;GAMG;AACH,MAAM,UAAU,QAAQ,CAAC,CAAS;IAChC,8FAA8F;IAC9F,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;IACvC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAChC,8BAA8B;QAC9B,iHAAiH;QACjH,8HAA8H;QAC9H,MAAM,QAAQ,GAAG,IAAI;aAClB,OAAO,CAAC,iBAAiB,EAAE,QAAQ,CAAC;aACpC,OAAO,CAAC,uBAAuB,EAAE,QAAQ,CAAC;aAC1C,KAAK,CAAC,IAAI,CAAC,CAAC;QACf,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;YAC/B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,8CAA8C;AAC9C,SAAS,WAAW,CAAC,KAAa;IAChC,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC;AACzB,CAAC;AAQD;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,MAAoB,EAAE,KAAa,EAAE,KAAa;IAC3E,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE;QAAE,OAAO,EAAE,CAAC;IAEpD,oBAAoB;IACpB,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IACpC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACxC,MAAM,iBAAiB,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC;IAEpD,qFAAqF;IACrF,MAAM,IAAI,GAAc,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC1C,MAAM,MAAM,GAAG,EAA6B,CAAC;QAC7C,MAAM,YAAY,GAAG,EAA2B,CAAC;QACjD,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,MAAM,KAAK,GAAI,IAA0C,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACnE,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;YAChC,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;YACjB,YAAY,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;QAChC,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,iDAAiD;IACjD,MAAM,WAAW,GAAG,EAA2B,CAAC;IAChD,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAClE,WAAW,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,CAAC;IAED,gEAAgE;IAChE,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,KAAK,MAAM,EAAE,IAAI,iBAAiB,EAAE,CAAC;QACnC,IAAI,EAAE,GAAG,CAAC,CAAC;QACX,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,kDAAkD;YAClD,IAAI,KAAK,GAAG,KAAK,CAAC;YAClB,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;gBACvB,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;oBAC/B,KAAK,GAAG,IAAI,CAAC;oBACb,MAAM;gBACR,CAAC;YACH,CAAC;YACD,IAAI,KAAK;gBAAE,EAAE,EAAE,CAAC;QAClB,CAAC;QACD,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACpB,CAAC;IAED,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;IAEtB,yBAAyB;IACzB,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;QAExC,KAAK,MAAM,EAAE,IAAI,iBAAiB,EAAE,CAAC;YACnC,MAAM,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;YAC9B,IAAI,EAAE,KAAK,CAAC;gBAAE,SAAS;YAEvB,yEAAyE;YACzE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;YAEtD,0CAA0C;YAC1C,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;gBACvB,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAC3B,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC;gBAC/C,IAAI,EAAE,KAAK,CAAC;oBAAE,SAAS;gBACvB,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACtB,MAAM,QAAQ,GAAG,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;gBACrC,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACvD,4CAA4C;gBAC5C,MAAM,SAAS,GAAG,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBAClF,UAAU,IAAI,OAAO,CAAC,CAAC,CAAE,GAAG,SAAS,CAAC;YACxC,CAAC;YAED,UAAU,IAAI,GAAG,GAAG,UAAU,CAAC;QACjC,CAAC;QAED,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;YACnB,oFAAoF;YACpF,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,eAAe,CAAC,GAAG,eAAe,CAAC;YAChF,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC,GAAG,aAAa,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC3F,CAAC;IACH,CAAC;IAED,yDAAyD;IACzD,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACnB,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;QAC/B,IAAI,IAAI,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAC5B,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACpE,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;AAChC,CAAC"}
+{"version":3,"file":"bm25f.js","sourceRoot":"","sources":["../../src/search/bm25f.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAUH,8BAA8B;AAC9B,MAAM,OAAO,GAA2B;IACtC,cAAc,EAAE,EAAE;IAClB,IAAI,EAAE,CAAC;IACP,OAAO,EAAE,CAAC;IACV,SAAS,EAAE,CAAC;IACZ,SAAS,EAAE,CAAC;CACb,CAAC;AAEF,2EAA2E;AAC3E,MAAM,MAAM,GAAG,CAAC,gBAAgB,EAAE,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,WAAW,CAAU,CAAC;AAGxF,MAAM,EAAE,GAAG,GAAG,CAAC;AACf,MAAM,CAAC,GAAG,IAAI,CAAC;AACf,MAAM,eAAe,GAAG,GAAG,CAAC,CAAC,4BAA4B;AAEzD;;;;;;GAMG;AACH,MAAM,UAAU,QAAQ,CAAC,CAAS;IAChC,8FAA8F;IAC9F,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;IACvC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAChC,8BAA8B;QAC9B,iHAAiH;QACjH,8HAA8H;QAC9H,MAAM,QAAQ,GAAG,IAAI;aAClB,OAAO,CAAC,iBAAiB,EAAE,QAAQ,CAAC;aACpC,OAAO,CAAC,uBAAuB,EAAE,QAAQ,CAAC;aAC1C,KAAK,CAAC,IAAI,CAAC,CAAC;QACf,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;YAC/B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,8CAA8C;AAC9C,SAAS,WAAW,CAAC,KAAa;IAChC,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC;AACzB,CAAC;AAED;;;4BAG4B;AAC5B,SAAS,gBAAgB,CAAC,IAAc,EAAE,MAAc;IACtD,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,KAAK,MAAM,CAAC,IAAI,IAAI;QAAE,IAAI,CAAC,KAAK,MAAM;YAAE,CAAC,EAAE,CAAC;IAC5C,OAAO,CAAC,CAAC;AACX,CAAC;AAED,sEAAsE;AACtE,SAAS,cAAc,CAAC,IAAgB;IACtC,MAAM,MAAM,GAAG,EAA6B,CAAC;IAC7C,MAAM,OAAO,GAAG,EAA2B,CAAC;IAC5C,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,WAAW,CAAE,IAA0C,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC/E,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;QACjB,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;IAC3B,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;AAC7B,CAAC;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,UAAU,UAAU,CAAC,MAA4B,EAAE,KAAa,EAAE,KAAa;IACnF,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE;QAAE,OAAO,EAAE,CAAC;IAE3C,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IACpC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACxC,MAAM,iBAAiB,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC;IAEpD,oEAAoE;IACpE,sEAAsE;IACtE,MAAM,aAAa,GAAG,EAA2B,CAAC;IAClD,KAAK,MAAM,CAAC,IAAI,MAAM;QAAE,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC7C,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,KAAK,MAAM,EAAE,IAAI,iBAAiB;QAAE,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IACrD,IAAI,CAAC,GAAG,CAAC,CAAC;IAEV,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,CAAC,EAAE,CAAC;QACJ,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACjD,KAAK,MAAM,CAAC,IAAI,MAAM;YAAE,aAAa,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;QACvD,KAAK,MAAM,EAAE,IAAI,iBAAiB,EAAE,CAAC;YACnC,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;gBACvB,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;oBAC3B,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAE,GAAG,CAAC,CAAC,CAAC;oBAClC,MAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEvB,MAAM,WAAW,GAAG,EAA2B,CAAC;IAChD,KAAK,MAAM,CAAC,IAAI,MAAM;QAAE,WAAW,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAE9D,iDAAiD;IACjD,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,cAAc,GAAG,CAAC,KAAa,EAAE,EAAU,EAAW,EAAE;QAC5D,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC;QACjC,IAAI,KAAK,KAAK,CAAC,CAAC,KAAK;YAAE,OAAO,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;QAC9C,OAAO,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,+CAA+C;IACxE,CAAC,CAAC;IAEF,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;QAExC,KAAK,MAAM,EAAE,IAAI,iBAAiB,EAAE,CAAC;YACnC,MAAM,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;YAC9B,IAAI,EAAE,KAAK,CAAC;gBAAE,SAAS;YAEvB,yEAAyE;YACzE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;YAEtD,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;gBACvB,MAAM,EAAE,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC3C,IAAI,EAAE,KAAK,CAAC;oBAAE,SAAS;gBACvB,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACtB,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;gBAC5B,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACvD,MAAM,SAAS,GAAG,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBAClF,UAAU,IAAI,OAAO,CAAC,CAAC,CAAE,GAAG,SAAS,CAAC;YACxC,CAAC;YAED,UAAU,IAAI,GAAG,GAAG,UAAU,CAAC;QACjC,CAAC;QAED,IAAI,UAAU,IAAI,CAAC;YAAE,SAAS;QAC9B,oEAAoE;QACpE,mBAAmB;QACnB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,eAAe,CAAC,GAAG,eAAe,CAAC;QACzE,IAAI,IAAI,CAAC,MAAM,IAAI,KAAK,IAAI,cAAc,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,CAAC;YAAE,SAAS;QAErE,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,GAAG,aAAa,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC/D,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACjB,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;YAC/B,IAAI,IAAI,KAAK,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC5B,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpE,CAAC,CAAC,CAAC;QACH,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK;YAAE,IAAI,CAAC,GAAG,EAAE,CAAC;IACtC,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/store/GraphStore.d.ts

@@ -74,8 +74,13 @@ export interface GraphStore {
     findByContentHash?(repoIds: string[], hash: string): Promise<TargetRow[]>;
     /** Raw read-only Cypher (read_cypher tool only). Stores without a Cypher
      *  engine throw CypherUnsupportedError. */
-    runRead(query: string, params?: Record<string, unknown>, opts?: {
+    runRead(query: string, params?: Record<string, unknown>, 
+    /** `maxRows` is a request to STOP STREAMING there, not a post-hoc filter.
+     *  An implementation that cannot stream may return everything, but one
+     *  that can must not materialise past the cap. */
+    opts?: {
         timeoutMs?: number;
+        maxRows?: number;
     }): Promise<Record<string, unknown>[]>;
     close(): Promise<void>;
 }

package/dist/store/JsonlGraphStore.d.ts

@@ -13,7 +13,12 @@ import type { TargetRow } from "../profile/types.js";
 export declare class JsonlGraphStore implements GraphStore {
     private readonly repoId;
     private graph;
-    private stamp;
+    private graphStamp;
+    private overlayStamp;
+    /** Per node, the prop values the overlay overwrote — its own undo log.
+     *  A key absent before the overlay is recorded as `undefined`, so removing a
+     *  summary removes the props it added instead of leaving them behind. */
+    private overlayUndo;
     private readonly repoPath;
     private readonly nodesPath;
     private readonly edgesPath;
@@ -27,17 +32,49 @@ export declare class JsonlGraphStore implements GraphStore {
      *  children (guarded, cycle-safe). Each child's repo_id comes from the proxy
      *  Repository node's federated_repo_id in the parent's nodes.jsonl. */
     private collectRepos;
-    /** Reloads the graph when anything it reads has changed on disk.
+    /** Two inputs, two keys.
      *
-     *  The key covers the derived JSONL, the committed summary shards, AND every
-     *  agent's sidecar. Keying on nodes/edges alone made a `git pull` that
-     *  brought in only a teammate's shards invisible for the life of the process
-     *  — the shards are committed and the derived graph is not, so that is the
-     *  ordinary case. Leaving the sidecars out did the same to a second agent
-     *  working the same checkout, which the overlay explicitly promises to
-     *  surface. Stat calls only, O(shards + sidecars). */
-    private freshnessKey;
+     *  The derived JSONL and the authored overlay change independently and cost
+     *  wildly different amounts to re-read. Keying both together meant every
+     *  `write_semantic_summary` — which rewrites this machine's sidecar — threw
+     *  the parsed graph away and re-read nodes.jsonl, edges.jsonl and every
+     *  shard on the next call. The node had already been updated in place, so
+     *  that rebuild bought nothing; a summarisation loop of K writes cost K full
+     *  reloads. Stat calls only, O(shards + sidecars). */
+    private graphKey;
+    /** The committed shards AND every agent's sidecar.
+     *
+     *  Keying on nodes/edges alone made a `git pull` that brought in only a
+     *  teammate's shards invisible for the life of the process — the shards are
+     *  committed and the derived graph is not, so that is the ordinary case.
+     *  Leaving the sidecars out did the same to a second agent working the same
+     *  checkout, which the overlay explicitly promises to surface. */
+    private overlayKey;
     private ensureFresh;
+    /** Write overlay props onto a node, remembering what they displaced.
+     *
+     *  Every in-place summary mutation goes through here — the overlay fold AND
+     *  writeSummary. A mutation that skipped it would be invisible to the undo
+     *  pass, and worse, the NEXT fold would record the value it wrote as the
+     *  pre-overlay original: removing that summary would then restore it. */
+    private overlayProps;
+    /** Fold the authored summaries onto the parsed graph, in place.
+     *
+     *  The committed shards first, then this machine's sidecars as a strictly
+     *  newer source, through the same rules the indexer uses. So what the server
+     *  serves is exactly what the next `index` will write — including which side
+     *  wins a divergence.
+     *
+     *  The shard overlay only matters between indexes (`index` already grafts
+     *  them into nodes.jsonl), but that is the ordinary case: a teammate's
+     *  summaries arrive with a `git pull` long before anyone re-indexes.
+     *  Sidecars are every agent's, not just this process's, so two agents on one
+     *  checkout see each other's work.
+     *
+     *  Re-applying is now routine rather than a side effect of a rebuild, so it
+     *  undoes the previous overlay first: a summary that has gone away must take
+     *  its props with it. */
+    private applyOverlay;
     getNode(repoIds: string[], id: string): Promise<TargetRow | null>;
     findByContentHash(repoIds: string[], hash: string): Promise<TargetRow[]>;
     resolveByPathAndName(repoIds: string[], filePath: string, name: string): Promise<TargetRow[]>;

package/dist/store/JsonlGraphStore.js

@@ -1,7 +1,8 @@
-import { statSync, readFileSync, existsSync } from "node:fs";
+import { statSync, existsSync } from "node:fs";
+import { readLines } from "./jsonlLines.js";
 import { join, resolve, isAbsolute, sep } from "node:path";
 import { CypherUnsupportedError, } from "./GraphStore.js";
-import { buildFederatedGraph, emptyGraph, } from "./jsonlGraph.js";
+import { buildFederatedGraphLines, emptyGraph, } from "./jsonlGraph.js";
 import { loadAllSidecars, sidecarPaths, sidecarPath, upsertSidecar } from "./sidecar.js";
 import { absorbSummarySource, emptyAccumulator, loadSummaryShards, recordSummaryConflicts, summaryShardsStamp, } from "./summaryShards.js";
 function str(v) {
@@ -73,7 +74,12 @@ const hasLabel = (n, l) => n.labels.includes(l);
 export class JsonlGraphStore {
     repoId;
     graph = emptyGraph();
-    stamp = "";
+    graphStamp = "";
+    overlayStamp = "";
+    /** Per node, the prop values the overlay overwrote — its own undo log.
+     *  A key absent before the overlay is recorded as `undefined`, so removing a
+     *  summary removes the props it added instead of leaving them behind. */
+    overlayUndo = new Map();
     repoPath;
     nodesPath;
     edgesPath;
@@ -103,22 +109,21 @@ export class JsonlGraphStore {
             seen.add(repoId);
             const nodesPath = join(dir, ".reposkein", "nodes.jsonl");
             const edgesPath = join(dir, ".reposkein", "edges.jsonl");
-            let nodesText = "";
-            let edgesText = "";
-            try {
-                if (existsSync(nodesPath))
-                    nodesText = readFileSync(nodesPath, "utf8");
-                if (existsSync(edgesPath))
-                    edgesText = readFileSync(edgesPath, "utf8");
-            }
-            catch {
+            if (!existsSync(nodesPath) && !existsSync(edgesPath))
                 return;
-            }
-            repos.push({ repoId, nodesText, edgesText });
-            // Discover children from this repo's Repository proxy nodes.
-            for (const line of nodesText.split("\n")) {
-                if (line.trim() === "")
-                    continue;
+            // Lazy: nothing is read until the build iterates them, and only one line
+            // is live at a time. This used to hold the COMPLETE text of every
+            // federated repo simultaneously — a root plus ten children meant the
+            // whole federation resident at once, before a single line was parsed.
+            repos.push({
+                repoId,
+                nodes: { [Symbol.iterator]: () => readLines(nodesPath) },
+                edges: { [Symbol.iterator]: () => readLines(edgesPath) },
+            });
+            // Discover children from this repo's Repository proxy nodes. A second
+            // streaming pass over the same file, which costs I/O rather than the
+            // second full line array the old code built.
+            for (const line of readLines(nodesPath)) {
                 let obj;
                 try {
                     obj = JSON.parse(line);
@@ -141,16 +146,16 @@ export class JsonlGraphStore {
         visit(this.repoPath, this.repoId);
         return repos;
     }
-    /** Reloads the graph when anything it reads has changed on disk.
+    /** Two inputs, two keys.
      *
-     *  The key covers the derived JSONL, the committed summary shards, AND every
-     *  agent's sidecar. Keying on nodes/edges alone made a `git pull` that
-     *  brought in only a teammate's shards invisible for the life of the process
-     *  — the shards are committed and the derived graph is not, so that is the
-     *  ordinary case. Leaving the sidecars out did the same to a second agent
-     *  working the same checkout, which the overlay explicitly promises to
-     *  surface. Stat calls only, O(shards + sidecars). */
-    freshnessKey() {
+     *  The derived JSONL and the authored overlay change independently and cost
+     *  wildly different amounts to re-read. Keying both together meant every
+     *  `write_semantic_summary` — which rewrites this machine's sidecar — threw
+     *  the parsed graph away and re-read nodes.jsonl, edges.jsonl and every
+     *  shard on the next call. The node had already been updated in place, so
+     *  that rebuild bought nothing; a summarisation loop of K writes cost K full
+     *  reloads. Stat calls only, O(shards + sidecars). */
+    graphKey() {
         let m = 0;
         try {
             if (existsSync(this.nodesPath))
@@ -161,25 +166,93 @@ export class JsonlGraphStore {
         catch {
             m = 0;
         }
-        return `${m}|${summaryShardsStamp(this.repoPath, sidecarPaths)}`;
+        return String(m);
+    }
+    /** The committed shards AND every agent's sidecar.
+     *
+     *  Keying on nodes/edges alone made a `git pull` that brought in only a
+     *  teammate's shards invisible for the life of the process — the shards are
+     *  committed and the derived graph is not, so that is the ordinary case.
+     *  Leaving the sidecars out did the same to a second agent working the same
+     *  checkout, which the overlay explicitly promises to surface. */
+    overlayKey() {
+        return summaryShardsStamp(this.repoPath, sidecarPaths);
     }
     ensureFresh() {
-        const key = this.freshnessKey();
-        if (key === this.stamp)
+        const graphKey = this.graphKey();
+        const overlayKey = this.overlayKey();
+        if (graphKey === this.graphStamp && overlayKey === this.overlayStamp)
             return;
-        this.stamp = key;
+        if (graphKey !== this.graphStamp) {
+            this.graphStamp = graphKey;
+            // A fresh graph carries no overlay, and the undo log refers to the node
+            // objects that were just discarded.
+            this.overlayUndo.clear();
+            this.overlayStamp = "";
+            try {
+                this.graph = buildFederatedGraphLines(this.collectRepos());
+            }
+            catch {
+                this.graph = emptyGraph();
+                return;
+            }
+        }
+        if (overlayKey !== this.overlayStamp) {
+            // Only bank the stamp when the overlay actually landed. Banking it first
+            // and then throwing would leave the graph with its previous overlay
+            // already undone and no new one applied — every summary silently gone
+            // until some unrelated change moved the key again.
+            this.overlayStamp = this.applyOverlay() ? overlayKey : "";
+        }
+    }
+    /** Write overlay props onto a node, remembering what they displaced.
+     *
+     *  Every in-place summary mutation goes through here — the overlay fold AND
+     *  writeSummary. A mutation that skipped it would be invisible to the undo
+     *  pass, and worse, the NEXT fold would record the value it wrote as the
+     *  pre-overlay original: removing that summary would then restore it. */
+    overlayProps(id, n, props) {
+        const saved = this.overlayUndo.get(id) ?? {};
+        for (const [k, v] of Object.entries(props)) {
+            // First writer wins the undo slot: it holds the pristine value, and a
+            // later overwrite would replace it with an already-overlaid one.
+            if (!Object.prototype.hasOwnProperty.call(saved, k)) {
+                saved[k] = Object.prototype.hasOwnProperty.call(n.props, k) ? n.props[k] : undefined;
+            }
+            n.props[k] = v;
+        }
+        this.overlayUndo.set(id, saved);
+    }
+    /** Fold the authored summaries onto the parsed graph, in place.
+     *
+     *  The committed shards first, then this machine's sidecars as a strictly
+     *  newer source, through the same rules the indexer uses. So what the server
+     *  serves is exactly what the next `index` will write — including which side
+     *  wins a divergence.
+     *
+     *  The shard overlay only matters between indexes (`index` already grafts
+     *  them into nodes.jsonl), but that is the ordinary case: a teammate's
+     *  summaries arrive with a `git pull` long before anyone re-indexes.
+     *  Sidecars are every agent's, not just this process's, so two agents on one
+     *  checkout see each other's work.
+     *
+     *  Re-applying is now routine rather than a side effect of a rebuild, so it
+     *  undoes the previous overlay first: a summary that has gone away must take
+     *  its props with it. */
+    applyOverlay() {
         try {
-            this.graph = buildFederatedGraph(this.collectRepos());
-            // Overlay authored summaries: the committed shards first, then this
-            // machine's sidecars as a strictly newer source, folded through the same
-            // rules the indexer uses. So what the server serves is exactly what the
-            // next `index` will write — including which side wins a divergence.
-            //
-            // The shard overlay only matters between indexes (`index` already grafts
-            // them into nodes.jsonl), but that is the ordinary case: a teammate's
-            // summaries arrive with a `git pull` long before anyone re-indexes.
-            // Sidecars are every agent's, not just this process's, so two agents on
-            // one checkout see each other's work.
+            for (const [id, saved] of this.overlayUndo) {
+                const n = this.graph.byId.get(id);
+                if (!n)
+                    continue;
+                for (const [k, v] of Object.entries(saved)) {
+                    if (v === undefined)
+                        delete n.props[k];
+                    else
+                        n.props[k] = v;
+                }
+            }
+            this.overlayUndo.clear();
             const shards = loadSummaryShards(this.repoPath);
             const overlay = emptyAccumulator();
             absorbSummarySource(overlay, {
@@ -199,16 +272,17 @@ export class JsonlGraphStore {
                 // node passes this the same way a shard record does.
                 if (rec.props.summary_of_hash !== n.props.content_hash)
                     continue;
-                for (const [k, v] of Object.entries(rec.props))
-                    n.props[k] = v;
+                this.overlayProps(id, n, rec.props);
             }
             // Divergence losers — from a merged shard, or from two agents writing the
             // same node — are authored prose. Preserve them for a human rather than
             // discarding them on the read path.
             recordSummaryConflicts(this.repoPath, [...shards.conflicts, ...overlay.conflicts]);
+            return true;
         }
         catch {
-            this.graph = emptyGraph();
+            // An unreadable overlay must not cost the caller the graph.
+            return false;
         }
     }
     async getNode(repoIds, id) {
@@ -286,11 +360,16 @@ export class JsonlGraphStore {
         const oldSummary = str(n.props.semantic_summary);
         const oldHash = str(n.props.summary_of_hash);
         const stale_replaced = oldSummary !== null && oldHash !== chash;
-        n.props.semantic_summary = fields.summary;
-        n.props.summary_of_hash = chash;
-        n.props.summary_model = fields.model;
-        n.props.summary_at = fields.at;
-        n.props.summary_by = fields.by;
+        // Through overlayProps, not raw assignment: this mutation has to be
+        // undoable like any other overlay write, or the next fold would mistake
+        // what it wrote for the node's pre-overlay state.
+        this.overlayProps(id, n, {
+            semantic_summary: fields.summary,
+            summary_of_hash: chash,
+            summary_model: fields.model,
+            summary_at: fields.at,
+            summary_by: fields.by,
+        });
         upsertSidecar(this.sidecarFile, {
             id,
             semantic_summary: fields.summary,