acp-kernel 0.0.28 → 0.0.29

package/dist/index.d.ts

@@ -29,6 +29,7 @@ export type { RenderStrategy } from "./render-refs.js";
 export { resolveTransformChannel } from "./transform-channel.js";
 export type { TransformChannel } from "./transform-channel.js";
 export { searchBlocks, searchBlocksAsync, blockDocs, messageDocs } from "./search.js";
+export { clearDocFeatures, docCacheInfo, docFeatures, setDocCacheCap } from "./search.js";
 export type { SearchResult, SearchOptions, SearchAlgorithm, AsyncSearchAlgorithm, AnySearchAlgorithm, SearchDoc, SearchDocKind, ScoredBlock, MessageRole, RoleWeights, MessageInput } from "./search.js";
 export { DEFAULT_ALGORITHM, DEFAULT_ROLE_WEIGHTS, registerSearchAlgorithm, getSearchAlgorithm, listSearchAlgorithms } from "./search.js";
 export { isMessageProtected, matchToolPattern } from "./protected.js";

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,YAAY,EACV,KAAK,EACL,eAAe,EACf,gBAAgB,EAChB,qBAAqB,GACtB,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,kBAAkB,EAClB,eAAe,EACf,aAAa,EACb,SAAS,EACT,YAAY,EACZ,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,GAChB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC5D,OAAO,EACL,UAAU,EACV,gBAAgB,EAChB,WAAW,EACX,UAAU,EACV,UAAU,EACV,SAAS,EACT,SAAS,EACT,WAAW,GACZ,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AAC1F,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAC3F,YAAY,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAChE,YAAY,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACjE,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAC/H,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9D,YAAY,EAAE,OAAO,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AACnE,OAAO,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAC/D,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC3E,OAAO,EACH,eAAe,EACf,6BAA6B,EAC7B,kBAAkB,EAClB,eAAe,EACf,2BAA2B,EAC3B,mBAAmB,GACtB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,iBAAiB,EAAE,sBAAsB,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACxG,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC5D,YAAY,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AAC/D,YAAY,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AACvD,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAC3F,YAAY,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACjE,YAAY,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACtF,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,eAAe,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,SAAS,EAAE,aAAa,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACzM,OAAO,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,uBAAuB,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACzI,OAAO,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACtE,OAAO,EACL,WAAW,EACX,MAAM,EACN,KAAK,YAAY,EACjB,KAAK,eAAe,EACpB,KAAK,MAAM,EACX,KAAK,WAAW,GACjB,MAAM,eAAe,CAAC;AACvB,cAAc,mBAAmB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,YAAY,EACV,KAAK,EACL,eAAe,EACf,gBAAgB,EAChB,qBAAqB,GACtB,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,kBAAkB,EAClB,eAAe,EACf,aAAa,EACb,SAAS,EACT,YAAY,EACZ,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,GAChB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC5D,OAAO,EACL,UAAU,EACV,gBAAgB,EAChB,WAAW,EACX,UAAU,EACV,UAAU,EACV,SAAS,EACT,SAAS,EACT,WAAW,GACZ,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AAC1F,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAC3F,YAAY,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAChE,YAAY,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACjE,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAC/H,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9D,YAAY,EAAE,OAAO,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AACnE,OAAO,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAC/D,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC3E,OAAO,EACH,eAAe,EACf,6BAA6B,EAC7B,kBAAkB,EAClB,eAAe,EACf,2BAA2B,EAC3B,mBAAmB,GACtB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,iBAAiB,EAAE,sBAAsB,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACxG,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC5D,YAAY,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AAC/D,YAAY,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AACvD,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAC3F,YAAY,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACjE,YAAY,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACtF,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC1F,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,eAAe,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,SAAS,EAAE,aAAa,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACzM,OAAO,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,uBAAuB,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACzI,OAAO,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACtE,OAAO,EACL,WAAW,EACX,MAAM,EACN,KAAK,YAAY,EACjB,KAAK,eAAe,EACpB,KAAK,MAAM,EACX,KAAK,WAAW,GACjB,MAAM,eAAe,CAAC;AACvB,cAAc,mBAAmB,CAAC"}

package/dist/index.js

@@ -2705,26 +2705,6 @@ function resolveTransformChannel(explicit, wireViable) {
   return explicit ?? (wireViable ? "wire" : "message");
 }
 
-// src/search/algorithms/substring.ts
-var substringAlgorithm = {
-  name: "substring",
-  description: "Exact substring counting (original baseline). Predictable, no normalization.",
-  score(docs, query) {
-    const terms = query.toLowerCase().trim().split(/\s+/).filter((t) => t.length > 0);
-    if (terms.length === 0) return docs.map((d) => ({ ref: d.ref, score: 0 }));
-    return docs.map((d) => {
-      const haystack = d.text.toLowerCase();
-      let score = 0;
-      for (const term of terms) score += countOccurrences2(haystack, term);
-      return { ref: d.ref, score };
-    });
-  }
-};
-function countOccurrences2(haystack, needle) {
-  if (!needle) return 0;
-  return haystack.split(needle).length - 1;
-}
-
 // src/search/stemmer.ts
 function stem(word) {
   let w = word;
@@ -2746,8 +2726,17 @@ function stem(word) {
 
 // src/search/tokenizer.ts
 var CJK = /[\u3400-\u9fff\uf900-\ufaff\u3040-\u30ff\uac00-\ud7af]/;
-var CJK_RUN = new RegExp(`${CJK.source}+`, "g");
 var LATIN_WORD = /[a-z][a-z0-9_]*[a-z0-9]|[a-z0-9]/g;
+var cjkSegmenter = new Intl.Segmenter("zh", { granularity: "word" });
+function cjkRunTokens(segs) {
+  const words = segs.filter((w) => w.length >= 2);
+  if (words.length > 0) return words;
+  const run = segs.join("");
+  const out = [];
+  for (let i = 0; i < run.length - 1; i++) out.push(run.slice(i, i + 2));
+  for (const ch of run) out.push(ch);
+  return out;
+}
 function tokenize(text, opts = {}) {
   const lower = text.toLowerCase();
   const tokens = [];
@@ -2758,15 +2747,23 @@ function tokenize(text, opts = {}) {
       tokens.push(w);
     }
   }
-  const cjkRuns = lower.match(CJK_RUN) ?? [];
-  for (const run of cjkRuns) {
-    if (run.length === 1) {
-      tokens.push(run);
-    } else {
-      for (let i = 0; i < run.length - 1; i++) tokens.push(run.slice(i, i + 2));
-      for (const ch of run) tokens.push(ch);
+  if (!CJK.test(lower)) return tokens;
+  const runSegs = [];
+  let cur = null;
+  for (const s of cjkSegmenter.segment(lower)) {
+    const t = s.segment;
+    if (t.length === 0) continue;
+    if (CJK.test(t)) {
+      (cur ??= []).push(t);
+    } else if (cur) {
+      runSegs.push(cur);
+      cur = null;
     }
   }
+  if (cur) runSegs.push(cur);
+  for (const segs of runSegs) {
+    tokens.push(...cjkRunTokens(segs));
+  }
   return tokens;
 }
 function charBigrams(text) {
@@ -2783,6 +2780,69 @@ function tfMap(text, stem2) {
   return m;
 }
 
+// src/search/doc-cache.ts
+var DEFAULT_CAP_CHARS = 8 * 1024 * 1024;
+var capChars = DEFAULT_CAP_CHARS;
+var cache = /* @__PURE__ */ new Map();
+var cachedChars = 0;
+function build(text) {
+  const tf = tfMap(text, true);
+  let len = 0;
+  for (const v of tf.values()) len += v;
+  const lower = text.toLowerCase();
+  return { tf, len, lower, grams: new Set(charBigrams(lower)) };
+}
+function docFeatures(text) {
+  const hit = cache.get(text);
+  if (hit) return hit;
+  const f = build(text);
+  if (text.length > 0 && text.length <= capChars) {
+    while (cachedChars + text.length > capChars && cache.size > 0) {
+      const k = cache.keys().next().value;
+      cachedChars -= k.length;
+      cache.delete(k);
+    }
+    cache.set(text, f);
+    cachedChars += text.length;
+  }
+  return f;
+}
+function clearDocFeatures() {
+  cache.clear();
+  cachedChars = 0;
+}
+function setDocCacheCap(chars) {
+  capChars = Math.max(1, chars);
+  while (cachedChars > capChars && cache.size > 0) {
+    const k = cache.keys().next().value;
+    cachedChars -= k.length;
+    cache.delete(k);
+  }
+}
+function docCacheInfo() {
+  return { entries: cache.size, chars: cachedChars };
+}
+
+// src/search/algorithms/substring.ts
+var substringAlgorithm = {
+  name: "substring",
+  description: "Exact substring counting (original baseline). Predictable, no normalization.",
+  score(docs, query) {
+    const terms = query.toLowerCase().trim().split(/\s+/).filter((t) => t.length > 0);
+    if (terms.length === 0) return docs.map((d) => ({ ref: d.ref, score: 0 }));
+    return docs.map((d) => {
+      const haystack = docFeatures(d.text).lower;
+      let score = 0;
+      for (const term of terms) score += countOccurrences2(haystack, term);
+      return { ref: d.ref, score };
+    });
+  }
+};
+function countOccurrences2(haystack, needle) {
+  if (!needle) return 0;
+  return haystack.split(needle).length - 1;
+}
+
 // src/search/algorithms/bm25.ts
 var bm25Algorithm = {
   name: "bm25",
@@ -2792,11 +2852,8 @@ var bm25Algorithm = {
     const k1 = 1.2;
     const b = 0.75;
     const parsed = docs.map((d) => {
-      const text = d.text;
-      const tf = tfMap(text, true);
-      let len = 0;
-      for (const v of tf.values()) len += v;
-      return { id: d.ref, tf, len };
+      const f = docFeatures(d.text);
+      return { id: d.ref, tf: f.tf, len: f.len };
     });
     const avgdl = parsed.reduce((s, d) => s + d.len, 0) / (N || 1);
     const qTerms = tokenize(query, { stem: true });
@@ -2825,14 +2882,13 @@ var fuzzyAlgorithm = {
   name: "fuzzy",
   description: "Character bigram overlap. Typo-tolerant, script-agnostic, high recall.",
   score(docs, query) {
-    const qTokens = query.toLowerCase().split(/[\s,]+/).filter((t) => t.length >= 4);
+    const qTokens = query.toLowerCase().split(/[\s,]+/).filter((t) => t.length >= 4 || t.length >= 2 && CJK.test(t));
     if (qTokens.length === 0) return docs.map((d) => ({ ref: d.ref, score: 0 }));
     const qGrams = /* @__PURE__ */ new Set();
     for (const t of qTokens) for (const g of charBigrams(t)) qGrams.add(g);
     if (qGrams.size === 0) return docs.map((d) => ({ ref: d.ref, score: 0 }));
     return docs.map((d) => {
-      const haystack = d.text.toLowerCase();
-      const docGrams = new Set(charBigrams(haystack));
+      const docGrams = docFeatures(d.text).grams;
       let hits = 0;
       for (const g of qGrams) if (docGrams.has(g)) hits++;
       return { ref: d.ref, score: hits / qGrams.size };
@@ -3006,6 +3062,7 @@ export {
   buildRecap,
   buildRestoredContentPreview,
   buildStatusReport,
+  clearDocFeatures,
   clearMessageFilters,
   collectBlockContent,
   coveredMessageIds,
@@ -3017,6 +3074,8 @@ export {
   defaultConfig,
   defaultCountTokens,
   defaultPrompts,
+  docCacheInfo,
+  docFeatures,
   emptyRefMap,
   estimateTokensFast,
   findActiveAncestor,
@@ -3052,6 +3111,7 @@ export {
   runPipeline,
   searchBlocks,
   searchBlocksAsync,
+  setDocCacheCap,
   syncBlocks,
   truncateLargeToolOutputs,
   validateConfig