acp-kernel 0.0.28 → 0.0.29

package/dist/search/algorithms/bm25.d.ts

@@ -7,9 +7,9 @@ import type { SearchAlgorithm } from "../types.js";
  * raw term count. Stemming collapses English morphology
  * (compress/compressed/compression → ~compress).
  *
- * On a 30-block mixed EN/CJK benchmark: MRR 0.812 vs 0.821 for substring
- * alone — not better in isolation, but a strong precision component when
- * combined with fuzzy recall (see hybrid.ts).
+ * On the 32-block mixed EN/CJK benchmark: MRR 0.833 / R@1 0.833 / R@3 0.833
+ * vs 0.797 / 0.792 / 0.792 for substring — better in isolation on every
+ * metric, and the precision component of the hybrid default (see hybrid.ts).
  */
 export declare const bm25Algorithm: SearchAlgorithm;
 //# sourceMappingURL=bm25.d.ts.map

package/dist/search/algorithms/bm25.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"bm25.d.ts","sourceRoot":"","sources":["../../../src/search/algorithms/bm25.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAA0B,MAAM,aAAa,CAAC;AAG3E;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,aAAa,EAAE,eAqC3B,CAAC"}
+{"version":3,"file":"bm25.d.ts","sourceRoot":"","sources":["../../../src/search/algorithms/bm25.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAA0B,MAAM,aAAa,CAAC;AAI3E;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,aAAa,EAAE,eAkC3B,CAAC"}

package/dist/search/algorithms/fuzzy.d.ts

@@ -4,11 +4,21 @@ import type { SearchAlgorithm } from "../types.js";
  *
  * Decomposes the query into character bigrams and measures overlap with
  * each doc. Robust to typos (tokan≈token), partial words, and works
- * uniformly across all scripts (CJK benefits most). Only considers query
- * tokens of length >= 4 to avoid noise from short common-character pairs.
+ * uniformly across all scripts (CJK benefits most).
  *
- * On benchmark: highest recall@3 (0.913) of any single algorithm — it is
- * the recall boost in the hybrid default.
+ * Query-token gate — CJK gets its own length rule; Latin is frozen:
+ *   length >= 4 (any script)   typo-tolerant bigram rescue needs a couple of
+ *       chars before it means anything; 2-3-char Latin tokens ("to", "of",
+ *       "us") are stop-word noise whose bigrams overlap nearly every doc.
+ *   length >= 2 && CJK         Chinese/Japanese/Korean words are mostly
+ *       2-character atomic units (登录/缓存/図表), so the Latin-style >= 4 rule
+ *       would lock the whole CJK query space out of this recall channel
+ *       (that gap is what bench "缓存 → nothing" exposed). Single CJK chars
+ *       stay excluded — one char cannot form a bigram, nothing to compare.
+ *
+ * On benchmark: lowest MRR of any single algorithm (0.795 — a hair under
+ * substring's 0.797) — precision is weak, but it is the recall boost in the
+ * hybrid default.
  */
 export declare const fuzzyAlgorithm: SearchAlgorithm;
 //# sourceMappingURL=fuzzy.d.ts.map

package/dist/search/algorithms/fuzzy.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fuzzy.d.ts","sourceRoot":"","sources":["../../../src/search/algorithms/fuzzy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAA0B,MAAM,aAAa,CAAC;AAG3E;;;;;;;;;;GAUG;AACH,eAAO,MAAM,cAAc,EAAE,eAmB5B,CAAC"}
+{"version":3,"file":"fuzzy.d.ts","sourceRoot":"","sources":["../../../src/search/algorithms/fuzzy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAA0B,MAAM,aAAa,CAAC;AAI3E;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,eAAO,MAAM,cAAc,EAAE,eAoB5B,CAAC"}

package/dist/search/algorithms/substring.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"substring.d.ts","sourceRoot":"","sources":["../../../src/search/algorithms/substring.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAA0B,MAAM,aAAa,CAAC;AAE3E;;;;;GAKG;AACH,eAAO,MAAM,kBAAkB,EAAE,eAahC,CAAC"}
+{"version":3,"file":"substring.d.ts","sourceRoot":"","sources":["../../../src/search/algorithms/substring.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAA0B,MAAM,aAAa,CAAC;AAG3E;;;;;GAKG;AACH,eAAO,MAAM,kBAAkB,EAAE,eAahC,CAAC"}

package/dist/search/doc-cache.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Per-doc derived features, memoized across search calls.
+ *
+ * A search over the compressed history re-scores the SAME immutable docs on
+ * every call — compressed block summaries and folded message text never
+ * change. Without this cache, every search_context call re-tokenized the
+ * entire corpus (segmenter CJK pass ≈ 0.3s/MB cold) plus re-lowercased it
+ * and rebuilt the bigram set for each channel: a 5MB session cost ~3s PER
+ * CALL, growing linearly with session length. With the cache the corpus is
+ * processed once; later searches are O(docs × query-terms).
+ *
+ * Keyed by doc text (immutable). Bounded by total cached source chars —
+ * oldest docs are evicted when the cap is exceeded, so a long-lived
+ * process serving many sessions cannot grow unboundedly. Hosts that want to
+ * release the memory eagerly on session shutdown/switch can call
+ * clearDocFeatures() (optional: the cap already bounds it).
+ */
+export interface DocFeatures {
+    /** Stemmed term frequencies (BM25 channel). */
+    tf: Map<string, number>;
+    /** Total term count (BM25 length normalization). */
+    len: number;
+    /** Lower-cased text (substring + fuzzy channels). */
+    lower: string;
+    /** Unique char bigrams of `lower` (fuzzy channel). */
+    grams: Set<string>;
+}
+export declare function docFeatures(text: string): DocFeatures;
+/** Drop all cached features (e.g. on session shutdown/switch). */
+export declare function clearDocFeatures(): void;
+/**
+ * Set the cache cap in source chars. Docs larger than the cap are never
+ * cached. Also used by tests to exercise eviction.
+ */
+export declare function setDocCacheCap(chars: number): void;
+/** Cache occupancy — for diagnostics. */
+export declare function docCacheInfo(): {
+    entries: number;
+    chars: number;
+};
+//# sourceMappingURL=doc-cache.d.ts.map

package/dist/search/doc-cache.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"doc-cache.d.ts","sourceRoot":"","sources":["../../src/search/doc-cache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAIH,MAAM,WAAW,WAAW;IACxB,+CAA+C;IAC/C,EAAE,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACxB,oDAAoD;IACpD,GAAG,EAAE,MAAM,CAAC;IACZ,qDAAqD;IACrD,KAAK,EAAE,MAAM,CAAC;IACd,sDAAsD;IACtD,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;CACtB;AAeD,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,CAcrD;AAED,kEAAkE;AAClE,wBAAgB,gBAAgB,IAAI,IAAI,CAGvC;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAOlD;AAED,yCAAyC;AACzC,wBAAgB,YAAY,IAAI;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAEjE"}

package/dist/search/index.d.ts

@@ -27,5 +27,7 @@ export declare function blockDocs(state: CompressionState): SearchDoc[];
 export declare function messageDocs(msgs: MessageInput[]): SearchDoc[];
 /** Sync entry — throws for async algorithms. Pass docs from blockDocs() + messageDocs(). */
 export declare function searchBlocks(docs: SearchDoc[], query: string, options?: SearchOptions): SearchResult[];
+export { clearDocFeatures, docCacheInfo, docFeatures, setDocCacheCap } from "./doc-cache.js";
+export type { DocFeatures } from "./doc-cache.js";
 export declare function searchBlocksAsync(docs: SearchDoc[], query: string, options?: SearchOptions): Promise<SearchResult[]>;
 //# sourceMappingURL=index.d.ts.map

package/dist/search/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/search/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAoB,MAAM,aAAa,CAAC;AAEtE,OAAO,KAAK,EAAE,SAAS,EAAe,YAAY,EAAE,MAAM,YAAY,CAAC;AACvE,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAe,MAAM,YAAY,CAAC;AAG3E,4EAA4E;AAC5E,wBAAgB,SAAS,CAAC,KAAK,EAAE,gBAAgB,GAAG,SAAS,EAAE,CAU9D;AAED;;;;;;;GAOG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,YAAY,EAAE,GAAG,SAAS,EAAE,CAW7D;AAkED,4FAA4F;AAC5F,wBAAgB,YAAY,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,YAAY,EAAE,CAQ1G;AAED,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAE9H"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/search/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAoB,MAAM,aAAa,CAAC;AAEtE,OAAO,KAAK,EAAE,SAAS,EAAe,YAAY,EAAE,MAAM,YAAY,CAAC;AACvE,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAe,MAAM,YAAY,CAAC;AAG3E,4EAA4E;AAC5E,wBAAgB,SAAS,CAAC,KAAK,EAAE,gBAAgB,GAAG,SAAS,EAAE,CAU9D;AAED;;;;;;;GAOG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,YAAY,EAAE,GAAG,SAAS,EAAE,CAW7D;AAkED,4FAA4F;AAC5F,wBAAgB,YAAY,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,YAAY,EAAE,CAQ1G;AAED,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAC7F,YAAY,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAElD,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAE9H"}

package/dist/search/tokenizer.d.ts

@@ -3,9 +3,20 @@
  *
  * Handles mixed Latin + CJK content — the single biggest quality lever
  * over plain substring search. Latin is split on non-word boundaries;
- * CJK (no spaces) is decomposed into overlapping bigrams so a query like
- * "身份验证" still scores against doc text "身份验证流程".
+ * CJK (no spaces) is word-segmented via Intl.Segmenter (CLDR dictionary),
+ * falling back to overlapping bigrams on out-of-vocabulary text so a query
+ * like "身份验证" still scores against doc text "身份验证流程".
+ *
+ * CJK segmentation is a SINGLE segment() pass over the whole text, not one
+ * call per CJK run: a segment() call has fixed overhead (~3µs), and
+ * run-heavy text (logs: dozens of short runs per line) made per-run calls
+ * 10-16× slower than one bulk pass. ICU never merges CJK words across
+ * non-CJK boundaries, so bulk segmentation yields the same words per run
+ * (differential-verified against the per-run implementation across a
+ * mixed-script stress corpus); run boundaries are re-derived below to keep
+ * the all-OOV bigram fallback.
  */
+export declare const CJK: RegExp;
 export interface TokenizeOptions {
     stem?: boolean;
 }

package/dist/search/tokenizer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../../src/search/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAQH,MAAM,WAAW,eAAe;IAC5B,IAAI,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,GAAE,eAAoB,GAAG,MAAM,EAAE,CAuB3E;AAED,sEAAsE;AACtE,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAOlD;AAED,0BAA0B;AAC1B,wBAAgB,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAItE"}
+{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../../src/search/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAYH,eAAO,MAAM,GAAG,QAA2D,CAAC;AAyB5E,MAAM,WAAW,eAAe;IAC5B,IAAI,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,GAAE,eAAoB,GAAG,MAAM,EAAE,CA+C3E;AAED,sEAAsE;AACtE,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAOlD;AAED,0BAA0B;AAC1B,wBAAgB,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAItE"}

package/dist/search.d.ts

@@ -2,6 +2,8 @@
  * Search — re-exports from the modular src/search/ implementation.
  */
 export { searchBlocks, searchBlocksAsync, blockDocs, messageDocs } from "./search/index.js";
+export { clearDocFeatures, docCacheInfo, docFeatures, setDocCacheCap } from "./search/index.js";
+export type { DocFeatures } from "./search/index.js";
 export type { SearchResult, SearchOptions, SearchAlgorithm, AsyncSearchAlgorithm, AnySearchAlgorithm, SearchDoc, SearchDocKind, ScoredBlock, MessageRole, RoleWeights, MessageInput, } from "./search/types.js";
 export { DEFAULT_ALGORITHM, DEFAULT_ROLE_WEIGHTS } from "./search/types.js";
 export { registerSearchAlgorithm, getSearchAlgorithm, listSearchAlgorithms } from "./search/registry.js";

package/dist/search.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAC5F,YAAY,EACR,YAAY,EACZ,aAAa,EACb,eAAe,EACf,oBAAoB,EACpB,kBAAkB,EAClB,SAAS,EACT,aAAa,EACb,WAAW,EACX,WAAW,EACX,WAAW,EACX,YAAY,GACf,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAC5E,OAAO,EAAE,uBAAuB,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC"}
+{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAC5F,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAChG,YAAY,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,YAAY,EACR,YAAY,EACZ,aAAa,EACb,eAAe,EACf,oBAAoB,EACpB,kBAAkB,EAClB,SAAS,EACT,aAAa,EACb,WAAW,EACX,WAAW,EACX,WAAW,EACX,YAAY,GACf,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAC5E,OAAO,EAAE,uBAAuB,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "acp-kernel",
-    "version": "0.0.28",
+    "version": "0.0.29",
     "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.",
     "license": "MIT",
     "author": "ranxianglei",