@avantf/dsh-mem 0.4.0 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/client.js +3 -1
- package/lib/engine/mem/config/config_files.d.ts +1 -1
- package/lib/engine/mem/db/dao/facts.d.ts +12 -0
- package/lib/engine/mem/db/vectors.d.ts +8 -2
- package/lib/engine/mem/store/common.d.ts +19 -9
- package/lib/engine/mem/store/floors.d.ts +28 -12
- package/lib/engine/mem/store/hybrid.d.ts +9 -0
- package/lib/engine/mem/store/knowledge.d.ts +2 -0
- package/lib/engine/mem/store/memory.d.ts +64 -4
- package/lib/engine/mem/store/self_query.d.ts +53 -0
- package/lib/engine/mem-contract/config.d.ts +5 -2
- package/lib/engine/mem-contract/types.d.ts +60 -0
- package/lib/engine/mem-retrieval/adapters/hnswlib.d.ts +11 -9
- package/lib/index.js +595 -104
- package/lib/types/client/index.d.ts.map +1 -1
- package/lib/types/envinit.d.ts +2 -1
- package/lib/types/envinit.d.ts.map +1 -1
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/render.d.ts.map +1 -1
- package/lib/types/vectorMigration.d.ts +44 -0
- package/lib/types/vectorMigration.d.ts.map +1 -0
- package/package.json +5 -5
package/lib/client.js
CHANGED
|
@@ -6457,10 +6457,12 @@ Set the \`cycles\` parameter to \`"ref"\` to resolve cyclical schemas with defs.
|
|
|
6457
6457
|
function HealthBlock(props) {
|
|
6458
6458
|
const r = props.stats.retrieval;
|
|
6459
6459
|
if (r === void 0) return h("div", { className: pages_module_css_default.status }, "检索健康度不可用。");
|
|
6460
|
+
const vectors = props.stats.vectors;
|
|
6461
|
+
const pendingVectors = vectors === void 0 ? 0 : vectors.stale + vectors.space_stale;
|
|
6460
6462
|
const pct = (v) => `${String(Math.round(v * 100))}%`;
|
|
6461
6463
|
const kinds = Object.entries(r.by_kind).map(([kind, v]) => `${kind} ${String(v.queries)} 次(空 ${String(v.zero_results)})`).join(" · ");
|
|
6462
6464
|
const truncated = r.embedding_truncated + r.rerank_truncated + r.output_truncated;
|
|
6463
|
-
return h("div", { className: pages_module_css_default.detail }, h("div", { className: pages_module_css_default.status }, `检索 ${String(r.queries)} 次 · 空结果 ${pct(r.zero_result_rate)} · 平均 ${r.avg_latency_ms.toFixed(1)}ms · 峰值 ${r.max_latency_ms.toFixed(1)}ms`), h("div", { className: pages_module_css_default.meta }, `语义腿在线 ${pct(r.semantic_live_rate)} · 平均命中 ${String(r.avg_results_per_query)} 条${r.rerank_used > 0 ? ` · 重排 ${String(r.rerank_used)} 次` : ""}`), kinds === "" ? null : h("div", { className: pages_module_css_default.meta }, kinds), h("div", { className: pages_module_css_default.meta }, truncated === 0 ? "未被任何预算截断。" : `截断:嵌入 ${String(r.embedding_truncated)} · 重排 ${String(r.rerank_truncated)} · 输出 ${String(r.output_truncated)}${r.output_truncated > 0 ? "(调大 retrieval.max_output_tokens 或用 max_tokens 参数)" : ""}`), r.updated_at === null ? null : h("div", { className: pages_module_css_default.meta }, `更新于 ${r.updated_at}`));
|
|
6465
|
+
return h("div", { className: pages_module_css_default.detail }, pendingVectors === 0 ? null : h("div", { className: pages_module_css_default.status }, `⚠ 向量空间待迁移:${String(pendingVectors)} 条 ACTIVE 向量来自旧嵌入空间(宽度不符 ${String(vectors?.stale ?? 0)} · 其他模型 ${String(vectors?.space_stale ?? 0)})——语义腿对它们失效,正在后台分批重算;手动入口 \`avantf-mem vectors --fix\``), h("div", { className: pages_module_css_default.status }, `检索 ${String(r.queries)} 次 · 空结果 ${pct(r.zero_result_rate)} · 平均 ${r.avg_latency_ms.toFixed(1)}ms · 峰值 ${r.max_latency_ms.toFixed(1)}ms`), h("div", { className: pages_module_css_default.meta }, `语义腿在线 ${pct(r.semantic_live_rate)} · 平均命中 ${String(r.avg_results_per_query)} 条${r.rerank_used > 0 ? ` · 重排 ${String(r.rerank_used)} 次` : ""}`), kinds === "" ? null : h("div", { className: pages_module_css_default.meta }, kinds), h("div", { className: pages_module_css_default.meta }, truncated === 0 ? "未被任何预算截断。" : `截断:嵌入 ${String(r.embedding_truncated)} · 重排 ${String(r.rerank_truncated)} · 输出 ${String(r.output_truncated)}${r.output_truncated > 0 ? "(调大 retrieval.max_output_tokens 或用 max_tokens 参数)" : ""}`), r.updated_at === null ? null : h("div", { className: pages_module_css_default.meta }, `更新于 ${r.updated_at}`));
|
|
6464
6466
|
}
|
|
6465
6467
|
/**
|
|
6466
6468
|
* Group cross-store hits for display: memory facts vs `domain → source`.
|
|
@@ -15,7 +15,7 @@ import type { AvantfLogger } from '../../mem-contract/index.js';
|
|
|
15
15
|
* UNCOMMENTED line is warned about by the loader, so a wrong name here would be a trap rather than a
|
|
16
16
|
* hint. `config_files.spec.ts` pins the ones this template names.
|
|
17
17
|
*/
|
|
18
|
-
export declare const DEFAULT_COMMON_CONFIG = "# avantf-mem \u516C\u5171\u914D\u7F6E\uFF08\u5206\u5C42\u7B2C\u2461\u5C42\uFF09\u3002\u8FD9\u91CC\u5168\u662F\u6CE8\u91CA\uFF1A\u5220\u6389/\u7559\u7A7A = \u7528\u5185\u5EFA\u9ED8\u8BA4\uFF0C\u53D6\u6D88\u6CE8\u91CA\u5373\u751F\u6548\u3002\n# \u5206\u5C42\uFF08\u4F4E\u2192\u9AD8\uFF09\uFF1A\u2460 \u5185\u5EFA\u9ED8\u8BA4 \u2192 \u2461 \u672C\u6587\u4EF6 \u2192 \u2462 \u540C\u76EE\u5F55\u7684 memory.yaml / knowledge.yaml\uFF08\u7F3A\u5931=\u4E0D\u8986\u76D6\uFF09\n# \u2192 \u2463 \u73AF\u5883\u53D8\u91CF \u2192 \u2464 \u63D2\u4EF6/CLI \u663E\u5F0F\u4F20\u53C2\n# \u6240\u6709\u53EF\u7F16\u8F91\u6587\u672C\u90FD\u5728 ~/.avantf \u4E0B\uFF1Aconfigs/*.yaml\uFF08\u672C\u76EE\u5F55\uFF09\u4E0E prompts/*.md\uFF08\u6A21\u578B\u63D0\u793A\u8BCD\uFF09\u3002\n\n# \u6570\u636E\u6839\u4E0D\u5728\u8FD9\u91CC\u914D\uFF1A\u5B83\u7531 \u2463 AVANTF_HOME\uFF08\u6216 CLI --data-home\u3001\u63D2\u4EF6 profile \u7684 dataHome\uFF09\u51B3\u5B9A \u2014\u2014\n# \u672C\u6587\u4EF6\u5C31\u5728\u6570\u636E\u6839\u91CC\u9762\uFF0C\u89E3\u6790\u6839\u7684\u65F6\u5019\u8FD8\u6CA1\u8BFB\u5230\u5B83\uFF0C\u6240\u4EE5\u5199\u5728\u8FD9\u91CC\u7684 dataHome \u5BF9\u6570\u636E\u6839\u65E0\u6548\uFF08DESIGN \u00A73\uFF09\u3002\n\n# semantic: # \u5D4C\u5165\u540E\u7AEF\uFF08\u8BB0\u5FC6\u4E0E\u77E5\u8BC6\u5171\u7528\uFF0C\u8DE8\u5E93\u5206\u6570\u624D\u53EF\u6BD4\uFF09\n# backend: local_bge\n# local_model: Xenova/bge-
|
|
18
|
+
export declare const DEFAULT_COMMON_CONFIG = "# avantf-mem \u516C\u5171\u914D\u7F6E\uFF08\u5206\u5C42\u7B2C\u2461\u5C42\uFF09\u3002\u8FD9\u91CC\u5168\u662F\u6CE8\u91CA\uFF1A\u5220\u6389/\u7559\u7A7A = \u7528\u5185\u5EFA\u9ED8\u8BA4\uFF0C\u53D6\u6D88\u6CE8\u91CA\u5373\u751F\u6548\u3002\n# \u5206\u5C42\uFF08\u4F4E\u2192\u9AD8\uFF09\uFF1A\u2460 \u5185\u5EFA\u9ED8\u8BA4 \u2192 \u2461 \u672C\u6587\u4EF6 \u2192 \u2462 \u540C\u76EE\u5F55\u7684 memory.yaml / knowledge.yaml\uFF08\u7F3A\u5931=\u4E0D\u8986\u76D6\uFF09\n# \u2192 \u2463 \u73AF\u5883\u53D8\u91CF \u2192 \u2464 \u63D2\u4EF6/CLI \u663E\u5F0F\u4F20\u53C2\n# \u6240\u6709\u53EF\u7F16\u8F91\u6587\u672C\u90FD\u5728 ~/.avantf \u4E0B\uFF1Aconfigs/*.yaml\uFF08\u672C\u76EE\u5F55\uFF09\u4E0E prompts/*.md\uFF08\u6A21\u578B\u63D0\u793A\u8BCD\uFF09\u3002\n\n# \u6570\u636E\u6839\u4E0D\u5728\u8FD9\u91CC\u914D\uFF1A\u5B83\u7531 \u2463 AVANTF_HOME\uFF08\u6216 CLI --data-home\u3001\u63D2\u4EF6 profile \u7684 dataHome\uFF09\u51B3\u5B9A \u2014\u2014\n# \u672C\u6587\u4EF6\u5C31\u5728\u6570\u636E\u6839\u91CC\u9762\uFF0C\u89E3\u6790\u6839\u7684\u65F6\u5019\u8FD8\u6CA1\u8BFB\u5230\u5B83\uFF0C\u6240\u4EE5\u5199\u5728\u8FD9\u91CC\u7684 dataHome \u5BF9\u6570\u636E\u6839\u65E0\u6548\uFF08DESIGN \u00A73\uFF09\u3002\n\n# semantic: # \u5D4C\u5165\u540E\u7AEF\uFF08\u8BB0\u5FC6\u4E0E\u77E5\u8BC6\u5171\u7528\uFF0C\u8DE8\u5E93\u5206\u6570\u624D\u53EF\u6BD4\uFF09\n# backend: local_bge\n# local_model: Xenova/bge-base-zh-v1.5 # \u5FC5\u987B\u662F ONNX \u4ED3\u5E93\n# dim: 768\n# auto_download: true\n# auto_migrate: true # \u6362\u6A21\u578B/\u5BBD\u5EA6\u540E\u81EA\u52A8\u5206\u6279\u91CD\u7B97\u65E7\u7A7A\u95F4\u7684\u5411\u91CF\uFF08\u9ED8\u8BA4\u5F00\uFF09\uFF1B\u5173\u6389\u53EA\u5269\u542F\u52A8\u544A\u8B66 + vectors --fix\n# \u26A0\uFE0F cache_dir / mirror \u7531\u5BB6\u65CF\u5E95\u5EA7 @avantf/dsh-plugin-base \u7BA1\u7406\uFF1A\u5199\u5728\u8FD9\u91CC\u4F1A\u88AB\u5FFD\u7565\u5E76\u544A\u8B66\u3002\n# \u7F13\u5B58\u76EE\u5F55\u7528\u73AF\u5883\u53D8\u91CF AVANTF_MEM_MODEL_CACHE\uFF0C\u955C\u50CF\u7528 AVANTF_MEM_MODEL_MIRROR\uFF08\u6216 HF_ENDPOINT\uFF09\u3002\n\n# rerank: # \u91CD\u6392\u7684\u552F\u4E00\u5F00\u5173\uFF1Bnone = \u4E0D\u52A0\u8F7D\u4EFB\u4F55\u91CD\u6392\u6A21\u578B\n# backend: none\n# local_model: Xenova/bge-reranker-base\n\n# vectorStore:\n# backend: auto # auto = \u8D8A\u8FC7\u9608\u503C\u81EA\u52A8\u4ECE local_numpy \u8FC1\u5230 hnswlib\n# auto_thresholds:\n# hnswlib: 2000\n# hnswlib_ef_search: 256 # ANN \u7684\u53EC\u56DE/\u901F\u5EA6\u65CB\u94AE\uFF0C\u8BED\u6599\u53D8\u5927\u5148\u8C03\u5B83\n\n# retriever: # \u4E09\u6761\u68C0\u7D22\u817F\u5728\u878D\u5408\u91CC\u7684\u6743\u91CD\u4E0E\"\u76F8\u5173\u6027\u95E8\u69DB\"\n# weight_semantic: 0.55\n# weight_fts: 0.30\n# weight_jaccard: 0.15\n# # \u7EDD\u5BF9\u95E8\u69DB\u6253\u5728\u6BCF\u6761\u817F\u7684\u539F\u59CB\u5206\u4E0A\u3001fuse() \u4E4B\u524D\uFF08\u878D\u5408\u5206\u53EA\u5728\u4E00\u6B21\u67E5\u8BE2\u5185\u53EF\u6BD4\uFF0C\u4E0D\u80FD\u5F53\u95E8\u69DB\uFF09\u3002\n# # 0 = \u5173\u95ED\u8BE5\u95E8\u69DB\uFF1B\u5206\u6570\u7B49\u4E8E\u95E8\u69DB\u4FDD\u7559\u3002\u8BED\u4E49\u540E\u7AEF\u4E0D\u53EF\u7528\u65F6 min_fts_terms \u7684\u751F\u6548\u503C\u653E\u5BBD\u5230 1\u3002\n# min_semantic_similarity: 0.5 # \u8BED\u4E49\u817F\u4F59\u5F26\uFF1B0.5 \u53EA\u5BF9 bge-base-zh-v1.5/768\uFF08mean pooling\uFF09\u6807\u5B9A\u8FC7\uFF0C\u6362\u6A21\u578B\u8981\u91CD\u6807\n# min_fts_terms: 2 # FTS \u817F\uFF1A\u8FD9\u4E00\u884C\u547D\u4E2D\u51E0\u4E2A\u4E0D\u540C\u7684\u67E5\u8BE2\u8BCD\u5143\uFF08\u62C9\u4E01\u8BCD\u22655\u5B57\u7B26 + CJK 3-gram\uFF09\n# min_jaccard: 0.2 # \u5B9E\u4F53\u817F Jaccard \u6BD4\u503C\n\n# lifecycle:\n# purge_after_archived_days: 365 # \u5F52\u6863\u4E8B\u5B9E\u7684\u7269\u7406\u6E05\u7406\u7A97\u53E3\uFF08\u6D3B\u8DC3\u65E5\uFF09\n# contradiction_threshold: 0.6\n\n# trust:\n# decay_per_day: 0.0055\u2026\n# presence:\n# heartbeat_minutes: 60 # \u5E38\u9A7B\u8FDB\u7A0B\u7684\u5FC3\u8DF3\uFF08\u6D3B\u8DC3\u65E5\u65F6\u949F\u4E0E\u751F\u547D\u5468\u671F\u626B\u63CF\uFF09\n\n# tools: # \u5916\u90E8\u4E8C\u8FDB\u5236\uFF08pandoc\uFF09\u7684\u53D7\u7BA1\u76EE\u5F55\n# dir: '' # \u7559\u7A7A = \u5BB6\u65CF\u6839 tools\uFF08~/.avantf/env/tools\uFF09\uFF1BAVANTF_TOOLS_DIR \u53EF\u8986\u76D6\n# auto_install: true\n";
|
|
19
19
|
/**
|
|
20
20
|
* Ensure the common config exists with content, and hand back the path either way.
|
|
21
21
|
*
|
|
@@ -100,6 +100,18 @@ export declare class FactsDao {
|
|
|
100
100
|
semantic_vector: Uint8Array;
|
|
101
101
|
embedding_model: string | null;
|
|
102
102
|
}[];
|
|
103
|
+
/**
|
|
104
|
+
* Width + recorded space of every ACTIVE persisted vector, WITHOUT reading the bytes.
|
|
105
|
+
*
|
|
106
|
+
* `length(blob)` is the byte count, so `bytes / 4` is the float width — enough to detect a
|
|
107
|
+
* changed `semantic.dim` without decoding a corpus. This is the cheap detection the startup
|
|
108
|
+
* warning and the `/mem` status surface read (see `MemoryStore.vectorSpaceHealth`).
|
|
109
|
+
*/
|
|
110
|
+
activeVectorSpaces(): {
|
|
111
|
+
fact_id: number;
|
|
112
|
+
bytes: number;
|
|
113
|
+
embedding_model: string | null;
|
|
114
|
+
}[];
|
|
103
115
|
/**
|
|
104
116
|
* Facts per archive reason — the trust diagnostics breakdown. One grouped scan rather than
|
|
105
117
|
* a count per reason, because the reasons are few but the table is not.
|
|
@@ -23,13 +23,19 @@ export declare function vectorSpaceId(backend: string, model: string, dim: numbe
|
|
|
23
23
|
* The rows come from the owning store's DAO (DESIGN §19): this helper decodes and
|
|
24
24
|
* filters them, it never runs SQL itself.
|
|
25
25
|
*
|
|
26
|
+
* A row whose width does not fit the store is skipped (never `add()`ed — the store
|
|
27
|
+
* would throw). The OWNING store reports that condition itself, as one actionable
|
|
28
|
+
* line with the count, the reason and the manual re-encode entry (`reportStaleVectors`),
|
|
29
|
+
* so this helper stays quiet: two warnings for one condition read as two problems.
|
|
30
|
+
*
|
|
26
31
|
* @param rows - `id` (fact/chunk id) + `vec` (BLOB), as read back from the store.
|
|
27
|
-
* @param
|
|
32
|
+
* @param _label - store name; unused here (the owning store owns the warning), kept so call sites read
|
|
33
|
+
* as "which store is reloading".
|
|
28
34
|
*/
|
|
29
35
|
export declare function reloadVectorIndex(vstore: VectorStore, rows: readonly {
|
|
30
36
|
id: number;
|
|
31
37
|
vec: Uint8Array | null;
|
|
32
|
-
}[],
|
|
38
|
+
}[], _label: string): {
|
|
33
39
|
loaded: number;
|
|
34
40
|
skipped: number;
|
|
35
41
|
};
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Small on purpose: `MemoryStore` and `KnowledgeStore` are separate stores over separate databases
|
|
5
5
|
* and should stay that way (AGENTS.md). What they must NOT have is two copies of the same
|
|
6
6
|
* three-line helper, because a fix then lands in one of them — which is exactly what happened to the
|
|
7
|
-
* retrieval orchestration (see `store/hybrid.ts`) and to the
|
|
7
|
+
* retrieval orchestration (see `store/hybrid.ts`) and to the stale-vector warning below: the
|
|
8
8
|
* memory store checked whether its persisted vectors came from the CURRENT vector space and told the
|
|
9
9
|
* operator how to fix it, while the knowledge store checked only the width, so a model swap left two
|
|
10
10
|
* spaces silently mixed in one index until someone happened to run `kb_reindex`.
|
|
@@ -40,19 +40,29 @@ export declare function vectorSpaceOf(semantic: SemanticBackend, model: string):
|
|
|
40
40
|
*/
|
|
41
41
|
export declare function evictVectors(vstore: VectorStore, ids: readonly number[]): void;
|
|
42
42
|
/**
|
|
43
|
-
* Tell the operator that persisted vectors came from ANOTHER vector space
|
|
43
|
+
* Tell the operator that persisted vectors came from ANOTHER vector space — loudly, and with the
|
|
44
|
+
* manual remedy.
|
|
44
45
|
*
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
-
*
|
|
48
|
-
*
|
|
46
|
+
* Changing the embedding space (model, width, pooling, normalization) is a DATA MIGRATION: the
|
|
47
|
+
* persisted bytes stay valid-looking but are no longer comparable with the current query vector, so
|
|
48
|
+
* the semantic leg must not serve them. Measured on the real library after the 512→768 default-model
|
|
49
|
+
* swap: 78 of 80 ACTIVE facts fell out of the semantic leg and retrieval silently degraded to
|
|
50
|
+
* lexical+entity (every probe query answered with the same unrelated short fact), with no error
|
|
51
|
+
* anywhere. The count, the reason and the ONE manual entry have to be visible at start.
|
|
52
|
+
*
|
|
53
|
+
* Both halves are reported in one message because they are one condition seen two ways: a changed
|
|
54
|
+
* WIDTH (`stale`, whose bytes cannot even be decoded into the current store) and a changed SPACE at
|
|
55
|
+
* the same width (`space_stale`, whose bytes decode but rank in another model's coordinates).
|
|
49
56
|
*
|
|
50
57
|
* @param kind store label for the message (`memory` / `knowledge`)
|
|
51
58
|
* @param space the vector space the store writes into now
|
|
52
|
-
* @param
|
|
53
|
-
* @param fix the
|
|
59
|
+
* @param counts how many persisted vectors are unusable in the current space, by kind
|
|
60
|
+
* @param fix the manual entry that re-encodes them, as the operator would type it
|
|
54
61
|
*/
|
|
55
|
-
export declare function
|
|
62
|
+
export declare function reportStaleVectors(kind: string, space: string, counts: {
|
|
63
|
+
stale: number;
|
|
64
|
+
space_stale: number;
|
|
65
|
+
}, fix: string): void;
|
|
56
66
|
/**
|
|
57
67
|
* Hand the event loop back between two synchronous batches.
|
|
58
68
|
*
|
|
@@ -23,8 +23,10 @@
|
|
|
23
23
|
* WHY THERE IS A RELAXED PROFILE AT ALL, AND WHY IT IS NOT "NO FLOOR". The floors are calibrated
|
|
24
24
|
* for paraphrase-style relevance, and there is a band where no threshold separates "answers the
|
|
25
25
|
* question" from "unrelated": measured on the live store, the fact answering 「我是谁」 scored cosine
|
|
26
|
-
* 0.444
|
|
27
|
-
*
|
|
26
|
+
* 0.444 under the original 512-dim default (0.459 under the shipped 768-dim `bge-base-zh-v1.5`)
|
|
27
|
+
* while four unrelated queries topped out at 0.384 (0.353 at 768) — a ~0.1 margin, and the
|
|
28
|
+
* answering fact was the query's own top-1. So the answer is not a lower default (which would trade
|
|
29
|
+
* precision for
|
|
28
30
|
* the whole corpus) but a SECOND pass with an absolute bottom line ({@link LOOSE_FLOORS}), run only
|
|
29
31
|
* when the strict pass returned nothing at all AND the floors are why (`dropped_by_floor > 0`).
|
|
30
32
|
*
|
|
@@ -50,7 +52,11 @@
|
|
|
50
52
|
* FTS REACHABILITY. `min_fts_terms` counts distinct query terms a row hits, but a query can only
|
|
51
53
|
* produce so many terms (`relevanceTerms`: latin words ≥5 chars + CJK 3-grams) — a 3-character CJK
|
|
52
54
|
* query yields exactly ONE trigram, so a configured bar of 2 is unreachable and that leg is empty
|
|
53
|
-
* for it by construction. The effective bar is therefore `min(configured, termCount)
|
|
55
|
+
* for it by construction. The effective bar is therefore `min(configured, termCount)`, and
|
|
56
|
+
* `termCount` is that of the text being GRADED — with 方案 A's augmentation the original and the
|
|
57
|
+
* rewrite are different texts with different counts, so each resolves its own bar
|
|
58
|
+
* (`store/hybrid.ts`). Resolving the rewrite from the original's count is the defect that let a
|
|
59
|
+
* single incidental trigram pass a bar it could not reach (2026-10-04).
|
|
54
60
|
*
|
|
55
61
|
* @module store/floors
|
|
56
62
|
*/
|
|
@@ -69,12 +75,14 @@ export type FloorLeg = 'semantic' | 'fts' | 'jaccard';
|
|
|
69
75
|
/**
|
|
70
76
|
* What the RELAXED pass lowers each floor TO, as absolute values.
|
|
71
77
|
*
|
|
72
|
-
* - semantic **0.40**: strictly BETWEEN the measured unrelated-query ceiling
|
|
73
|
-
* pronoun-query top-1
|
|
78
|
+
* - semantic **0.40**: strictly BETWEEN the measured unrelated-query ceiling and the
|
|
79
|
+
* pronoun-query top-1. Both bounds are load-bearing. Above the ceiling, so a genuinely
|
|
74
80
|
* unrelated question still comes back EMPTY from the relaxed pass — the four measured samples
|
|
75
|
-
* topped out at 0.300/0.384/0.315/0.351
|
|
76
|
-
* 0.
|
|
77
|
-
*
|
|
81
|
+
* topped out at 0.300/0.384/0.315/0.351 under the original 512-dim default and at
|
|
82
|
+
* 0.271/0.353/0.350/0.269 under the shipped 768-dim `bge-base-zh-v1.5` (a 0.35 bar would already
|
|
83
|
+
* admit two of them in either scale). Below the top-1, so 「我是谁」 finds its fact (0.444 at 512,
|
|
84
|
+
* 0.459 at 768). 0.40 keeps the larger margin on the noise side, which is where a false positive
|
|
85
|
+
* costs the caller a wrong answer rather than a missing one.
|
|
78
86
|
* - fts: NOT a constant — {@link looseTermFloor}, half the query's distinct terms with a floor of
|
|
79
87
|
* 1. `1` here is the "no terms to scale by" baseline (the previously constant value); a query
|
|
80
88
|
* that cannot produce terms is not graded at all, and one that produces a single term (every
|
|
@@ -113,16 +121,24 @@ export declare function emptyFloorDrops(): RetrievalFloorDrops;
|
|
|
113
121
|
* and changing the REPORTED value there would misdescribe what the caller is reading. `0` stays `0`
|
|
114
122
|
* (the operator's off switch).
|
|
115
123
|
*
|
|
116
|
-
* This is the ONE place the clamp lives; `resolveFloors` (what the result reports)
|
|
117
|
-
* `applyTermFloor` (what the leg does) both call it, so the number
|
|
118
|
-
*
|
|
124
|
+
* This is the ONE place the clamp lives; `resolveFloors` (what the result reports / a run is handed)
|
|
125
|
+
* and `applyTermFloor` (what the leg does) both call it, so the number a run reads is the number its
|
|
126
|
+
* leg used. It is a `min`, i.e. one-directional: it can LOWER an unreachable configured bar but never
|
|
127
|
+
* RAISE a bar inherited from another text, which is why the caller must resolve per graded text
|
|
128
|
+
* rather than reusing one text's resolution for another.
|
|
119
129
|
*/
|
|
120
130
|
export declare function effectiveTermFloor(floor: number, termCount?: number): number;
|
|
121
131
|
/** The per-query floor resolution inputs beyond the configuration. */
|
|
122
132
|
export interface FloorResolution {
|
|
123
133
|
/** `loose` lowers each configured floor (see {@link LOOSE_FLOORS}); absent means `strict`. */
|
|
124
134
|
profile?: FloorProfile;
|
|
125
|
-
/**
|
|
135
|
+
/**
|
|
136
|
+
* The distinct term count of the text being GRADED (`relevanceTerms(text).length`), for the FTS
|
|
137
|
+
* clamp. It must describe the query the legs will actually run on: under 方案 A's augmentation the
|
|
138
|
+
* rewrite is a DIFFERENT text with its own term count, and resolving both runs from the original's
|
|
139
|
+
* count is the defect that let one incidental trigram through (see `store/hybrid.ts`). Callers with
|
|
140
|
+
* several graded texts resolve once per text.
|
|
141
|
+
*/
|
|
126
142
|
termCount?: number;
|
|
127
143
|
/**
|
|
128
144
|
* Which legs a `'loose'` profile may lower, from {@link droppedLegs} of the pass that made the
|
|
@@ -144,6 +144,15 @@ export interface HybridPlan {
|
|
|
144
144
|
/** Per-call output token budget; `0` = unlimited, omitted = `retriever.max_output_tokens`. */
|
|
145
145
|
maxTokens?: number;
|
|
146
146
|
queryVector?: Float32Array;
|
|
147
|
+
/**
|
|
148
|
+
* Overrides the self-reference rewriter for this search (方案 A, `store/self_query.ts`).
|
|
149
|
+
*
|
|
150
|
+
* Default = the intent table. A TEST SEAM, not a second pluggability path: the frozen sentinel
|
|
151
|
+
* passes `() => undefined` to prove its assertions rest on the table (turn it off and the
|
|
152
|
+
* self-referential questions fall back to their pre-A outcomes), and `() => '<anything>'` to
|
|
153
|
+
* reproduce a MISCONFIGURED table and prove the augmentation only ever ADDS recall.
|
|
154
|
+
*/
|
|
155
|
+
rewriteQuery?: (query: string) => string | undefined;
|
|
147
156
|
/**
|
|
148
157
|
* Which relevance-floor policy governs this query (see the contract's `FLOOR_PROFILES`):
|
|
149
158
|
* omitted = the configured floors plus ONE relaxed pass when they empty the result, `'strict'` =
|
|
@@ -31,6 +31,8 @@ export interface KnowledgeSearchOptions {
|
|
|
31
31
|
* dropping legs on its second call.
|
|
32
32
|
*/
|
|
33
33
|
relaxLegs?: readonly FloorLeg[];
|
|
34
|
+
/** See `SearchInput.rewriteQuery` (方案 A test seam); default = the intent table. */
|
|
35
|
+
rewriteQuery?: (query: string) => string | undefined;
|
|
34
36
|
/** See `SearchInput.recordStats`: the cross-store router records the merged query once. */
|
|
35
37
|
recordStats?: boolean;
|
|
36
38
|
/**
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type Config, type ContradictionRecord, type FactDetail, type FactPage, type FloorProfile, type RecallResult, type RememberResult, type TrustDiagnostic, type VectorsDiagnostic, type VectorsFixReport } from '../../mem-contract/index.js';
|
|
1
|
+
import { type Config, type ContradictionRecord, type FactDetail, type FactPage, type FloorProfile, type RecallResult, type RememberResult, type TrustDiagnostic, type VectorMigrationOutcome, type VectorMigrationProgress, type VectorSpaceHealth, type VectorsDiagnostic, type VectorsFixReport } from '../../mem-contract/index.js';
|
|
2
2
|
import type { Reranker, SemanticBackend, VectorStore } from '../../mem-retrieval/index.js';
|
|
3
3
|
import type { Db } from '../db/conn.js';
|
|
4
4
|
import { type LexicalProbe } from './lexical.js';
|
|
@@ -21,6 +21,16 @@ export type { DetectedContradiction, RememberResult as AddResult } from '../../m
|
|
|
21
21
|
* until `deferred === 0` instead.
|
|
22
22
|
*/
|
|
23
23
|
export declare const ENTITY_SWEEP_BATCH = 2000;
|
|
24
|
+
/**
|
|
25
|
+
* Rows one background vector-space migration slice may drop + re-encode.
|
|
26
|
+
*
|
|
27
|
+
* Small on purpose: the point of batching is that a query arriving mid-migration waits for at most
|
|
28
|
+
* one ONNX forward pass batch, not for a whole corpus. At ~30 ms per bge-base-zh encode a batch of 16
|
|
29
|
+
* is well under a second of contested model time, and the event loop is yielded between batches
|
|
30
|
+
* (`yieldToEventLoop`). The value is a default, not a promise — `migrateVectorsBatch(batchSize)` takes
|
|
31
|
+
* whatever the caller needs.
|
|
32
|
+
*/
|
|
33
|
+
export declare const DEFAULT_VECTOR_MIGRATION_BATCH = 16;
|
|
24
34
|
/** What one derived-state sweep pass did (see `MemoryStore.reindexEntities`). */
|
|
25
35
|
export interface EntitySweepReport {
|
|
26
36
|
/**
|
|
@@ -111,6 +121,12 @@ export interface SearchInput {
|
|
|
111
121
|
* knows which legs were the problem for the user's one question.
|
|
112
122
|
*/
|
|
113
123
|
relaxLegs?: readonly FloorLeg[];
|
|
124
|
+
/**
|
|
125
|
+
* Overrides the self-reference rewriter for this search (方案 A). Default = the intent table in
|
|
126
|
+
* `store/self_query.ts`; a TEST SEAM (see `HybridPlan.rewriteQuery`), not a second pluggability
|
|
127
|
+
* path — `runtime` and the cross-store router never set it.
|
|
128
|
+
*/
|
|
129
|
+
rewriteQuery?: (query: string) => string | undefined;
|
|
114
130
|
/**
|
|
115
131
|
* Emit a `kind: 'memory'` health event for this search (default true). The cross-store router
|
|
116
132
|
* sets it false: it fuses this leg with the knowledge leg and records ONE `kind: 'cross'`
|
|
@@ -167,6 +183,16 @@ export declare class MemoryStore {
|
|
|
167
183
|
* `archive()` evicts and `restore()` re-adds, keeping the invariant.
|
|
168
184
|
*/
|
|
169
185
|
private reloadIndex;
|
|
186
|
+
/**
|
|
187
|
+
* Cheapest possible DETECTION of "the embedding space changed under this store": counts of ACTIVE
|
|
188
|
+
* persisted vectors whose width or recorded space id no longer matches the one this store writes
|
|
189
|
+
* into. Reads blob LENGTHS only — never decodes — so the `/mem` status surface and the startup
|
|
190
|
+
* warning can afford it at any corpus size.
|
|
191
|
+
*
|
|
192
|
+
* The counts are the same two `VectorsDiagnostic` already separates (`stale` / `space_stale`); this
|
|
193
|
+
* is the read model those surfaces use instead of the full diagnose pass.
|
|
194
|
+
*/
|
|
195
|
+
vectorSpaceHealth(): VectorSpaceHealth;
|
|
170
196
|
add(content: string, category?: string, ttlDays?: number): Promise<RememberResult>;
|
|
171
197
|
/**
|
|
172
198
|
* Write-path contradiction detection (DESIGN §11): check the fact that was just
|
|
@@ -311,7 +337,7 @@ export declare class MemoryStore {
|
|
|
311
337
|
*/
|
|
312
338
|
private classifyVectors;
|
|
313
339
|
/**
|
|
314
|
-
* Repair the semantic read model:
|
|
340
|
+
* Repair the semantic read model, in ONE bounded slice or in full:
|
|
315
341
|
* (a) reload persisted, dim-valid vectors missing from the live index
|
|
316
342
|
* (restart / index loss — needs no model),
|
|
317
343
|
* (b) drop persisted vectors whose dim no longer matches `semantic.dim` so a
|
|
@@ -319,12 +345,46 @@ export declare class MemoryStore {
|
|
|
319
345
|
* throws on dim mismatch),
|
|
320
346
|
* (c) encode active facts that have no vector yet (needs the semantic backend),
|
|
321
347
|
* (d) re-encode vectors written in ANOTHER vector space (a model swap, or the one-time
|
|
322
|
-
* adoption of a store predating the space id).
|
|
323
|
-
*
|
|
348
|
+
* adoption of a store predating the space id).
|
|
349
|
+
*
|
|
350
|
+
* `limit` is what makes the SAME logic usable as a background migration: at most that many rows are
|
|
351
|
+
* dropped and at most that many are re-encoded, so one call is a bounded unit of work the caller can
|
|
352
|
+
* yield between. `vectorsFix` passes no limit (the explicit operator action drains everything);
|
|
353
|
+
* `migrateVectorsBatch` passes the migration's batch size.
|
|
354
|
+
*/
|
|
355
|
+
private repairVectors;
|
|
356
|
+
/**
|
|
357
|
+
* The explicit, unbounded repair behind `mem_admin vectors_fix` / `avantf-mem vectors --fix`.
|
|
324
358
|
* `dry_run` previews every count without writing — and without loading the model, which is why
|
|
325
359
|
* the preview also reports `would_warm` (see the contract's `VectorsFixReport`).
|
|
326
360
|
*/
|
|
327
361
|
vectorsFix(dryRun?: boolean): Promise<VectorsFixReport>;
|
|
362
|
+
/**
|
|
363
|
+
* ONE bounded slice of the vector-space migration: at most `batchSize` rows are dropped and
|
|
364
|
+
* re-encoded, so the caller keeps control of the event loop between slices. This is the unit the
|
|
365
|
+
* background migration ({@link migrateVectors}) and its tests drive.
|
|
366
|
+
*
|
|
367
|
+
* Resumability is a property of the DATABASE, not of this process: every slice re-derives what is
|
|
368
|
+
* stale from the persisted width/space, so a restart simply continues where the last slice stopped.
|
|
369
|
+
*/
|
|
370
|
+
migrateVectorsBatch(batchSize?: number): Promise<VectorMigrationProgress>;
|
|
371
|
+
/**
|
|
372
|
+
* Drive the bounded background migration until the store is current, the model is unavailable, or
|
|
373
|
+
* `shouldStop()` turns true (plugin unmount / process shutdown). A batch FAILURE never rejects: it is
|
|
374
|
+
* logged and reported as `remaining > 0`, which is what makes the next heartbeat a retry (the caller
|
|
375
|
+
* still owns the outer catch for a closed database, e.g. a shutdown racing the first read). Never
|
|
376
|
+
* blocks the event loop for longer than one batch.
|
|
377
|
+
*
|
|
378
|
+
* Honours `semantic.auto_migrate` (default on) — the switch exists so a host that must not spend CPU
|
|
379
|
+
* on background re-encoding can leave the loud warning and the manual entry as the only path.
|
|
380
|
+
*/
|
|
381
|
+
migrateVectors(opts?: {
|
|
382
|
+
batchSize?: number;
|
|
383
|
+
shouldStop?: () => boolean;
|
|
384
|
+
onProgress?: (progress: VectorMigrationOutcome) => void;
|
|
385
|
+
}): Promise<VectorMigrationOutcome>;
|
|
386
|
+
/** Active rows the semantic leg still cannot use (old width, old space, or no vector at all). */
|
|
387
|
+
private migrationRemaining;
|
|
328
388
|
/**
|
|
329
389
|
* One lifecycle pass, owned by the store so the DB writes and the live-index
|
|
330
390
|
* invariant (ACTIVE vectors only) are maintained in ONE place — the runtime no
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Query-side SELF-REFERENCE rewriting (方案 A, `mem/docs/SELF_QUERY_RELEVANCE.md` §4-A).
|
|
3
|
+
*
|
|
4
|
+
* THE PROBLEM. The corpus writes the user in the THIRD person ("用户的名字是…"), the user asks in
|
|
5
|
+
* the FIRST ("我是谁?" / "我的名字"). On the measured model that costs 0.15–0.24 of cosine and,
|
|
6
|
+
* worse, the two sides share NO lexical term — so the FTS and entity legs contribute exactly
|
|
7
|
+
* nothing and the semantic leg has to carry the whole decision. Rewriting the query onto the
|
|
8
|
+
* canonical third-person form the corpus is written in is the one measured change that lifts this
|
|
9
|
+
* family of questions over the strict bar (0.685 / 0.677 / 0.626 vs 0.476 / 0.459 / 0.420).
|
|
10
|
+
*
|
|
11
|
+
* WHAT THIS MODULE IS, AND IS NOT. A pure string table: cues (substrings) → ONE canonical rewrite.
|
|
12
|
+
* No model, no tokenizer, no await, no I/O — that is what lets the SAME table serve the synchronous
|
|
13
|
+
* conditional hint (`runtime.relevance`, 方案 A②) and the async retrieval path (`store/hybrid.ts`).
|
|
14
|
+
*
|
|
15
|
+
* WHY THE TABLE IS CLOSED AND WHY ONLY ONE REWRITE. The first match wins and produces exactly one
|
|
16
|
+
* canonical string. Chinese self-reference has unbounded variants ("我叫啥" / "本人是" / "我是干啥的"),
|
|
17
|
+
* so no closed table can recognize all of them; keeping the OUTPUT set closed (one canonical form per
|
|
18
|
+
* intent) bounds the cost and keeps the fused numbers comparable across queries. Recognizing more
|
|
19
|
+
* variants is a job for the (future) slot/structured work, not for an ever-growing cue list.
|
|
20
|
+
*
|
|
21
|
+
* WHY A WRONG MATCH IS AFFORDABLE. The rewrite is applied as AUGMENTATION, never replacement
|
|
22
|
+
* (`store/hybrid.ts`): the original query still runs and its candidates are still kept, so a false
|
|
23
|
+
* positive can only ADD recall — it can never change what the user actually asked. That is also why
|
|
24
|
+
* the cues can be a little generous: the measured cost of over-matching is "one more leg run", where
|
|
25
|
+
* the cost of under-matching is the failure this whole file exists to remove.
|
|
26
|
+
*
|
|
27
|
+
* @module store/self_query
|
|
28
|
+
*/
|
|
29
|
+
/** One intent: any of `cues` present ⇒ this intent, rewritten to `rewrite`. */
|
|
30
|
+
export interface SelfQueryRule {
|
|
31
|
+
/**
|
|
32
|
+
* Substring cues, matched case-sensitively against the raw query. Deliberately plain strings
|
|
33
|
+
* (no regex) — the table stays readable and cheap to extend, and matching cannot backtrack.
|
|
34
|
+
*/
|
|
35
|
+
readonly cues: readonly string[];
|
|
36
|
+
/** The ONE canonical third-person form this intent rewrites to. */
|
|
37
|
+
readonly rewrite: string;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* The intent table. ORDER IS LOAD-BEARING: the first rule with a matching cue wins, so a more
|
|
41
|
+
* specific intent must precede a broader one that would also match (e.g. the occupation forms sit
|
|
42
|
+
* before the bare identity cue). All cues are first-person ("我") or the formal self-reference
|
|
43
|
+
* "本人"; nothing here can match a question about somebody else.
|
|
44
|
+
*/
|
|
45
|
+
export declare const SELF_QUERY_RULES: readonly SelfQueryRule[];
|
|
46
|
+
/**
|
|
47
|
+
* The ONE canonical rewrite for `text`, or `undefined` when the input is not self-referential.
|
|
48
|
+
*
|
|
49
|
+
* `undefined` (not `text`) is the "nothing to add" answer on purpose: the retrieval path keys
|
|
50
|
+
* "run a second leg set" off it, and returning the input unchanged would make every query pay for
|
|
51
|
+
* an augmentation that cannot add anything.
|
|
52
|
+
*/
|
|
53
|
+
export declare function selfQueryRewrite(text: string): string | undefined;
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
/**
|
|
3
|
-
* Input window of the SHIPPED models: `Xenova/bge-
|
|
4
|
-
* `Xenova/bge-reranker-base` (reranker) both read 512 tokens.
|
|
3
|
+
* Input window of the SHIPPED models: `Xenova/bge-base-zh-v1.5` (embedder) and
|
|
4
|
+
* `Xenova/bge-reranker-base` (reranker) both read 512 tokens. (The previous default embedder,
|
|
5
|
+
* `Xenova/bge-small-zh-v1.5`, shares the same 512-token window — this constant did NOT move when
|
|
6
|
+
* the embedder changed; only its vector width did, 512 → 768.)
|
|
5
7
|
*
|
|
6
8
|
* Exported because more than one default derives from it: the embedder's fallback window, the
|
|
7
9
|
* reranker's pair budget (retrieval-core), and `knowledge.chunk_size` below. transformers.js
|
|
@@ -19,6 +21,7 @@ export declare const ConfigSchema: z.ZodObject<{
|
|
|
19
21
|
mirror: z.ZodDefault<z.ZodString>;
|
|
20
22
|
cache_dir: z.ZodDefault<z.ZodString>;
|
|
21
23
|
auto_download: z.ZodDefault<z.ZodBoolean>;
|
|
24
|
+
auto_migrate: z.ZodDefault<z.ZodBoolean>;
|
|
22
25
|
}, z.core.$strip>>;
|
|
23
26
|
rerank: z.ZodPrefault<z.ZodObject<{
|
|
24
27
|
backend: z.ZodDefault<z.ZodString>;
|
|
@@ -189,6 +189,16 @@ export interface RecallResult {
|
|
|
189
189
|
* same literal lived in three files and could silently desync.
|
|
190
190
|
*/
|
|
191
191
|
export declare const DEGRADED_WEIGHTS: RecallResult['weights'];
|
|
192
|
+
/**
|
|
193
|
+
* The human/model-facing sentence a caller renders when {@link RecallResult.degraded} is true (方案 G).
|
|
194
|
+
*
|
|
195
|
+
* The signal itself is the EXISTING `degraded` flag — this constant adds NO field and no new state,
|
|
196
|
+
* it only spells out what that flag means so the tool output, the CLI and the panel say the same
|
|
197
|
+
* thing instead of each keeping its own copy. Deliberately not awaited on a model warm-up: the
|
|
198
|
+
* three-stage prewarm gate keeps the model off the first screen, so a cold semantic leg is annotated,
|
|
199
|
+
* never waited for.
|
|
200
|
+
*/
|
|
201
|
+
export declare const DEGRADED_LEG_NOTE = "\u672C\u6B21\u4EC5\u8BCD\u6CD5\u817F\uFF0C\u8BED\u4E49\u817F\u672A\u5C31\u7EEA\uFF08\u8BED\u4E49\u540E\u7AEF\u4E0D\u53EF\u7528\uFF0C\u7ED3\u679C\u53EA\u6765\u81EA FTS \u4E0E\u5B9E\u4F53\u817F\uFF09\u3002";
|
|
192
202
|
export interface ContradictionRecord {
|
|
193
203
|
contradiction_id: number;
|
|
194
204
|
fact_a: number;
|
|
@@ -289,11 +299,29 @@ export interface RetrievalHealthSummary {
|
|
|
289
299
|
updated_at: string | null;
|
|
290
300
|
by_kind: Record<string, KindHealth>;
|
|
291
301
|
}
|
|
302
|
+
/**
|
|
303
|
+
* Persisted-vector health of the ACTIVE corpus, relative to the vector space the store writes into
|
|
304
|
+
* NOW — the DETECTION half of "changing the embedding space is a data migration" (DESIGN §20).
|
|
305
|
+
*
|
|
306
|
+
* Deliberately cheap: it compares the recorded width and space id only, never decoding the blobs, so
|
|
307
|
+
* the `/mem` status surface can read it on every poll without loading a whole corpus.
|
|
308
|
+
*/
|
|
309
|
+
export interface VectorSpaceHealth {
|
|
310
|
+
/** Active facts whose persisted vector has a different width than `semantic.dim`. */
|
|
311
|
+
stale: number;
|
|
312
|
+
/** Active facts whose persisted vector is provably from another model space (same width). */
|
|
313
|
+
space_stale: number;
|
|
314
|
+
}
|
|
292
315
|
/** `mem_admin stats`: store counts plus the retrieval-health counters. */
|
|
293
316
|
export interface StatsSummary {
|
|
294
317
|
active: number;
|
|
295
318
|
archived: number;
|
|
296
319
|
retrieval: RetrievalHealthSummary;
|
|
320
|
+
/**
|
|
321
|
+
* Vector-space health, so "retrieval silently degraded after a model upgrade" is visible from the
|
|
322
|
+
* status surface instead of only from a one-shot startup warning.
|
|
323
|
+
*/
|
|
324
|
+
vectors: VectorSpaceHealth;
|
|
297
325
|
}
|
|
298
326
|
/** One page of facts (`mem_admin list`). */
|
|
299
327
|
export interface FactPage {
|
|
@@ -337,10 +365,42 @@ export interface VectorsDiagnostic {
|
|
|
337
365
|
space_stale: number;
|
|
338
366
|
indexed: number;
|
|
339
367
|
unindexed: number;
|
|
368
|
+
/**
|
|
369
|
+
* Distinct recorded spaces across the WHOLE table (archived rows included), while `stale` /
|
|
370
|
+
* `space_stale` count only ACTIVE rows — an archived row legitimately keeps its old vector, so the
|
|
371
|
+
* two must not be compared directly.
|
|
372
|
+
*/
|
|
340
373
|
models: Record<string, number>;
|
|
341
374
|
/** Which backend is actually serving reads (`auto:local_numpy` → `auto:hnswlib`). */
|
|
342
375
|
store: string;
|
|
343
376
|
}
|
|
377
|
+
/**
|
|
378
|
+
* One BOUNDED slice of the vector-space migration (`MemoryStore.migrateVectorsBatch`).
|
|
379
|
+
*
|
|
380
|
+
* The migration is what turns "the default embedder changed" from a silent degradation into a
|
|
381
|
+
* resumable background repair: each call re-encodes at most `batchSize` rows, and the remaining
|
|
382
|
+
* count is what the next call (or the next process, after a restart) resumes from.
|
|
383
|
+
*/
|
|
384
|
+
export interface VectorMigrationProgress {
|
|
385
|
+
/** Active rows still not usable by the semantic leg after this slice (old width/space, or no vector). */
|
|
386
|
+
remaining: number;
|
|
387
|
+
/** Rows re-encoded into the current space by this slice. */
|
|
388
|
+
migrated: number;
|
|
389
|
+
/** Rows whose old-space bytes were dropped so a re-encode could replace them. */
|
|
390
|
+
dropped: number;
|
|
391
|
+
/** Usable persisted vectors re-added to the live index (no model needed). */
|
|
392
|
+
reindexed: number;
|
|
393
|
+
/** `false` when the model was not loaded: nothing could be encoded this slice. */
|
|
394
|
+
semantic_available: boolean;
|
|
395
|
+
}
|
|
396
|
+
/**
|
|
397
|
+
* The outcome of a whole migration drive (`MemoryStore.migrateVectors`): the last slice plus
|
|
398
|
+
* cumulative counters, and whether the `semantic.auto_migrate` switch allowed any work at all.
|
|
399
|
+
*/
|
|
400
|
+
export interface VectorMigrationOutcome extends VectorMigrationProgress {
|
|
401
|
+
/** `false` when `semantic.auto_migrate` is off — nothing was migrated on purpose. */
|
|
402
|
+
enabled: boolean;
|
|
403
|
+
}
|
|
344
404
|
/**
|
|
345
405
|
* `mem_admin vectors_fix`: what a repair did, or — with `dry_run` — what it WOULD do.
|
|
346
406
|
*
|
|
@@ -2,18 +2,20 @@ import type { VectorStore } from '../interfaces.js';
|
|
|
2
2
|
/**
|
|
3
3
|
* Search-time beam width, when the config does not say otherwise.
|
|
4
4
|
*
|
|
5
|
-
* hnswlib
|
|
6
|
-
*
|
|
7
|
-
*
|
|
5
|
+
* hnswlib's own narrow default beam is a QUALITY CLIFF rather than a neutral choice. Measured with
|
|
6
|
+
* the raw binding at the shipped width (2000 random unit vectors, dim 768, true top-10 as ground
|
|
7
|
+
* truth):
|
|
8
8
|
*
|
|
9
9
|
* ef 10 64 128 256 400
|
|
10
|
-
* R@10 0.
|
|
11
|
-
* ms 0.
|
|
10
|
+
* R@10 0.29 0.79 0.94 0.99 1.00
|
|
11
|
+
* ms 0.20 0.43 0.56 0.72 0.82
|
|
12
12
|
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
13
|
+
* (At dim 512 the same measurement was R@10 0.36 / 0.81 / 0.93 / 1.00 / 1.00 at 0.13 / 0.27 / 0.39 /
|
|
14
|
+
* 0.51 / 0.57 ms; the row of 0.45 … 1.00 this comment used to carry was an earlier run.) Recall
|
|
15
|
+
* falls as the corpus grows (at n=8000 / dim 768, R@10 was 0.86 at `ef = 256` and 0.91 at
|
|
16
|
+
* `ef = 400`), and random vectors are the WORST case — real embeddings cluster, so production
|
|
17
|
+
* recall is higher than these numbers. 256 keeps the small/medium corpora near-exact without
|
|
18
|
+
* giving up the speedup; the knob is `vectorStore.hnswlib_ef_search`.
|
|
17
19
|
*/
|
|
18
20
|
export declare const DEFAULT_EF_SEARCH = 256;
|
|
19
21
|
/**
|