@avantf/dsh-mem 0.4.0 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/client.js +3 -1
- package/lib/engine/mem/config/config_files.d.ts +1 -1
- package/lib/engine/mem/db/dao/facts.d.ts +12 -0
- package/lib/engine/mem/db/vectors.d.ts +8 -2
- package/lib/engine/mem/store/common.d.ts +19 -9
- package/lib/engine/mem/store/floors.d.ts +28 -12
- package/lib/engine/mem/store/hybrid.d.ts +9 -0
- package/lib/engine/mem/store/knowledge.d.ts +2 -0
- package/lib/engine/mem/store/memory.d.ts +64 -4
- package/lib/engine/mem/store/self_query.d.ts +53 -0
- package/lib/engine/mem-contract/config.d.ts +5 -2
- package/lib/engine/mem-contract/types.d.ts +60 -0
- package/lib/engine/mem-retrieval/adapters/hnswlib.d.ts +11 -9
- package/lib/index.js +595 -104
- package/lib/types/client/index.d.ts.map +1 -1
- package/lib/types/envinit.d.ts +2 -1
- package/lib/types/envinit.d.ts.map +1 -1
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/render.d.ts.map +1 -1
- package/lib/types/vectorMigration.d.ts +44 -0
- package/lib/types/vectorMigration.d.ts.map +1 -0
- package/package.json +5 -5
package/lib/index.js
CHANGED
|
@@ -96,6 +96,16 @@ const DEGRADED_WEIGHTS = {
|
|
|
96
96
|
fts: .65,
|
|
97
97
|
jaccard: .35
|
|
98
98
|
};
|
|
99
|
+
/**
|
|
100
|
+
* The human/model-facing sentence a caller renders when {@link RecallResult.degraded} is true (方案 G).
|
|
101
|
+
*
|
|
102
|
+
* The signal itself is the EXISTING `degraded` flag — this constant adds NO field and no new state,
|
|
103
|
+
* it only spells out what that flag means so the tool output, the CLI and the panel say the same
|
|
104
|
+
* thing instead of each keeping its own copy. Deliberately not awaited on a model warm-up: the
|
|
105
|
+
* three-stage prewarm gate keeps the model off the first screen, so a cold semantic leg is annotated,
|
|
106
|
+
* never waited for.
|
|
107
|
+
*/
|
|
108
|
+
const DEGRADED_LEG_NOTE = "本次仅词法腿,语义腿未就绪(语义后端不可用,结果只来自 FTS 与实体腿)。";
|
|
99
109
|
//#endregion
|
|
100
110
|
//#region ../contract/lib/config.js
|
|
101
111
|
/**
|
|
@@ -106,12 +116,24 @@ const DEGRADED_WEIGHTS = {
|
|
|
106
116
|
const maxInputTokens = z.number().int().nonnegative().default(0);
|
|
107
117
|
const semanticSchema = z.object({
|
|
108
118
|
backend: z.string().default("local_bge"),
|
|
109
|
-
local_model: z.string().default("Xenova/bge-
|
|
110
|
-
dim: z.number().int().positive().default(
|
|
119
|
+
local_model: z.string().default("Xenova/bge-base-zh-v1.5"),
|
|
120
|
+
dim: z.number().int().positive().default(768),
|
|
111
121
|
max_input_tokens: maxInputTokens,
|
|
112
122
|
mirror: z.string().default("https://hf-mirror.com"),
|
|
113
123
|
cache_dir: z.string().default(""),
|
|
114
|
-
auto_download: z.boolean().default(true)
|
|
124
|
+
auto_download: z.boolean().default(true),
|
|
125
|
+
/**
|
|
126
|
+
* Migrate persisted vectors that belong to an OLDER vector space (a changed model or width) in a
|
|
127
|
+
* bounded background pass. Default ON: changing the embedding space IS a data migration, and
|
|
128
|
+
* without this the semantic leg silently skips every old-space vector while lexical+entity keep
|
|
129
|
+
* answering — measured on the real library after the 512→768 default-model swap, 78 of 80 ACTIVE
|
|
130
|
+
* facts became semantically unreachable with no error anywhere.
|
|
131
|
+
*
|
|
132
|
+
* Off = the loud startup warning and the manual entry (`vectors --fix` / `mem_admin vectors_fix`)
|
|
133
|
+
* stay, but nothing re-encodes on its own. Set to `false` on a metered/embedded host that must not
|
|
134
|
+
* spend CPU on background re-encoding.
|
|
135
|
+
*/
|
|
136
|
+
auto_migrate: z.boolean().default(true)
|
|
115
137
|
});
|
|
116
138
|
const rerankSchema = z.object({
|
|
117
139
|
backend: z.string().default("none"),
|
|
@@ -126,23 +148,26 @@ const vectorStoreSchema = z.object({
|
|
|
126
148
|
/**
|
|
127
149
|
* Vector counts at which `backend: auto` migrates to an ANN index.
|
|
128
150
|
*
|
|
129
|
-
* Measured with `node scripts/bench-vstore.mjs`
|
|
130
|
-
*
|
|
131
|
-
*
|
|
132
|
-
*
|
|
133
|
-
*
|
|
134
|
-
*
|
|
135
|
-
*
|
|
136
|
-
*
|
|
151
|
+
* Measured with `node scripts/bench-vstore.mjs` at the shipped width (**dim 768**, k=50,
|
|
152
|
+
* uniform-random vectors = the ANN worst case): at 2000 the brute-force store answers in 3.3 ms
|
|
153
|
+
* and hnswlib in 1.0 ms (recall 0.9987) while the native index takes **1.17 s** to BUILD; at
|
|
154
|
+
* 8000 it is 14.3 ms → 2.5 ms at recall 0.864 (build 10.7 s). The earlier dim-512 run was
|
|
155
|
+
* 2.5 ms → 0.7 ms at recall 0.9993 / build 0.83 s — the RATIO moves with the CPU, so compare your
|
|
156
|
+
* own run of the script rather than these pairs. Migrating at 2000 is both worthwhile and nearly
|
|
157
|
+
* exact. Two reasons not to lower
|
|
158
|
+
* it: the native index build is paid on the upgrade and on every batch eviction,
|
|
159
|
+
* i.e. on the write path, and below a few hundred vectors brute force is already
|
|
160
|
+
* millisecond-scale. Recall falls as the corpus grows, so for a larger corpus raise
|
|
137
161
|
* `hnswlib_ef_search` (or accept the loss) instead of moving this threshold.
|
|
138
162
|
*/
|
|
139
163
|
auto_thresholds: z.object({ hnswlib: z.number().int().positive().default(2e3) }).default({ hnswlib: 2e3 }),
|
|
140
164
|
/**
|
|
141
165
|
* ANN search beam width (`ef`). This is the recall/speed knob of the hnswlib index, and it is
|
|
142
|
-
* NOT optional in practice: the library's own default
|
|
143
|
-
* a measured benchmark at
|
|
144
|
-
*
|
|
145
|
-
*
|
|
166
|
+
* NOT optional in practice: the library's own narrow default returned only 0.29 of the true
|
|
167
|
+
* top-10 in a measured benchmark at the shipped dim 768 (0.36 at dim 512), while 256 returned
|
|
168
|
+
* 0.99 at 0.72 ms per query (brute force: 3.3 ms at n=2000, dim 768). Recall falls as the corpus
|
|
169
|
+
* grows (at n=8000 / dim 768, R@10 is 0.86 at `ef=256` and 0.91 at `ef=400`), so raise this before
|
|
170
|
+
* raising `auto_thresholds.hnswlib`. The store uses `max(ef_search, 8 × k)`.
|
|
146
171
|
*/
|
|
147
172
|
hnswlib_ef_search: z.number().int().min(16).max(2048).default(256)
|
|
148
173
|
});
|
|
@@ -165,7 +190,7 @@ const retrieverSchema = z.object({
|
|
|
165
190
|
* the fused number is relative to the query and cannot carry an absolute cutoff (DESIGN §7 /
|
|
166
191
|
* §20.19). A score EQUAL to the floor is kept — only strictly-lower candidates are dropped.
|
|
167
192
|
*
|
|
168
|
-
* The default is calibrated for the shipped embedder (`Xenova/bge-
|
|
193
|
+
* The default is calibrated for the shipped embedder (`Xenova/bge-base-zh-v1.5`, dim 768,
|
|
169
194
|
* mean-pooled AND normalized). A different model is a different cosine scale: re-measure before
|
|
170
195
|
* keeping 0.5 (see DESIGN §20.19).
|
|
171
196
|
*/
|
|
@@ -353,7 +378,7 @@ const KnowledgeConfigSchema = z.object({
|
|
|
353
378
|
]),
|
|
354
379
|
/**
|
|
355
380
|
* Characters per chunk, derived from the SHIPPED model's window rather than chosen by feel:
|
|
356
|
-
* `bge-
|
|
381
|
+
* `bge-base-zh-v1.5` reads {@link DEFAULT_MODEL_WINDOW_TOKENS} tokens and Chinese costs
|
|
357
382
|
* ~1 token per character, so ~510 characters fit minus the two wrapper tokens; 500 leaves
|
|
358
383
|
* room and keeps the number round. Longer chunks are not
|
|
359
384
|
* rejected — the embedder bounds them and warns once — but their tail would only reach the
|
|
@@ -1400,9 +1425,10 @@ const DEFAULT_COMMON_CONFIG = `# avantf-mem 公共配置(分层第②层)。
|
|
|
1400
1425
|
|
|
1401
1426
|
# semantic: # 嵌入后端(记忆与知识共用,跨库分数才可比)
|
|
1402
1427
|
# backend: local_bge
|
|
1403
|
-
# local_model: Xenova/bge-
|
|
1404
|
-
# dim:
|
|
1428
|
+
# local_model: Xenova/bge-base-zh-v1.5 # 必须是 ONNX 仓库
|
|
1429
|
+
# dim: 768
|
|
1405
1430
|
# auto_download: true
|
|
1431
|
+
# auto_migrate: true # 换模型/宽度后自动分批重算旧空间的向量(默认开);关掉只剩启动告警 + vectors --fix
|
|
1406
1432
|
# ⚠️ cache_dir / mirror 由家族底座 @avantf/dsh-plugin-base 管理:写在这里会被忽略并告警。
|
|
1407
1433
|
# 缓存目录用环境变量 AVANTF_MEM_MODEL_CACHE,镜像用 AVANTF_MEM_MODEL_MIRROR(或 HF_ENDPOINT)。
|
|
1408
1434
|
|
|
@@ -1422,7 +1448,7 @@ const DEFAULT_COMMON_CONFIG = `# avantf-mem 公共配置(分层第②层)。
|
|
|
1422
1448
|
# weight_jaccard: 0.15
|
|
1423
1449
|
# # 绝对门槛打在每条腿的原始分上、fuse() 之前(融合分只在一次查询内可比,不能当门槛)。
|
|
1424
1450
|
# # 0 = 关闭该门槛;分数等于门槛保留。语义后端不可用时 min_fts_terms 的生效值放宽到 1。
|
|
1425
|
-
# min_semantic_similarity: 0.5 # 语义腿余弦;0.5 只对 bge-
|
|
1451
|
+
# min_semantic_similarity: 0.5 # 语义腿余弦;0.5 只对 bge-base-zh-v1.5/768(mean pooling)标定过,换模型要重标
|
|
1426
1452
|
# min_fts_terms: 2 # FTS 腿:这一行命中几个不同的查询词元(拉丁词≥5字符 + CJK 3-gram)
|
|
1427
1453
|
# min_jaccard: 0.2 # 实体腿 Jaccard 比值
|
|
1428
1454
|
|
|
@@ -3799,7 +3825,7 @@ function round(value, digits) {
|
|
|
3799
3825
|
* Token budgets for every MODEL-FACING text path (embedder input, reranker pair, retrieval
|
|
3800
3826
|
* output). See DESIGN §20.
|
|
3801
3827
|
*
|
|
3802
|
-
* Why this exists: the shipped ONNX models have a hard input window (`bge-
|
|
3828
|
+
* Why this exists: the shipped ONNX models have a hard input window (`bge-base-zh-v1.5` and
|
|
3803
3829
|
* `bge-reranker-base`: 512 tokens) and transformers.js truncates SILENTLY past it —
|
|
3804
3830
|
* `feature-extraction` and `text-classification` both tokenize with `truncation: true` and
|
|
3805
3831
|
* default `max_length` to the tokenizer's `model_max_length`. A Chinese chunk of 800
|
|
@@ -6734,16 +6760,117 @@ function probeTerms(text, holds, stopAt = Number.POSITIVE_INFINITY) {
|
|
|
6734
6760
|
};
|
|
6735
6761
|
}
|
|
6736
6762
|
//#endregion
|
|
6763
|
+
//#region ../core/lib/store/self_query.js
|
|
6764
|
+
/**
|
|
6765
|
+
* Query-side SELF-REFERENCE rewriting (方案 A, `mem/docs/SELF_QUERY_RELEVANCE.md` §4-A).
|
|
6766
|
+
*
|
|
6767
|
+
* THE PROBLEM. The corpus writes the user in the THIRD person ("用户的名字是…"), the user asks in
|
|
6768
|
+
* the FIRST ("我是谁?" / "我的名字"). On the measured model that costs 0.15–0.24 of cosine and,
|
|
6769
|
+
* worse, the two sides share NO lexical term — so the FTS and entity legs contribute exactly
|
|
6770
|
+
* nothing and the semantic leg has to carry the whole decision. Rewriting the query onto the
|
|
6771
|
+
* canonical third-person form the corpus is written in is the one measured change that lifts this
|
|
6772
|
+
* family of questions over the strict bar (0.685 / 0.677 / 0.626 vs 0.476 / 0.459 / 0.420).
|
|
6773
|
+
*
|
|
6774
|
+
* WHAT THIS MODULE IS, AND IS NOT. A pure string table: cues (substrings) → ONE canonical rewrite.
|
|
6775
|
+
* No model, no tokenizer, no await, no I/O — that is what lets the SAME table serve the synchronous
|
|
6776
|
+
* conditional hint (`runtime.relevance`, 方案 A②) and the async retrieval path (`store/hybrid.ts`).
|
|
6777
|
+
*
|
|
6778
|
+
* WHY THE TABLE IS CLOSED AND WHY ONLY ONE REWRITE. The first match wins and produces exactly one
|
|
6779
|
+
* canonical string. Chinese self-reference has unbounded variants ("我叫啥" / "本人是" / "我是干啥的"),
|
|
6780
|
+
* so no closed table can recognize all of them; keeping the OUTPUT set closed (one canonical form per
|
|
6781
|
+
* intent) bounds the cost and keeps the fused numbers comparable across queries. Recognizing more
|
|
6782
|
+
* variants is a job for the (future) slot/structured work, not for an ever-growing cue list.
|
|
6783
|
+
*
|
|
6784
|
+
* WHY A WRONG MATCH IS AFFORDABLE. The rewrite is applied as AUGMENTATION, never replacement
|
|
6785
|
+
* (`store/hybrid.ts`): the original query still runs and its candidates are still kept, so a false
|
|
6786
|
+
* positive can only ADD recall — it can never change what the user actually asked. That is also why
|
|
6787
|
+
* the cues can be a little generous: the measured cost of over-matching is "one more leg run", where
|
|
6788
|
+
* the cost of under-matching is the failure this whole file exists to remove.
|
|
6789
|
+
*
|
|
6790
|
+
* @module store/self_query
|
|
6791
|
+
*/
|
|
6792
|
+
/**
|
|
6793
|
+
* The intent table. ORDER IS LOAD-BEARING: the first rule with a matching cue wins, so a more
|
|
6794
|
+
* specific intent must precede a broader one that would also match (e.g. the occupation forms sit
|
|
6795
|
+
* before the bare identity cue). All cues are first-person ("我") or the formal self-reference
|
|
6796
|
+
* "本人"; nothing here can match a question about somebody else.
|
|
6797
|
+
*/
|
|
6798
|
+
const SELF_QUERY_RULES = [
|
|
6799
|
+
{
|
|
6800
|
+
cues: [
|
|
6801
|
+
"我是做什么的",
|
|
6802
|
+
"我是干什么的",
|
|
6803
|
+
"我是干啥的",
|
|
6804
|
+
"我是做哪行的",
|
|
6805
|
+
"我的职业",
|
|
6806
|
+
"我的工作"
|
|
6807
|
+
],
|
|
6808
|
+
rewrite: "用户是做什么的"
|
|
6809
|
+
},
|
|
6810
|
+
{
|
|
6811
|
+
cues: [
|
|
6812
|
+
"我的名字",
|
|
6813
|
+
"我叫什么",
|
|
6814
|
+
"我叫啥",
|
|
6815
|
+
"我的姓名",
|
|
6816
|
+
"我叫什么名字"
|
|
6817
|
+
],
|
|
6818
|
+
rewrite: "用户的名字"
|
|
6819
|
+
},
|
|
6820
|
+
{
|
|
6821
|
+
cues: [
|
|
6822
|
+
"我是谁",
|
|
6823
|
+
"本人是谁",
|
|
6824
|
+
"我是什么人",
|
|
6825
|
+
"本人是什么人"
|
|
6826
|
+
],
|
|
6827
|
+
rewrite: "用户是谁"
|
|
6828
|
+
},
|
|
6829
|
+
{
|
|
6830
|
+
cues: [
|
|
6831
|
+
"我在哪",
|
|
6832
|
+
"我在哪里",
|
|
6833
|
+
"我住在哪",
|
|
6834
|
+
"我的位置",
|
|
6835
|
+
"我的所在地"
|
|
6836
|
+
],
|
|
6837
|
+
rewrite: "用户在哪里"
|
|
6838
|
+
},
|
|
6839
|
+
{
|
|
6840
|
+
cues: [
|
|
6841
|
+
"我的偏好",
|
|
6842
|
+
"我喜欢什么",
|
|
6843
|
+
"我的喜好",
|
|
6844
|
+
"我的习惯"
|
|
6845
|
+
],
|
|
6846
|
+
rewrite: "用户的偏好"
|
|
6847
|
+
}
|
|
6848
|
+
];
|
|
6849
|
+
/**
|
|
6850
|
+
* The ONE canonical rewrite for `text`, or `undefined` when the input is not self-referential.
|
|
6851
|
+
*
|
|
6852
|
+
* `undefined` (not `text`) is the "nothing to add" answer on purpose: the retrieval path keys
|
|
6853
|
+
* "run a second leg set" off it, and returning the input unchanged would make every query pay for
|
|
6854
|
+
* an augmentation that cannot add anything.
|
|
6855
|
+
*/
|
|
6856
|
+
function selfQueryRewrite(text) {
|
|
6857
|
+
const query = text.trim();
|
|
6858
|
+
if (query === "") return void 0;
|
|
6859
|
+
for (const rule of SELF_QUERY_RULES) for (const cue of rule.cues) if (query.includes(cue)) return rule.rewrite;
|
|
6860
|
+
}
|
|
6861
|
+
//#endregion
|
|
6737
6862
|
//#region ../core/lib/store/floors.js
|
|
6738
6863
|
/**
|
|
6739
6864
|
* What the RELAXED pass lowers each floor TO, as absolute values.
|
|
6740
6865
|
*
|
|
6741
|
-
* - semantic **0.40**: strictly BETWEEN the measured unrelated-query ceiling
|
|
6742
|
-
* pronoun-query top-1
|
|
6866
|
+
* - semantic **0.40**: strictly BETWEEN the measured unrelated-query ceiling and the
|
|
6867
|
+
* pronoun-query top-1. Both bounds are load-bearing. Above the ceiling, so a genuinely
|
|
6743
6868
|
* unrelated question still comes back EMPTY from the relaxed pass — the four measured samples
|
|
6744
|
-
* topped out at 0.300/0.384/0.315/0.351
|
|
6745
|
-
* 0.
|
|
6746
|
-
*
|
|
6869
|
+
* topped out at 0.300/0.384/0.315/0.351 under the original 512-dim default and at
|
|
6870
|
+
* 0.271/0.353/0.350/0.269 under the shipped 768-dim `bge-base-zh-v1.5` (a 0.35 bar would already
|
|
6871
|
+
* admit two of them in either scale). Below the top-1, so 「我是谁」 finds its fact (0.444 at 512,
|
|
6872
|
+
* 0.459 at 768). 0.40 keeps the larger margin on the noise side, which is where a false positive
|
|
6873
|
+
* costs the caller a wrong answer rather than a missing one.
|
|
6747
6874
|
* - fts: NOT a constant — {@link looseTermFloor}, half the query's distinct terms with a floor of
|
|
6748
6875
|
* 1. `1` here is the "no terms to scale by" baseline (the previously constant value); a query
|
|
6749
6876
|
* that cannot produce terms is not graded at all, and one that produces a single term (every
|
|
@@ -6796,9 +6923,11 @@ function emptyFloorDrops() {
|
|
|
6796
6923
|
* and changing the REPORTED value there would misdescribe what the caller is reading. `0` stays `0`
|
|
6797
6924
|
* (the operator's off switch).
|
|
6798
6925
|
*
|
|
6799
|
-
* This is the ONE place the clamp lives; `resolveFloors` (what the result reports)
|
|
6800
|
-
* `applyTermFloor` (what the leg does) both call it, so the number
|
|
6801
|
-
*
|
|
6926
|
+
* This is the ONE place the clamp lives; `resolveFloors` (what the result reports / a run is handed)
|
|
6927
|
+
* and `applyTermFloor` (what the leg does) both call it, so the number a run reads is the number its
|
|
6928
|
+
* leg used. It is a `min`, i.e. one-directional: it can LOWER an unreachable configured bar but never
|
|
6929
|
+
* RAISE a bar inherited from another text, which is why the caller must resolve per graded text
|
|
6930
|
+
* rather than reusing one text's resolution for another.
|
|
6802
6931
|
*/
|
|
6803
6932
|
function effectiveTermFloor(floor, termCount) {
|
|
6804
6933
|
if (!(floor > 0)) return floor;
|
|
@@ -7009,6 +7138,48 @@ async function hybridSearch(deps, plan) {
|
|
|
7009
7138
|
const legCap = legCapFor(deps.config, overFetch);
|
|
7010
7139
|
const termCount = relevanceTerms(query).length;
|
|
7011
7140
|
/**
|
|
7141
|
+
* 方案 A — the self-reference AUGMENTATION (docs/SELF_QUERY_RELEVANCE.md §4-A).
|
|
7142
|
+
*
|
|
7143
|
+
* The corpus writes the user in the third person and the user asks in the first, so a
|
|
7144
|
+
* first-person question shares no lexical term with the fact that answers it and the semantic leg
|
|
7145
|
+
* has to carry the whole decision. The intent table (`store/self_query.ts`) supplies ONE canonical
|
|
7146
|
+
* third-person rewrite and the ORIGINAL query and the rewrite each run the full leg set.
|
|
7147
|
+
*
|
|
7148
|
+
* THE FLOOR IS RESOLVED PER RUN (0.4.2). Each run is graded by ITS OWN reachability clamp —
|
|
7149
|
+
* `min(configured, relevanceTerms(variant).length)` — because a variant with MORE terms has a
|
|
7150
|
+
* HIGHER bar, and grading it at the original's lower bar is what let a single incidental trigram
|
|
7151
|
+
* from the rewrite become that leg's head. Measured on the live store: `我是谁?` yields ONE term
|
|
7152
|
+
* (`我是谁`) while its rewrite `用户是谁` yields TWO (`用户是` / `户是谁`), and the rewrite's
|
|
7153
|
+
* `用户是` is a literal substring of an unrelated long note — that note then took the FTS leg's
|
|
7154
|
+
* head (weight × 1.0) and outvoted the identity fact, which the rewrite had correctly put first.
|
|
7155
|
+
* `applyTermFloor` re-clamps to the query it is grading, but the clamp is one-directional
|
|
7156
|
+
* (`min`): it can only LOWER an unreachable bar, never RAISE a bar inherited from another text, so
|
|
7157
|
+
* the resolution has to happen per variant (see {@link resolveFloors}). Each variant's own bar is
|
|
7158
|
+
* then floored at the reported one, so every run is graded at or ABOVE what the envelope says: the
|
|
7159
|
+
* envelope reports the ORIGINAL query's floors (the question the user actually asked) and can never
|
|
7160
|
+
* overstate the bar a run was held to.
|
|
7161
|
+
*
|
|
7162
|
+
* WHY AUGMENT, NOT REPLACE. The table is a closed cue list over a language with unbounded
|
|
7163
|
+
* self-reference variants, so a wrong match is inevitable. Under augmentation a false positive can
|
|
7164
|
+
* only add candidates; the original query's candidates are always kept, exactly like the relaxed
|
|
7165
|
+
* floor policy ("only ever widens"). Under replacement it would change what the user actually
|
|
7166
|
+
* asked. `queries.length === 1` (the overwhelming majority) leaves the run below bit-identical to
|
|
7167
|
+
* the pre-A path: one `deps.legs` call, same context, same vector wiring.
|
|
7168
|
+
*
|
|
7169
|
+
* FLOORS, `relaxed` AND `dropped_by_floor` UNDER AUGMENTATION. The PROFILE is resolved once per
|
|
7170
|
+
* pass from the ORIGINAL query and handed to both runs; only the per-variant reachability clamp
|
|
7171
|
+
* above differs, and it can only raise. Every leg of every run is therefore judged by AT LEAST the
|
|
7172
|
+
* numbers the result reports — the envelope can never say "strict" while a rewrite run answered from
|
|
7173
|
+
* the relaxed band (the distortion this design exists to rule out). `relaxed` is decided exactly as
|
|
7174
|
+
* it was: the automatic loose pass fires only when the strict pass returned NOTHING with drops, and
|
|
7175
|
+
* it re-runs BOTH variants under the loosened floors. Because a leg only ever REMOVES entries, the
|
|
7176
|
+
* per-leg merge of the two runs' drop counts uses `max` (summing would count the same candidate
|
|
7177
|
+
* twice) and `capped` uses OR — both conservative, neither inflates the report that feeds the retry
|
|
7178
|
+
* decision.
|
|
7179
|
+
*/
|
|
7180
|
+
const augment = (plan.rewriteQuery ?? selfQueryRewrite)(query);
|
|
7181
|
+
const queries = augment !== void 0 && augment !== query ? [query, augment] : [query];
|
|
7182
|
+
/**
|
|
7012
7183
|
* The query vector in force, ACROSS both passes.
|
|
7013
7184
|
*
|
|
7014
7185
|
* The single-store path is handed no vector by its caller (only the cross-store router encodes
|
|
@@ -7019,6 +7190,12 @@ async function hybridSearch(deps, plan) {
|
|
|
7019
7190
|
*/
|
|
7020
7191
|
let passVector = plan.queryVector;
|
|
7021
7192
|
/**
|
|
7193
|
+
* The rewrite's own vector, ACROSS both passes, for the same reason as {@link passVector}: the
|
|
7194
|
+
* rewrite is a different text, so it needs its own encode, and the relaxed retry must not pay for
|
|
7195
|
+
* it twice. Keyed by variant text (there is at most one).
|
|
7196
|
+
*/
|
|
7197
|
+
const augmentVectors = /* @__PURE__ */ new Map();
|
|
7198
|
+
/**
|
|
7022
7199
|
* One full pass under ONE floor profile: legs → fuse → live filter → rerank → slice → budget.
|
|
7023
7200
|
*
|
|
7024
7201
|
* `onReturn` (the store's reinforcement) is deliberately NOT called here: the strict pass is a
|
|
@@ -7034,19 +7211,34 @@ async function hybridSearch(deps, plan) {
|
|
|
7034
7211
|
termCount,
|
|
7035
7212
|
...relaxLegs === void 0 ? {} : { relaxLegs }
|
|
7036
7213
|
});
|
|
7037
|
-
const
|
|
7038
|
-
|
|
7039
|
-
|
|
7040
|
-
|
|
7041
|
-
|
|
7042
|
-
|
|
7043
|
-
|
|
7044
|
-
|
|
7045
|
-
|
|
7046
|
-
|
|
7047
|
-
|
|
7048
|
-
}
|
|
7049
|
-
|
|
7214
|
+
const runs = await Promise.all(queries.map(async (variant, index) => {
|
|
7215
|
+
const isOriginal = index === 0;
|
|
7216
|
+
const vector = isOriginal ? passVector : augmentVectors.get(variant);
|
|
7217
|
+
const ownFloors = isOriginal ? floors : resolveFloors(retriever, semAvail, {
|
|
7218
|
+
profile,
|
|
7219
|
+
termCount: relevanceTerms(variant).length,
|
|
7220
|
+
...relaxLegs === void 0 ? {} : { relaxLegs }
|
|
7221
|
+
});
|
|
7222
|
+
const variantFloors = ownFloors === floors ? floors : {
|
|
7223
|
+
...ownFloors,
|
|
7224
|
+
fts: Math.max(floors.fts, ownFloors.fts)
|
|
7225
|
+
};
|
|
7226
|
+
return runLegs(deps, {
|
|
7227
|
+
query: variant,
|
|
7228
|
+
limit,
|
|
7229
|
+
overFetch,
|
|
7230
|
+
legCap,
|
|
7231
|
+
semAvail,
|
|
7232
|
+
weights,
|
|
7233
|
+
floors: variantFloors,
|
|
7234
|
+
...vector === void 0 ? {} : { queryVector: vector },
|
|
7235
|
+
onQueryVector: (vec) => {
|
|
7236
|
+
if (isOriginal) passVector = vec;
|
|
7237
|
+
else augmentVectors.set(variant, vec);
|
|
7238
|
+
}
|
|
7239
|
+
});
|
|
7240
|
+
}));
|
|
7241
|
+
const legs = runs.length === 1 ? runs[0] : unionLegs(runs);
|
|
7050
7242
|
let capped = 0;
|
|
7051
7243
|
for (const leg of legs) if (leg.capped === true) capped += 1;
|
|
7052
7244
|
const droppedByFloor = emptyFloorDrops();
|
|
@@ -7102,6 +7294,48 @@ async function hybridSearch(deps, plan) {
|
|
|
7102
7294
|
};
|
|
7103
7295
|
}
|
|
7104
7296
|
/**
|
|
7297
|
+
* Union the per-leg raw scores of the augmented runs (方案 A) — original first, then the rewrite(s).
|
|
7298
|
+
*
|
|
7299
|
+
* Same leg INDEX, not "same `leg` label": both runs come from ONE store with an identical context
|
|
7300
|
+
* except for the query text, so the leg array has the same shape and order (memory: semantic,
|
|
7301
|
+
* jaccard, fts[, hrr]; knowledge: semantic, fts, jaccard). Merging by index is what keeps this from
|
|
7302
|
+
* becoming a second leg with its own weight — the weight, `leg` label and `capped` bookkeeping stay
|
|
7303
|
+
* the ORIGINAL run's.
|
|
7304
|
+
*
|
|
7305
|
+
* A candidate present in both runs keeps its MAX score, and `capped` is the OR. `droppedByFloor`
|
|
7306
|
+
* is the MAX too: both runs commonly drop the SAME candidate (the rewrite is a third-person
|
|
7307
|
+
* paraphrase of the same question), and we do not carry the pre-floor raw sets, so summing would
|
|
7308
|
+
* double-count it. The max is the conservative number that never inflates the drop report — which
|
|
7309
|
+
* is what `feeds` the relaxed-retry decision and the health counters. A store returning a different
|
|
7310
|
+
* leg count for one variant is a contract violation; the extra legs are appended rather than
|
|
7311
|
+
* silently discarded, so it cannot narrow the union either.
|
|
7312
|
+
*/
|
|
7313
|
+
function unionLegs(runs) {
|
|
7314
|
+
const [first, ...rest] = runs;
|
|
7315
|
+
const out = (first ?? []).map((leg) => ({
|
|
7316
|
+
...leg,
|
|
7317
|
+
scores: new Map(leg.scores)
|
|
7318
|
+
}));
|
|
7319
|
+
for (const run of rest) run.forEach((leg, index) => {
|
|
7320
|
+
const current = out[index];
|
|
7321
|
+
if (current === void 0) {
|
|
7322
|
+
out.push({
|
|
7323
|
+
...leg,
|
|
7324
|
+
scores: new Map(leg.scores)
|
|
7325
|
+
});
|
|
7326
|
+
return;
|
|
7327
|
+
}
|
|
7328
|
+
for (const [id, score] of leg.scores) {
|
|
7329
|
+
const previous = current.scores.get(id);
|
|
7330
|
+
if (previous === void 0 || score > previous) current.scores.set(id, score);
|
|
7331
|
+
}
|
|
7332
|
+
if (leg.capped === true) current.capped = true;
|
|
7333
|
+
if (leg.leg !== void 0) current.leg = leg.leg;
|
|
7334
|
+
if (leg.droppedByFloor !== void 0) current.droppedByFloor = Math.max(current.droppedByFloor ?? 0, leg.droppedByFloor);
|
|
7335
|
+
});
|
|
7336
|
+
return out;
|
|
7337
|
+
}
|
|
7338
|
+
/**
|
|
7105
7339
|
* Await the store's legs, isolating a failure to the leg that caused it.
|
|
7106
7340
|
*
|
|
7107
7341
|
* A rejected leg contributes nothing and is reported; the query still answers from the others. The
|
|
@@ -7171,10 +7405,16 @@ function vectorSpaceId(backend, model, dim) {
|
|
|
7171
7405
|
* The rows come from the owning store's DAO (DESIGN §19): this helper decodes and
|
|
7172
7406
|
* filters them, it never runs SQL itself.
|
|
7173
7407
|
*
|
|
7408
|
+
* A row whose width does not fit the store is skipped (never `add()`ed — the store
|
|
7409
|
+
* would throw). The OWNING store reports that condition itself, as one actionable
|
|
7410
|
+
* line with the count, the reason and the manual re-encode entry (`reportStaleVectors`),
|
|
7411
|
+
* so this helper stays quiet: two warnings for one condition read as two problems.
|
|
7412
|
+
*
|
|
7174
7413
|
* @param rows - `id` (fact/chunk id) + `vec` (BLOB), as read back from the store.
|
|
7175
|
-
* @param
|
|
7414
|
+
* @param _label - store name; unused here (the owning store owns the warning), kept so call sites read
|
|
7415
|
+
* as "which store is reloading".
|
|
7176
7416
|
*/
|
|
7177
|
-
function reloadVectorIndex(vstore, rows,
|
|
7417
|
+
function reloadVectorIndex(vstore, rows, _label) {
|
|
7178
7418
|
const entries = [];
|
|
7179
7419
|
let skipped = 0;
|
|
7180
7420
|
for (const r of rows) {
|
|
@@ -7185,7 +7425,6 @@ function reloadVectorIndex(vstore, rows, label) {
|
|
|
7185
7425
|
});
|
|
7186
7426
|
else skipped++;
|
|
7187
7427
|
}
|
|
7188
|
-
if (skipped > 0) retrievalLogger().warn(`${label} index reload: skipped ${skipped} vector(s) with dim != ${vstore.dim}`);
|
|
7189
7428
|
vstore.rebuild(entries);
|
|
7190
7429
|
return {
|
|
7191
7430
|
loaded: entries.length,
|
|
@@ -7250,21 +7489,32 @@ function evictVectors(vstore, ids) {
|
|
|
7250
7489
|
else for (const id of ids) vstore.remove(id);
|
|
7251
7490
|
}
|
|
7252
7491
|
/**
|
|
7253
|
-
* Tell the operator that persisted vectors came from ANOTHER vector space
|
|
7492
|
+
* Tell the operator that persisted vectors came from ANOTHER vector space — loudly, and with the
|
|
7493
|
+
* manual remedy.
|
|
7494
|
+
*
|
|
7495
|
+
* Changing the embedding space (model, width, pooling, normalization) is a DATA MIGRATION: the
|
|
7496
|
+
* persisted bytes stay valid-looking but are no longer comparable with the current query vector, so
|
|
7497
|
+
* the semantic leg must not serve them. Measured on the real library after the 512→768 default-model
|
|
7498
|
+
* swap: 78 of 80 ACTIVE facts fell out of the semantic leg and retrieval silently degraded to
|
|
7499
|
+
* lexical+entity (every probe query answered with the same unrelated short fact), with no error
|
|
7500
|
+
* anywhere. The count, the reason and the ONE manual entry have to be visible at start.
|
|
7254
7501
|
*
|
|
7255
|
-
*
|
|
7256
|
-
*
|
|
7257
|
-
*
|
|
7258
|
-
* re-encoding is a real cost (and they should see the count first).
|
|
7502
|
+
* Both halves are reported in one message because they are one condition seen two ways: a changed
|
|
7503
|
+
* WIDTH (`stale`, whose bytes cannot even be decoded into the current store) and a changed SPACE at
|
|
7504
|
+
* the same width (`space_stale`, whose bytes decode but rank in another model's coordinates).
|
|
7259
7505
|
*
|
|
7260
7506
|
* @param kind store label for the message (`memory` / `knowledge`)
|
|
7261
7507
|
* @param space the vector space the store writes into now
|
|
7262
|
-
* @param
|
|
7263
|
-
* @param fix the
|
|
7508
|
+
* @param counts how many persisted vectors are unusable in the current space, by kind
|
|
7509
|
+
* @param fix the manual entry that re-encodes them, as the operator would type it
|
|
7264
7510
|
*/
|
|
7265
|
-
function
|
|
7266
|
-
|
|
7267
|
-
|
|
7511
|
+
function reportStaleVectors(kind, space, counts, fix) {
|
|
7512
|
+
const total = counts.stale + counts.space_stale;
|
|
7513
|
+
if (total <= 0) return;
|
|
7514
|
+
const reasons = [];
|
|
7515
|
+
if (counts.stale > 0) reasons.push(`${String(counts.stale)} with a width that no longer matches`);
|
|
7516
|
+
if (counts.space_stale > 0) reasons.push(`${String(counts.space_stale)} written by another model`);
|
|
7517
|
+
retrievalLogger().warn(`${kind} index: ${String(total)} persisted vector(s) belong to an OLDER embedding space than ${space} (${reasons.join("; ")}) — the semantic leg cannot use them, so retrieval degrades to lexical+entity (this is a data migration, not a transient error); run ${fix} to re-encode them`);
|
|
7268
7518
|
}
|
|
7269
7519
|
/**
|
|
7270
7520
|
* Hand the event loop back between two synchronous batches.
|
|
@@ -7846,6 +8096,17 @@ var FactsDao = class FactsDao {
|
|
|
7846
8096
|
WHERE semantic_vector IS NOT NULL AND status = 'active'`).all();
|
|
7847
8097
|
}
|
|
7848
8098
|
/**
|
|
8099
|
+
* Width + recorded space of every ACTIVE persisted vector, WITHOUT reading the bytes.
|
|
8100
|
+
*
|
|
8101
|
+
* `length(blob)` is the byte count, so `bytes / 4` is the float width — enough to detect a
|
|
8102
|
+
* changed `semantic.dim` without decoding a corpus. This is the cheap detection the startup
|
|
8103
|
+
* warning and the `/mem` status surface read (see `MemoryStore.vectorSpaceHealth`).
|
|
8104
|
+
*/
|
|
8105
|
+
activeVectorSpaces() {
|
|
8106
|
+
return this.db.prepare(`SELECT fact_id, length(semantic_vector) AS bytes, embedding_model FROM facts
|
|
8107
|
+
WHERE semantic_vector IS NOT NULL AND status = 'active'`).all();
|
|
8108
|
+
}
|
|
8109
|
+
/**
|
|
7849
8110
|
* Facts per archive reason — the trust diagnostics breakdown. One grouped scan rather than
|
|
7850
8111
|
* a count per reason, because the reasons are few but the table is not.
|
|
7851
8112
|
*/
|
|
@@ -9393,13 +9654,38 @@ var MemoryStore = class {
|
|
|
9393
9654
|
*/
|
|
9394
9655
|
reloadIndex() {
|
|
9395
9656
|
const space = this.vectorSpace();
|
|
9396
|
-
const
|
|
9397
|
-
const rows = persisted.map((r) => ({
|
|
9657
|
+
const rows = this.facts.activeVectorRows().map((r) => ({
|
|
9398
9658
|
id: r.fact_id,
|
|
9399
9659
|
vec: r.semantic_vector
|
|
9400
9660
|
}));
|
|
9401
9661
|
reloadVectorIndex(this.vstore, rows, "memory");
|
|
9402
|
-
|
|
9662
|
+
reportStaleVectors("memory", space, this.vectorSpaceHealth(), "`avantf-mem vectors --fix` (MCP/plugin: `mem_admin vectors_fix`)");
|
|
9663
|
+
}
|
|
9664
|
+
/**
|
|
9665
|
+
* Cheapest possible DETECTION of "the embedding space changed under this store": counts of ACTIVE
|
|
9666
|
+
* persisted vectors whose width or recorded space id no longer matches the one this store writes
|
|
9667
|
+
* into. Reads blob LENGTHS only — never decodes — so the `/mem` status surface and the startup
|
|
9668
|
+
* warning can afford it at any corpus size.
|
|
9669
|
+
*
|
|
9670
|
+
* The counts are the same two `VectorsDiagnostic` already separates (`stale` / `space_stale`); this
|
|
9671
|
+
* is the read model those surfaces use instead of the full diagnose pass.
|
|
9672
|
+
*/
|
|
9673
|
+
vectorSpaceHealth() {
|
|
9674
|
+
const space = this.vectorSpace();
|
|
9675
|
+
const expectedBytes = this.vstore.dim * 4;
|
|
9676
|
+
let stale = 0;
|
|
9677
|
+
let spaceStale = 0;
|
|
9678
|
+
for (const row of this.facts.activeVectorSpaces()) {
|
|
9679
|
+
if (row.bytes !== expectedBytes) {
|
|
9680
|
+
stale++;
|
|
9681
|
+
continue;
|
|
9682
|
+
}
|
|
9683
|
+
if (row.embedding_model !== space) spaceStale++;
|
|
9684
|
+
}
|
|
9685
|
+
return {
|
|
9686
|
+
stale,
|
|
9687
|
+
space_stale: spaceStale
|
|
9688
|
+
};
|
|
9403
9689
|
}
|
|
9404
9690
|
async add(content, category, ttlDays) {
|
|
9405
9691
|
const normalized = normalizeWrite(content).trim();
|
|
@@ -9828,7 +10114,7 @@ var MemoryStore = class {
|
|
|
9828
10114
|
};
|
|
9829
10115
|
}
|
|
9830
10116
|
/**
|
|
9831
|
-
* Repair the semantic read model:
|
|
10117
|
+
* Repair the semantic read model, in ONE bounded slice or in full:
|
|
9832
10118
|
* (a) reload persisted, dim-valid vectors missing from the live index
|
|
9833
10119
|
* (restart / index loss — needs no model),
|
|
9834
10120
|
* (b) drop persisted vectors whose dim no longer matches `semantic.dim` so a
|
|
@@ -9836,64 +10122,162 @@ var MemoryStore = class {
|
|
|
9836
10122
|
* throws on dim mismatch),
|
|
9837
10123
|
* (c) encode active facts that have no vector yet (needs the semantic backend),
|
|
9838
10124
|
* (d) re-encode vectors written in ANOTHER vector space (a model swap, or the one-time
|
|
9839
|
-
* adoption of a store predating the space id).
|
|
9840
|
-
*
|
|
9841
|
-
* `
|
|
9842
|
-
*
|
|
10125
|
+
* adoption of a store predating the space id).
|
|
10126
|
+
*
|
|
10127
|
+
* `limit` is what makes the SAME logic usable as a background migration: at most that many rows are
|
|
10128
|
+
* dropped and at most that many are re-encoded, so one call is a bounded unit of work the caller can
|
|
10129
|
+
* yield between. `vectorsFix` passes no limit (the explicit operator action drains everything);
|
|
10130
|
+
* `migrateVectorsBatch` passes the migration's batch size.
|
|
9843
10131
|
*/
|
|
9844
|
-
async
|
|
10132
|
+
async repairVectors(opts) {
|
|
10133
|
+
const limit = opts.limit ?? Number.POSITIVE_INFINITY;
|
|
9845
10134
|
const c = this.classifyVectors();
|
|
9846
10135
|
const loadedNow = this.semantic.isAvailable();
|
|
9847
10136
|
let semAvailable = loadedNow;
|
|
9848
|
-
if (!semAvailable && !dryRun) semAvailable = await this.warmupSemantic();
|
|
10137
|
+
if (!semAvailable && !opts.dryRun) semAvailable = await this.warmupSemantic();
|
|
9849
10138
|
const wouldWarm = !loadedNow;
|
|
9850
|
-
|
|
9851
|
-
|
|
9852
|
-
|
|
9853
|
-
|
|
9854
|
-
|
|
9855
|
-
|
|
9856
|
-
|
|
9857
|
-
|
|
9858
|
-
|
|
9859
|
-
|
|
9860
|
-
|
|
10139
|
+
const remainingOf = (fixed) => Math.max(0, c.staleIds.length + c.spaceStaleIds.length + c.missing - fixed);
|
|
10140
|
+
if (opts.dryRun) return {
|
|
10141
|
+
report: {
|
|
10142
|
+
missing: c.missing,
|
|
10143
|
+
stale: c.staleIds.length,
|
|
10144
|
+
space_stale: c.spaceStaleIds.length,
|
|
10145
|
+
unindexed: c.unindexed.length,
|
|
10146
|
+
reindexed: 0,
|
|
10147
|
+
dropped: 0,
|
|
10148
|
+
fixed: 0,
|
|
10149
|
+
semantic_available: semAvailable,
|
|
10150
|
+
would_warm: wouldWarm,
|
|
10151
|
+
dry_run: true
|
|
10152
|
+
},
|
|
10153
|
+
remaining: remainingOf(0)
|
|
9861
10154
|
};
|
|
9862
10155
|
let reindexed = 0;
|
|
9863
|
-
for (const u of c.unindexed) {
|
|
10156
|
+
for (const u of c.unindexed.slice(0, limit)) {
|
|
9864
10157
|
this.vstore.add(u.id, u.vec);
|
|
9865
10158
|
reindexed++;
|
|
9866
10159
|
}
|
|
9867
10160
|
let dropped = 0;
|
|
9868
10161
|
if (semAvailable) {
|
|
9869
|
-
const toDrop = [...c.staleIds, ...c.spaceStaleIds];
|
|
10162
|
+
const toDrop = [...c.staleIds, ...c.spaceStaleIds].slice(0, limit);
|
|
9870
10163
|
if (toDrop.length) {
|
|
10164
|
+
const spaceStale = new Set(c.spaceStaleIds);
|
|
9871
10165
|
dropped = this.facts.clearVectors(toDrop);
|
|
9872
|
-
this.evictVectors(
|
|
10166
|
+
this.evictVectors(toDrop.filter((id) => spaceStale.has(id)));
|
|
9873
10167
|
}
|
|
9874
10168
|
}
|
|
9875
10169
|
let fixed = 0;
|
|
9876
10170
|
if (semAvailable) {
|
|
9877
|
-
const missingRows = this.facts.missingVectorRows();
|
|
10171
|
+
const missingRows = this.facts.missingVectorRows().slice(0, limit);
|
|
9878
10172
|
for (const m of missingRows) {
|
|
9879
10173
|
if (!await this.maybeIndexSemantic(m.fact_id, m.content)) continue;
|
|
9880
10174
|
fixed++;
|
|
9881
10175
|
}
|
|
9882
10176
|
}
|
|
9883
10177
|
return {
|
|
9884
|
-
|
|
9885
|
-
|
|
9886
|
-
|
|
9887
|
-
|
|
9888
|
-
|
|
9889
|
-
|
|
9890
|
-
|
|
9891
|
-
|
|
9892
|
-
|
|
9893
|
-
|
|
10178
|
+
report: {
|
|
10179
|
+
missing: c.missing,
|
|
10180
|
+
stale: c.staleIds.length,
|
|
10181
|
+
space_stale: c.spaceStaleIds.length,
|
|
10182
|
+
unindexed: c.unindexed.length,
|
|
10183
|
+
reindexed,
|
|
10184
|
+
dropped,
|
|
10185
|
+
fixed,
|
|
10186
|
+
semantic_available: semAvailable,
|
|
10187
|
+
would_warm: wouldWarm,
|
|
10188
|
+
dry_run: false
|
|
10189
|
+
},
|
|
10190
|
+
remaining: remainingOf(fixed)
|
|
9894
10191
|
};
|
|
9895
10192
|
}
|
|
9896
10193
|
/**
|
|
10194
|
+
* The explicit, unbounded repair behind `mem_admin vectors_fix` / `avantf-mem vectors --fix`.
|
|
10195
|
+
* `dry_run` previews every count without writing — and without loading the model, which is why
|
|
10196
|
+
* the preview also reports `would_warm` (see the contract's `VectorsFixReport`).
|
|
10197
|
+
*/
|
|
10198
|
+
async vectorsFix(dryRun = false) {
|
|
10199
|
+
return (await this.repairVectors({ dryRun })).report;
|
|
10200
|
+
}
|
|
10201
|
+
/**
|
|
10202
|
+
* ONE bounded slice of the vector-space migration: at most `batchSize` rows are dropped and
|
|
10203
|
+
* re-encoded, so the caller keeps control of the event loop between slices. This is the unit the
|
|
10204
|
+
* background migration ({@link migrateVectors}) and its tests drive.
|
|
10205
|
+
*
|
|
10206
|
+
* Resumability is a property of the DATABASE, not of this process: every slice re-derives what is
|
|
10207
|
+
* stale from the persisted width/space, so a restart simply continues where the last slice stopped.
|
|
10208
|
+
*/
|
|
10209
|
+
async migrateVectorsBatch(batchSize = 16) {
|
|
10210
|
+
const size = Math.max(1, Math.floor(batchSize));
|
|
10211
|
+
const { report, remaining } = await this.repairVectors({
|
|
10212
|
+
dryRun: false,
|
|
10213
|
+
limit: size
|
|
10214
|
+
});
|
|
10215
|
+
return {
|
|
10216
|
+
remaining,
|
|
10217
|
+
migrated: report.fixed,
|
|
10218
|
+
dropped: report.dropped,
|
|
10219
|
+
reindexed: report.reindexed,
|
|
10220
|
+
semantic_available: report.semantic_available
|
|
10221
|
+
};
|
|
10222
|
+
}
|
|
10223
|
+
/**
|
|
10224
|
+
* Drive the bounded background migration until the store is current, the model is unavailable, or
|
|
10225
|
+
* `shouldStop()` turns true (plugin unmount / process shutdown). A batch FAILURE never rejects: it is
|
|
10226
|
+
* logged and reported as `remaining > 0`, which is what makes the next heartbeat a retry (the caller
|
|
10227
|
+
* still owns the outer catch for a closed database, e.g. a shutdown racing the first read). Never
|
|
10228
|
+
* blocks the event loop for longer than one batch.
|
|
10229
|
+
*
|
|
10230
|
+
* Honours `semantic.auto_migrate` (default on) — the switch exists so a host that must not spend CPU
|
|
10231
|
+
* on background re-encoding can leave the loud warning and the manual entry as the only path.
|
|
10232
|
+
*/
|
|
10233
|
+
async migrateVectors(opts = {}) {
|
|
10234
|
+
const enabled = this.config.semantic.auto_migrate !== false;
|
|
10235
|
+
const snapshot = () => ({
|
|
10236
|
+
enabled,
|
|
10237
|
+
remaining: this.migrationRemaining(),
|
|
10238
|
+
migrated: 0,
|
|
10239
|
+
dropped: 0,
|
|
10240
|
+
reindexed: 0,
|
|
10241
|
+
semantic_available: this.semantic.isAvailable()
|
|
10242
|
+
});
|
|
10243
|
+
if (!enabled) return snapshot();
|
|
10244
|
+
let migrated = 0;
|
|
10245
|
+
let dropped = 0;
|
|
10246
|
+
let reindexed = 0;
|
|
10247
|
+
let last = snapshot();
|
|
10248
|
+
if (last.remaining === 0) return last;
|
|
10249
|
+
for (;;) {
|
|
10250
|
+
if (opts.shouldStop?.() === true) break;
|
|
10251
|
+
let step;
|
|
10252
|
+
try {
|
|
10253
|
+
step = await this.migrateVectorsBatch(opts.batchSize);
|
|
10254
|
+
} catch (error) {
|
|
10255
|
+
retrievalLogger().warn(`memory vector migration: batch failed (${describeError$2(error)}) — ${String(this.migrationRemaining())} vector(s) still belong to an older embedding space; the next pass retries`);
|
|
10256
|
+
break;
|
|
10257
|
+
}
|
|
10258
|
+
migrated += step.migrated;
|
|
10259
|
+
dropped += step.dropped;
|
|
10260
|
+
reindexed += step.reindexed;
|
|
10261
|
+
last = {
|
|
10262
|
+
...step,
|
|
10263
|
+
enabled,
|
|
10264
|
+
migrated,
|
|
10265
|
+
dropped,
|
|
10266
|
+
reindexed
|
|
10267
|
+
};
|
|
10268
|
+
opts.onProgress?.(last);
|
|
10269
|
+
if (step.migrated === 0 && step.dropped === 0 && step.reindexed === 0) break;
|
|
10270
|
+
if (last.remaining === 0) break;
|
|
10271
|
+
await yieldToEventLoop();
|
|
10272
|
+
}
|
|
10273
|
+
return last;
|
|
10274
|
+
}
|
|
10275
|
+
/** Active rows the semantic leg still cannot use (old width, old space, or no vector at all). */
|
|
10276
|
+
migrationRemaining() {
|
|
10277
|
+
const health = this.vectorSpaceHealth();
|
|
10278
|
+
return health.stale + health.space_stale + this.facts.vectorCounts().missing;
|
|
10279
|
+
}
|
|
10280
|
+
/**
|
|
9897
10281
|
* One lifecycle pass, owned by the store so the DB writes and the live-index
|
|
9898
10282
|
* invariant (ACTIVE vectors only) are maintained in ONE place — the runtime no
|
|
9899
10283
|
* longer has to patch the index behind the store's back.
|
|
@@ -10048,7 +10432,8 @@ var MemoryStore = class {
|
|
|
10048
10432
|
queryVector: input.queryVector,
|
|
10049
10433
|
recordStats: input.recordStats,
|
|
10050
10434
|
floors: input.floors,
|
|
10051
|
-
...input.relaxLegs === void 0 ? {} : { relaxLegs: input.relaxLegs }
|
|
10435
|
+
...input.relaxLegs === void 0 ? {} : { relaxLegs: input.relaxLegs },
|
|
10436
|
+
...input.rewriteQuery === void 0 ? {} : { rewriteQuery: input.rewriteQuery }
|
|
10052
10437
|
});
|
|
10053
10438
|
return {
|
|
10054
10439
|
hits: result.hits,
|
|
@@ -12224,7 +12609,20 @@ var KnowledgeStore = class {
|
|
|
12224
12609
|
const space = this.vectorSpace();
|
|
12225
12610
|
const persisted = this.chunks.vectorRows();
|
|
12226
12611
|
reloadVectorIndex(this.vstore, persisted, "knowledge");
|
|
12227
|
-
|
|
12612
|
+
let stale = 0;
|
|
12613
|
+
let spaceStale = 0;
|
|
12614
|
+
const expectedBytes = this.vstore.dim * 4;
|
|
12615
|
+
for (const row of persisted) {
|
|
12616
|
+
if (!row.vec || row.vec.byteLength !== expectedBytes) {
|
|
12617
|
+
stale++;
|
|
12618
|
+
continue;
|
|
12619
|
+
}
|
|
12620
|
+
if (row.embedding_model !== space) spaceStale++;
|
|
12621
|
+
}
|
|
12622
|
+
reportStaleVectors("knowledge", space, {
|
|
12623
|
+
stale,
|
|
12624
|
+
space_stale: spaceStale
|
|
12625
|
+
}, "`avantf-mem kb reindex` (MCP/plugin: `kb_reindex`)");
|
|
12228
12626
|
}
|
|
12229
12627
|
/**
|
|
12230
12628
|
* The write-side domain allowlist AND the 知识页 picker's data, in one place: the configured
|
|
@@ -13061,7 +13459,8 @@ var KnowledgeStore = class {
|
|
|
13061
13459
|
queryVector: opts?.queryVector,
|
|
13062
13460
|
recordStats: opts?.recordStats,
|
|
13063
13461
|
floors: opts?.floors,
|
|
13064
|
-
...opts?.relaxLegs === void 0 ? {} : { relaxLegs: opts.relaxLegs }
|
|
13462
|
+
...opts?.relaxLegs === void 0 ? {} : { relaxLegs: opts.relaxLegs },
|
|
13463
|
+
...opts?.rewriteQuery === void 0 ? {} : { rewriteQuery: opts.rewriteQuery }
|
|
13065
13464
|
});
|
|
13066
13465
|
opts?.onResult?.(result);
|
|
13067
13466
|
return result.hits;
|
|
@@ -13451,7 +13850,10 @@ function buildRuntime(opts) {
|
|
|
13451
13850
|
knowledge,
|
|
13452
13851
|
logger,
|
|
13453
13852
|
relevance(text) {
|
|
13454
|
-
|
|
13853
|
+
if (looksRelevant(memory.lexicalProbe(text, 2)) || looksRelevant(knowledge.lexicalProbe(text, 2))) return true;
|
|
13854
|
+
const rewrite = selfQueryRewrite(text);
|
|
13855
|
+
if (rewrite === void 0) return false;
|
|
13856
|
+
return looksRelevant(memory.lexicalProbe(rewrite, 2)) || looksRelevant(knowledge.lexicalProbe(rewrite, 2));
|
|
13455
13857
|
},
|
|
13456
13858
|
async remember(req) {
|
|
13457
13859
|
switch (req.action) {
|
|
@@ -13501,7 +13903,8 @@ function buildRuntime(opts) {
|
|
|
13501
13903
|
switch (req.action) {
|
|
13502
13904
|
case "stats": return {
|
|
13503
13905
|
...this.memory.countByStatus(),
|
|
13504
|
-
retrieval: retrievalHealthSummary()
|
|
13906
|
+
retrieval: retrievalHealthSummary(),
|
|
13907
|
+
vectors: this.memory.vectorSpaceHealth()
|
|
13505
13908
|
};
|
|
13506
13909
|
case "list": return this.memory.list(req.category, req.status ?? "active", req.limit ?? 50, req.offset ?? 0);
|
|
13507
13910
|
case "detail": return this.memory.get(req.fact_id) ?? { error: `fact_id=${req.fact_id} not found` };
|
|
@@ -14631,16 +15034,18 @@ const MODEL_ITEM = "mem:model";
|
|
|
14631
15034
|
* {@link MODEL_FILES} describes. A DIFFERENT repo is a different file set, which is why the item
|
|
14632
15035
|
* omits `spec.files` for it (see {@link managedModelSpec}).
|
|
14633
15036
|
*/
|
|
14634
|
-
const DEFAULT_MODEL_REPO = "Xenova/bge-
|
|
15037
|
+
const DEFAULT_MODEL_REPO = "Xenova/bge-base-zh-v1.5";
|
|
14635
15038
|
/**
|
|
14636
15039
|
* The files the local embedding runtime actually requests for the DEFAULT ONNX model.
|
|
14637
15040
|
*
|
|
14638
15041
|
* Measured, not guessed: with an empty cache, `@huggingface/transformers@4.x` resolving
|
|
14639
|
-
* `feature-extraction` for `Xenova/bge-
|
|
15042
|
+
* `feature-extraction` for `Xenova/bge-base-zh-v1.5` fetches exactly these four — `config.json`,
|
|
14640
15043
|
* `tokenizer.json`, `tokenizer_config.json` and the fp32 `onnx/model.onnx`. The repo ALSO carries
|
|
14641
15044
|
* `special_tokens_map.json`, `vocab.txt` and seven other ONNX quantisations the default run never
|
|
14642
15045
|
* opens; declaring them would download hundreds of megabytes of dead weight. The list stays narrow
|
|
14643
|
-
* to what is read.
|
|
15046
|
+
* to what is read. (It is UNCHANGED from the previous default `Xenova/bge-small-zh-v1.5`: both
|
|
15047
|
+
* repos are Xenova BGE ports with the same four runtime files — verified against the local cache of
|
|
15048
|
+
* each. Only the bytes behind `onnx/model.onnx` differ, ~95 MB for small vs ~389 MB for base.)
|
|
14644
15049
|
*/
|
|
14645
15050
|
const MODEL_FILES = [
|
|
14646
15051
|
"config.json",
|
|
@@ -14677,7 +15082,8 @@ const OWN_PREFIX = "envinit:";
|
|
|
14677
15082
|
* 2. **The runtime reads its models from elsewhere** — `semantic.cache_dir` is not the framework's
|
|
14678
15083
|
* `<home>/models`. That happens through the operator escape hatch `AVANTF_MEM_MODEL_CACHE`
|
|
14679
15084
|
* (env layer ④): the runtime loads from that directory, so a `mem:model` item would install a
|
|
14680
|
-
* full second copy under `<home>/models` that nothing ever reads (~
|
|
15085
|
+
* full second copy under `<home>/models` that nothing ever reads (~389 MB for the default
|
|
15086
|
+
* `Xenova/bge-base-zh-v1.5`). Declaring nothing
|
|
14681
15087
|
* is the fix; the caller warms from the override directory instead.
|
|
14682
15088
|
*
|
|
14683
15089
|
* `files` is the narrow {@link MODEL_FILES} set only for the DEFAULT repo; a user-changed repo has a
|
|
@@ -15197,6 +15603,58 @@ function openDocumentPath(documentPath, target, configured, deps = {}) {
|
|
|
15197
15603
|
throw new Error(`无法启动编辑器(试过:${tried.join("、")}${detail})`);
|
|
15198
15604
|
}
|
|
15199
15605
|
//#endregion
|
|
15606
|
+
//#region lib/types/vectorMigration.js
|
|
15607
|
+
function createVectorMigration(opts) {
|
|
15608
|
+
const { rt, logger } = opts;
|
|
15609
|
+
let running = false;
|
|
15610
|
+
let stopped = false;
|
|
15611
|
+
let warnedDisabled = false;
|
|
15612
|
+
const run = async () => {
|
|
15613
|
+
const health = rt.memory.vectorSpaceHealth();
|
|
15614
|
+
const pending = health.stale + health.space_stale;
|
|
15615
|
+
if (pending === 0) return;
|
|
15616
|
+
if (rt.closed || !opts.isActive()) return;
|
|
15617
|
+
if (rt.config.common.semantic.auto_migrate === false) {
|
|
15618
|
+
if (!warnedDisabled) {
|
|
15619
|
+
warnedDisabled = true;
|
|
15620
|
+
logger.warn(`vector migration: ${String(pending)} ACTIVE vector(s) belong to an older embedding space and \`semantic.auto_migrate\` is off — the semantic leg stays blind to them; run \`avantf-mem vectors --fix\` (MCP/plugin: \`mem_admin vectors_fix\`) or set \`semantic.auto_migrate: true\``);
|
|
15621
|
+
}
|
|
15622
|
+
return;
|
|
15623
|
+
}
|
|
15624
|
+
const startedAt = Date.now();
|
|
15625
|
+
logger.info(`vector migration: ${String(pending)} ACTIVE vector(s) belong to an older embedding space (${String(health.space_stale)} same-width other-model, ${String(health.stale)} wrong-width) — re-encoding in the background`);
|
|
15626
|
+
const outcome = await rt.memory.migrateVectors({
|
|
15627
|
+
...opts.batchSize === void 0 ? {} : { batchSize: opts.batchSize },
|
|
15628
|
+
shouldStop: () => stopped || rt.closed || !opts.isActive(),
|
|
15629
|
+
onProgress: (progress) => {
|
|
15630
|
+
logger.info(`vector migration: re-encoded ${String(progress.migrated)}/${String(pending)} (${String(progress.remaining)} left)`);
|
|
15631
|
+
}
|
|
15632
|
+
});
|
|
15633
|
+
if (outcome.remaining === 0) {
|
|
15634
|
+
logger.info(`vector migration: complete — ${String(outcome.migrated)} vector(s) re-encoded in ${String(Date.now() - startedAt)}ms; the semantic leg is current`);
|
|
15635
|
+
return;
|
|
15636
|
+
}
|
|
15637
|
+
if (!stopped && !rt.closed && opts.isActive()) logger.warn(`vector migration: ${String(outcome.remaining)} vector(s) still belong to an older embedding space (the embedding model may be unavailable) — the next heartbeat retries; manual entry \`avantf-mem vectors --fix\``);
|
|
15638
|
+
};
|
|
15639
|
+
return {
|
|
15640
|
+
start() {
|
|
15641
|
+
if (stopped || running) return;
|
|
15642
|
+
running = true;
|
|
15643
|
+
run().catch((error) => {
|
|
15644
|
+
logger.warn(`vector migration failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
15645
|
+
}).finally(() => {
|
|
15646
|
+
running = false;
|
|
15647
|
+
});
|
|
15648
|
+
},
|
|
15649
|
+
stop() {
|
|
15650
|
+
stopped = true;
|
|
15651
|
+
},
|
|
15652
|
+
get running() {
|
|
15653
|
+
return running;
|
|
15654
|
+
}
|
|
15655
|
+
};
|
|
15656
|
+
}
|
|
15657
|
+
//#endregion
|
|
15200
15658
|
//#region lib/types/tool_schema.js
|
|
15201
15659
|
/**
|
|
15202
15660
|
* zod → DSH parameter schema derivation for the agent tools.
|
|
@@ -15334,16 +15792,37 @@ function flattenToolSpec(spec) {
|
|
|
15334
15792
|
}
|
|
15335
15793
|
//#endregion
|
|
15336
15794
|
//#region lib/types/render.js
|
|
15795
|
+
/**
|
|
15796
|
+
* 方案 G — surface the EXISTING `degraded` flag as a sentence, without inventing a field.
|
|
15797
|
+
*
|
|
15798
|
+
* A recall/query result already carries `degraded` (with `weights`/`floors`) in its JSON, so a
|
|
15799
|
+
* caller COULD read it; the annotation names the consequence in plain words ("本次仅词法腿") for the
|
|
15800
|
+
* model and the operator who does not parse the envelope. It is a SECOND text block, never text
|
|
15801
|
+
* appended to the JSON block: the JSON block stays strictly parseable (see the module note). Absent
|
|
15802
|
+
* whenever the flag is absent/false — a graph-only answer reports `degraded: false`, so no note.
|
|
15803
|
+
*/
|
|
15804
|
+
function degradedLegNote(value) {
|
|
15805
|
+
const result = value?.result;
|
|
15806
|
+
return result !== null && typeof result === "object" && result.degraded === true ? DEGRADED_LEG_NOTE : void 0;
|
|
15807
|
+
}
|
|
15337
15808
|
/** The tool output spec registered on every mem tool. */
|
|
15338
15809
|
const OUTPUT = {
|
|
15339
15810
|
schema: {
|
|
15340
15811
|
type: "object",
|
|
15341
15812
|
additionalProperties: true
|
|
15342
15813
|
},
|
|
15343
|
-
render: (_args, value) =>
|
|
15344
|
-
|
|
15345
|
-
|
|
15346
|
-
|
|
15814
|
+
render: (_args, value) => {
|
|
15815
|
+
const blocks = [{
|
|
15816
|
+
type: "text",
|
|
15817
|
+
text: JSON.stringify(toWellFormedDeep(value), null, 2)
|
|
15818
|
+
}];
|
|
15819
|
+
const note = degradedLegNote(value);
|
|
15820
|
+
if (note !== void 0) blocks.push({
|
|
15821
|
+
type: "text",
|
|
15822
|
+
text: note
|
|
15823
|
+
});
|
|
15824
|
+
return blocks;
|
|
15825
|
+
}
|
|
15347
15826
|
};
|
|
15348
15827
|
//#endregion
|
|
15349
15828
|
//#region lib/types/inject.js
|
|
@@ -15922,6 +16401,14 @@ async function apply(ctx, config) {
|
|
|
15922
16401
|
...waitFor === void 0 ? {} : { waitFor },
|
|
15923
16402
|
...verdict === void 0 ? {} : { dshCompat: verdict }
|
|
15924
16403
|
});
|
|
16404
|
+
const vectorMigration = createVectorMigration({
|
|
16405
|
+
rt,
|
|
16406
|
+
logger,
|
|
16407
|
+
isActive: () => stillActive(ctx)
|
|
16408
|
+
});
|
|
16409
|
+
ctx.effect(() => () => {
|
|
16410
|
+
vectorMigration.stop();
|
|
16411
|
+
});
|
|
15925
16412
|
const sweepEntities = () => {
|
|
15926
16413
|
rt.memory.reindexEntities(ENTITY_SWEEP_PER_HEARTBEAT).then((swept) => {
|
|
15927
16414
|
if (swept.rebuilt > 0) logger.info(`entity sweep: rebuilt ${swept.rebuilt} fact(s) written by older rules (${swept.deferred} still stale)`);
|
|
@@ -15933,7 +16420,10 @@ async function apply(ctx, config) {
|
|
|
15933
16420
|
...waitFor === void 0 ? {} : { waitFor },
|
|
15934
16421
|
deferTokenizerUntilIdle: true
|
|
15935
16422
|
}).then(() => {
|
|
15936
|
-
if (!rt.closed && stillActive(ctx))
|
|
16423
|
+
if (!rt.closed && stillActive(ctx)) {
|
|
16424
|
+
sweepEntities();
|
|
16425
|
+
vectorMigration.start();
|
|
16426
|
+
}
|
|
15937
16427
|
}).catch((error) => {
|
|
15938
16428
|
logger.warn(`entity sweep gate failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
15939
16429
|
});
|
|
@@ -15946,6 +16436,7 @@ async function apply(ctx, config) {
|
|
|
15946
16436
|
logger.warn(`trust tick failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
15947
16437
|
}
|
|
15948
16438
|
sweepEntities();
|
|
16439
|
+
vectorMigration.start();
|
|
15949
16440
|
}, heartbeatMinutes * 6e4);
|
|
15950
16441
|
const unref = timer.unref;
|
|
15951
16442
|
if (typeof unref === "function") unref.call(timer);
|