@kira905/dsh-memory-toolkit 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +130 -1
- package/README.en.md +225 -2
- package/README.md +208 -2
- package/cordis.patch.yml +32 -0
- package/lib/deploy-guard.mjs +463 -275
- package/lib/guard-http.mjs +59 -62
- package/lib/pitfall-form-map.mjs +94 -84
- package/lib/pitfall-score.mjs +262 -185
- package/lib/session-file-name.mjs +116 -100
- package/lib/zstd-frames.mjs +88 -67
- package/memory-db-tools/README.md +29 -1
- package/memory-db-tools/audit-analyze.mjs +1 -1
- package/memory-db-tools/audit-export.mjs +60 -16
- package/memory-db-tools/audit-struct-probe.mjs +29 -8
- package/memory-db-tools/lib/known-devices.mjs +137 -0
- package/memory-engine/README.md +58 -4
- package/memory-engine/bench/bench.mjs +1 -1
- package/memory-engine/cli/band-sentinel.mjs +164 -0
- package/memory-engine/cli/distill-rollback.mjs +112 -44
- package/memory-engine/cli/distill-sessions.mjs +237 -0
- package/memory-engine/cli/distill.mjs +196 -95
- package/memory-engine/cli/doc-index-rebuild.mjs +142 -108
- package/memory-engine/cli/doc-index.mjs +47 -38
- package/memory-engine/cli/domain-tags.mjs +246 -162
- package/memory-engine/cli/layer-migrate.mjs +176 -0
- package/memory-engine/cli/layer-move.mjs +189 -0
- package/memory-engine/cli/pitfall-hook.mjs +136 -70
- package/memory-engine/cli/recall.mjs +243 -110
- package/memory-engine/cli/selfcheck.mjs +93 -44
- package/memory-engine/cli/serve-start.mjs +221 -140
- package/memory-engine/cli/sweep.mjs +83 -44
- package/memory-engine/cli/sync.mjs +93 -57
- package/memory-engine/config/audit-devices.json +6 -0
- package/memory-engine/config/bands.json +7 -0
- package/memory-engine/config/layers.json +7 -0
- package/memory-engine/dict/LICENSE-/350/257/215/345/205/270.md +53 -0
- package/memory-engine/dict/lexicon-config.json +11 -0
- package/memory-engine/dict//350/257/215/345/205/270.txt +109750 -0
- package/memory-engine/dict//350/257/215/351/242/221.txt +20000 -0
- package/memory-engine/lib/band-sentinel.mjs +417 -0
- package/memory-engine/lib/cli-strict.mjs +128 -0
- package/memory-engine/lib/cli-util.mjs +147 -125
- package/memory-engine/lib/config.mjs +83 -96
- package/memory-engine/lib/dirty-ledger.mjs +127 -94
- package/memory-engine/lib/distill-rollback.mjs +173 -133
- package/memory-engine/lib/distill.mjs +878 -585
- package/memory-engine/lib/doc-index-store.mjs +208 -235
- package/memory-engine/lib/doc-index.mjs +351 -350
- package/memory-engine/lib/doc-scan.mjs +178 -156
- package/memory-engine/lib/doc-summarize.mjs +220 -205
- package/memory-engine/lib/domain-tags.mjs +358 -277
- package/memory-engine/lib/embedder.mjs +175 -194
- package/memory-engine/lib/hashes.mjs +22 -7
- package/memory-engine/lib/layer-migrate.mjs +330 -0
- package/memory-engine/lib/layer-move.mjs +307 -0
- package/memory-engine/lib/layers.mjs +191 -0
- package/memory-engine/lib/lexicon-recall.mjs +473 -0
- package/memory-engine/lib/memory-source.mjs +78 -80
- package/memory-engine/lib/pitfall-source.mjs +115 -114
- package/memory-engine/lib/policy-source.mjs +279 -261
- package/memory-engine/lib/recall-log.mjs +285 -0
- package/memory-engine/lib/recall.mjs +565 -240
- package/memory-engine/lib/selfcheck.mjs +142 -106
- package/memory-engine/lib/session-distill.mjs +603 -0
- package/memory-engine/lib/session-source.mjs +421 -0
- package/memory-engine/lib/sweep.mjs +107 -130
- package/memory-engine/lib/sync.mjs +110 -140
- package/memory-engine/lib/tick-hook.mjs +123 -148
- package/memory-engine/lib/vectors-store.mjs +213 -188
- package/memory-engine/test/12-layering.test.mjs +539 -0
- package/memory-engine/test/13-session-distill.test.mjs +616 -0
- package/memory-engine/tools/bridge-smoke.mjs +94 -55
- package/memory-engine/tools/fetch-model.mjs +167 -87
- package/memory-engine/tools/smoke-recall.mjs +94 -38
- package/memory-engine/tools/verify-recall-chain.mjs +413 -0
- package/package.json +11 -1
|
@@ -1,11 +1,21 @@
|
|
|
1
|
-
//
|
|
1
|
+
// 召回入口:向量召回 + FTS 补位 + 倒数排名融合,pinned 条目全程排除。
|
|
2
|
+
// 这里只管参数整形与结果排版;召回算法在 ../lib/recall.mjs。
|
|
2
3
|
//
|
|
3
|
-
//
|
|
4
|
-
// node cli/recall.mjs --query "
|
|
4
|
+
// 常用姿势:
|
|
5
|
+
// node cli/recall.mjs --query "某句暗号" --top-k 8
|
|
5
6
|
// node cli/recall.mjs --query "task-board 看板" --json
|
|
6
|
-
// node cli/recall.mjs --query "..." --repeat 100
|
|
7
|
-
// node cli/recall.mjs --query "..." --source pitfall #
|
|
8
|
-
// node cli/recall.mjs --query "..." --sources memory,pitfall #
|
|
7
|
+
// node cli/recall.mjs --query "..." --repeat 100 # 连跑取分位数
|
|
8
|
+
// node cli/recall.mjs --query "..." --source pitfall # 换来源(缺省 memory)
|
|
9
|
+
// node cli/recall.mjs --query "..." --sources memory,pitfall # 多来源跨源融合
|
|
10
|
+
//
|
|
11
|
+
// 本版新增(都是可选开关,不改变既有默认行为):
|
|
12
|
+
// node cli/recall.mjs --query "..." --lexicon on # 打开词表第二路(默认读 dict/lexicon-config.json)
|
|
13
|
+
// node cli/recall.mjs --query "..." --fallback-top-k 8 # 编码失败时的固定兜底条数
|
|
14
|
+
// node cli/recall.mjs --query "..." --encode-timeout-ms 1500 # 编码预算
|
|
15
|
+
// node cli/recall.mjs --query "..." --recall-log <目录> # 旁路留痕(默认不写)
|
|
16
|
+
//
|
|
17
|
+
// 退出码:0 = 正常跑完(未触发也算正常)· 2 = 起不来/跑炸(--json 时错误也只上 stdout 的 JSON)。
|
|
18
|
+
import path from 'node:path';
|
|
9
19
|
import { isMainEntry, parseArgs, makeLogger, openRuntime, num } from '../lib/cli-util.mjs';
|
|
10
20
|
import {
|
|
11
21
|
recall,
|
|
@@ -14,118 +24,241 @@ import {
|
|
|
14
24
|
shouldTrigger,
|
|
15
25
|
extractFtsTokens,
|
|
16
26
|
buildFtsMatch,
|
|
27
|
+
truncateQuery,
|
|
28
|
+
QUERY_CHAR_LIMIT,
|
|
29
|
+
ENCODE_TIMEOUT_MS,
|
|
30
|
+
FALLBACK_TOP_K,
|
|
17
31
|
} from '../lib/recall.mjs';
|
|
32
|
+
import { getLexiconIndex, resolveLexiconSwitch } from '../lib/lexicon-recall.mjs';
|
|
33
|
+
import { openRecallLog } from '../lib/recall-log.mjs';
|
|
18
34
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
const list = String(raw)
|
|
23
|
-
.split(',')
|
|
24
|
-
.map((s) => s.trim())
|
|
25
|
-
.filter(Boolean);
|
|
26
|
-
return list.length ? list : null;
|
|
27
|
-
}
|
|
35
|
+
const CLEAN = 0;
|
|
36
|
+
const CRASH = 2;
|
|
37
|
+
const COLUMNS = 12;
|
|
28
38
|
|
|
29
|
-
|
|
39
|
+
/** 统一的启动三件套:原始 flag、是否机器可读、以及分好流的 logger */
|
|
40
|
+
function boot() {
|
|
30
41
|
const { flags } = parseArgs();
|
|
31
42
|
const json = Boolean(flags.json);
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
43
|
+
return { flags, json, log: makeLogger({ json, quiet: Boolean(flags.quiet) }) };
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** 开一个运行时,交给回调用,出了什么事都把库关掉 */
|
|
47
|
+
async function withRuntime(flags, work) {
|
|
48
|
+
const rt = await openRuntime(flags);
|
|
36
49
|
try {
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
50
|
+
return await work(rt);
|
|
51
|
+
} finally {
|
|
52
|
+
rt.close();
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** 起来就炸:json 走 stdout,文本走 logger,退出码固定 2 */
|
|
57
|
+
function oops(json, log, err) {
|
|
58
|
+
const detail = String(err?.stack ?? err);
|
|
59
|
+
if (json) process.stdout.write(`${JSON.stringify({ ok: false, error: detail }, null, 2)}\n`);
|
|
60
|
+
else log(`❌ ${detail}`);
|
|
61
|
+
process.exitCode = CRASH;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** 一列里中文占两格 ⇒ 自己算显示宽度再补空格,冒号才能对齐 */
|
|
65
|
+
function field(text) {
|
|
66
|
+
let used = 0;
|
|
67
|
+
for (const ch of String(text)) used += ch.codePointAt(0) > 0xff ? 2 : 1;
|
|
68
|
+
return String(text) + ' '.repeat(Math.max(0, COLUMNS - used));
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const row = (log, name, value) => log(`${field(name)}: ${value}`);
|
|
72
|
+
|
|
73
|
+
const readQuery = (flags) => (flags.query === undefined || flags.query === true ? '' : String(flags.query));
|
|
74
|
+
|
|
75
|
+
/** `--sources a,b` → ['a','b'];没给 / 给了空 ⇒ null(退回单来源那条路) */
|
|
76
|
+
function pickNames(flags) {
|
|
77
|
+
const raw = flags.sources;
|
|
78
|
+
if (raw === undefined || raw === true || raw === '') return null;
|
|
79
|
+
const names = String(raw).split(',').map((s) => s.trim()).filter(Boolean);
|
|
80
|
+
return names.length ? names : null;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** 分位数取最接近那条(不插值,和原来一致) */
|
|
84
|
+
const at = (sorted, pct) => sorted[Math.min(sorted.length - 1, Math.floor((pct / 100) * sorted.length))];
|
|
85
|
+
|
|
86
|
+
/** 三态开关:`--lexicon`(裸)= 开;`--lexicon on/off` 两种写法都认;没给 ⇒ null(读配置文件)。 */
|
|
87
|
+
function triState(raw, name) {
|
|
88
|
+
if (raw === undefined) return null;
|
|
89
|
+
if (raw === true) return true;
|
|
90
|
+
const text = String(raw).trim().toLowerCase();
|
|
91
|
+
if (['on', '1', 'true', 'yes', 'enable', 'enabled'].includes(text)) return true;
|
|
92
|
+
if (['off', '0', 'false', 'no', 'disable', 'disabled'].includes(text)) return false;
|
|
93
|
+
throw new Error(`--${name} 只认 on / off(收到 ${JSON.stringify(String(raw))})`);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** 单来源一轮:建一次向量索引(词表路开着就顺手建好词表索引,省一次预热等待) */
|
|
97
|
+
async function singleRound(rt, opts, log) {
|
|
98
|
+
const { store, source, sourceName, embedder } = rt;
|
|
99
|
+
const index = buildVectorIndex(store.db, { source: sourceName });
|
|
100
|
+
const lexiconIndex = opts.lexiconEnabled ? getLexiconIndex(source, { allowBuild: true }) : null;
|
|
101
|
+
const res = await recall({
|
|
102
|
+
query: opts.query, store, source, embedder, index, sourceName,
|
|
103
|
+
vectorTopK: opts.vectorTopK, ftsTopK: opts.ftsTopK, rrfK: opts.rrfK, resultTopK: opts.topK,
|
|
104
|
+
queryCharLimit: opts.queryCharLimit, encodeTimeoutMs: opts.encodeTimeoutMs, fallbackTopK: opts.fallbackTopK,
|
|
105
|
+
lexiconEnabled: opts.lexiconEnabled, lexiconTopK: opts.lexiconTopK, lexiconFloor: opts.lexiconFloor,
|
|
106
|
+
lexiconIndex, recallLog: opts.recallLog,
|
|
107
|
+
log,
|
|
108
|
+
});
|
|
109
|
+
return { size: index.count, res };
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** 多来源一轮:每源各建索引、各出候选,融合交给 recallMulti */
|
|
113
|
+
async function fusedRound(rt, opts, log) {
|
|
114
|
+
const { store, embedder } = rt;
|
|
115
|
+
const entries = opts.names.map((name) => ({
|
|
116
|
+
name, source: rt.openSource(name), index: buildVectorIndex(store.db, { source: name }),
|
|
117
|
+
}));
|
|
118
|
+
const res = await recallMulti({
|
|
119
|
+
query: opts.query, store, entries, embedder,
|
|
120
|
+
perSourceTopK: opts.perSource, rrfK: opts.rrfK, resultTopK: opts.topK,
|
|
121
|
+
queryCharLimit: opts.queryCharLimit, encodeTimeoutMs: opts.encodeTimeoutMs,
|
|
122
|
+
lexiconEnabled: opts.lexiconEnabled, lexiconTopK: opts.lexiconTopK, lexiconFloor: opts.lexiconFloor,
|
|
123
|
+
recallLog: opts.recallLog,
|
|
124
|
+
log,
|
|
125
|
+
});
|
|
126
|
+
return { size: entries.reduce((sum, e) => sum + (e.index.count || 0), 0), res };
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function printHit(log, hit, multi) {
|
|
130
|
+
const owner = multi ? ` [${hit.source}]` : '';
|
|
131
|
+
const vec = hit.vectorRank ?? '-';
|
|
132
|
+
const fts = hit.ftsRank ?? '-';
|
|
133
|
+
const lex = hit.lexiconRank == null ? '' : ` lexRank=${hit.lexiconRank}`;
|
|
134
|
+
const fallback = hit.fallbackRank == null ? '' : ` fallback=${hit.fallbackRank}`;
|
|
135
|
+
log(` #${hit.refId}${owner} score=${hit.score.toFixed(5)} 来源=${hit.sources.join('+')} vecRank=${vec} ftsRank=${fts}${lex}${fallback}`);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function printReport(log, out, last, names, repeat, times) {
|
|
139
|
+
const multi = Boolean(names);
|
|
140
|
+
const timings = last.timings;
|
|
141
|
+
log('=== 召回 ===');
|
|
142
|
+
row(log, '来源', out.source);
|
|
143
|
+
row(log, '查询', JSON.stringify(out.query));
|
|
144
|
+
row(log, '触发', last.triggered ? `是(${last.reason})` : `否(${last.reason})`);
|
|
145
|
+
if (last.triggered) {
|
|
146
|
+
const stats = last.stats || {};
|
|
147
|
+
if (stats.queryTruncated) {
|
|
148
|
+
row(log, '查询截断', `${stats.queryChars} → ${stats.queryCharsUsed} 字符(上限 ${stats.queryCharLimit})`);
|
|
79
149
|
}
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
log(`=== 召回 ===`);
|
|
97
|
-
log(`来源 : ${out.source}`);
|
|
98
|
-
log(`查询 : ${JSON.stringify(query)}`);
|
|
99
|
-
log(`触发 : ${last.triggered ? `是(${last.reason})` : `否(${last.reason})`}`);
|
|
100
|
-
if (last.triggered) {
|
|
101
|
-
if (multiSources) {
|
|
102
|
-
for (const [name, rec] of Object.entries(last.perSource || {})) {
|
|
103
|
-
log(
|
|
104
|
-
` · ${name}: 索引 ${rec.indexSize} 条 / 排除 pinned ${rec.pinnedExcluded} / 向量命中 ${rec.vectorHits} / FTS 命中 ${rec.ftsHits}${rec.error ? ` / ⚠️ ${rec.error}` : ''}`,
|
|
105
|
-
);
|
|
106
|
-
}
|
|
107
|
-
} else {
|
|
108
|
-
log(`索引规模 : ${last.stats.indexSize} 条向量 / 排除 pinned ${last.stats.pinnedExcluded} 条`);
|
|
109
|
-
log(`FTS 补位 : tokens=${JSON.stringify(last.stats.ftsTokens)} → 命中 ${last.stats.ftsMatched}`);
|
|
110
|
-
}
|
|
111
|
-
log(`耗时 : 编码 ${last.timings.encodeMs.toFixed(1)}ms / 向量 ${last.timings.vectorMs.toFixed(1)}ms / 融合 ${last.timings.fuseMs.toFixed(1)}ms / 合计 ${last.timings.totalMs.toFixed(1)}ms`);
|
|
112
|
-
log(`命中 top-${last.hits.length}:`);
|
|
113
|
-
for (const h of last.hits) {
|
|
114
|
-
log(
|
|
115
|
-
` #${h.refId}${multiSources ? ` [${h.source}]` : ''} score=${h.score.toFixed(5)} 来源=${h.sources.join('+')} vecRank=${h.vectorRank ?? '-'} ftsRank=${h.ftsRank ?? '-'}`,
|
|
116
|
-
);
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
if (repeat > 1) log(`重复 ${times.length} 次:P50 ${pct(50).toFixed(1)}ms / P95 ${pct(95).toFixed(1)}ms`);
|
|
150
|
+
// 降级是「这一轮记忆从哪来的」的判据:显眼打出来,别让 SLI 只看到"有命中"
|
|
151
|
+
row(log, '降级', last.degraded
|
|
152
|
+
? `是(${last.degraded.mode} / ${last.degraded.reason})→ 固定 Top-${last.hits.length}`
|
|
153
|
+
: '否');
|
|
154
|
+
if (multi) {
|
|
155
|
+
Object.entries(last.perSource || {}).forEach(([name, rec]) => log(
|
|
156
|
+
` · ${name}: 索引 ${rec.indexSize} 条 / 排除 pinned ${rec.pinnedExcluded} / 向量命中 ${rec.vectorHits} / FTS 命中 ${rec.ftsHits}${rec.lexiconHits ? ` / 词表命中 ${rec.lexiconHits}` : ''}${rec.fallbackHits ? ` / 兜底 ${rec.fallbackHits}` : ''}${rec.error ? ` / ⚠️ ${rec.error}` : ''}`,
|
|
157
|
+
));
|
|
158
|
+
} else {
|
|
159
|
+
row(log, '索引规模', `${stats.indexSize} 条向量 / 排除 pinned ${stats.pinnedExcluded} 条`);
|
|
160
|
+
row(log, 'FTS 补位', stats.ftsSkipped
|
|
161
|
+
? '本轮跳过(编码降级)'
|
|
162
|
+
: `tokens=${JSON.stringify(stats.ftsTokens)} → 命中 ${stats.ftsMatched}`);
|
|
163
|
+
row(log, '词表路', stats.lexiconEnabled
|
|
164
|
+
? `开(${stats.lexiconStatus}${stats.lexiconHits ? ` / 命中 ${stats.lexiconHits}` : ''}${stats.lexiconBuildMs != null ? ` / 建索引 ${stats.lexiconBuildMs}ms` : ''})`
|
|
165
|
+
: `关(${stats.lexiconSwitchSource || 'default'})`);
|
|
120
166
|
}
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
167
|
+
const lexiconMs = timings.lexiconMs == null ? '' : `词表 ${timings.lexiconMs.toFixed(1)}ms / `;
|
|
168
|
+
row(
|
|
169
|
+
log,
|
|
170
|
+
'耗时',
|
|
171
|
+
`编码 ${timings.encodeMs.toFixed(1)}ms / 向量 ${timings.vectorMs.toFixed(1)}ms / ${lexiconMs}融合 ${timings.fuseMs.toFixed(1)}ms / 合计 ${timings.totalMs.toFixed(1)}ms`,
|
|
172
|
+
);
|
|
173
|
+
log('命中 top-' + last.hits.length + ':');
|
|
174
|
+
last.hits.forEach((hit) => printHit(log, hit, multi));
|
|
175
|
+
}
|
|
176
|
+
if (out.recallLog) row(log, '留痕', out.recallLog.file);
|
|
177
|
+
if (repeat > 1) {
|
|
178
|
+
log(`重复 ${times.length} 次:P50 ${at(times, 50).toFixed(1)}ms / P95 ${at(times, 95).toFixed(1)}ms`);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
async function work(rt, flags, log, json) {
|
|
183
|
+
const { config, sourceName } = rt;
|
|
184
|
+
const query = readQuery(flags);
|
|
185
|
+
const names = pickNames(flags);
|
|
186
|
+
const repeat = num(flags.repeat, 1);
|
|
187
|
+
// 词表开关:命令行 > 环境变量 > dict/lexicon-config.json > 默认(关)
|
|
188
|
+
const lexiconSetting = resolveLexiconSwitch({
|
|
189
|
+
enabled: triState(flags.lexicon, 'lexicon'),
|
|
190
|
+
topK: num(flags['lexicon-top-k'], null),
|
|
191
|
+
floor: num(flags['lexicon-floor'], null),
|
|
192
|
+
});
|
|
193
|
+
// 留痕:给了 --recall-log 才写;--no-recall-log 或 DSH_MEMORY_RECALL_LOG=off 一键关回
|
|
194
|
+
const wantedLog = flags['no-recall-log'] ? false : flags['recall-log'] === undefined ? null : true;
|
|
195
|
+
const logDir = flags['recall-log'] === true
|
|
196
|
+
? path.join(config.dshHome, 'logs', 'memory-recall')
|
|
197
|
+
: typeof flags['recall-log'] === 'string' ? flags['recall-log'] : null;
|
|
198
|
+
const recallLog = wantedLog === false ? null : openRecallLog({ dir: logDir, enabled: wantedLog, env: process.env });
|
|
199
|
+
|
|
200
|
+
const opts = {
|
|
201
|
+
query,
|
|
202
|
+
names,
|
|
203
|
+
perSource: num(flags['per-source-top-k'], 5),
|
|
204
|
+
vectorTopK: num(flags['vector-top-k'], config.vectorTopK),
|
|
205
|
+
ftsTopK: num(flags['fts-top-k'], config.ftsTopK),
|
|
206
|
+
rrfK: num(flags['rrf-k'], config.rrfK),
|
|
207
|
+
topK: num(flags['top-k'], config.resultTopK),
|
|
208
|
+
queryCharLimit: num(flags['query-max-chars'], QUERY_CHAR_LIMIT),
|
|
209
|
+
encodeTimeoutMs: num(flags['encode-timeout-ms'], ENCODE_TIMEOUT_MS),
|
|
210
|
+
fallbackTopK: num(flags['fallback-top-k'], FALLBACK_TOP_K),
|
|
211
|
+
lexiconEnabled: lexiconSetting.enabled,
|
|
212
|
+
lexiconTopK: lexiconSetting.topK,
|
|
213
|
+
lexiconFloor: lexiconSetting.floor,
|
|
214
|
+
recallLog,
|
|
215
|
+
};
|
|
216
|
+
// 连跑 --repeat 次只为拿分位数:每轮的结果都覆盖,只留最后一轮做展示
|
|
217
|
+
const run = { times: [], last: null, indexSize: 0 };
|
|
218
|
+
while (run.times.length < repeat) {
|
|
219
|
+
const done = await (names ? fusedRound : singleRound)(rt, opts, log);
|
|
220
|
+
run.indexSize = done.size;
|
|
221
|
+
run.last = done.res;
|
|
222
|
+
run.times.push(done.res.timings.totalMs);
|
|
223
|
+
}
|
|
224
|
+
run.times.sort((a, b) => a - b);
|
|
225
|
+
const shaped = truncateQuery(query, opts.queryCharLimit);
|
|
226
|
+
const tokens = extractFtsTokens(shaped.text);
|
|
227
|
+
const out = {
|
|
228
|
+
ok: true,
|
|
229
|
+
source: names ? names.join(',') : sourceName,
|
|
230
|
+
vectorsDb: config.vectorsDb,
|
|
231
|
+
indexSize: run.indexSize,
|
|
232
|
+
gate: shouldTrigger(shaped.text),
|
|
233
|
+
ftsTokens: tokens,
|
|
234
|
+
ftsMatchExpr: buildFtsMatch(tokens),
|
|
235
|
+
lexicon: {
|
|
236
|
+
enabled: lexiconSetting.enabled,
|
|
237
|
+
switchSource: lexiconSetting.source,
|
|
238
|
+
topK: lexiconSetting.topK,
|
|
239
|
+
floor: lexiconSetting.floor,
|
|
240
|
+
},
|
|
241
|
+
recallLog: recallLog ? { file: recallLog.file, stats: recallLog.stats() } : null,
|
|
242
|
+
...run.last,
|
|
243
|
+
repeat: {
|
|
244
|
+
n: run.times.length,
|
|
245
|
+
p50: at(run.times, 50),
|
|
246
|
+
p95: at(run.times, 95),
|
|
247
|
+
min: run.times[0],
|
|
248
|
+
max: run.times[run.times.length - 1],
|
|
249
|
+
},
|
|
250
|
+
};
|
|
251
|
+
if (json) process.stdout.write(`${JSON.stringify(out, null, 2)}\n`);
|
|
252
|
+
else printReport(log, out, run.last, names, repeat, run.times);
|
|
253
|
+
process.exitCode = CLEAN;
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
async function main() {
|
|
257
|
+
const { flags, json, log } = boot();
|
|
258
|
+
try {
|
|
259
|
+
await withRuntime(flags, (rt) => work(rt, flags, log, json));
|
|
260
|
+
} catch (err) {
|
|
261
|
+
oops(json, log, err);
|
|
129
262
|
}
|
|
130
263
|
}
|
|
131
264
|
|
|
@@ -1,56 +1,105 @@
|
|
|
1
|
-
//
|
|
1
|
+
// 启动自检入口:把「向量表在不在 / schema 对不对 / 编码器活不活 / 维度合不合 / 库是不是本机的」一次问清,
|
|
2
|
+
// 顺带报一行各来源的向量覆盖度。warn 不算失败 —— 自检结果只是给人看和给调用方判断降级用。
|
|
2
3
|
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
4
|
+
// node cli/selfcheck.mjs [--json] [--stub-embedder] [--memory-db X] [--vectors-db Y] [--model-dir Z]
|
|
5
|
+
//
|
|
6
|
+
// 退出码:0 = 没有 fail 项(warn 照旧 0)· 1 = 有 fail 项 · 2 = 跑不起来。
|
|
5
7
|
import { isMainEntry, parseArgs, makeLogger, openRuntime } from '../lib/cli-util.mjs';
|
|
6
8
|
import { runSelfcheck } from '../lib/selfcheck.mjs';
|
|
7
9
|
|
|
8
|
-
const
|
|
10
|
+
const OK = 0;
|
|
11
|
+
const FAIL_CHECK = 1;
|
|
12
|
+
const CRASH = 2;
|
|
13
|
+
const COLUMNS = 12;
|
|
9
14
|
|
|
10
|
-
|
|
15
|
+
const MARKS = new Map([
|
|
16
|
+
['ok', '✅'],
|
|
17
|
+
['warn', '⚠️ '],
|
|
18
|
+
['fail', '❌'],
|
|
19
|
+
['info', 'ℹ️ '],
|
|
20
|
+
]);
|
|
21
|
+
|
|
22
|
+
/** 统一的启动三件套:原始 flag、是否机器可读、以及分好流的 logger */
|
|
23
|
+
function boot() {
|
|
11
24
|
const { flags } = parseArgs();
|
|
12
25
|
const json = Boolean(flags.json);
|
|
13
|
-
|
|
14
|
-
|
|
26
|
+
return { flags, json, log: makeLogger({ json, quiet: Boolean(flags.quiet) }) };
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** 开一个运行时,交给回调用,出了什么事都把库关掉 */
|
|
30
|
+
async function withRuntime(flags, work) {
|
|
31
|
+
const rt = await openRuntime(flags);
|
|
15
32
|
try {
|
|
16
|
-
|
|
17
|
-
const result = await runSelfcheck({
|
|
18
|
-
store: rt.store,
|
|
19
|
-
source: rt.source,
|
|
20
|
-
embedder: rt.embedder,
|
|
21
|
-
config: rt.config,
|
|
22
|
-
allowHostMismatch: !flags['strict-host'],
|
|
23
|
-
});
|
|
24
|
-
const out = {
|
|
25
|
-
ok: result.ok,
|
|
26
|
-
host: rt.config.hostId,
|
|
27
|
-
memoryDb: rt.config.memoryDb,
|
|
28
|
-
vectorsDb: rt.config.vectorsDb,
|
|
29
|
-
modelId: rt.config.modelId,
|
|
30
|
-
modelDir: rt.config.modelDir,
|
|
31
|
-
embedder: rt.embedder.describe(),
|
|
32
|
-
...result,
|
|
33
|
-
};
|
|
34
|
-
if (json) process.stdout.write(JSON.stringify(out, null, 2) + '\n');
|
|
35
|
-
else {
|
|
36
|
-
log('=== 启动自检 ===');
|
|
37
|
-
log(`主机 : ${rt.config.hostId}`);
|
|
38
|
-
log(`记忆库 : ${rt.config.memoryDb}`);
|
|
39
|
-
log(`向量库 : ${rt.config.vectorsDb}`);
|
|
40
|
-
log(`编码器 : ${JSON.stringify(rt.embedder.describe())}`);
|
|
41
|
-
log('');
|
|
42
|
-
for (const c of result.checks) log(`${ICON[c.status] || '·'} ${c.name}:${c.detail}`);
|
|
43
|
-
log('');
|
|
44
|
-
log(result.ok ? `✅ 自检通过(warn ${result.warned})` : `❌ 自检失败(fail ${result.failed})`);
|
|
45
|
-
}
|
|
46
|
-
process.exitCode = result.ok ? 0 : 1;
|
|
47
|
-
} catch (e) {
|
|
48
|
-
const msg = String(e && e.stack ? e.stack : e);
|
|
49
|
-
if (json) process.stdout.write(JSON.stringify({ ok: false, error: msg }, null, 2) + '\n');
|
|
50
|
-
else log(`❌ ${msg}`);
|
|
51
|
-
process.exitCode = 2;
|
|
33
|
+
return await work(rt);
|
|
52
34
|
} finally {
|
|
53
|
-
|
|
35
|
+
rt.close();
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** 起来就炸:json 走 stdout,文本走 logger,退出码固定 2 */
|
|
40
|
+
function oops(json, log, err) {
|
|
41
|
+
const detail = String(err?.stack ?? err);
|
|
42
|
+
if (json) process.stdout.write(`${JSON.stringify({ ok: false, error: detail }, null, 2)}\n`);
|
|
43
|
+
else log(`❌ ${detail}`);
|
|
44
|
+
process.exitCode = CRASH;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** 中文占两格:按显示宽度补空格,冒号右对齐 */
|
|
48
|
+
function field(text) {
|
|
49
|
+
let used = 0;
|
|
50
|
+
for (const ch of String(text)) used += ch.codePointAt(0) > 0xff ? 2 : 1;
|
|
51
|
+
return String(text) + ' '.repeat(Math.max(0, COLUMNS - used));
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function report(log, view, result) {
|
|
55
|
+
const row = (name, value) => log(`${field(name)}: ${value}`);
|
|
56
|
+
log('=== 启动自检 ===');
|
|
57
|
+
row('主机', view.host);
|
|
58
|
+
row('记忆库', view.memoryDb);
|
|
59
|
+
row('向量库', view.vectorsDb);
|
|
60
|
+
row('编码器', JSON.stringify(view.embedder));
|
|
61
|
+
log('');
|
|
62
|
+
result.checks.forEach((c) => log(`${MARKS.get(c.status) || '·'} ${c.name}:${c.detail}`));
|
|
63
|
+
log('');
|
|
64
|
+
log(result.ok ? `✅ 自检通过(warn ${result.warned})` : `❌ 自检失败(fail ${result.failed})`);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
async function work(rt, flags, log, json) {
|
|
68
|
+
const { config, embedder, source, store } = rt;
|
|
69
|
+
const result = await runSelfcheck({
|
|
70
|
+
store,
|
|
71
|
+
source,
|
|
72
|
+
embedder,
|
|
73
|
+
config,
|
|
74
|
+
// 缺省容忍 hostId 不一致(只告警);--strict-host 才当失败
|
|
75
|
+
allowHostMismatch: !flags['strict-host'],
|
|
76
|
+
});
|
|
77
|
+
const described = embedder.describe();
|
|
78
|
+
const out = {
|
|
79
|
+
ok: result.ok,
|
|
80
|
+
host: config.hostId,
|
|
81
|
+
memoryDb: config.memoryDb,
|
|
82
|
+
vectorsDb: config.vectorsDb,
|
|
83
|
+
modelId: config.modelId,
|
|
84
|
+
modelDir: config.modelDir,
|
|
85
|
+
embedder: described,
|
|
86
|
+
...result,
|
|
87
|
+
};
|
|
88
|
+
if (json) process.stdout.write(`${JSON.stringify(out, null, 2)}\n`);
|
|
89
|
+
else {
|
|
90
|
+
report(log, {
|
|
91
|
+
host: config.hostId, memoryDb: config.memoryDb, vectorsDb: config.vectorsDb, embedder: described,
|
|
92
|
+
}, result);
|
|
93
|
+
}
|
|
94
|
+
process.exitCode = result.ok ? OK : FAIL_CHECK;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
async function main() {
|
|
98
|
+
const { flags, json, log } = boot();
|
|
99
|
+
try {
|
|
100
|
+
await withRuntime(flags, (rt) => work(rt, flags, log, json));
|
|
101
|
+
} catch (err) {
|
|
102
|
+
oops(json, log, err);
|
|
54
103
|
}
|
|
55
104
|
}
|
|
56
105
|
|