@kira905/dsh-memory-toolkit 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +92 -0
- package/README.en.md +225 -2
- package/README.md +180 -2
- package/lib/deploy-guard.mjs +463 -275
- package/lib/guard-http.mjs +59 -62
- package/lib/pitfall-form-map.mjs +94 -84
- package/lib/pitfall-score.mjs +262 -185
- package/lib/session-file-name.mjs +116 -100
- package/lib/zstd-frames.mjs +88 -67
- package/memory-db-tools/README.md +29 -1
- package/memory-db-tools/audit-analyze.mjs +1 -1
- package/memory-db-tools/audit-export.mjs +60 -16
- package/memory-db-tools/audit-struct-probe.mjs +29 -8
- package/memory-db-tools/lib/known-devices.mjs +137 -0
- package/memory-engine/README.md +58 -4
- package/memory-engine/bench/bench.mjs +1 -1
- package/memory-engine/cli/band-sentinel.mjs +164 -0
- package/memory-engine/cli/distill-rollback.mjs +112 -44
- package/memory-engine/cli/distill-sessions.mjs +237 -0
- package/memory-engine/cli/distill.mjs +196 -95
- package/memory-engine/cli/doc-index-rebuild.mjs +142 -108
- package/memory-engine/cli/doc-index.mjs +47 -38
- package/memory-engine/cli/domain-tags.mjs +246 -162
- package/memory-engine/cli/layer-migrate.mjs +176 -0
- package/memory-engine/cli/layer-move.mjs +189 -0
- package/memory-engine/cli/pitfall-hook.mjs +136 -70
- package/memory-engine/cli/recall.mjs +243 -110
- package/memory-engine/cli/selfcheck.mjs +93 -44
- package/memory-engine/cli/serve-start.mjs +221 -140
- package/memory-engine/cli/sweep.mjs +83 -44
- package/memory-engine/cli/sync.mjs +93 -57
- package/memory-engine/config/audit-devices.json +6 -0
- package/memory-engine/config/bands.json +7 -0
- package/memory-engine/config/layers.json +7 -0
- package/memory-engine/dict/LICENSE-/350/257/215/345/205/270.md +53 -0
- package/memory-engine/dict/lexicon-config.json +11 -0
- package/memory-engine/dict//350/257/215/345/205/270.txt +109750 -0
- package/memory-engine/dict//350/257/215/351/242/221.txt +20000 -0
- package/memory-engine/lib/band-sentinel.mjs +417 -0
- package/memory-engine/lib/cli-strict.mjs +128 -0
- package/memory-engine/lib/cli-util.mjs +147 -125
- package/memory-engine/lib/config.mjs +83 -96
- package/memory-engine/lib/dirty-ledger.mjs +127 -94
- package/memory-engine/lib/distill-rollback.mjs +173 -133
- package/memory-engine/lib/distill.mjs +878 -585
- package/memory-engine/lib/doc-index-store.mjs +208 -235
- package/memory-engine/lib/doc-index.mjs +351 -350
- package/memory-engine/lib/doc-scan.mjs +178 -156
- package/memory-engine/lib/doc-summarize.mjs +220 -205
- package/memory-engine/lib/domain-tags.mjs +358 -277
- package/memory-engine/lib/embedder.mjs +175 -194
- package/memory-engine/lib/hashes.mjs +22 -7
- package/memory-engine/lib/layer-migrate.mjs +330 -0
- package/memory-engine/lib/layer-move.mjs +307 -0
- package/memory-engine/lib/layers.mjs +191 -0
- package/memory-engine/lib/lexicon-recall.mjs +473 -0
- package/memory-engine/lib/memory-source.mjs +78 -80
- package/memory-engine/lib/pitfall-source.mjs +115 -114
- package/memory-engine/lib/policy-source.mjs +279 -261
- package/memory-engine/lib/recall-log.mjs +285 -0
- package/memory-engine/lib/recall.mjs +565 -240
- package/memory-engine/lib/selfcheck.mjs +142 -106
- package/memory-engine/lib/session-distill.mjs +603 -0
- package/memory-engine/lib/session-source.mjs +421 -0
- package/memory-engine/lib/sweep.mjs +107 -130
- package/memory-engine/lib/sync.mjs +110 -140
- package/memory-engine/lib/tick-hook.mjs +123 -148
- package/memory-engine/lib/vectors-store.mjs +213 -188
- package/memory-engine/test/12-layering.test.mjs +539 -0
- package/memory-engine/test/13-session-distill.test.mjs +616 -0
- package/memory-engine/tools/bridge-smoke.mjs +94 -55
- package/memory-engine/tools/fetch-model.mjs +167 -87
- package/memory-engine/tools/smoke-recall.mjs +94 -38
- package/memory-engine/tools/verify-recall-chain.mjs +413 -0
- package/package.json +1 -1
|
@@ -1,10 +1,137 @@
|
|
|
1
|
-
//
|
|
1
|
+
// 启动自检:把「引擎现在到底能不能用」拆成几条独立判据,逐条给结论。
|
|
2
2
|
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
3
|
+
// 设计取舍:自检**永不阻断启动**。任何一条判死,也只是往清单里写一条 fail,
|
|
4
|
+
// 由调用方自己决定是降级跑还是重建 —— 因为自检本身挂掉比带病启动更糟。
|
|
5
5
|
import { SCHEMA_VERSION } from './config.mjs';
|
|
6
6
|
import { META_KEYS, getMeta, getMetaFor, setMeta, countVectors } from './vectors-store.mjs';
|
|
7
7
|
|
|
8
|
+
const errorText = (e) => String(e && e.message ? e.message : e);
|
|
9
|
+
|
|
10
|
+
// ------------------------------------------------------------------ 各判据
|
|
11
|
+
|
|
12
|
+
/** ① 表存在性:缺表时 open 阶段应已重建,这里只核对重建结果。 */
|
|
13
|
+
function checkTables(store) {
|
|
14
|
+
const have = store.schema.tables;
|
|
15
|
+
if (!have.vectors || !have.meta) {
|
|
16
|
+
return { name: '表存在性', status: 'fail', detail: 'vectors 或 meta 表缺失且重建失败' };
|
|
17
|
+
}
|
|
18
|
+
const created = store.schema.created;
|
|
19
|
+
return {
|
|
20
|
+
name: '表存在性',
|
|
21
|
+
status: 'ok',
|
|
22
|
+
detail: created.length ? `缺表已重建:${created.join(', ')}` : 'vectors / meta 均在位',
|
|
23
|
+
created,
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** ② 库内 schemaVersion 与代码期望值是否一致(不一致只 warn,不做迁移)。 */
|
|
28
|
+
function checkSchemaVersion(db) {
|
|
29
|
+
const found = getMeta(db, META_KEYS.schemaVersion);
|
|
30
|
+
const same = found === SCHEMA_VERSION;
|
|
31
|
+
return {
|
|
32
|
+
name: 'schemaVersion',
|
|
33
|
+
status: same ? 'ok' : 'warn',
|
|
34
|
+
detail: same ? `=${found}` : `库内为 ${found ?? '(空)'},代码期望 ${SCHEMA_VERSION}`,
|
|
35
|
+
schemaVersion: found,
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** ③ 模型可用性:探针本身可能抛,抛了也算「不可用」而不是让自检崩。 */
|
|
40
|
+
async function probeModel(embedder) {
|
|
41
|
+
try {
|
|
42
|
+
return await embedder.health();
|
|
43
|
+
} catch (e) {
|
|
44
|
+
return { ready: false, error: errorText(e) };
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function checkModel(health, embedder) {
|
|
49
|
+
const model = health.model ?? embedder.modelId;
|
|
50
|
+
const where = health.modelDir ?? health.bridge ?? embedder.kind;
|
|
51
|
+
return {
|
|
52
|
+
name: '模型可用性',
|
|
53
|
+
status: health.ready ? 'ok' : 'fail',
|
|
54
|
+
detail: health.ready ? `${model} @ ${where}` : `不可用:${health.error || 'unknown'}`,
|
|
55
|
+
health,
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** ④ 维度三方比对:实测维 vs 配置维 vs 库内 meta。 */
|
|
60
|
+
async function probeDimension(embedder) {
|
|
61
|
+
try {
|
|
62
|
+
const vector = await embedder.encode('维度自检');
|
|
63
|
+
return { dim: vector.length, error: null };
|
|
64
|
+
} catch (e) {
|
|
65
|
+
return { dim: null, error: errorText(e) };
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function checkDimension(probe, db, config, sourceName) {
|
|
70
|
+
if (probe.error) {
|
|
71
|
+
return { name: '维度一致', status: 'fail', detail: `编码探测失败:${probe.error}` };
|
|
72
|
+
}
|
|
73
|
+
const stored = getMetaFor(db, META_KEYS.dim, sourceName);
|
|
74
|
+
const fitsConfig = Number(probe.dim) === Number(config.dim);
|
|
75
|
+
const fitsStore = stored == null || Number(stored) === Number(probe.dim);
|
|
76
|
+
const status = fitsConfig && fitsStore ? 'ok' : 'fail';
|
|
77
|
+
const hint = status === 'ok' ? '' : ' ⇒ 需全量重建(--full)';
|
|
78
|
+
return {
|
|
79
|
+
name: '维度一致',
|
|
80
|
+
status,
|
|
81
|
+
detail: `实测 ${probe.dim} 维 / 配置 ${config.dim} / 库内 ${stored ?? '(空)'}${hint}`,
|
|
82
|
+
probeDim: probe.dim,
|
|
83
|
+
configDim: config.dim,
|
|
84
|
+
metaDim: stored,
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** ⑤ hostId:向量库明确不跨机同步,库里的 hostId 与当前机器不符 = 库不是本机产物。 */
|
|
89
|
+
function checkHostId(db, config, allowHostMismatch) {
|
|
90
|
+
const stored = getMeta(db, META_KEYS.hostId);
|
|
91
|
+
if (!stored) {
|
|
92
|
+
setMeta(db, META_KEYS.hostId, String(config.hostId));
|
|
93
|
+
return { name: 'hostId 校验', status: 'ok', detail: `首次初始化,已写入 ${config.hostId}` };
|
|
94
|
+
}
|
|
95
|
+
if (stored === String(config.hostId)) {
|
|
96
|
+
return { name: 'hostId 校验', status: 'ok', detail: `=${stored}` };
|
|
97
|
+
}
|
|
98
|
+
return {
|
|
99
|
+
name: 'hostId 校验',
|
|
100
|
+
status: allowHostMismatch ? 'warn' : 'fail',
|
|
101
|
+
detail: `库内 hostId=${stored},当前=${config.hostId} ⇒ 该库非本机产物,建议全量重建(--full)`,
|
|
102
|
+
metaHost: stored,
|
|
103
|
+
current: config.hostId,
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** ⑥ 覆盖度(信息项,不是判据):每个来源各有自己的对照物,不能用同一个分母。 */
|
|
108
|
+
function coverageTargets(source, sources, sourceName) {
|
|
109
|
+
if (sources && sources.length) return sources.map((s) => ({ name: String(s.name), source: s.source }));
|
|
110
|
+
if (source) return [{ name: String(sourceName), source }];
|
|
111
|
+
return [];
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function checkCoverage(target, db, multi) {
|
|
115
|
+
const name = multi ? `向量覆盖度(${target.name})` : '向量覆盖度';
|
|
116
|
+
try {
|
|
117
|
+
const total = target.source.count();
|
|
118
|
+
const encoded = countVectors(db, target.name);
|
|
119
|
+
const pct = total ? (encoded / total) * 100 : 0;
|
|
120
|
+
return {
|
|
121
|
+
name,
|
|
122
|
+
status: 'info',
|
|
123
|
+
detail: `${target.name} 源:向量 ${encoded} 条 / 源条数 ${total} 条(${pct.toFixed(1)}%)`,
|
|
124
|
+
sourceName: target.name,
|
|
125
|
+
vecCount: encoded,
|
|
126
|
+
memCount: total,
|
|
127
|
+
};
|
|
128
|
+
} catch (e) {
|
|
129
|
+
return { name, status: 'warn', detail: `统计失败:${errorText(e)}` };
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// ------------------------------------------------------------------ 入口
|
|
134
|
+
|
|
8
135
|
export async function runSelfcheck({
|
|
9
136
|
store,
|
|
10
137
|
source = null,
|
|
@@ -16,114 +143,23 @@ export async function runSelfcheck({
|
|
|
16
143
|
}) {
|
|
17
144
|
const db = store.db;
|
|
18
145
|
const checks = [];
|
|
19
|
-
const
|
|
20
|
-
checks.push(
|
|
21
|
-
return status;
|
|
146
|
+
const record = (entry) => {
|
|
147
|
+
checks.push(entry);
|
|
148
|
+
return entry.status;
|
|
22
149
|
};
|
|
23
150
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
if (t.vectors && t.meta) {
|
|
27
|
-
add(
|
|
28
|
-
'表存在性',
|
|
29
|
-
'ok',
|
|
30
|
-
store.schema.created.length
|
|
31
|
-
? `缺表已重建:${store.schema.created.join(', ')}`
|
|
32
|
-
: 'vectors / meta 均在位',
|
|
33
|
-
{ created: store.schema.created },
|
|
34
|
-
);
|
|
35
|
-
} else {
|
|
36
|
-
add('表存在性', 'fail', 'vectors 或 meta 表缺失且重建失败');
|
|
37
|
-
}
|
|
151
|
+
record(checkTables(store));
|
|
152
|
+
record(checkSchemaVersion(db));
|
|
38
153
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
add(
|
|
42
|
-
'schemaVersion',
|
|
43
|
-
sv === SCHEMA_VERSION ? 'ok' : 'warn',
|
|
44
|
-
sv === SCHEMA_VERSION ? `=${sv}` : `库内为 ${sv ?? '(空)'},代码期望 ${SCHEMA_VERSION}`,
|
|
45
|
-
{ schemaVersion: sv },
|
|
46
|
-
);
|
|
47
|
-
|
|
48
|
-
// ③ 模型可用性
|
|
49
|
-
let health = { ready: false, error: 'not probed' };
|
|
50
|
-
try {
|
|
51
|
-
health = await embedder.health();
|
|
52
|
-
} catch (e) {
|
|
53
|
-
health = { ready: false, error: String(e && e.message ? e.message : e) };
|
|
54
|
-
}
|
|
55
|
-
add(
|
|
56
|
-
'模型可用性',
|
|
57
|
-
health.ready ? 'ok' : 'fail',
|
|
58
|
-
health.ready
|
|
59
|
-
? `${health.model ?? embedder.modelId} @ ${health.modelDir ?? health.bridge ?? embedder.kind}`
|
|
60
|
-
: `不可用:${health.error || 'unknown'}`,
|
|
61
|
-
{ health },
|
|
62
|
-
);
|
|
63
|
-
|
|
64
|
-
// ④ 维度一致(编码探测 + 与库内 meta / 配置三方比对)
|
|
65
|
-
let probeDim = null;
|
|
66
|
-
let probeError = null;
|
|
67
|
-
try {
|
|
68
|
-
const v = await embedder.encode('维度自检');
|
|
69
|
-
probeDim = v.length;
|
|
70
|
-
} catch (e) {
|
|
71
|
-
probeError = String(e && e.message ? e.message : e);
|
|
72
|
-
}
|
|
73
|
-
const metaDim = getMetaFor(db, META_KEYS.dim, sourceName);
|
|
74
|
-
if (probeError) {
|
|
75
|
-
add('维度一致', 'fail', `编码探测失败:${probeError}`);
|
|
76
|
-
} else {
|
|
77
|
-
const configOk = Number(probeDim) === Number(config.dim);
|
|
78
|
-
const metaOk = metaDim == null || Number(metaDim) === Number(probeDim);
|
|
79
|
-
const status = configOk && metaOk ? 'ok' : 'fail';
|
|
80
|
-
add(
|
|
81
|
-
'维度一致',
|
|
82
|
-
status,
|
|
83
|
-
`实测 ${probeDim} 维 / 配置 ${config.dim} / 库内 ${metaDim ?? '(空)'}` +
|
|
84
|
-
(status === 'ok' ? '' : ' ⇒ 需全量重建(--full)'),
|
|
85
|
-
{ probeDim, configDim: config.dim, metaDim },
|
|
86
|
-
);
|
|
87
|
-
}
|
|
154
|
+
const health = await probeModel(embedder);
|
|
155
|
+
record(checkModel(health, embedder));
|
|
88
156
|
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
if (!metaHost) {
|
|
92
|
-
setMeta(db, META_KEYS.hostId, String(config.hostId));
|
|
93
|
-
add('hostId 校验', 'ok', `首次初始化,已写入 ${config.hostId}`);
|
|
94
|
-
} else if (metaHost === String(config.hostId)) {
|
|
95
|
-
add('hostId 校验', 'ok', `=${metaHost}`);
|
|
96
|
-
} else {
|
|
97
|
-
add(
|
|
98
|
-
'hostId 校验',
|
|
99
|
-
allowHostMismatch ? 'warn' : 'fail',
|
|
100
|
-
`库内 hostId=${metaHost},当前=${config.hostId} ⇒ 该库非本机产物,建议全量重建(--full)`,
|
|
101
|
-
{ metaHost, current: config.hostId },
|
|
102
|
-
);
|
|
103
|
-
}
|
|
157
|
+
record(checkDimension(await probeDimension(embedder), db, config, sourceName));
|
|
158
|
+
record(checkHostId(db, config, allowHostMismatch));
|
|
104
159
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
const
|
|
108
|
-
? sources.map((s) => ({ name: String(s.name), source: s.source }))
|
|
109
|
-
: source
|
|
110
|
-
? [{ name: String(sourceName), source }]
|
|
111
|
-
: [];
|
|
112
|
-
for (const s of srcList) {
|
|
113
|
-
const label = srcList.length > 1 ? `向量覆盖度(${s.name})` : '向量覆盖度';
|
|
114
|
-
try {
|
|
115
|
-
const memCount = s.source.count();
|
|
116
|
-
const vecCount = countVectors(db, s.name);
|
|
117
|
-
const ratio = memCount ? (vecCount / memCount) * 100 : 0;
|
|
118
|
-
add(label, 'info', `${s.name} 源:向量 ${vecCount} 条 / 源条数 ${memCount} 条(${ratio.toFixed(1)}%)`, {
|
|
119
|
-
sourceName: s.name,
|
|
120
|
-
vecCount,
|
|
121
|
-
memCount,
|
|
122
|
-
});
|
|
123
|
-
} catch (e) {
|
|
124
|
-
add(label, 'warn', `统计失败:${String(e && e.message ? e.message : e)}`);
|
|
125
|
-
}
|
|
126
|
-
}
|
|
160
|
+
const targets = coverageTargets(source, sources, sourceName);
|
|
161
|
+
const multi = targets.length > 1;
|
|
162
|
+
for (const target of targets) record(checkCoverage(target, db, multi));
|
|
127
163
|
|
|
128
164
|
const failed = checks.filter((c) => c.status === 'fail');
|
|
129
165
|
const warned = checks.filter((c) => c.status === 'warn');
|