@kira905/dsh-memory-toolkit 0.1.3 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/CHANGELOG.md +92 -0
  2. package/README.en.md +225 -2
  3. package/README.md +180 -2
  4. package/lib/deploy-guard.mjs +463 -275
  5. package/lib/guard-http.mjs +59 -62
  6. package/lib/pitfall-form-map.mjs +94 -84
  7. package/lib/pitfall-score.mjs +262 -185
  8. package/lib/session-file-name.mjs +116 -100
  9. package/lib/zstd-frames.mjs +88 -67
  10. package/memory-db-tools/README.md +29 -1
  11. package/memory-db-tools/audit-analyze.mjs +1 -1
  12. package/memory-db-tools/audit-export.mjs +60 -16
  13. package/memory-db-tools/audit-struct-probe.mjs +29 -8
  14. package/memory-db-tools/lib/known-devices.mjs +137 -0
  15. package/memory-engine/README.md +58 -4
  16. package/memory-engine/bench/bench.mjs +1 -1
  17. package/memory-engine/cli/band-sentinel.mjs +164 -0
  18. package/memory-engine/cli/distill-rollback.mjs +112 -44
  19. package/memory-engine/cli/distill-sessions.mjs +237 -0
  20. package/memory-engine/cli/distill.mjs +196 -95
  21. package/memory-engine/cli/doc-index-rebuild.mjs +142 -108
  22. package/memory-engine/cli/doc-index.mjs +47 -38
  23. package/memory-engine/cli/domain-tags.mjs +246 -162
  24. package/memory-engine/cli/layer-migrate.mjs +176 -0
  25. package/memory-engine/cli/layer-move.mjs +189 -0
  26. package/memory-engine/cli/pitfall-hook.mjs +136 -70
  27. package/memory-engine/cli/recall.mjs +243 -110
  28. package/memory-engine/cli/selfcheck.mjs +93 -44
  29. package/memory-engine/cli/serve-start.mjs +221 -140
  30. package/memory-engine/cli/sweep.mjs +83 -44
  31. package/memory-engine/cli/sync.mjs +93 -57
  32. package/memory-engine/config/audit-devices.json +6 -0
  33. package/memory-engine/config/bands.json +7 -0
  34. package/memory-engine/config/layers.json +7 -0
  35. package/memory-engine/dict/LICENSE-/350/257/215/345/205/270.md +53 -0
  36. package/memory-engine/dict/lexicon-config.json +11 -0
  37. package/memory-engine/dict//350/257/215/345/205/270.txt +109750 -0
  38. package/memory-engine/dict//350/257/215/351/242/221.txt +20000 -0
  39. package/memory-engine/lib/band-sentinel.mjs +417 -0
  40. package/memory-engine/lib/cli-strict.mjs +128 -0
  41. package/memory-engine/lib/cli-util.mjs +147 -125
  42. package/memory-engine/lib/config.mjs +83 -96
  43. package/memory-engine/lib/dirty-ledger.mjs +127 -94
  44. package/memory-engine/lib/distill-rollback.mjs +173 -133
  45. package/memory-engine/lib/distill.mjs +878 -585
  46. package/memory-engine/lib/doc-index-store.mjs +208 -235
  47. package/memory-engine/lib/doc-index.mjs +351 -350
  48. package/memory-engine/lib/doc-scan.mjs +178 -156
  49. package/memory-engine/lib/doc-summarize.mjs +220 -205
  50. package/memory-engine/lib/domain-tags.mjs +358 -277
  51. package/memory-engine/lib/embedder.mjs +175 -194
  52. package/memory-engine/lib/hashes.mjs +22 -7
  53. package/memory-engine/lib/layer-migrate.mjs +330 -0
  54. package/memory-engine/lib/layer-move.mjs +307 -0
  55. package/memory-engine/lib/layers.mjs +191 -0
  56. package/memory-engine/lib/lexicon-recall.mjs +473 -0
  57. package/memory-engine/lib/memory-source.mjs +78 -80
  58. package/memory-engine/lib/pitfall-source.mjs +115 -114
  59. package/memory-engine/lib/policy-source.mjs +279 -261
  60. package/memory-engine/lib/recall-log.mjs +285 -0
  61. package/memory-engine/lib/recall.mjs +565 -240
  62. package/memory-engine/lib/selfcheck.mjs +142 -106
  63. package/memory-engine/lib/session-distill.mjs +603 -0
  64. package/memory-engine/lib/session-source.mjs +421 -0
  65. package/memory-engine/lib/sweep.mjs +107 -130
  66. package/memory-engine/lib/sync.mjs +110 -140
  67. package/memory-engine/lib/tick-hook.mjs +123 -148
  68. package/memory-engine/lib/vectors-store.mjs +213 -188
  69. package/memory-engine/test/12-layering.test.mjs +539 -0
  70. package/memory-engine/test/13-session-distill.test.mjs +616 -0
  71. package/memory-engine/tools/bridge-smoke.mjs +94 -55
  72. package/memory-engine/tools/fetch-model.mjs +167 -87
  73. package/memory-engine/tools/smoke-recall.mjs +94 -38
  74. package/memory-engine/tools/verify-recall-chain.mjs +413 -0
  75. package/package.json +1 -1
@@ -1,10 +1,137 @@
1
- // 记忆引擎 v2 · 离线层 —— 启动自检(方案 §8.3 交付物 #9)
1
+ // 启动自检:把「引擎现在到底能不能用」拆成几条独立判据,逐条给结论。
2
2
  //
3
- // 四判据:模型可用性 / 维度一致 / 表存在性(缺则重建)/ hostId 校验。
4
- // 自检失败绝不阻断启动:返回结构化结果,由调用方决定是否降级。
3
+ // 设计取舍:自检**永不阻断启动**。任何一条判死,也只是往清单里写一条 fail,
4
+ // 由调用方自己决定是降级跑还是重建 —— 因为自检本身挂掉比带病启动更糟。
5
5
  import { SCHEMA_VERSION } from './config.mjs';
6
6
  import { META_KEYS, getMeta, getMetaFor, setMeta, countVectors } from './vectors-store.mjs';
7
7
 
8
+ const errorText = (e) => String(e && e.message ? e.message : e);
9
+
10
+ // ------------------------------------------------------------------ 各判据
11
+
12
+ /** ① 表存在性:缺表时 open 阶段应已重建,这里只核对重建结果。 */
13
+ function checkTables(store) {
14
+ const have = store.schema.tables;
15
+ if (!have.vectors || !have.meta) {
16
+ return { name: '表存在性', status: 'fail', detail: 'vectors 或 meta 表缺失且重建失败' };
17
+ }
18
+ const created = store.schema.created;
19
+ return {
20
+ name: '表存在性',
21
+ status: 'ok',
22
+ detail: created.length ? `缺表已重建:${created.join(', ')}` : 'vectors / meta 均在位',
23
+ created,
24
+ };
25
+ }
26
+
27
+ /** ② 库内 schemaVersion 与代码期望值是否一致(不一致只 warn,不做迁移)。 */
28
+ function checkSchemaVersion(db) {
29
+ const found = getMeta(db, META_KEYS.schemaVersion);
30
+ const same = found === SCHEMA_VERSION;
31
+ return {
32
+ name: 'schemaVersion',
33
+ status: same ? 'ok' : 'warn',
34
+ detail: same ? `=${found}` : `库内为 ${found ?? '(空)'},代码期望 ${SCHEMA_VERSION}`,
35
+ schemaVersion: found,
36
+ };
37
+ }
38
+
39
+ /** ③ 模型可用性:探针本身可能抛,抛了也算「不可用」而不是让自检崩。 */
40
+ async function probeModel(embedder) {
41
+ try {
42
+ return await embedder.health();
43
+ } catch (e) {
44
+ return { ready: false, error: errorText(e) };
45
+ }
46
+ }
47
+
48
+ function checkModel(health, embedder) {
49
+ const model = health.model ?? embedder.modelId;
50
+ const where = health.modelDir ?? health.bridge ?? embedder.kind;
51
+ return {
52
+ name: '模型可用性',
53
+ status: health.ready ? 'ok' : 'fail',
54
+ detail: health.ready ? `${model} @ ${where}` : `不可用:${health.error || 'unknown'}`,
55
+ health,
56
+ };
57
+ }
58
+
59
+ /** ④ 维度三方比对:实测维 vs 配置维 vs 库内 meta。 */
60
+ async function probeDimension(embedder) {
61
+ try {
62
+ const vector = await embedder.encode('维度自检');
63
+ return { dim: vector.length, error: null };
64
+ } catch (e) {
65
+ return { dim: null, error: errorText(e) };
66
+ }
67
+ }
68
+
69
+ function checkDimension(probe, db, config, sourceName) {
70
+ if (probe.error) {
71
+ return { name: '维度一致', status: 'fail', detail: `编码探测失败:${probe.error}` };
72
+ }
73
+ const stored = getMetaFor(db, META_KEYS.dim, sourceName);
74
+ const fitsConfig = Number(probe.dim) === Number(config.dim);
75
+ const fitsStore = stored == null || Number(stored) === Number(probe.dim);
76
+ const status = fitsConfig && fitsStore ? 'ok' : 'fail';
77
+ const hint = status === 'ok' ? '' : ' ⇒ 需全量重建(--full)';
78
+ return {
79
+ name: '维度一致',
80
+ status,
81
+ detail: `实测 ${probe.dim} 维 / 配置 ${config.dim} / 库内 ${stored ?? '(空)'}${hint}`,
82
+ probeDim: probe.dim,
83
+ configDim: config.dim,
84
+ metaDim: stored,
85
+ };
86
+ }
87
+
88
+ /** ⑤ hostId:向量库明确不跨机同步,库里的 hostId 与当前机器不符 = 库不是本机产物。 */
89
+ function checkHostId(db, config, allowHostMismatch) {
90
+ const stored = getMeta(db, META_KEYS.hostId);
91
+ if (!stored) {
92
+ setMeta(db, META_KEYS.hostId, String(config.hostId));
93
+ return { name: 'hostId 校验', status: 'ok', detail: `首次初始化,已写入 ${config.hostId}` };
94
+ }
95
+ if (stored === String(config.hostId)) {
96
+ return { name: 'hostId 校验', status: 'ok', detail: `=${stored}` };
97
+ }
98
+ return {
99
+ name: 'hostId 校验',
100
+ status: allowHostMismatch ? 'warn' : 'fail',
101
+ detail: `库内 hostId=${stored},当前=${config.hostId} ⇒ 该库非本机产物,建议全量重建(--full)`,
102
+ metaHost: stored,
103
+ current: config.hostId,
104
+ };
105
+ }
106
+
107
+ /** ⑥ 覆盖度(信息项,不是判据):每个来源各有自己的对照物,不能用同一个分母。 */
108
+ function coverageTargets(source, sources, sourceName) {
109
+ if (sources && sources.length) return sources.map((s) => ({ name: String(s.name), source: s.source }));
110
+ if (source) return [{ name: String(sourceName), source }];
111
+ return [];
112
+ }
113
+
114
+ function checkCoverage(target, db, multi) {
115
+ const name = multi ? `向量覆盖度(${target.name})` : '向量覆盖度';
116
+ try {
117
+ const total = target.source.count();
118
+ const encoded = countVectors(db, target.name);
119
+ const pct = total ? (encoded / total) * 100 : 0;
120
+ return {
121
+ name,
122
+ status: 'info',
123
+ detail: `${target.name} 源:向量 ${encoded} 条 / 源条数 ${total} 条(${pct.toFixed(1)}%)`,
124
+ sourceName: target.name,
125
+ vecCount: encoded,
126
+ memCount: total,
127
+ };
128
+ } catch (e) {
129
+ return { name, status: 'warn', detail: `统计失败:${errorText(e)}` };
130
+ }
131
+ }
132
+
133
+ // ------------------------------------------------------------------ 入口
134
+
8
135
  export async function runSelfcheck({
9
136
  store,
10
137
  source = null,
@@ -16,114 +143,23 @@ export async function runSelfcheck({
16
143
  }) {
17
144
  const db = store.db;
18
145
  const checks = [];
19
- const add = (name, status, detail, extra = {}) => {
20
- checks.push({ name, status, detail, ...extra });
21
- return status;
146
+ const record = (entry) => {
147
+ checks.push(entry);
148
+ return entry.status;
22
149
  };
23
150
 
24
- // ① 表存在性(缺则重建)
25
- const t = store.schema.tables;
26
- if (t.vectors && t.meta) {
27
- add(
28
- '表存在性',
29
- 'ok',
30
- store.schema.created.length
31
- ? `缺表已重建:${store.schema.created.join(', ')}`
32
- : 'vectors / meta 均在位',
33
- { created: store.schema.created },
34
- );
35
- } else {
36
- add('表存在性', 'fail', 'vectors 或 meta 表缺失且重建失败');
37
- }
151
+ record(checkTables(store));
152
+ record(checkSchemaVersion(db));
38
153
 
39
- // ② schemaVersion
40
- const sv = getMeta(db, META_KEYS.schemaVersion);
41
- add(
42
- 'schemaVersion',
43
- sv === SCHEMA_VERSION ? 'ok' : 'warn',
44
- sv === SCHEMA_VERSION ? `=${sv}` : `库内为 ${sv ?? '(空)'},代码期望 ${SCHEMA_VERSION}`,
45
- { schemaVersion: sv },
46
- );
47
-
48
- // ③ 模型可用性
49
- let health = { ready: false, error: 'not probed' };
50
- try {
51
- health = await embedder.health();
52
- } catch (e) {
53
- health = { ready: false, error: String(e && e.message ? e.message : e) };
54
- }
55
- add(
56
- '模型可用性',
57
- health.ready ? 'ok' : 'fail',
58
- health.ready
59
- ? `${health.model ?? embedder.modelId} @ ${health.modelDir ?? health.bridge ?? embedder.kind}`
60
- : `不可用:${health.error || 'unknown'}`,
61
- { health },
62
- );
63
-
64
- // ④ 维度一致(编码探测 + 与库内 meta / 配置三方比对)
65
- let probeDim = null;
66
- let probeError = null;
67
- try {
68
- const v = await embedder.encode('维度自检');
69
- probeDim = v.length;
70
- } catch (e) {
71
- probeError = String(e && e.message ? e.message : e);
72
- }
73
- const metaDim = getMetaFor(db, META_KEYS.dim, sourceName);
74
- if (probeError) {
75
- add('维度一致', 'fail', `编码探测失败:${probeError}`);
76
- } else {
77
- const configOk = Number(probeDim) === Number(config.dim);
78
- const metaOk = metaDim == null || Number(metaDim) === Number(probeDim);
79
- const status = configOk && metaOk ? 'ok' : 'fail';
80
- add(
81
- '维度一致',
82
- status,
83
- `实测 ${probeDim} 维 / 配置 ${config.dim} / 库内 ${metaDim ?? '(空)'}` +
84
- (status === 'ok' ? '' : ' ⇒ 需全量重建(--full)'),
85
- { probeDim, configDim: config.dim, metaDim },
86
- );
87
- }
154
+ const health = await probeModel(embedder);
155
+ record(checkModel(health, embedder));
88
156
 
89
- // ⑤ hostId 校验(向量库明确不同步;hostId 不符说明库来自别的机器)
90
- const metaHost = getMeta(db, META_KEYS.hostId);
91
- if (!metaHost) {
92
- setMeta(db, META_KEYS.hostId, String(config.hostId));
93
- add('hostId 校验', 'ok', `首次初始化,已写入 ${config.hostId}`);
94
- } else if (metaHost === String(config.hostId)) {
95
- add('hostId 校验', 'ok', `=${metaHost}`);
96
- } else {
97
- add(
98
- 'hostId 校验',
99
- allowHostMismatch ? 'warn' : 'fail',
100
- `库内 hostId=${metaHost},当前=${config.hostId} ⇒ 该库非本机产物,建议全量重建(--full)`,
101
- { metaHost, current: config.hostId },
102
- );
103
- }
157
+ record(checkDimension(await probeDimension(embedder), db, config, sourceName));
158
+ record(checkHostId(db, config, allowHostMismatch));
104
159
 
105
- // ⑥ 覆盖度(信息项;非判据)——【2026-10-10 · 补坑库联动A】改成**逐 source** 报告
106
- // ⚠️ 判据不是"向量数 vs 记忆数"这一种:坑源的对照物是**坑库条数**(`butler-pitfalls.json` 的条目数)。
107
- const srcList = sources && sources.length
108
- ? sources.map((s) => ({ name: String(s.name), source: s.source }))
109
- : source
110
- ? [{ name: String(sourceName), source }]
111
- : [];
112
- for (const s of srcList) {
113
- const label = srcList.length > 1 ? `向量覆盖度(${s.name})` : '向量覆盖度';
114
- try {
115
- const memCount = s.source.count();
116
- const vecCount = countVectors(db, s.name);
117
- const ratio = memCount ? (vecCount / memCount) * 100 : 0;
118
- add(label, 'info', `${s.name} 源:向量 ${vecCount} 条 / 源条数 ${memCount} 条(${ratio.toFixed(1)}%)`, {
119
- sourceName: s.name,
120
- vecCount,
121
- memCount,
122
- });
123
- } catch (e) {
124
- add(label, 'warn', `统计失败:${String(e && e.message ? e.message : e)}`);
125
- }
126
- }
160
+ const targets = coverageTargets(source, sources, sourceName);
161
+ const multi = targets.length > 1;
162
+ for (const target of targets) record(checkCoverage(target, db, multi));
127
163
 
128
164
  const failed = checks.filter((c) => c.status === 'fail');
129
165
  const warned = checks.filter((c) => c.status === 'warn');