@kira905/dsh-memory-toolkit 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/CHANGELOG.md +130 -1
  2. package/README.en.md +225 -2
  3. package/README.md +208 -2
  4. package/cordis.patch.yml +32 -0
  5. package/lib/deploy-guard.mjs +463 -275
  6. package/lib/guard-http.mjs +59 -62
  7. package/lib/pitfall-form-map.mjs +94 -84
  8. package/lib/pitfall-score.mjs +262 -185
  9. package/lib/session-file-name.mjs +116 -100
  10. package/lib/zstd-frames.mjs +88 -67
  11. package/memory-db-tools/README.md +29 -1
  12. package/memory-db-tools/audit-analyze.mjs +1 -1
  13. package/memory-db-tools/audit-export.mjs +60 -16
  14. package/memory-db-tools/audit-struct-probe.mjs +29 -8
  15. package/memory-db-tools/lib/known-devices.mjs +137 -0
  16. package/memory-engine/README.md +58 -4
  17. package/memory-engine/bench/bench.mjs +1 -1
  18. package/memory-engine/cli/band-sentinel.mjs +164 -0
  19. package/memory-engine/cli/distill-rollback.mjs +112 -44
  20. package/memory-engine/cli/distill-sessions.mjs +237 -0
  21. package/memory-engine/cli/distill.mjs +196 -95
  22. package/memory-engine/cli/doc-index-rebuild.mjs +142 -108
  23. package/memory-engine/cli/doc-index.mjs +47 -38
  24. package/memory-engine/cli/domain-tags.mjs +246 -162
  25. package/memory-engine/cli/layer-migrate.mjs +176 -0
  26. package/memory-engine/cli/layer-move.mjs +189 -0
  27. package/memory-engine/cli/pitfall-hook.mjs +136 -70
  28. package/memory-engine/cli/recall.mjs +243 -110
  29. package/memory-engine/cli/selfcheck.mjs +93 -44
  30. package/memory-engine/cli/serve-start.mjs +221 -140
  31. package/memory-engine/cli/sweep.mjs +83 -44
  32. package/memory-engine/cli/sync.mjs +93 -57
  33. package/memory-engine/config/audit-devices.json +6 -0
  34. package/memory-engine/config/bands.json +7 -0
  35. package/memory-engine/config/layers.json +7 -0
  36. package/memory-engine/dict/LICENSE-/350/257/215/345/205/270.md +53 -0
  37. package/memory-engine/dict/lexicon-config.json +11 -0
  38. package/memory-engine/dict//350/257/215/345/205/270.txt +109750 -0
  39. package/memory-engine/dict//350/257/215/351/242/221.txt +20000 -0
  40. package/memory-engine/lib/band-sentinel.mjs +417 -0
  41. package/memory-engine/lib/cli-strict.mjs +128 -0
  42. package/memory-engine/lib/cli-util.mjs +147 -125
  43. package/memory-engine/lib/config.mjs +83 -96
  44. package/memory-engine/lib/dirty-ledger.mjs +127 -94
  45. package/memory-engine/lib/distill-rollback.mjs +173 -133
  46. package/memory-engine/lib/distill.mjs +878 -585
  47. package/memory-engine/lib/doc-index-store.mjs +208 -235
  48. package/memory-engine/lib/doc-index.mjs +351 -350
  49. package/memory-engine/lib/doc-scan.mjs +178 -156
  50. package/memory-engine/lib/doc-summarize.mjs +220 -205
  51. package/memory-engine/lib/domain-tags.mjs +358 -277
  52. package/memory-engine/lib/embedder.mjs +175 -194
  53. package/memory-engine/lib/hashes.mjs +22 -7
  54. package/memory-engine/lib/layer-migrate.mjs +330 -0
  55. package/memory-engine/lib/layer-move.mjs +307 -0
  56. package/memory-engine/lib/layers.mjs +191 -0
  57. package/memory-engine/lib/lexicon-recall.mjs +473 -0
  58. package/memory-engine/lib/memory-source.mjs +78 -80
  59. package/memory-engine/lib/pitfall-source.mjs +115 -114
  60. package/memory-engine/lib/policy-source.mjs +279 -261
  61. package/memory-engine/lib/recall-log.mjs +285 -0
  62. package/memory-engine/lib/recall.mjs +565 -240
  63. package/memory-engine/lib/selfcheck.mjs +142 -106
  64. package/memory-engine/lib/session-distill.mjs +603 -0
  65. package/memory-engine/lib/session-source.mjs +421 -0
  66. package/memory-engine/lib/sweep.mjs +107 -130
  67. package/memory-engine/lib/sync.mjs +110 -140
  68. package/memory-engine/lib/tick-hook.mjs +123 -148
  69. package/memory-engine/lib/vectors-store.mjs +213 -188
  70. package/memory-engine/test/12-layering.test.mjs +539 -0
  71. package/memory-engine/test/13-session-distill.test.mjs +616 -0
  72. package/memory-engine/tools/bridge-smoke.mjs +94 -55
  73. package/memory-engine/tools/fetch-model.mjs +167 -87
  74. package/memory-engine/tools/smoke-recall.mjs +94 -38
  75. package/memory-engine/tools/verify-recall-chain.mjs +413 -0
  76. package/package.json +11 -1
@@ -1,78 +1,117 @@
1
- // 记忆引擎 v2 · 离线层 —— 编码桥实测(方案 §3.1 编码通道契约)
1
+ // 记忆引擎 · 离线层 —— 编码桥联调(编码通道契约)
2
2
  //
3
- // 同进程内起桥 → 打 /health 与 /encode → 校验端口顺延 → 关桥。
4
- // 为什么不走 ssh 后台进程:SSH 会话结束会带走 detached 进程(跨机技能坑 3),
5
- // 起与测必须同一条命令;本脚本把「起 + 测 + 关」放进同一个 node 进程,最省事也最可靠。
3
+ // 一个进程管全流程:起桥 → 轮询 /health 直到模型就绪 → 打两个 /encode →
4
+ // 确认第二个实例顺延到下一个端口 → 确认非回环绑定被拒 → 关桥。
5
+ // 为什么不起后台进程:SSH 会话一结束会把 detached 进程一起带走(跨机踩过的坑),
6
+ // 「起 + 测 + 关」必须落在同一条命令里 —— 本脚本就是那条命令。
6
7
  import { startBridge } from '../serve.mjs';
7
8
  import { parseArgs, isMainEntry } from '../lib/cli-util.mjs';
8
9
 
9
- async function waitReady(port, timeoutMs = 60000) {
10
- const t0 = Date.now();
10
+ const FALLBACK_PORT = 8201;
11
+ const POLL_MS = 400;
12
+ const LOAD_BUDGET_MS = 60000;
13
+ const LOOPBACK = '127.0.0.1';
14
+
15
+ const delay = (ms) => new Promise((done) => setTimeout(done, ms));
16
+
17
+ /** 按要打印的键投影一份快照,保持声明顺序。 */
18
+ const pick = (snapshot, keys) => Object.fromEntries(keys.map((key) => [key, snapshot[key]]));
19
+
20
+ const meanSquare = (vector) => vector.reduce((sum, value) => sum + value * value, 0);
21
+
22
+ async function callJson(port, route, init) {
23
+ const response = await fetch(`http://${LOOPBACK}:${port}${route}`, init);
24
+ return response.json();
25
+ }
26
+
27
+ const healthOf = (port) => callJson(port, '/health');
28
+
29
+ /** 正常请求的 /encode;只有传输层或 JSON 坏掉才会抛。 */
30
+ const encodeTexts = (port, texts) =>
31
+ callJson(port, '/encode', {
32
+ method: 'POST',
33
+ headers: { 'content-type': 'application/json' },
34
+ body: JSON.stringify({ texts }),
35
+ });
36
+
37
+ /** 故意打坏请求的 /encode;状态码先取下来再解析正文。 */
38
+ async function encodeBroken(port, payload) {
39
+ const response = await fetch(`http://${LOOPBACK}:${port}/encode`, {
40
+ method: 'POST',
41
+ headers: { 'content-type': 'application/json' },
42
+ body: JSON.stringify(payload),
43
+ });
44
+ return { status: response.status, body: await response.json() };
45
+ }
46
+
47
+ /**
48
+ * 轮询 /health 直到模型就绪。
49
+ * 桥报了加载错误也只等到预算耗尽才抛 —— 与桥的懒加载行为对齐,期间的加载失败
50
+ * 不算终局(模型可能重试后起来)。
51
+ */
52
+ async function awaitReady(port, budget = LOAD_BUDGET_MS) {
53
+ const startedAt = Date.now();
54
+ const overdue = () => Date.now() - startedAt > budget;
11
55
  for (;;) {
12
56
  try {
13
- const h = await fetch(`http://127.0.0.1:${port}/health`).then((r) => r.json());
14
- if (h.ready) return h;
15
- if (h.lastError) throw new Error(`模型加载失败:${h.lastError}`);
16
- } catch (e) {
17
- if (Date.now() - t0 > timeoutMs) throw e;
57
+ const snapshot = await healthOf(port);
58
+ if (snapshot.ready) return snapshot;
59
+ if (snapshot.lastError) throw new Error(`模型加载失败:${snapshot.lastError}`);
60
+ } catch (err) {
61
+ if (overdue()) throw err;
18
62
  }
19
- if (Date.now() - t0 > timeoutMs) throw new Error('等待模型就绪超时');
20
- await new Promise((r) => setTimeout(r, 400));
63
+ if (overdue()) throw new Error('等待模型就绪超时');
64
+ await delay(POLL_MS);
65
+ }
66
+ }
67
+
68
+ function reportPortShift(first, second) {
69
+ const verdict = second.port === first.port + 1 ? ' ✅' : ' ❌';
70
+ console.log(`端口顺延检查:首个=${first.port},第二个=${second.port}${verdict}`);
71
+ }
72
+
73
+ /** 除了回环,桥不许绑任何地址。 */
74
+ async function probeLoopbackGuard(modelDir) {
75
+ try {
76
+ await startBridge({ flags: { 'model-dir': modelDir, bind: '0.0.0.0' }, log: () => {} });
77
+ return '未拦截 ❌';
78
+ } catch (err) {
79
+ return `已拦截 ✅(${String(err.message).slice(0, 40)}…)`;
21
80
  }
22
81
  }
23
82
 
24
83
  async function main() {
25
84
  const { flags } = parseArgs();
26
85
  const modelDir = flags['model-dir'];
27
- const wantPort = Number(flags.port || 8201);
28
- const logs = [];
29
- const log = (m) => {
30
- logs.push(m);
31
- console.log(m);
32
- };
86
+ const wantPort = Number(flags.port || FALLBACK_PORT);
87
+ const log = (message) => console.log(message);
33
88
 
34
89
  console.log('=== 编码桥实测(仅回环) ===');
35
- const b1 = await startBridge({ flags: { 'model-dir': modelDir, port: String(wantPort) }, log });
36
- const b2 = await startBridge({ flags: { 'model-dir': modelDir, port: String(wantPort), warm: false }, log });
37
- console.log(`端口顺延检查:首个=${b1.port},第二个=${b2.port}${b2.port === b1.port + 1 ? ' ✅' : ' ❌'}`);
90
+ const primary = await startBridge({ flags: { 'model-dir': modelDir, port: String(wantPort) }, log });
91
+ const secondary = await startBridge({ flags: { 'model-dir': modelDir, port: String(wantPort), warm: false }, log });
92
+ reportPortShift(primary, secondary);
38
93
 
39
- const health0 = await fetch(`http://127.0.0.1:${b1.port}/health`).then((r) => r.json());
40
- console.log(`/health(加载中):${JSON.stringify({ ready: health0.ready, modelLoading: health0.modelLoading, model: health0.model, dim: health0.dim })}`);
94
+ const whileLoading = await healthOf(primary.port);
95
+ console.log(`/health(加载中):${JSON.stringify(pick(whileLoading, ['ready', 'modelLoading', 'model', 'dim']))}`);
41
96
 
42
- const ready = await waitReady(b1.port);
43
- console.log(`/health(就绪):${JSON.stringify({ ready: ready.ready, model: ready.model, dim: ready.dim, port: ready.port })}`);
97
+ const ready = await awaitReady(primary.port);
98
+ console.log(`/health(就绪):${JSON.stringify(pick(ready, ['ready', 'model', 'dim', 'port']))}`);
44
99
 
45
- const t0 = performance.now();
46
- const enc = await fetch(`http://127.0.0.1:${b1.port}/encode`, {
47
- method: 'POST',
48
- headers: { 'content-type': 'application/json' },
49
- body: JSON.stringify({ texts: ['跨机同步的挡推裁决', '整夜干活'] }),
50
- }).then((r) => r.json());
51
- const ms = performance.now() - t0;
52
- console.log(`/encode 2 条:${ms.toFixed(1)}ms · model=${enc.model} dim=${enc.dim} · 首向量前 5 维 ${enc.vectors[0].slice(0, 5).map((x) => x.toFixed(4)).join(', ')}`);
53
- const norm = Math.sqrt(enc.vectors[0].reduce((s, x) => s + x * x, 0));
54
- console.log(`首向量 L2 范数:${norm.toFixed(6)}(契约要求归一化 ⇒ 应 ≈1)`);
55
-
56
- const bad = await fetch(`http://127.0.0.1:${b1.port}/encode`, {
57
- method: 'POST',
58
- headers: { 'content-type': 'application/json' },
59
- body: JSON.stringify({ nope: 1 }),
60
- }).then((r) => ({ status: r.status, body: r.json() }));
61
- const badRes = await bad.body;
62
- console.log(`坏请求处理:HTTP ${bad.status} ${JSON.stringify(badRes)}`);
100
+ const startedAt = performance.now();
101
+ const encoded = await encodeTexts(primary.port, ['跨机同步的挡推裁决', '整夜干活']);
102
+ const elapsed = performance.now() - startedAt;
103
+ const firstVector = encoded.vectors[0];
104
+ const headDims = firstVector.slice(0, 5).map((value) => value.toFixed(4)).join(', ');
105
+ console.log(`/encode 2 条:${elapsed.toFixed(1)}ms · model=${encoded.model} dim=${encoded.dim} · 首向量前 5 维 ${headDims}`);
106
+ console.log(`首向量 L2 范数:${Math.sqrt(meanSquare(firstVector)).toFixed(6)}(契约要求归一化 ⇒ 应 ≈1)`);
63
107
 
64
- const nonLoop = await (async () => {
65
- try {
66
- await startBridge({ flags: { 'model-dir': modelDir, bind: '0.0.0.0' }, log: () => {} });
67
- return '未拦截 ❌';
68
- } catch (e) {
69
- return `已拦截 ✅(${String(e.message).slice(0, 40)}…)`;
70
- }
71
- })();
72
- console.log(`非回环绑定拦截:${nonLoop}`);
108
+ const broken = await encodeBroken(primary.port, { nope: 1 });
109
+ console.log(`坏请求处理:HTTP ${broken.status} ${JSON.stringify(broken.body)}`);
110
+
111
+ console.log(`非回环绑定拦截:${await probeLoopbackGuard(modelDir)}`);
73
112
 
74
- await b2.close();
75
- await b1.close();
113
+ await secondary.close();
114
+ await primary.close();
76
115
  console.log('桥已关闭');
77
116
  process.exitCode = 0;
78
117
  }
@@ -1,92 +1,179 @@
1
- // 记忆引擎 v2 · 离线层 —— 模型权重获取器(node 版,不用 curl)
1
+ // 记忆引擎 · 离线层 —— 模型权重获取器(纯 Node,不依赖 curl)
2
2
  //
3
- // 为什么要有这个脚本(Windows + schannel 环境实测坑):
4
- // Windows 自带 curl.exe 走 schannel;HF 的 LFS 大文件会 302 到 Xet CAS 桥
5
- // (cas-bridge.xethub.hf.co),部分 Windows 主机上 schannel 对该域名 TLS 握手直接失败
6
- // (curl: (35) SEC_E_ILLEGAL_MESSAGE),而小文件(走 hf-mirror 直连)没问题。
7
- // node 自带 TLS 栈不受 schannel 影响 ⇒ 用本脚本取权重。
3
+ // 为什么不用 curl:Windows 自带的 curl.exe 走 schannel 做 TLS,而 Hub 上的大文件
4
+ // 会 302 到 Xet CAS 桥(cas-bridge.xethub.hf.co);部分 Windows 主机对这个域名的
5
+ // 握手直接失败(curl: (35) SEC_E_ILLEGAL_MESSAGE),反倒是直连下发的小文件没事。
6
+ // Node 自带独立 TLS 栈,能绕开 schannel 这条路。
8
7
  //
9
8
  // 用法:
10
9
  // node tools/fetch-model.mjs --out <模型目录> [--base https://hf-mirror.com]
11
- // [--repo Xenova/bge-small-zh-v1.5] [--variant fp32|int8] [--force]
12
- // 幂等:已存在且大小与远端一致 → 跳过(--force 强制重下)。
13
- import fs from 'node:fs';
14
- import path from 'node:path';
15
- import crypto from 'node:crypto';
16
- import { Readable } from 'node:stream';
17
- import { pipeline } from 'node:stream/promises';
10
+ // [--repo Xenova/bge-small-zh-v1.5] [--variant fp32|int8|fp16] [--force]
11
+ // 幂等:本地文件的字节数与服务端 content-length 相同就不动它;--force 要求重下。
12
+ import { createHash } from 'node:crypto';
13
+ import { existsSync, mkdirSync, readFileSync, renameSync, statSync } from 'node:fs';
14
+ import { open } from 'node:fs/promises';
15
+ import { basename, dirname, join, resolve, sep } from 'node:path';
18
16
  import { parseArgs, isMainEntry } from '../lib/cli-util.mjs';
19
17
 
20
- const COMMON = [
18
+ const MIB = 1024 * 1024;
19
+
20
+ /** 所有变体都要的配套文件,按这个顺序取。 */
21
+ const SHARED_FILES = [
21
22
  'config.json',
22
23
  'tokenizer.json',
23
24
  'tokenizer_config.json',
24
25
  'special_tokens_map.json',
25
26
  'vocab.txt',
26
27
  ];
27
- const VARIANTS = {
28
+
29
+ /** 变体名 → 仓内的权重文件(键顺序会出现在报错文案里,别乱动)。 */
30
+ const WEIGHTS = {
28
31
  fp32: 'onnx/model.onnx',
29
32
  int8: 'onnx/model_quantized.onnx',
30
33
  fp16: 'onnx/model_fp16.onnx',
31
34
  };
32
35
 
33
- async function fetchOne({ url, dest, force, log }) {
34
- const head = await fetch(url, { method: 'GET', redirect: 'follow' });
35
- if (!head.ok) throw new Error(`HTTP ${head.status} ${head.statusText} (${url})`);
36
- const total = Number(head.headers.get('content-length') || 0);
37
- const name = path.basename(dest);
38
- if (!force && fs.existsSync(dest) && total > 0 && fs.statSync(dest).size === total) {
39
- await head.body?.cancel?.();
40
- return { file: name, status: 'skip', size: total, sha: shortSha(dest) };
36
+ /** 命令行没给时的兜底值。 */
37
+ const BAREFALLS = {
38
+ base: 'https://hf-mirror.com',
39
+ repo: 'Xenova/bge-small-zh-v1.5',
40
+ variant: 'fp32',
41
+ };
42
+
43
+ const USAGE =
44
+ '用法: node tools/fetch-model.mjs --out <dir> [--base <url>] [--repo <owner/name>] [--variant fp32|int8|fp16] [--force]';
45
+
46
+ const pause = (ms) => new Promise((done) => setTimeout(done, ms));
47
+
48
+ /** sha256 前 32 位 hex:够人对一眼,又不至于刷出 64 位。 */
49
+ const shortDigest = (filePath) => createHash('sha256').update(readFileSync(filePath)).digest('hex').slice(0, 32);
50
+
51
+ const asMegabytes = (bytes) => (bytes / MIB).toFixed(1);
52
+
53
+ /** 每跨过 10 % 才打一行进度,免得刷屏。 */
54
+ function announce(name, received, total, cursor, say) {
55
+ if (!total) return;
56
+ const percent = Math.floor((received / total) * 100);
57
+ if (percent < cursor.last + 10) return;
58
+ cursor.last = percent;
59
+ say(` ${name}: ${percent}%(${asMegabytes(received)}/${asMegabytes(total)} MB)`);
60
+ }
61
+
62
+ /** 把一块 Uint8Array 全写进去;短写就接着写剩下的。 */
63
+ async function writeAll(handle, chunk) {
64
+ let written = 0;
65
+ while (written < chunk.byteLength) {
66
+ const { bytesWritten } = await handle.write(chunk, written, chunk.byteLength - written);
67
+ if (!bytesWritten) throw new Error('写入中断(0 字节)');
68
+ written += bytesWritten;
41
69
  }
42
- fs.mkdirSync(path.dirname(dest), { recursive: true });
43
- const part = `${dest}.part`;
44
- let got = 0;
45
- let lastPct = -1;
46
- const src = Readable.fromWeb(head.body);
47
- src.on('data', (c) => {
48
- got += c.length;
49
- if (total) {
50
- const pct = Math.floor((got / total) * 100);
51
- if (pct >= lastPct + 10) {
52
- lastPct = pct;
53
- log(` ${name}: ${pct}%(${(got / 1048576).toFixed(1)}/${(total / 1048576).toFixed(1)} MB)`);
54
- }
70
+ }
71
+
72
+ /**
73
+ * 把响应体逐块落到临时文件里,每块都 await ⇒ 套接字永远不会跑到磁盘前面。
74
+ * 落完怎么处理(改名 / 丢弃)由调用方决定。
75
+ */
76
+ async function spoolTo(partialPath, body, expected, name, say) {
77
+ if (!body) throw new Error(`${name}:响应没有正文`);
78
+ const reader = body.getReader();
79
+ const handle = await open(partialPath, 'w');
80
+ const cursor = { last: -1 };
81
+ let received = 0;
82
+ try {
83
+ for (;;) {
84
+ const { done, value } = await reader.read();
85
+ if (done) break;
86
+ if (!value) continue;
87
+ await writeAll(handle, value);
88
+ received += value.byteLength;
89
+ announce(name, received, expected, cursor, say);
55
90
  }
56
- });
57
- await pipeline(src, fs.createWriteStream(part));
58
- if (total && fs.statSync(part).size !== total) {
59
- throw new Error(`${name} 大小不符:期望 ${total},实际 ${fs.statSync(part).size}`);
91
+ } finally {
92
+ await handle.close();
60
93
  }
61
- fs.renameSync(part, dest);
62
- return { file: name, status: 'fetched', size: fs.statSync(dest).size, sha: shortSha(dest) };
63
94
  }
64
95
 
65
- function shortSha(p) {
66
- return crypto.createHash('sha256').update(fs.readFileSync(p)).digest('hex').slice(0, 32);
96
+ /**
97
+ * 取一个远端路径到 destination。
98
+ * 返回 {file,status:'skip'|'fetched',size,sha};传输层出问题就抛。
99
+ */
100
+ async function pullFile({ url, destination, overwrite, say }) {
101
+ const response = await fetch(url, { method: 'GET', redirect: 'follow' });
102
+ if (!response.ok) throw new Error(`HTTP ${response.status} ${response.statusText} (${url})`);
103
+
104
+ const expected = Number(response.headers.get('content-length') || 0);
105
+ const name = basename(destination);
106
+ const localSize = existsSync(destination) ? statSync(destination).size : -1;
107
+
108
+ if (!overwrite && expected > 0 && localSize === expected) {
109
+ await response.body?.cancel?.();
110
+ return { file: name, status: 'skip', size: expected, sha: shortDigest(destination) };
111
+ }
112
+
113
+ mkdirSync(dirname(destination), { recursive: true });
114
+ const partialPath = `${destination}.part`;
115
+ await spoolTo(partialPath, response.body, expected, name, say);
116
+
117
+ const landed = statSync(partialPath).size;
118
+ if (expected && landed !== expected) throw new Error(`${name} 大小不符:期望 ${expected},实际 ${landed}`);
119
+ renameSync(partialPath, destination);
120
+ return { file: name, status: 'fetched', size: statSync(destination).size, sha: shortDigest(destination) };
67
121
  }
68
122
 
69
- export async function fetchModel({ out, base, repo, variant, force, log, retries = 3 }) {
70
- const files = [...COMMON, VARIANTS[variant]];
71
- const results = [];
72
- for (const f of files) {
73
- const url = `${base}/${repo}/resolve/main/${f}`;
74
- let lastErr = null;
75
- for (let attempt = 1; attempt <= retries; attempt += 1) {
76
- try {
77
- log(` → ${f}${attempt > 1 ? `(第 ${attempt} 次尝试)` : ''}`);
78
- results.push(await fetchOne({ url, dest: path.join(out, f.replace(/\//g, path.sep)), force, log }));
79
- lastErr = null;
80
- break;
81
- } catch (e) {
82
- lastErr = e;
83
- log(` ⚠ ${String(e && e.message ? e.message : e)}`);
84
- await new Promise((r) => setTimeout(r, 1500 * attempt));
85
- }
123
+ /**
124
+ * 一个远端路径 + 重试预算。
125
+ * 调用方把 retries 设成 0 时返回 null(一条记录都不写,保持原语义)。
126
+ */
127
+ async function attemptFile({ url, destination, force, log, retries, name }) {
128
+ let failure = null;
129
+ for (let round = 1; round <= retries; round += 1) {
130
+ log(` → ${name}${round > 1 ? `(第 ${round} 次尝试)` : ''}`);
131
+ try {
132
+ return await pullFile({ url, destination, overwrite: force, say: log });
133
+ } catch (err) {
134
+ failure = err;
135
+ log(` ⚠ ${String(err && err.message ? err.message : err)}`);
136
+ await pause(1500 * round);
86
137
  }
87
- if (lastErr) {
88
- results.push({ file: f, status: 'FAILED', error: String(lastErr.message || lastErr) });
138
+ }
139
+ if (!failure) return null;
140
+ return { file: name, status: 'FAILED', error: String(failure.message || failure) };
141
+ }
142
+
143
+ /** 逐条打结果,并定进程退出码。 */
144
+ function summarize(results) {
145
+ console.log('\n=== 结果 ===');
146
+ let failed = 0;
147
+ for (const item of results) {
148
+ if (item.status === 'FAILED') {
149
+ failed += 1;
150
+ console.log(`❌ ${item.file}:${item.error}`);
151
+ continue;
89
152
  }
153
+ const mark = item.status === 'skip' ? '⏭️ ' : '✅';
154
+ console.log(`${mark} ${item.file.padEnd(28)} ${(item.size / MIB).toFixed(2)} MB sha256=${item.sha}`);
155
+ }
156
+ console.log(failed ? `\n❌ ${failed} 个文件失败` : '\n✅ 全部就绪');
157
+ process.exitCode = failed ? 1 : 0;
158
+ }
159
+
160
+ /**
161
+ * 把某个变体需要的文件全取到 out 里。
162
+ * 普通下载失败不会让本函数 reject —— 那些都变成 FAILED 记录。
163
+ */
164
+ export async function fetchModel({ out, base, repo, variant, force, log, retries = 3 }) {
165
+ const remotePaths = [...SHARED_FILES, WEIGHTS[variant]];
166
+ const results = [];
167
+ for (const remotePath of remotePaths) {
168
+ const settled = await attemptFile({
169
+ url: `${base}/${repo}/resolve/main/${remotePath}`,
170
+ destination: join(out, remotePath.replace(/\//g, sep)),
171
+ force,
172
+ log,
173
+ retries,
174
+ name: remotePath,
175
+ });
176
+ if (settled) results.push(settled);
90
177
  }
91
178
  return results;
92
179
  }
@@ -94,35 +181,28 @@ export async function fetchModel({ out, base, repo, variant, force, log, retries
94
181
  async function main() {
95
182
  const { flags } = parseArgs();
96
183
  if (flags.help) {
97
- console.log(`用法: node tools/fetch-model.mjs --out <dir> [--base <url>] [--repo <owner/name>] [--variant fp32|int8|fp16] [--force]`);
184
+ console.log(USAGE);
98
185
  return;
99
186
  }
100
- const out = flags.out || process.env.MEMORY_ENGINE_MODEL_DIR;
101
- if (!out) throw new Error('必须指定 --out(模型落盘目录)');
102
- const opts = {
103
- out: path.resolve(out),
104
- base: flags.base || 'https://hf-mirror.com',
105
- repo: flags.repo || 'Xenova/bge-small-zh-v1.5',
106
- variant: flags.variant || 'fp32',
187
+ const requestedOut = flags.out || process.env.MEMORY_ENGINE_MODEL_DIR;
188
+ if (!requestedOut) throw new Error('必须指定 --out(模型落盘目录)');
189
+
190
+ const settings = {
191
+ out: resolve(requestedOut),
192
+ base: flags.base || BAREFALLS.base,
193
+ repo: flags.repo || BAREFALLS.repo,
194
+ variant: flags.variant || BAREFALLS.variant,
107
195
  force: Boolean(flags.force),
108
196
  };
109
- console.log(`模型获取器:${opts.repo}(${opts.variant})`);
110
- console.log(`镜像 :${opts.base}`);
111
- console.log(`目标目录 :${opts.out}`);
112
- if (!VARIANTS[opts.variant]) throw new Error(`未知 variant=${opts.variant}(可选 ${Object.keys(VARIANTS).join('/')})`);
113
- const results = await fetchModel({ ...opts, log: (m) => console.log(m) });
114
- console.log('\n=== 结果 ===');
115
- let failed = 0;
116
- for (const r of results) {
117
- if (r.status === 'FAILED') {
118
- failed += 1;
119
- console.log(`❌ ${r.file}:${r.error}`);
120
- } else {
121
- console.log(`${r.status === 'skip' ? '⏭️ ' : '✅'} ${r.file.padEnd(28)} ${(r.size / 1048576).toFixed(2)} MB sha256=${r.sha}`);
122
- }
197
+ console.log(`模型获取器:${settings.repo}(${settings.variant})`);
198
+ console.log(`镜像 :${settings.base}`);
199
+ console.log(`目标目录 :${settings.out}`);
200
+
201
+ if (!WEIGHTS[settings.variant]) {
202
+ throw new Error(`未知 variant=${settings.variant}(可选 ${Object.keys(WEIGHTS).join('/')})`);
123
203
  }
124
- console.log(failed ? `\n❌ ${failed} 个文件失败` : '\n✅ 全部就绪');
125
- process.exitCode = failed ? 1 : 0;
204
+
205
+ summarize(await fetchModel({ ...settings, log: (message) => console.log(message) }));
126
206
  }
127
207
 
128
208
  if (isMainEntry(import.meta.url)) await main();
@@ -1,8 +1,9 @@
1
- // 记忆引擎 v2 · 离线层 —— 真实模型召回冒烟(人读证据)
1
+ // 记忆引擎 · 离线层 —— 真实模型召回冒烟(给人读的证据)
2
2
  //
3
- // 为什么单独一个脚本:命令行传中文经 cmd/ssh 会乱码,这里把查询集写死在文件里,
4
- // 在目标机上直接跑,得到「真实模型 + 真实语料」的可读召回证据(含短暗号)。
5
- import fs from 'node:fs';
3
+ // 单独成一个脚本的理由:中文查询经 cmd/ssh 传过去会变乱码,所以把探针查询集写死在
4
+ // 这个文件里当数据用;在目标机上跑一条命令,就能拿到「真模型 + 真语料」的可读召回
5
+ // 证据(含短暗号)。
6
+ import { existsSync } from 'node:fs';
6
7
  import { parseArgs, isMainEntry } from '../lib/cli-util.mjs';
7
8
  import { resolveConfig } from '../lib/config.mjs';
8
9
  import { openVectorsStore, countVectors, getMetaAll } from '../lib/vectors-store.mjs';
@@ -10,7 +11,8 @@ import { openMemorySource } from '../lib/memory-source.mjs';
10
11
  import { createLocalEmbedder } from '../lib/embedder.mjs';
11
12
  import { recall, buildVectorIndex, shouldTrigger } from '../lib/recall.mjs';
12
13
 
13
- const QUERIES = [
14
+ /** 固定探针集:日常暗号、工程问题,外加一个裸端口号。 */
15
+ const PROBES = [
14
16
  '退下吧',
15
17
  '整夜干活',
16
18
  '去深水区',
@@ -23,16 +25,84 @@ const QUERIES = [
23
25
  '3080 端口',
24
26
  ];
25
27
 
28
+ /** CLI 里能改配置路径的几个开关。 */
29
+ const PATH_FLAGS = [
30
+ ['memoryDb', 'memory-db'],
31
+ ['vectorsDb', 'vectors-db'],
32
+ ['modelDir', 'model-dir'],
33
+ ];
34
+
35
+ const LABEL_COLUMNS = 9;
36
+ const HIT_INDENT = ' ';
37
+ const MAX_BRIEF_CHARS = 58;
38
+ const TOP_K = 5;
39
+
40
+ /** 字符串的终端宽度:CJK 码点按两列算(首列才能对齐)。 */
41
+ const widthOf = (text) => [...text].reduce((cols, ch) => cols + (ch.codePointAt(0) > 0x2e7f ? 2 : 1), 0);
42
+
43
+ const padLabel = (text) => text + ' '.repeat(Math.max(0, LABEL_COLUMNS - widthOf(text)));
44
+
45
+ const collapse = (text) => String(text).replace(/\s+/g, ' ');
46
+
47
+ /** 一条记忆的单行、截断版摘要。 */
48
+ const excerpt = (text) => collapse(text).slice(0, MAX_BRIEF_CHARS);
49
+
50
+ const gateVerdict = (gate) => (gate.trigger ? '是' : `否(${gate.reason})`);
51
+
52
+ /** 把命令行开关翻成 resolveConfig 的覆盖项;没给的键一律不出现。 */
53
+ function overridesFrom(flags) {
54
+ const overrides = {};
55
+ for (const [key, flag] of PATH_FLAGS) {
56
+ if (flags[flag]) overrides[key] = flags[flag];
57
+ }
58
+ if (flags.model) overrides.modelId = flags.model;
59
+ if (flags.dim) overrides.dim = Number(flags.dim);
60
+ return overrides;
61
+ }
62
+
63
+ /** 表头段:查之前先把语料形状交代清楚。 */
64
+ function printCorpora({ config, source, store, embedder }) {
65
+ const rows = [
66
+ ['主机', process.env.COMPUTERNAME || '?'],
67
+ ['记忆库', `${config.memoryDb}(${source.count()} 条 / pinned ${source.pinnedIds().length} 条)`],
68
+ ['向量库', `${config.vectorsDb}(${countVectors(store.db, 'memory')} 条向量)`],
69
+ ['meta', JSON.stringify(getMetaAll(store.db))],
70
+ ['模型', `${config.modelId} dim=${embedder.dim}`],
71
+ ];
72
+ console.log('=== 真实模型召回冒烟 ===');
73
+ for (const [label, value] of rows) console.log(`${padLabel(label)}: ${value}`);
74
+ console.log('');
75
+ }
76
+
77
+ function printHit(hit, row) {
78
+ const stored = row ? row.pinned : '?';
79
+ const text = excerpt(row ? row.text : '(已删除)');
80
+ console.log(`${HIT_INDENT}#${hit.refId} score=${hit.score.toFixed(5)} [${hit.sources.join('+')}] pinned=${stored} · ${text}…`);
81
+ }
82
+
83
+ /** 跑一条探针;返回这条命中里漏进了几条 pinned。 */
84
+ async function runProbe({ query, store, source, embedder, index, pinnedIds }) {
85
+ const gate = shouldTrigger(query);
86
+ const startedAt = performance.now();
87
+ const outcome = await recall({ query, store, source, embedder, index, resultTopK: TOP_K });
88
+ const elapsed = performance.now() - startedAt;
89
+
90
+ const ftsMatched = outcome.stats.ftsMatched ?? 0;
91
+ console.log(`【${query}】触发=${gateVerdict(gate)} 命中 ${outcome.hits.length} 条 · ${elapsed.toFixed(1)}ms · FTS命中=${ftsMatched}`);
92
+
93
+ let leaked = 0;
94
+ for (const hit of outcome.hits) {
95
+ if (pinnedIds.has(String(hit.refId))) leaked += 1;
96
+ printHit(hit, source.getById(hit.id));
97
+ }
98
+ console.log('');
99
+ return leaked;
100
+ }
101
+
26
102
  async function main() {
27
103
  const { flags } = parseArgs();
28
- const config = resolveConfig({
29
- memoryDb: flags['memory-db'] || undefined,
30
- vectorsDb: flags['vectors-db'] || undefined,
31
- modelDir: flags['model-dir'] || undefined,
32
- modelId: flags.model || undefined,
33
- dim: flags.dim ? Number(flags.dim) : undefined,
34
- });
35
- if (!fs.existsSync(config.memoryDb)) throw new Error(`记忆库不存在:${config.memoryDb}`);
104
+ const config = resolveConfig(overridesFrom(flags));
105
+ if (!existsSync(config.memoryDb)) throw new Error(`记忆库不存在:${config.memoryDb}`);
36
106
 
37
107
  const embedder = await createLocalEmbedder({
38
108
  modelDir: config.modelDir,
@@ -43,34 +113,20 @@ async function main() {
43
113
  const source = openMemorySource(config.memoryDb);
44
114
  const index = buildVectorIndex(store.db, { source: 'memory' });
45
115
 
46
- console.log('=== 真实模型召回冒烟 ===');
47
- console.log(`主机 : ${process.env.COMPUTERNAME || '?'}`);
48
- console.log(`记忆库 : ${config.memoryDb}(${source.count()} 条 / pinned ${source.pinnedIds().length} 条)`);
49
- console.log(`向量库 : ${config.vectorsDb}(${countVectors(store.db, 'memory')} 条向量)`);
50
- console.log(`meta : ${JSON.stringify(getMetaAll(store.db))}`);
51
- console.log(`模型 : ${config.modelId} dim=${embedder.dim}`);
52
- console.log('');
116
+ printCorpora({ config, source, store, embedder });
53
117
 
54
- const pinned = new Set(source.pinnedIds().map(String));
55
- let pinnedLeak = 0;
56
- for (const q of QUERIES) {
57
- const gate = shouldTrigger(q);
58
- const t0 = performance.now();
59
- const r = await recall({ query: q, store, source, embedder, index, resultTopK: 5 });
60
- const ms = performance.now() - t0;
61
- console.log(`【${q}】触发=${gate.trigger ? '是' : `否(${gate.reason})`} 命中 ${r.hits.length} 条 · ${ms.toFixed(1)}ms · FTS命中=${r.stats.ftsMatched ?? 0}`);
62
- for (const h of r.hits) {
63
- if (pinned.has(String(h.refId))) pinnedLeak += 1;
64
- const row = source.getById(h.id);
65
- const brief = (row ? row.text : '(已删除)').replace(/\s+/g, ' ').slice(0, 58);
66
- console.log(` #${h.refId} score=${h.score.toFixed(5)} [${h.sources.join('+')}] pinned=${row ? row.pinned : '?'} · ${brief}…`);
118
+ const pinnedIds = new Set(source.pinnedIds().map(String));
119
+ let leakedTotal = 0;
120
+ try {
121
+ for (const query of PROBES) {
122
+ leakedTotal += await runProbe({ query, store, source, embedder, index, pinnedIds });
67
123
  }
68
- console.log('');
124
+ console.log(`pinned 泄漏检查:${leakedTotal === 0 ? '✅ 0 条(排除生效)' : `❌ ${leakedTotal} 条`}`);
125
+ } finally {
126
+ source.close();
127
+ store.db.close();
69
128
  }
70
- console.log(`pinned 泄漏检查:${pinnedLeak === 0 ? '✅ 0 条(排除生效)' : `❌ ${pinnedLeak} 条`}`);
71
- source.close();
72
- store.db.close();
73
- process.exitCode = pinnedLeak === 0 ? 0 : 4;
129
+ process.exitCode = leakedTotal === 0 ? 0 : 4;
74
130
  }
75
131
 
76
132
  if (isMainEntry(import.meta.url)) await main();