@kira905/dsh-memory-toolkit 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/CHANGELOG.md +130 -1
  2. package/README.en.md +225 -2
  3. package/README.md +208 -2
  4. package/cordis.patch.yml +32 -0
  5. package/lib/deploy-guard.mjs +463 -275
  6. package/lib/guard-http.mjs +59 -62
  7. package/lib/pitfall-form-map.mjs +94 -84
  8. package/lib/pitfall-score.mjs +262 -185
  9. package/lib/session-file-name.mjs +116 -100
  10. package/lib/zstd-frames.mjs +88 -67
  11. package/memory-db-tools/README.md +29 -1
  12. package/memory-db-tools/audit-analyze.mjs +1 -1
  13. package/memory-db-tools/audit-export.mjs +60 -16
  14. package/memory-db-tools/audit-struct-probe.mjs +29 -8
  15. package/memory-db-tools/lib/known-devices.mjs +137 -0
  16. package/memory-engine/README.md +58 -4
  17. package/memory-engine/bench/bench.mjs +1 -1
  18. package/memory-engine/cli/band-sentinel.mjs +164 -0
  19. package/memory-engine/cli/distill-rollback.mjs +112 -44
  20. package/memory-engine/cli/distill-sessions.mjs +237 -0
  21. package/memory-engine/cli/distill.mjs +196 -95
  22. package/memory-engine/cli/doc-index-rebuild.mjs +142 -108
  23. package/memory-engine/cli/doc-index.mjs +47 -38
  24. package/memory-engine/cli/domain-tags.mjs +246 -162
  25. package/memory-engine/cli/layer-migrate.mjs +176 -0
  26. package/memory-engine/cli/layer-move.mjs +189 -0
  27. package/memory-engine/cli/pitfall-hook.mjs +136 -70
  28. package/memory-engine/cli/recall.mjs +243 -110
  29. package/memory-engine/cli/selfcheck.mjs +93 -44
  30. package/memory-engine/cli/serve-start.mjs +221 -140
  31. package/memory-engine/cli/sweep.mjs +83 -44
  32. package/memory-engine/cli/sync.mjs +93 -57
  33. package/memory-engine/config/audit-devices.json +6 -0
  34. package/memory-engine/config/bands.json +7 -0
  35. package/memory-engine/config/layers.json +7 -0
  36. package/memory-engine/dict/LICENSE-/350/257/215/345/205/270.md +53 -0
  37. package/memory-engine/dict/lexicon-config.json +11 -0
  38. package/memory-engine/dict//350/257/215/345/205/270.txt +109750 -0
  39. package/memory-engine/dict//350/257/215/351/242/221.txt +20000 -0
  40. package/memory-engine/lib/band-sentinel.mjs +417 -0
  41. package/memory-engine/lib/cli-strict.mjs +128 -0
  42. package/memory-engine/lib/cli-util.mjs +147 -125
  43. package/memory-engine/lib/config.mjs +83 -96
  44. package/memory-engine/lib/dirty-ledger.mjs +127 -94
  45. package/memory-engine/lib/distill-rollback.mjs +173 -133
  46. package/memory-engine/lib/distill.mjs +878 -585
  47. package/memory-engine/lib/doc-index-store.mjs +208 -235
  48. package/memory-engine/lib/doc-index.mjs +351 -350
  49. package/memory-engine/lib/doc-scan.mjs +178 -156
  50. package/memory-engine/lib/doc-summarize.mjs +220 -205
  51. package/memory-engine/lib/domain-tags.mjs +358 -277
  52. package/memory-engine/lib/embedder.mjs +175 -194
  53. package/memory-engine/lib/hashes.mjs +22 -7
  54. package/memory-engine/lib/layer-migrate.mjs +330 -0
  55. package/memory-engine/lib/layer-move.mjs +307 -0
  56. package/memory-engine/lib/layers.mjs +191 -0
  57. package/memory-engine/lib/lexicon-recall.mjs +473 -0
  58. package/memory-engine/lib/memory-source.mjs +78 -80
  59. package/memory-engine/lib/pitfall-source.mjs +115 -114
  60. package/memory-engine/lib/policy-source.mjs +279 -261
  61. package/memory-engine/lib/recall-log.mjs +285 -0
  62. package/memory-engine/lib/recall.mjs +565 -240
  63. package/memory-engine/lib/selfcheck.mjs +142 -106
  64. package/memory-engine/lib/session-distill.mjs +603 -0
  65. package/memory-engine/lib/session-source.mjs +421 -0
  66. package/memory-engine/lib/sweep.mjs +107 -130
  67. package/memory-engine/lib/sync.mjs +110 -140
  68. package/memory-engine/lib/tick-hook.mjs +123 -148
  69. package/memory-engine/lib/vectors-store.mjs +213 -188
  70. package/memory-engine/test/12-layering.test.mjs +539 -0
  71. package/memory-engine/test/13-session-distill.test.mjs +616 -0
  72. package/memory-engine/tools/bridge-smoke.mjs +94 -55
  73. package/memory-engine/tools/fetch-model.mjs +167 -87
  74. package/memory-engine/tools/smoke-recall.mjs +94 -38
  75. package/memory-engine/tools/verify-recall-chain.mjs +413 -0
  76. package/package.json +11 -1
@@ -1,16 +1,19 @@
1
- // 记忆引擎 v2 · S2c —— 编码桥**幂等启动器**(方案 §3.1「谁起」+ §3.5 第 5 条)
1
+ // 编码桥「幂等启动器」
2
2
  //
3
- // 契约(照方案原文,本文件是它的落地件):
4
- // · 幂等:先 `GET /health` 探活;**已在跑 → 直接退出 0 并打印端口**(不重复拉起、不抢端口);
5
- // · 不在跑 → 拉起 `serve.mjs`(detached),轮询 `/health` 直到 `ready:true`(默认 30s 上限);
6
- // · 端口顺延:`serve.mjs` 自己 +1 顺延(最多 10 次);启动器**按子进程 pid 在
7
- // [port, port+10] 里定位真实端口**,顺势落 `bridge-port.json` ⇒ 顺延后调用方仍查得到;
8
- // · 崩溃 / 冷启:桥不在 / 模型仍在加载 / 返回异常 ⇒ **调用方 fail-open**(S2b 已做),
9
- // 启动器**只负责把桥拉起来**,不做无限重试(每轮 tick 幂等重启,靠 job 节奏兜底);
10
- // · 仅绑回环:`--bind` 非回环一律拒绝(与 serve.mjs 同款铁律,双层防呆)。
3
+ // 一句话:调用方想用桥,就喊本脚本;桥在就跑个探活收工,桥不在才拉起来,拉完等到就绪为止。
11
4
  //
12
- // 退出码:0 = 已在跑(跳过)或拉起成功且 ready / 1 = 拉起失败(超时 / spawn 失败 / 非回环)
13
- // 3 = `--no-start` 且桥不在(只探活模式,供调用方判让路)/ 2 = 用法错
5
+ // 行为约定:
6
+ // · 幂等 —— 权威判据是 `GET /health`(不是 pid 文件、不是端口台账)。已在跑就 exit 0,只把端口报出来,
7
+ // 绝不重复拉起、不抢端口;
8
+ // · 冷启 —— spawn `serve.mjs`(detached:本进程退出后桥照样活),然后轮询 `/health` 到 `ready:true`
9
+ // (默认最多等 30s);
10
+ // · 端口顺延 —— `serve.mjs` 自己 +1 找空位(最多 10 次)。启动器按**子进程 pid** 在 [port, port+10]
11
+ // 里认领真实端口,并把结果落到端口台账,下游照着台账也能找到;
12
+ // · 不做无限重试 —— 崩了/模型还在加载就当这轮失败,下一轮 tick 会再幂等调一次;
13
+ // · 只许回环 —— `--bind` 给非回环一律拒(和 serve.mjs 同一条铁律,双层防呆)。
14
+ //
15
+ // 退出码:0 = 已在跑或拉起成功且就绪 / 1 = 拉起失败(超时、spawn 失败、非回环)
16
+ // 3 = `--no-start` 且桥不在(纯探活模式,调用方据此让路)/ 2 = 用法错
14
17
  //
15
18
  // 用法:
16
19
  // node cli/serve-start.mjs # 幂等确保 127.0.0.1:8201 在跑
@@ -27,106 +30,118 @@ import { parseArgs, num, isMainEntry } from '../lib/cli-util.mjs';
27
30
 
28
31
  export const SERVE_START_VERSION = '0.1.0';
29
32
 
30
- /** 与 serve.mjs 的 MAX_PORT_TRIES 保持一致(顺延搜索范围 = 同款 +1 逻辑的镜像)。 */
33
+ /** 与 serve.mjs 的 MAX_PORT_TRIES 同值:顺延窗 = 它 +1 搜索范围的镜像。 */
31
34
  export const MAX_PORT_TRIES = 10;
32
- const PROBE_TIMEOUT_MS = 1200;
33
- const PROBE_TIMEOUT_FAST_MS = 400;
34
- const POLL_INTERVAL_MS = 500;
35
- const DEFAULT_WAIT_MS = 30_000;
36
- const LOG_MAX_BYTES = 2 * 1024 * 1024;
37
- const PORT_FILE_SCHEMA = 1;
38
35
 
39
- export const EXIT = { OK: 0, FAIL: 1, USAGE: 2, NOT_RUNNING: 3 };
36
+ const TIMEOUT_PROBE_MS = 1200;
37
+ const TIMEOUT_WAIT_MS = 400;
38
+ const POLL_GAP_MS = 500;
39
+ const WAIT_BUDGET_MS = 30_000;
40
+ const LOG_ROTATE_BYTES = 2 * 1024 * 1024;
41
+ const LEDGER_SCHEMA = 1;
42
+ const LOOPBACK_HOSTS = new Set(['127.0.0.1', 'localhost', '::1']);
40
43
 
41
- const LOOPBACK_RE = /^(127\.0\.0\.1|localhost|::1)$/;
44
+ export const EXIT = { OK: 0, FAIL: 1, USAGE: 2, NOT_RUNNING: 3 };
42
45
 
43
- /** `/health` 响应是否为**本编码桥**(而不是恰好占着端口的路人服务,如 3080 的 web)。 */
46
+ /** 这条 `/health` 响应到底是不是本编码桥?(3080 那类路人服务也占端口,不能见到 200 就认) */
44
47
  export function isBridgeHealth(body) {
45
- return Boolean(body) && typeof body === 'object'
46
- && typeof body.ready === 'boolean' && typeof body.modelLoading === 'boolean'
47
- && (typeof body.model === 'string' || typeof body.dim === 'number');
48
+ const isObj = Boolean(body) && typeof body === 'object';
49
+ if (!isObj) return false;
50
+ const flagsOk = typeof body.ready === 'boolean' && typeof body.modelLoading === 'boolean';
51
+ const identityOk = typeof body.model === 'string' || typeof body.dim === 'number';
52
+ return flagsOk && identityOk;
48
53
  }
49
54
 
50
- /** 单端口探活:命中返回 { ok:true, port, body };否则 { ok:false, port, error }。 */
51
- export async function probeHealth(port, { host = '127.0.0.1', timeoutMs = PROBE_TIMEOUT_MS } = {}) {
52
- const ac = new AbortController();
53
- const t = setTimeout(() => ac.abort(), timeoutMs);
55
+ /** 探一个端口:活着且是桥 → {ok:true, port, body};否则 {ok:false, port, error}(**不抛**)。 */
56
+ export async function probeHealth(port, { host = '127.0.0.1', timeoutMs = TIMEOUT_PROBE_MS } = {}) {
54
57
  try {
55
- const res = await fetch(`http://${host}:${port}/health`, { signal: ac.signal });
58
+ const res = await fetch(`http://${host}:${port}/health`, { signal: AbortSignal.timeout(timeoutMs) });
56
59
  if (!res.ok) return { ok: false, port, error: `HTTP ${res.status}` };
57
60
  const body = await res.json();
58
- if (!isBridgeHealth(body)) return { ok: false, port, error: 'not-a-bridge' };
59
- return { ok: true, port, body };
61
+ return isBridgeHealth(body) ? { ok: true, port, body } : { ok: false, port, error: 'not-a-bridge' };
60
62
  } catch (e) {
61
- const name = e && e.name;
62
- return { ok: false, port, error: name === 'AbortError' || name === 'TimeoutError' ? 'timeout' : String(e?.message ?? e) };
63
- } finally {
64
- clearTimeout(t);
63
+ const timedOut = e?.name === 'TimeoutError' || e?.name === 'AbortError';
64
+ return { ok: false, port, error: timedOut ? 'timeout' : String(e?.message ?? e) };
65
65
  }
66
66
  }
67
67
 
68
- /** 端口台账(bridge-port.json)读取:读不到/坏 JSON 一律 null(台账是派生物,不阻断)。 */
68
+ /** 读端口台账:台账是派生物,读不到 / 坏 JSON 一律当没有,绝不因此阻断启动。 */
69
69
  export function readPortFile(file) {
70
70
  try {
71
- const txt = fs.readFileSync(file, 'utf8');
72
- const obj = JSON.parse(txt);
73
- return obj && typeof obj === 'object' ? obj : null;
71
+ const parsed = JSON.parse(fs.readFileSync(file, 'utf8'));
72
+ return parsed && typeof parsed === 'object' ? parsed : null;
74
73
  } catch {
75
74
  return null;
76
75
  }
77
76
  }
78
77
 
79
- /** 原子写端口台账(同目录 tmp → rename;失败不致命)。 */
78
+ /** 写端口台账:先写同目录临时件再 rename,读者永远看不到半截 JSON。失败返回 false(不致命)。 */
80
79
  export function writePortFile(file, obj) {
80
+ const tmp = `${file}.tmp-${process.pid}`;
81
81
  try {
82
82
  fs.mkdirSync(path.dirname(file), { recursive: true });
83
- const tmp = `${file}.tmp-${process.pid}`;
84
- fs.writeFileSync(tmp, JSON.stringify(obj, null, 2) + '\n', 'utf8');
83
+ fs.writeFileSync(tmp, `${JSON.stringify(obj, null, 2)}\n`, 'utf8');
85
84
  fs.renameSync(tmp, file);
86
85
  return true;
87
86
  } catch {
87
+ try {
88
+ fs.rmSync(tmp, { force: true });
89
+ } catch {
90
+ /* 临时件都删不掉也无所谓,反正别把异常漏出去 */
91
+ }
88
92
  return false;
89
93
  }
90
94
  }
91
95
 
92
- /** 候选端口(去重、保序):显式 `--port` / 台账记录 / 顺延窗。 */
96
+ /** 候选端口:显式端口 → 台账记的端口 → 顺延窗;去重保序,非法值直接丢。 */
93
97
  export function candidatePorts(basePort, recorded = null) {
94
- const out = [];
95
- const push = (p) => {
96
- const n = Number(p);
97
- if (!Number.isInteger(n) || n <= 0 || n > 65535) return;
98
- if (!out.includes(n)) out.push(n);
99
- };
100
- push(basePort);
101
- push(recorded);
102
- for (let i = 1; i <= MAX_PORT_TRIES; i += 1) push(Number(basePort) + i);
103
- return out;
98
+ const wanted = [basePort, recorded];
99
+ const base = Number(basePort);
100
+ for (let step = 1; step <= MAX_PORT_TRIES; step += 1) wanted.push(base + step);
101
+ const seen = new Set();
102
+ for (const raw of wanted) {
103
+ const port = Number(raw);
104
+ const legal = Number.isInteger(port) && port > 0 && port <= 65535;
105
+ if (legal) seen.add(port);
106
+ }
107
+ return [...seen];
104
108
  }
105
109
 
106
- /** 在候选端口里找**已在跑的桥**(顺延后仍能查到)。 */
107
- export async function findRunningBridge({ host = '127.0.0.1', ports = [], timeoutMs = PROBE_TIMEOUT_MS } = {}) {
110
+ /** 在候选端口里找已经在跑的桥(顺延之后也照样能认出来)。 */
111
+ export async function findRunningBridge({ host = '127.0.0.1', ports = [], timeoutMs = TIMEOUT_PROBE_MS } = {}) {
108
112
  for (const port of ports) {
109
113
  // eslint-disable-next-line no-await-in-loop
110
- const r = await probeHealth(port, { host, timeoutMs });
111
- if (r.ok) return r;
114
+ const hit = await probeHealth(port, { host, timeoutMs });
115
+ if (hit.ok) return hit;
112
116
  }
113
117
  return null;
114
118
  }
115
119
 
116
- function truncateLog(file) {
120
+ /** 日志文件太大就先清空,免得它无限长。 */
121
+ function capLog(file) {
117
122
  try {
118
- const st = fs.existsSync(file) ? fs.statSync(file) : null;
119
- if (st && st.size > LOG_MAX_BYTES) fs.writeFileSync(file, '', 'utf8');
120
- } catch { /* 忽略 */ }
123
+ if (!fs.existsSync(file)) return;
124
+ if (fs.statSync(file).size > LOG_ROTATE_BYTES) fs.writeFileSync(file, '', 'utf8');
125
+ } catch {
126
+ /* 清不动就算了 */
127
+ }
121
128
  }
122
129
 
123
- /**
124
- * 幂等确保桥在跑(job / 人工都走这一个入口)。
125
- * @param {{flags?:object, log?:(msg:string)=>void}} opts
126
- * @returns {Promise<{ok:boolean, started:boolean, port:number|null, pid:number|null, ready:boolean,
127
- * skippedReason?:string, error?:string, portFile:string, bypass?:string}>}
128
- */
129
- export async function ensureBridge({ flags = {}, log = (m) => process.stdout.write(m + '\n') } = {}) {
130
+ /** 一轮把候选端口全扫一遍:既拿到「最后见到的自家 pid」,也拿到「第一个就绪的」。 */
131
+ async function sweepPorts({ ports, host, pid }) {
132
+ let seen = null;
133
+ for (const port of ports) {
134
+ // eslint-disable-next-line no-await-in-loop
135
+ const hit = await probeHealth(port, { host, timeoutMs: TIMEOUT_WAIT_MS });
136
+ if (!hit.ok || hit.body.pid !== pid) continue; // 端口上有东西但不是我们拉的那只 ⇒ 不认
137
+ if (hit.body.ready === true) return { ready: { ...hit, port }, seen };
138
+ seen = { ...hit, port };
139
+ }
140
+ return { ready: null, seen };
141
+ }
142
+
143
+ /** 解析本次要用的入口/端口/台账/日志四件套。 */
144
+ function planTargets(flags) {
130
145
  const config = resolveConfig({
131
146
  modelDir: flags['model-dir'] || undefined,
132
147
  modelId: flags.model || undefined,
@@ -135,30 +150,57 @@ export async function ensureBridge({ flags = {}, log = (m) => process.stdout.wri
135
150
  bridgePort: flags.port ? Number(flags.port) : undefined,
136
151
  });
137
152
  const host = flags.bind || config.bridgeHost;
138
- if (!LOOPBACK_RE.test(String(host))) {
139
- return { ok: false, started: false, port: null, pid: null, ready: false, portFile: '', error: `只允许回环绑定(方案 §3.1),收到 ${host}` };
140
- }
141
- const basePort = Number(flags.port || config.bridgePort);
142
153
  const portFile = flags['port-file']
143
154
  ? path.resolve(String(flags['port-file']))
144
155
  : path.join(config.vectorsDir, 'bridge-port.json');
145
- const logFile = flags.log ? path.resolve(String(flags.log)) : path.join(path.dirname(portFile), 'bridge.log');
146
- const recorded = readPortFile(portFile);
147
- const ports = candidatePorts(basePort, recorded?.port ?? null);
148
-
149
- // ① 探活:已在跑 → 幂等跳过(唯一权威判据 = /health,不是 pid 文件)
150
- const found = await findRunningBridge({ host, ports });
151
- if (found) {
152
- const b = found.body;
156
+ return {
157
+ config,
158
+ host,
159
+ basePort: Number(flags.port || config.bridgePort),
160
+ portFile,
161
+ logFile: flags.log ? path.resolve(String(flags.log)) : path.join(path.dirname(portFile), 'bridge.log'),
162
+ };
163
+ }
164
+
165
+ /**
166
+ * 幂等确保桥在跑 —— job 与人工都走这一个口。
167
+ * @param {{flags?:object, log?:(msg:string)=>void}} opts
168
+ * @returns {Promise<{ok:boolean, started:boolean, port:number|null, pid:number|null, ready:boolean,
169
+ * skippedReason?:string, error?:string, portFile:string, health?:object}>}
170
+ */
171
+ export async function ensureBridge({ flags = {}, log = (m) => process.stdout.write(`${m}\n`) } = {}) {
172
+ const target = planTargets(flags);
173
+ const { config, host, basePort, portFile, logFile } = target;
174
+
175
+ if (!LOOPBACK_HOSTS.has(String(host))) {
176
+ return {
177
+ ok: false, started: false, port: null, pid: null, ready: false, portFile: '',
178
+ error: `只允许回环绑定(收到 ${host})`,
179
+ };
180
+ }
181
+
182
+ const ledger = readPortFile(portFile);
183
+ const ports = candidatePorts(basePort, ledger?.port ?? null);
184
+
185
+ // ① 先探活:已经在跑就收工(幂等点就在这一句里)
186
+ const alive = await findRunningBridge({ host, ports });
187
+ if (alive) {
188
+ const h = alive.body;
153
189
  writePortFile(portFile, {
154
- schemaVersion: PORT_FILE_SCHEMA, host, port: found.port, pid: b.pid ?? null,
155
- modelId: b.model ?? config.modelId, modelDir: b.modelDir ?? config.modelDir,
156
- ready: b.ready === true, modelLoading: b.modelLoading === true,
157
- startedAt: b.uptimeMs ? new Date(Date.now() - Number(b.uptimeMs)).toISOString() : null,
158
- checkedAt: new Date().toISOString(), via: 'serve-start(probe)',
190
+ schemaVersion: LEDGER_SCHEMA,
191
+ host,
192
+ port: alive.port,
193
+ pid: h.pid ?? null,
194
+ modelId: h.model ?? config.modelId,
195
+ modelDir: h.modelDir ?? config.modelDir,
196
+ ready: h.ready === true,
197
+ modelLoading: h.modelLoading === true,
198
+ startedAt: h.uptimeMs ? new Date(Date.now() - Number(h.uptimeMs)).toISOString() : null,
199
+ checkedAt: new Date().toISOString(),
200
+ via: 'serve-start(probe)',
159
201
  });
160
- log(`[serve-start] 已在跑,跳过拉起:http://${host}:${found.port} pid=${b.pid ?? '?'} ready=${b.ready} model=${b.model ?? '?'}(台账 ${portFile})`);
161
- return { ok: true, started: false, port: found.port, pid: b.pid ?? null, ready: b.ready === true, portFile, health: b };
202
+ log(`[serve-start] 已在跑,跳过拉起:http://${host}:${alive.port} pid=${h.pid ?? '?'} ready=${h.ready} model=${h.model ?? '?'}(台账 ${portFile})`);
203
+ return { ok: true, started: false, port: alive.port, pid: h.pid ?? null, ready: h.ready === true, portFile, health: h };
162
204
  }
163
205
 
164
206
  if (flags['no-start']) {
@@ -166,80 +208,119 @@ export async function ensureBridge({ flags = {}, log = (m) => process.stdout.wri
166
208
  return { ok: false, started: false, port: null, pid: null, ready: false, portFile, skippedReason: 'not-running' };
167
209
  }
168
210
 
169
- // ② 拉起(detached:本进程退出后桥继续活着;日志走文件,不污染调用方 stdout)
170
- const serveScript = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', 'serve.mjs');
171
- fs.mkdirSync(path.dirname(logFile), { recursive: true });
172
- truncateLog(logFile);
173
- const fd = fs.openSync(logFile, 'a');
174
- let child;
175
- try {
176
- child = spawn(process.execPath, [
177
- serveScript,
178
- '--model-dir', config.modelDir,
179
- '--port', String(basePort),
180
- '--bind', host,
181
- ], { detached: true, stdio: ['ignore', fd, fd], windowsHide: true });
182
- } catch (e) {
183
- try { fs.closeSync(fd); } catch { /* 忽略 */ }
184
- return { ok: false, started: false, port: null, pid: null, ready: false, portFile, error: `spawn 失败:${e?.message ?? e}` };
211
+ // ② 拉起:detached 运行,stdout/stderr 全进日志文件,绝不污染调用方的输出
212
+ const child = launchBridge({ target, host, basePort });
213
+ if (child.error) {
214
+ return { ok: false, started: false, port: null, pid: null, ready: false, portFile, error: child.error };
185
215
  }
186
- child.unref();
187
- try { fs.closeSync(fd); } catch { /* 忽略 */ }
216
+ const pid = child.pid;
188
217
 
189
- // ③ 等 ready:按 pid 在顺延窗里定位真实端口
190
- const waitMs = num(flags['wait-ms'], DEFAULT_WAIT_MS);
191
- const deadline = Date.now() + waitMs;
218
+ // ③ 等就绪:顺延之后真实端口靠 pid 认领
219
+ const waitMs = num(flags['wait-ms'], WAIT_BUDGET_MS);
220
+ const giveUpAt = Date.now() + waitMs;
192
221
  let lastSeen = null;
193
- while (Date.now() < deadline) {
222
+ while (Date.now() < giveUpAt) {
194
223
  // eslint-disable-next-line no-await-in-loop
195
- await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
196
- for (const port of ports) {
197
- // eslint-disable-next-line no-await-in-loop
198
- const r = await probeHealth(port, { host, timeoutMs: PROBE_TIMEOUT_FAST_MS });
199
- if (!r.ok) continue;
200
- if (r.body.pid !== child.pid) continue; // 别人占着这个端口 → 不是我们拉的那只
201
- lastSeen = r;
202
- if (r.body.ready === true) {
203
- writePortFile(portFile, {
204
- schemaVersion: PORT_FILE_SCHEMA, host, port, pid: child.pid,
205
- modelId: r.body.model ?? config.modelId, modelDir: r.body.modelDir ?? config.modelDir,
206
- ready: true, modelLoading: false,
207
- startedAt: new Date().toISOString(), readyAt: new Date().toISOString(), checkedAt: new Date().toISOString(),
208
- via: 'serve-start(spawn)', logFile,
209
- });
210
- log(`[serve-start] 已拉起并就绪:http://${host}:${port} pid=${child.pid} model=${r.body.model ?? '?'}`
211
- + `${port !== basePort ? `(端口 ${basePort} 被占,顺延到 ${port})` : ''} 日志 ${logFile}`);
212
- return { ok: true, started: true, port, pid: child.pid, ready: true, portFile, health: r.body };
213
- }
224
+ await new Promise((done) => setTimeout(done, POLL_GAP_MS));
225
+ // eslint-disable-next-line no-await-in-loop
226
+ const round = await sweepPorts({ ports, host, pid });
227
+ if (round.seen) lastSeen = round.seen;
228
+ if (round.ready) {
229
+ const hit = round.ready;
230
+ const nowIso = new Date().toISOString();
231
+ writePortFile(portFile, {
232
+ schemaVersion: LEDGER_SCHEMA,
233
+ host,
234
+ port: hit.port,
235
+ pid,
236
+ modelId: hit.body.model ?? config.modelId,
237
+ modelDir: hit.body.modelDir ?? config.modelDir,
238
+ ready: true,
239
+ modelLoading: false,
240
+ startedAt: nowIso,
241
+ readyAt: nowIso,
242
+ checkedAt: nowIso,
243
+ via: 'serve-start(spawn)',
244
+ logFile,
245
+ });
246
+ const shiftNote = hit.port !== basePort ? `(端口 ${basePort} 被占,顺延到 ${hit.port})` : '';
247
+ log(`[serve-start] 已拉起并就绪:http://${host}:${hit.port} pid=${pid} model=${hit.body.model ?? '?'}${shiftNote} 日志 ${logFile}`);
248
+ return { ok: true, started: true, port: hit.port, pid, ready: true, portFile, health: hit.body };
214
249
  }
215
250
  }
216
251
 
217
252
  writePortFile(portFile, {
218
- schemaVersion: PORT_FILE_SCHEMA, host, port: lastSeen?.port ?? basePort, pid: child.pid,
219
- modelId: lastSeen?.body?.model ?? config.modelId, modelDir: config.modelDir,
220
- ready: false, modelLoading: lastSeen?.body?.modelLoading ?? null,
221
- startedAt: new Date().toISOString(), checkedAt: new Date().toISOString(),
222
- via: 'serve-start(spawn)', logFile, error: 'not-ready-in-wait-window',
253
+ schemaVersion: LEDGER_SCHEMA,
254
+ host,
255
+ port: lastSeen?.port ?? basePort,
256
+ pid,
257
+ modelId: lastSeen?.body?.model ?? config.modelId,
258
+ modelDir: config.modelDir,
259
+ ready: false,
260
+ modelLoading: lastSeen?.body?.modelLoading ?? null,
261
+ startedAt: new Date().toISOString(),
262
+ checkedAt: new Date().toISOString(),
263
+ via: 'serve-start(spawn)',
264
+ logFile,
265
+ error: 'not-ready-in-wait-window',
223
266
  });
267
+ const why = lastSeen ? `(pid 已探到,modelLoading=${lastSeen.body.modelLoading})` : '(未探到 pid)';
224
268
  return {
225
- ok: false, started: true, port: lastSeen?.port ?? null, pid: child.pid, ready: false, portFile,
226
- error: `拉起后 ${waitMs}ms 内未就绪${lastSeen ? `(pid 已探到,modelLoading=${lastSeen.body.modelLoading})` : '(未探到 pid)'};日志 ${logFile}`,
269
+ ok: false, started: true, port: lastSeen?.port ?? null, pid, ready: false, portFile,
270
+ error: `拉起后 ${waitMs}ms 内未就绪${why};日志 ${logFile}`,
227
271
  };
228
272
  }
229
273
 
274
+ /** 真正 spawn 子进程(返回 pid 或 error,不外抛)。 */
275
+ function launchBridge({ target, host, basePort }) {
276
+ const serveScript = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', 'serve.mjs');
277
+ const { config, logFile } = target;
278
+ let fd = null;
279
+ try {
280
+ fs.mkdirSync(path.dirname(logFile), { recursive: true });
281
+ capLog(logFile);
282
+ fd = fs.openSync(logFile, 'a');
283
+ const child = spawn(process.execPath, [
284
+ serveScript,
285
+ '--model-dir', config.modelDir,
286
+ '--port', String(basePort),
287
+ '--bind', host,
288
+ ], { detached: true, stdio: ['ignore', fd, fd], windowsHide: true });
289
+ child.unref();
290
+ return { pid: child.pid };
291
+ } catch (e) {
292
+ return { error: `spawn 失败:${e?.message ?? e}` };
293
+ } finally {
294
+ if (fd !== null) {
295
+ try {
296
+ fs.closeSync(fd);
297
+ } catch {
298
+ /* ignore */
299
+ }
300
+ }
301
+ }
302
+ }
303
+
304
+ const USAGE_PARTS = [
305
+ '用法:node cli/serve-start.mjs',
306
+ '[--port N]', '[--bind 127.0.0.1]', '[--model-dir P]', '[--port-file P]',
307
+ '[--log P]', '[--wait-ms N]', '[--no-start]', '[--json]',
308
+ ];
309
+
230
310
  async function main() {
231
311
  const { flags } = parseArgs();
232
312
  const json = Boolean(flags.json);
233
- const log = (m) => { if (!json) process.stdout.write(m + '\n'); };
313
+ const log = (m) => {
314
+ if (!json) process.stdout.write(`${m}\n`);
315
+ };
234
316
  if (flags.help) {
235
- process.stdout.write('用法:node cli/serve-start.mjs [--port N] [--bind 127.0.0.1] [--model-dir P] '
236
- + '[--port-file P] [--log P] [--wait-ms N] [--no-start] [--json]\n');
317
+ process.stdout.write(`${USAGE_PARTS.join(' ')}\n`);
237
318
  return EXIT.OK;
238
319
  }
239
320
  const r = await ensureBridge({ flags, log });
240
- if (json) process.stdout.write(JSON.stringify(r, null, 2) + '\n');
321
+ if (json) process.stdout.write(`${JSON.stringify(r, null, 2)}\n`);
241
322
  if (r.ok) return EXIT.OK;
242
- // 失败原因必须说人话(否则 `--bind 0.0.0.0` 这类用法错只会得到一个静默的 exit 1)
323
+ // 失败一定要说人话,否则 `--bind 0.0.0.0` 这类用法错只会换来一个闷声 exit 1
243
324
  if (r.error) log(`[serve-start] 失败:${r.error}`);
244
325
  return r.skippedReason === 'not-running' ? EXIT.NOT_RUNNING : EXIT.FAIL;
245
326
  }
@@ -1,57 +1,96 @@
1
- // 记忆引擎 v2 · 离线层 —— 兜底 hash 段轮转 CLI(方案 §3.1 兜底)
1
+ // 兜底轮转入口:按 id 段一圈一圈扫全表,重编码「正文哈希与向量库不符」和「库里压根没有」的条目。
2
+ // 主增量路径靠 updated_at 水位,摸不到那种「改了正文却没动 updated_at」的勘误;本命令就是补它的。
2
3
  //
3
- // 用法:
4
- // node cli/sweep.mjs # 跑 1 段(默认 20 段轮转;每周跑一次即可)
4
+ // node cli/sweep.mjs # 走 1 段(默认把全表切 20 段,一周跑一次足够)
5
5
  // node cli/sweep.mjs --rounds 5
6
- // node cli/sweep.mjs --all # 一轮完整覆盖(= --rounds <segments>)
6
+ // node cli/sweep.mjs --all # 一次走完一整圈(= --rounds <段数>)
7
7
  // node cli/sweep.mjs --dry-run --json
8
- // node cli/sweep.mjs --source pitfall # 坑源兜底轮转(默认 memory;坑源按 lastSeen 排序切片)
8
+ // node cli/sweep.mjs --source pitfall # 换来源(坑源按 lastSeen 排序后切片)
9
+ //
10
+ // 退出码:0 = 跑完 · 1 = 中途失败 · 2 = 启动即炸。
9
11
  import { isMainEntry, parseArgs, makeLogger, openRuntime, num } from '../lib/cli-util.mjs';
10
12
  import { runHashSweep } from '../lib/sweep.mjs';
11
13
 
12
- async function main() {
14
+ const OK = 0;
15
+ const FAIL_RUN = 1;
16
+ const CRASH = 2;
17
+ const COLUMNS = 13;
18
+
19
+ /** 统一的启动三件套:原始 flag、是否机器可读、以及分好流的 logger */
20
+ function boot() {
13
21
  const { flags } = parseArgs();
14
22
  const json = Boolean(flags.json);
15
- const log = makeLogger({ json, quiet: Boolean(flags.quiet) });
16
- let rt = null;
23
+ return { flags, json, log: makeLogger({ json, quiet: Boolean(flags.quiet) }) };
24
+ }
25
+
26
+ /** 开一个运行时,交给回调用,出了什么事都把库关掉 */
27
+ async function withRuntime(flags, work) {
28
+ const rt = await openRuntime(flags);
17
29
  try {
18
- rt = await openRuntime(flags);
19
- const segments = num(flags.segments, rt.config.sweepSegments);
20
- const rounds = flags.all ? segments : num(flags.rounds, 1);
21
- const stats = await runHashSweep({
22
- source: rt.source,
23
- store: rt.store,
24
- embedder: rt.embedder,
25
- modelId: rt.config.modelId,
26
- dim: rt.config.dim,
27
- sourceName: rt.sourceName,
28
- segments,
29
- rounds,
30
- dryRun: Boolean(flags['dry-run']),
31
- log,
32
- });
33
- const out = { ok: !stats.error, source: rt.sourceName, vectorsDb: rt.config.vectorsDb, ...stats };
34
- if (json) process.stdout.write(JSON.stringify(out, null, 2) + '\n');
35
- else {
36
- log('');
37
- log('=== 兜底 hash 段轮转结果 ===');
38
- log(`来源 : ${rt.sourceName}`);
39
- log(`id 段 : ${stats.range.lo} - ${stats.range.hi}(${stats.range.n} 条)`);
40
- log(`轮转游标 : ${stats.cursorStart} → ${stats.cursorEnd} / ${stats.segments}`);
41
- log(`扫描/重编码 : ${stats.scanned} / ${stats.fixed}(缺失 ${stats.missing} / 哈希不符 ${stats.mismatched})`);
42
- log(`一轮完整覆盖 : ${stats.completedFullSweep ? '是(已更新 lastFullHashSweepAt)' : '否'}`);
43
- log(`耗时 : ${(stats.elapsedMs / 1000).toFixed(2)}s`);
44
- if (stats.error) log(`❌ 失败 : ${stats.error}`);
45
- else log('✅ 完成');
46
- }
47
- process.exitCode = stats.error ? 1 : 0;
48
- } catch (e) {
49
- const msg = String(e && e.stack ? e.stack : e);
50
- if (json) process.stdout.write(JSON.stringify({ ok: false, error: msg }, null, 2) + '\n');
51
- else log(`❌ ${msg}`);
52
- process.exitCode = 2;
30
+ return await work(rt);
53
31
  } finally {
54
- if (rt) rt.close();
32
+ rt.close();
33
+ }
34
+ }
35
+
36
+ /** 起来就炸:json 走 stdout,文本走 logger,退出码固定 2 */
37
+ function oops(json, log, err) {
38
+ const detail = String(err?.stack ?? err);
39
+ if (json) process.stdout.write(`${JSON.stringify({ ok: false, error: detail }, null, 2)}\n`);
40
+ else log(`❌ ${detail}`);
41
+ process.exitCode = CRASH;
42
+ }
43
+
44
+ /** 中文占两格:按显示宽度补空格,冒号右对齐 */
45
+ function field(text) {
46
+ let used = 0;
47
+ for (const ch of String(text)) used += ch.codePointAt(0) > 0xff ? 2 : 1;
48
+ return String(text) + ' '.repeat(Math.max(0, COLUMNS - used));
49
+ }
50
+
51
+ function report(log, sourceName, stats) {
52
+ const row = (name, value) => log(`${field(name)}: ${value}`);
53
+ log('');
54
+ log('=== 兜底 hash 段轮转结果 ===');
55
+ row('来源', sourceName);
56
+ row('id 段', `${stats.range.lo} - ${stats.range.hi}(${stats.range.n} 条)`);
57
+ row('轮转游标', `${stats.cursorStart} → ${stats.cursorEnd} / ${stats.segments}`);
58
+ row('扫描/重编码', `${stats.scanned} / ${stats.fixed}(缺失 ${stats.missing} / 哈希不符 ${stats.mismatched})`);
59
+ row('一轮完整覆盖', stats.completedFullSweep ? '是(已更新 lastFullHashSweepAt)' : '否');
60
+ row('耗时', `${(stats.elapsedMs / 1000).toFixed(2)}s`);
61
+ if (stats.error) log(`❌ 失败 : ${stats.error}`);
62
+ else log('✅ 完成');
63
+ }
64
+
65
+ async function work(rt, flags, log, json) {
66
+ const { config, embedder, source, sourceName, store } = rt;
67
+ const segments = num(flags.segments, config.sweepSegments);
68
+ // --all 就是把段数当轮数(一次覆盖全表);否则默认只走 1 段
69
+ const rounds = flags.all ? segments : num(flags.rounds, 1);
70
+ const stats = await runHashSweep({
71
+ source,
72
+ store,
73
+ embedder,
74
+ modelId: config.modelId,
75
+ dim: config.dim,
76
+ sourceName,
77
+ segments,
78
+ rounds,
79
+ dryRun: Boolean(flags['dry-run']),
80
+ log,
81
+ });
82
+ const out = { ok: !stats.error, source: sourceName, vectorsDb: config.vectorsDb, ...stats };
83
+ if (json) process.stdout.write(`${JSON.stringify(out, null, 2)}\n`);
84
+ else report(log, sourceName, stats);
85
+ process.exitCode = stats.error ? FAIL_RUN : OK;
86
+ }
87
+
88
+ async function main() {
89
+ const { flags, json, log } = boot();
90
+ try {
91
+ await withRuntime(flags, (rt) => work(rt, flags, log, json));
92
+ } catch (err) {
93
+ oops(json, log, err);
55
94
  }
56
95
  }
57
96