@kira905/dsh-memory-toolkit 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/CHANGELOG.md +130 -1
  2. package/README.en.md +225 -2
  3. package/README.md +208 -2
  4. package/cordis.patch.yml +32 -0
  5. package/lib/deploy-guard.mjs +463 -275
  6. package/lib/guard-http.mjs +59 -62
  7. package/lib/pitfall-form-map.mjs +94 -84
  8. package/lib/pitfall-score.mjs +262 -185
  9. package/lib/session-file-name.mjs +116 -100
  10. package/lib/zstd-frames.mjs +88 -67
  11. package/memory-db-tools/README.md +29 -1
  12. package/memory-db-tools/audit-analyze.mjs +1 -1
  13. package/memory-db-tools/audit-export.mjs +60 -16
  14. package/memory-db-tools/audit-struct-probe.mjs +29 -8
  15. package/memory-db-tools/lib/known-devices.mjs +137 -0
  16. package/memory-engine/README.md +58 -4
  17. package/memory-engine/bench/bench.mjs +1 -1
  18. package/memory-engine/cli/band-sentinel.mjs +164 -0
  19. package/memory-engine/cli/distill-rollback.mjs +112 -44
  20. package/memory-engine/cli/distill-sessions.mjs +237 -0
  21. package/memory-engine/cli/distill.mjs +196 -95
  22. package/memory-engine/cli/doc-index-rebuild.mjs +142 -108
  23. package/memory-engine/cli/doc-index.mjs +47 -38
  24. package/memory-engine/cli/domain-tags.mjs +246 -162
  25. package/memory-engine/cli/layer-migrate.mjs +176 -0
  26. package/memory-engine/cli/layer-move.mjs +189 -0
  27. package/memory-engine/cli/pitfall-hook.mjs +136 -70
  28. package/memory-engine/cli/recall.mjs +243 -110
  29. package/memory-engine/cli/selfcheck.mjs +93 -44
  30. package/memory-engine/cli/serve-start.mjs +221 -140
  31. package/memory-engine/cli/sweep.mjs +83 -44
  32. package/memory-engine/cli/sync.mjs +93 -57
  33. package/memory-engine/config/audit-devices.json +6 -0
  34. package/memory-engine/config/bands.json +7 -0
  35. package/memory-engine/config/layers.json +7 -0
  36. package/memory-engine/dict/LICENSE-/350/257/215/345/205/270.md +53 -0
  37. package/memory-engine/dict/lexicon-config.json +11 -0
  38. package/memory-engine/dict//350/257/215/345/205/270.txt +109750 -0
  39. package/memory-engine/dict//350/257/215/351/242/221.txt +20000 -0
  40. package/memory-engine/lib/band-sentinel.mjs +417 -0
  41. package/memory-engine/lib/cli-strict.mjs +128 -0
  42. package/memory-engine/lib/cli-util.mjs +147 -125
  43. package/memory-engine/lib/config.mjs +83 -96
  44. package/memory-engine/lib/dirty-ledger.mjs +127 -94
  45. package/memory-engine/lib/distill-rollback.mjs +173 -133
  46. package/memory-engine/lib/distill.mjs +878 -585
  47. package/memory-engine/lib/doc-index-store.mjs +208 -235
  48. package/memory-engine/lib/doc-index.mjs +351 -350
  49. package/memory-engine/lib/doc-scan.mjs +178 -156
  50. package/memory-engine/lib/doc-summarize.mjs +220 -205
  51. package/memory-engine/lib/domain-tags.mjs +358 -277
  52. package/memory-engine/lib/embedder.mjs +175 -194
  53. package/memory-engine/lib/hashes.mjs +22 -7
  54. package/memory-engine/lib/layer-migrate.mjs +330 -0
  55. package/memory-engine/lib/layer-move.mjs +307 -0
  56. package/memory-engine/lib/layers.mjs +191 -0
  57. package/memory-engine/lib/lexicon-recall.mjs +473 -0
  58. package/memory-engine/lib/memory-source.mjs +78 -80
  59. package/memory-engine/lib/pitfall-source.mjs +115 -114
  60. package/memory-engine/lib/policy-source.mjs +279 -261
  61. package/memory-engine/lib/recall-log.mjs +285 -0
  62. package/memory-engine/lib/recall.mjs +565 -240
  63. package/memory-engine/lib/selfcheck.mjs +142 -106
  64. package/memory-engine/lib/session-distill.mjs +603 -0
  65. package/memory-engine/lib/session-source.mjs +421 -0
  66. package/memory-engine/lib/sweep.mjs +107 -130
  67. package/memory-engine/lib/sync.mjs +110 -140
  68. package/memory-engine/lib/tick-hook.mjs +123 -148
  69. package/memory-engine/lib/vectors-store.mjs +213 -188
  70. package/memory-engine/test/12-layering.test.mjs +539 -0
  71. package/memory-engine/test/13-session-distill.test.mjs +616 -0
  72. package/memory-engine/tools/bridge-smoke.mjs +94 -55
  73. package/memory-engine/tools/fetch-model.mjs +167 -87
  74. package/memory-engine/tools/smoke-recall.mjs +94 -38
  75. package/memory-engine/tools/verify-recall-chain.mjs +413 -0
  76. package/package.json +11 -1
@@ -1,69 +1,156 @@
1
- // 记忆引擎 v2 · S3b —— 域标签补齐 CLI(方案 §3.6 / §5.4 / §7 / §3.9 D2)
1
+ // 记忆域标签补齐 CLI
2
2
  //
3
- // 用法:
4
- // node cli/domain-tags.mjs --db <库路径> # dry-run(默认;只读规划,零写入)
5
- // node cli/domain-tags.mjs --db <库路径> --apply # 真跑(副本库;分批 + 可续跑)
6
- // node cli/domain-tags.mjs --db <库路径> --apply --snapshot <dir> # 跑前 VACUUM INTO 快照
7
- // node cli/domain-tags.mjs --db <库路径> --limit 25 --batch 25 # 只跑 1 批(试跑)
8
- // node cli/domain-tags.mjs --db <库路径> --apply --json # 机器可读(stdout 只有 JSON)
3
+ // 干的事:把库里待处理的记忆条目按批送给分类模型,模型只准从固定的域标签集合里挑,
4
+ // 挑完写回库、并把「这批问过了」记进幂等台账(所以中断重跑不会重复花钱)。
9
5
  //
10
- // 关键开关:`--batch`(每批条数,默认 25)· `--max-chars-per-item`(送模型的单条截断,默认 240)
11
- // `--owner-user`(本机 owner 的 userid;缺省由身份表解析、再回落常量)· `--state <file>`
12
- // `--max-calls`(默认 20)· `--max-tokens`(默认 150000)· `--model` · `--base-url`
13
- // `--no-daily-report`(不写日报段)· `--daily-report <path>`(覆盖日报路径)
6
+ // 命令行形态(对外契约,参数名 / 默认值 / 退出码一律照旧):
7
+ // --db <path> 必填。本工具没有默认库路径 —— 写死库路径是明令禁止的
8
+ // --apply 真跑(缺省是 dry-run:只规划、不调模型、不写库)
9
+ // --snapshot <dir> 真跑前先 VACUUM INTO 到 <dir>/memory.db
10
+ // --batch <n> 每批条数(25)
11
+ // --limit <n> 本次最多取多少条(0 = 不限)
12
+ // --max-chars-per-item <n> 送模型时单条正文截断长度(240)
13
+ // --max-calls / --max-tokens 每日成本闸(缺省取库里的 COST_LIMITS)
14
+ // --owner-user <id> 本机 owner 的 userid;缺省先查身份表,再不行用内置常量
15
+ // --state <file> 幂等台账落点(缺省 <home>/memory/domain-tags-state.json)
16
+ // --model / --base-url 分类模型档
17
+ // --no-daily-report 不写日报段
18
+ // --daily-report <file> 覆盖日报落点
19
+ // --json stdout 只留 JSON,人看的话全走 stderr
14
20
  //
15
- // 退出码:0 = 完成/无待处理(含"成本闸停机"这种受控停止);1 = 有异常;2 = 用法错;3 = 环境不可用(无 key / 库不存在)
21
+ // 退出码:0 = 正常收尾(包含「成本闸到点、主动停」这种受控停止)
22
+ // 1 = 有异常 / 2 = 用法错 / 3 = 环境不可用(没 key、库不在)
16
23
  import fs from 'node:fs';
17
- import os from 'node:os';
18
24
  import path from 'node:path';
19
- import { DatabaseSync } from 'node:sqlite';
20
25
  import { isMainEntry, parseArgs, num, makeLogger } from '../lib/cli-util.mjs';
21
26
  import * as D from '../lib/domain-tags.mjs';
22
27
 
23
- const argv = process.argv.slice(2);
28
+ const EXIT = { OK: 0, FAIL: 1, USAGE: 2, ENV: 3 };
29
+ const OWNER_FALLBACK = 'LiJianXiao';
30
+ const OWNER_SOURCES = ['../../memory-db-tools/lib/memory-scope.mjs', '../dsh-memory-local/src/schema-migrate.mjs'];
31
+
32
+ /** 数值型开关:源长什么样、落到哪个字段、不给时取什么。 */
33
+ const NUMERIC_OPTIONS = [
34
+ { from: 'batch', to: 'batchSize', fallback: 25 },
35
+ { from: 'limit', to: 'limit', fallback: 0 },
36
+ { from: 'max-chars-per-item', to: 'maxCharsPerItem', fallback: 240 },
37
+ { from: 'max-calls', to: 'callsPerDay', fallback: D.COST_LIMITS.callsPerDay },
38
+ { from: 'max-tokens', to: 'tokensPerDay', fallback: D.COST_LIMITS.tokensPerDay },
39
+ ];
40
+
24
41
  if (isMainEntry(import.meta.url)) {
25
42
  main().catch((e) => {
26
- console.error(`未捕获异常:${e && e.stack ? e.stack : e}`);
27
- process.exitCode = 1;
43
+ console.error(`未捕获异常:${e?.stack ?? e}`);
44
+ process.exitCode = EXIT.FAIL;
28
45
  });
29
46
  }
30
47
 
31
- /** 解析「本机 owner 的 userid」:优先 ⑤-3 的权威实现,其次 fork 的自检镜像,最后内置回落。 */
48
+ /**
49
+ * 本机 owner 的 userid:先问权威实现,再问自检镜像,最后才用内置常量。
50
+ * 顺序不能反 —— 常量只是兜底,不是口径。
51
+ */
32
52
  async function resolveOwnerUser(explicit, home) {
33
53
  if (explicit) return { user: explicit, source: 'cli' };
34
- const cands = ['../../memory-db-tools/lib/memory-scope.mjs', '../dsh-memory-local/src/schema-migrate.mjs'];
35
- for (const rel of cands) {
36
- try {
37
- const mod = await import(new URL(rel, import.meta.url).href);
38
- const identity = mod.readIdentityTable ? mod.readIdentityTable(home) : null;
39
- const r = mod.resolveDefaultOwner ? mod.resolveDefaultOwner(identity) : null;
40
- if (r && r.ownerUser) return { user: r.ownerUser, source: `${rel} (${r.source})`, notes: r.notes ?? [] };
41
- } catch { /* 试下一个候选 */ }
54
+ for (const rel of OWNER_SOURCES) {
55
+ const hit = await askOwnerSource(rel, home);
56
+ if (hit) return hit;
42
57
  }
43
- return { user: 'LiJianXiao', source: '内置回落(两条权威源都不可用)', notes: ['⚠️ 身份表解析不可用 ⇒ 回落常量'] };
58
+ return {
59
+ user: OWNER_FALLBACK,
60
+ source: '内置回落(两条权威源都不可用)',
61
+ notes: ['⚠️ 身份表解析不可用 ⇒ 回落常量'],
62
+ };
44
63
  }
45
64
 
46
- async function main() {
47
- const { flags } = parseArgs(argv);
48
- const json = Boolean(flags.json);
49
- const log = makeLogger({ json });
50
- const dbPath = flags.db ? path.resolve(String(flags.db)) : null;
51
- if (!dbPath) {
52
- console.error('用法错:必须给 --db <库路径>(本脚本没有默认库路径 —— 写死路径是明令禁止的)');
53
- process.exitCode = 2;
54
- return;
65
+ async function askOwnerSource(rel, home) {
66
+ try {
67
+ const mod = await import(new URL(rel, import.meta.url).href);
68
+ const table = mod.readIdentityTable ? mod.readIdentityTable(home) : null;
69
+ const resolved = mod.resolveDefaultOwner ? mod.resolveDefaultOwner(table) : null;
70
+ if (resolved?.ownerUser) {
71
+ return { user: resolved.ownerUser, source: `${rel} (${resolved.source})`, notes: resolved.notes ?? [] };
72
+ }
73
+ } catch {
74
+ /* 这一路不可用,换下一路 */
55
75
  }
56
- if (!fs.existsSync(dbPath)) {
57
- console.error(`环境不可用:库不存在 ${dbPath}`);
58
- process.exitCode = 3;
59
- return;
76
+ return null;
77
+ }
78
+
79
+ /** 数值开关一次性取齐(表驱动,省得逐个 if)。 */
80
+ function takeNumbers(flags) {
81
+ const bag = {};
82
+ for (const opt of NUMERIC_OPTIONS) bag[opt.to] = num(flags[opt.from], opt.fallback);
83
+ return bag;
84
+ }
85
+
86
+ /** 选工作集:非本机 owner 的先摘出去(归属从严,绝不出机),台账里盖过章的也不重复问。 */
87
+ function planWork(items, { ownerColumn, ownerUser, processed }) {
88
+ const queue = [];
89
+ const foreign = [];
90
+ for (const row of items) {
91
+ const holder = ownerColumn ? row.ownerUser : ownerUser;
92
+ if (holder !== ownerUser) {
93
+ foreign.push({ id: row.id, ownerUser: holder ?? '(空)' });
94
+ continue;
95
+ }
96
+ if (processed[String(row.id)]) continue;
97
+ queue.push(row);
60
98
  }
99
+ return { queue, foreign };
100
+ }
101
+
102
+ /** 模型输出落库前的清洗:只认本批的 id、只认固定集合里的标签(防模型自由发挥)。 */
103
+ function sanitizeModelItems(raw, batch) {
104
+ const wanted = new Map(batch.map((row) => [Number(row.id), row]));
105
+ const keep = [];
106
+ let dropped = 0;
107
+ for (const item of raw) {
108
+ const id = Number(item?.id);
109
+ if (!Number.isFinite(id) || !wanted.has(id)) continue;
110
+ const labels = Array.isArray(item?.domains) ? item.domains : Array.isArray(item?.domain) ? item.domain : [];
111
+ const valid = labels.filter((label) => D.DOMAIN_TOKEN_BY_LABEL[String(label).trim()] !== undefined);
112
+ dropped += labels.length - valid.length;
113
+ keep.push({ id, domains: valid });
114
+ }
115
+ // 模型漏答的条目按「没有标签」记 —— 写库时就当空集合,别让它们卡住后面的批
116
+ for (const row of batch) {
117
+ const id = Number(row.id);
118
+ if (!keep.some((r) => r.id === id)) keep.push({ id, domains: [] });
119
+ }
120
+ return { rows: keep, dropped };
121
+ }
122
+
123
+ function bump(state, field, delta) {
124
+ state.stats[field] = Number(state.stats[field] ?? 0) + delta;
125
+ }
126
+
127
+ /** 收尾:把 JSON 或人话吐出来,再把退出码钉住。 */
128
+ function seal(result, code, opts) {
129
+ result.exitCode = code;
130
+ if (opts.asJson) process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
131
+ else opts.log(`\n[${code === EXIT.OK ? 'OK' : 'STOP'}] exit=${code}`);
132
+ process.exitCode = code;
133
+ }
134
+
135
+ function refuse(code, message, asJson) {
136
+ process.stderr.write(`${message}\n`);
137
+ process.exitCode = code;
138
+ void asJson;
139
+ return code;
140
+ }
141
+
142
+ async function main() {
143
+ const { flags } = parseArgs();
144
+ const asJson = Boolean(flags.json);
145
+ const log = makeLogger({ json: asJson });
61
146
 
147
+ const dbPath = flags.db ? path.resolve(String(flags.db)) : '';
148
+ if (!dbPath) return refuse(EXIT.USAGE, '用法错:必须给 --db <库路径>(本脚本没有默认库路径 —— 写死路径是明令禁止的)', asJson);
149
+ if (!fs.existsSync(dbPath)) return refuse(EXIT.ENV, `环境不可用:库不存在 ${dbPath}`, asJson);
150
+
151
+ const conf = takeNumbers(flags);
152
+ const limits = { callsPerDay: conf.callsPerDay, tokensPerDay: conf.tokensPerDay };
62
153
  const apply = Boolean(flags.apply);
63
- const batchSize = num(flags.batch, 25);
64
- const limit = num(flags.limit, 0);
65
- const maxCharsPerItem = num(flags['max-chars-per-item'], 240);
66
- const limits = { callsPerDay: num(flags['max-calls'], D.COST_LIMITS.callsPerDay), tokensPerDay: num(flags['max-tokens'], D.COST_LIMITS.tokensPerDay) };
67
154
  const home = path.dirname(path.dirname(dbPath)); // <home>/memory/memory.db → <home>
68
155
  const stateFile = flags.state ? path.resolve(String(flags.state)) : path.join(home, 'memory', 'domain-tags-state.json');
69
156
  const snapshotDir = flags.snapshot ? path.resolve(String(flags.snapshot)) : null;
@@ -73,179 +160,176 @@ async function main() {
73
160
 
74
161
  const owner = await resolveOwnerUser(flags['owner-user'] ? String(flags['owner-user']) : null, home);
75
162
  const result = {
76
- at: nowIso, mode: apply ? 'apply' : 'dry-run', db: dbPath, model, baseUrl,
77
- ownerUser: owner.user, ownerSource: owner.source, limits, batchSize, maxCharsPerItem,
78
- stats: {}, cost: null, stopping: null, ok: false, exitCode: 0,
163
+ at: nowIso,
164
+ mode: apply ? 'apply' : 'dry-run',
165
+ db: dbPath,
166
+ model,
167
+ baseUrl,
168
+ ownerUser: owner.user,
169
+ ownerSource: owner.source,
170
+ limits,
171
+ batchSize: conf.batchSize,
172
+ maxCharsPerItem: conf.maxCharsPerItem,
173
+ stats: {},
174
+ cost: null,
175
+ stopping: null,
176
+ ok: false,
177
+ exitCode: EXIT.OK,
79
178
  };
80
179
 
81
- log(`库 : ${dbPath}`);
180
+ log(`目标库 : ${dbPath}`);
82
181
  log(`本机 owner: ${owner.user}(来源 ${owner.source})`);
83
- log(`模型 : ${model} @ ${baseUrl}(成本闸 ≤${limits.callsPerDay} 次 / ≤${limits.tokensPerDay} token 每日)`);
84
- for (const n of owner.notes ?? []) log(`⚠️ ${n}`);
182
+ log(`分类模型 : ${model} @ ${baseUrl}(成本闸 ≤${limits.callsPerDay} 次 / ≤${limits.tokensPerDay} token 每日)`);
183
+ for (const note of owner.notes ?? []) log(`⚠️ ${note}`);
85
184
 
86
- // ── 只读取样 + 过滤 ────────────────────────────────────────────────────────
87
- const scanned = D.readCandidates(dbPath, { limit, maxCharsPerItem });
88
- const state = D.readState(stateFile, dbPath, nowIso);
185
+ const scanned = D.readCandidates(dbPath, { limit: conf.limit, maxCharsPerItem: conf.maxCharsPerItem });
186
+ const book = D.readState(stateFile, dbPath, nowIso);
89
187
  if (!scanned.ownerColumn) log('⚠️ 库无 owner_user 列(未迁移的老库形态)⇒ 全部条目按「本机 owner」处理');
90
188
 
91
- const pending = [];
92
- const skippedOwner = [];
93
- for (const it of scanned.items) {
94
- const ownerUser = scanned.ownerColumn ? it.ownerUser : owner.user;
95
- if (ownerUser !== owner.user) {
96
- // D2 从严:归属不明 / 非本机 owner ⇒ **一律不送第三方模型**,跳过并记账
97
- skippedOwner.push({ id: it.id, ownerUser: ownerUser ?? '(空)' });
98
- continue;
99
- }
100
- if (state.processed[String(it.id)]) continue; // 幂等台账:已处理过的不再送模型
101
- pending.push(it);
102
- }
103
- const batches = Math.ceil(pending.length / batchSize);
104
- const estimated = pending.slice(0, batchSize).length ? D.estimateTokens(pending.slice(0, batchSize)) : 0;
189
+ const { queue, foreign } = planWork(scanned.items, {
190
+ ownerColumn: scanned.ownerColumn,
191
+ ownerUser: owner.user,
192
+ processed: book.processed,
193
+ });
194
+ const batchCount = Math.ceil(queue.length / conf.batchSize);
195
+ const firstBatchTokens = queue.length ? D.estimateTokens(queue.slice(0, conf.batchSize)) : 0;
196
+
105
197
  result.stats = {
106
- scanned: scanned.items.length, skippedOwner: skippedOwner.length, alreadyProcessed: scanned.items.length - pending.length - skippedOwner.length,
107
- pending: pending.length, plannedBatches: batches, estimatedTokensPerBatch: estimated,
198
+ scanned: scanned.items.length,
199
+ skippedOwner: foreign.length,
200
+ alreadyProcessed: scanned.items.length - queue.length - foreign.length,
201
+ pending: queue.length,
202
+ plannedBatches: batchCount,
203
+ estimatedTokensPerBatch: firstBatchTokens,
108
204
  };
109
- log(`扫描 ${scanned.items.length} 条 · 跳过(非本机 owner) ${skippedOwner.length} · 已处理(台账) ${result.stats.alreadyProcessed} · 待处理 ${pending.length}(${batches} 批)`);
110
- if (skippedOwner.length) log(` ⚠️ 跳过条目(D2 从严:不进第三方)id=[${skippedOwner.map((s) => s.id).slice(0, 20).join(',')}${skippedOwner.length > 20 ? '…' : ''}]`);
205
+ log(`扫描 ${scanned.items.length} 条 · 跳过(非本机 owner) ${foreign.length} · 已处理(台账) ${result.stats.alreadyProcessed} · 待处理 ${queue.length}(${batchCount} 批)`);
206
+ if (foreign.length) {
207
+ const preview = foreign.map((x) => x.id).slice(0, 20).join(',');
208
+ log(` ⚠️ 跳过条目(归属从严:不进第三方)id=[${preview}${foreign.length > 20 ? '…' : ''}]`);
209
+ }
111
210
 
112
211
  if (!apply) {
113
212
  log('\n[DRY-RUN] 未写任何内容(未调模型、未写 state);要真跑加 --apply');
114
213
  result.ok = true;
115
- result.state = { file: stateFile, cost: state.cost };
116
- return finish(result, 0, json, log);
214
+ result.state = { file: stateFile, cost: book.cost };
215
+ return seal(result, EXIT.OK, { asJson, log });
117
216
  }
118
217
 
119
- // ── 成本闸(调用前总检查) ─────────────────────────────────────────────────
120
- if (pending.length === 0) {
218
+ if (!queue.length) {
121
219
  log('\n无待处理条目(幂等台账已覆盖)—— 零写入');
122
220
  result.ok = true;
123
- result.state = { file: stateFile, cost: state.cost, processed: Object.keys(state.processed).length };
124
- return finish(result, 0, json, log);
221
+ result.state = { file: stateFile, cost: book.cost, processed: Object.keys(book.processed).length };
222
+ return seal(result, EXIT.OK, { asJson, log });
125
223
  }
126
- const gate0 = D.costGate(state, estimated, limits);
127
- if (!gate0.ok) {
128
- result.stopping = gate0.reason;
129
- log(`\n[STOP] 成本闸:${gate0.reason}(超限即停,不重试)`);
224
+
225
+ // 动钱之前先算总账:首批都过不去就干脆一批都不跑
226
+ const preflight = D.costGate(book, firstBatchTokens, limits);
227
+ if (!preflight.ok) {
228
+ result.stopping = preflight.reason;
229
+ log(`\n[STOP] 成本闸:${preflight.reason}(超限即停,不重试)`);
130
230
  result.ok = true;
131
- return finish(result, 0, json, log);
231
+ return seal(result, EXIT.OK, { asJson, log });
132
232
  }
133
233
 
134
- // ── 快照(§3.6 前置) ──────────────────────────────────────────────────────
135
234
  if (snapshotDir) {
136
- const dest = path.join(snapshotDir, 'memory.db');
137
- const file = D.vacuumInto(dbPath, dest);
138
- const st = fs.statSync(file);
139
- result.snapshot = { file, bytes: st.size };
140
- log(`✓ 快照:${file}(${st.size} B)`);
235
+ const file = D.vacuumInto(dbPath, path.join(snapshotDir, 'memory.db'));
236
+ result.snapshot = { file, bytes: fs.statSync(file).size };
237
+ log(`✓ 快照:${result.snapshot.file}(${result.snapshot.bytes} B)`);
141
238
  } else {
142
239
  log('⚠️ 未给 --snapshot:本次无快照(副本演练可接受,生产前置必须有)');
143
240
  result.snapshot = null;
144
241
  }
145
242
 
146
- // ── 分类模型 key ──────────────────────────────────────────────────────────
147
243
  const apiKey = process.env[D.DEFAULT_LLM.credentialKey] || D.readCredential(D.DEFAULT_LLM.credentialKey);
148
244
  if (!apiKey) {
149
245
  log(`环境不可用:找不到 ${D.DEFAULT_LLM.credentialKey}(env 与 .credentials.yaml 都没有)`);
150
- result.exitCode = 3;
151
- return finish(result, 3, json, log);
246
+ result.exitCode = EXIT.ENV;
247
+ return seal(result, EXIT.ENV, { asJson, log });
152
248
  }
153
249
 
154
- // ── 分批跑 ────────────────────────────────────────────────────────────────
250
+ const startedAt = Date.now();
155
251
  const db = D.openDb(dbPath, { write: true });
156
- state.stats.runs = Number(state.stats.runs ?? 0) + 1;
157
- const runStart = Date.now();
252
+ book.stats.runs = Number(book.stats.runs ?? 0) + 1;
158
253
  try {
159
- for (let b = 0; b < batches; b += 1) {
160
- const slice = pending.slice(b * batchSize, (b + 1) * batchSize);
161
- if (slice.length === 0) break;
162
- const est = D.estimateTokens(slice);
163
- const gate = D.costGate(state, est, limits);
254
+ for (let round = 0; round < batchCount; round += 1) {
255
+ const batch = queue.slice(round * conf.batchSize, (round + 1) * conf.batchSize);
256
+ if (!batch.length) break;
257
+ const gate = D.costGate(book, D.estimateTokens(batch), limits);
164
258
  if (!gate.ok) {
165
259
  result.stopping = gate.reason;
166
260
  log(`\n[STOP] 成本闸:${gate.reason}(超限即停,不重试)`);
167
261
  break;
168
262
  }
169
- const callStart = Date.now();
170
- const resp = await D.callClassifier({ items: slice, apiKey, baseUrl, model });
171
- if (!resp.ok) {
172
- result.stopping = `模型调用失败(第 ${b + 1} 批):${resp.error}`;
263
+
264
+ const roundStart = Date.now();
265
+ const reply = await D.callClassifier({ items: batch, apiKey, baseUrl, model });
266
+ if (!reply.ok) {
267
+ result.stopping = `模型调用失败(第 ${round + 1} 批):${reply.error}`;
173
268
  log(`\n[STOP] ${result.stopping}`);
174
269
  break;
175
270
  }
176
- state.cost.calls = Number(state.cost.calls ?? 0) + 1;
177
- state.cost.promptTokens = Number(state.cost.promptTokens ?? 0) + Number(resp.usage?.prompt_tokens ?? 0);
178
- state.cost.completionTokens = Number(state.cost.completionTokens ?? 0) + Number(resp.usage?.completion_tokens ?? 0);
179
- state.cost.date = nowIso.slice(0, 10);
180
-
181
- // 模型输出 → 只保留本批条目的 id(防模型编 id);域标签只从固定集合里认(appendDomainTags 里过滤)
182
- const inBatch = new Map(slice.map((it) => [Number(it.id), it]));
183
- const results = [];
184
- let unknownDomainDropped = 0;
185
- for (const item of resp.items) {
186
- const id = Number(item?.id);
187
- if (!Number.isFinite(id) || !inBatch.has(id)) continue;
188
- const rawDomains = Array.isArray(item?.domains) ? item.domains : (Array.isArray(item?.domain) ? item.domain : []);
189
- const valid = rawDomains.filter((d) => D.DOMAIN_TOKEN_BY_LABEL[String(d).trim()] !== undefined);
190
- unknownDomainDropped += rawDomains.length - valid.length;
191
- results.push({ id, domains: valid });
192
- }
193
- const missing = slice.filter((it) => !results.some((r) => r.id === Number(it.id)));
194
- for (const m of missing) results.push({ id: Number(m.id), domains: [] }); // 模型漏答 ⇒ 记为"无标签",不写库
271
+ book.cost.calls = Number(book.cost.calls ?? 0) + 1;
272
+ book.cost.promptTokens = Number(book.cost.promptTokens ?? 0) + Number(reply.usage?.prompt_tokens ?? 0);
273
+ book.cost.completionTokens = Number(book.cost.completionTokens ?? 0) + Number(reply.usage?.completion_tokens ?? 0);
274
+ book.cost.date = nowIso.slice(0, 10);
195
275
 
196
- const wr = await D.applyResults(db, results);
197
- if (!wr.ok) {
198
- result.stopping = `写库失败(第 ${b + 1} 批):${wr.error}`;
276
+ const cleaned = sanitizeModelItems(reply.items, batch);
277
+ const written = await D.applyResults(db, cleaned.rows);
278
+ if (!written.ok) {
279
+ result.stopping = `写库失败(第 ${round + 1} 批):${written.error}`;
199
280
  log(`\n[STOP] ${result.stopping}`);
200
281
  break;
201
282
  }
202
- // 幂等台账:本批已处理(含"无标签"——它们也算处理过,避免每轮重复问模型)
203
- for (const it of slice) {
204
- const r = results.find((x) => x.id === Number(it.id));
205
- state.processed[String(it.id)] = D.tagSetHash(r ? r.domains : []);
283
+
284
+ // 「问过了」入账:连「模型说没标签」的也算 —— 否则下一轮又会花钱问同一批
285
+ for (const row of batch) {
286
+ const hit = cleaned.rows.find((r) => r.id === Number(row.id));
287
+ book.processed[String(row.id)] = D.tagSetHash(hit ? hit.domains : []);
206
288
  }
207
- state.stats.batches = Number(state.stats.batches ?? 0) + 1;
208
- state.stats.tagged = Number(state.stats.tagged ?? 0) + wr.written;
209
- state.stats.noTag = Number(state.stats.noTag ?? 0) + results.filter((r) => r.domains.length === 0).length;
210
- state.stats.skippedOwner = skippedOwner.length;
211
- state.stats.skippedUnknownDomain = Number(state.stats.skippedUnknownDomain ?? 0) + unknownDomainDropped;
212
- state.updatedAt = new Date().toISOString();
213
- D.writeState(stateFile, state); // **每批落盘** ⇒ 可中断续跑
214
- log(` 批 ${b + 1}/${batches}: 调用 ${state.cost.calls} 次 · 写 ${wr.written} 条 · 已是 ${wr.unchanged} 条 · 耗时 ${Date.now() - callStart}ms · 当日 token ${Number(state.cost.promptTokens) + Number(state.cost.completionTokens)}`);
289
+ book.stats.batches = Number(book.stats.batches ?? 0) + 1;
290
+ bump(book, 'tagged', written.written);
291
+ bump(book, 'noTag', cleaned.rows.filter((r) => r.domains.length === 0).length);
292
+ book.stats.skippedOwner = foreign.length;
293
+ bump(book, 'skippedUnknownDomain', cleaned.dropped);
294
+ book.updatedAt = new Date().toISOString();
295
+ D.writeState(stateFile, book); // 每批落盘 ⇒ 断电也能接着跑
296
+ const dayTokens = Number(book.cost.promptTokens) + Number(book.cost.completionTokens);
297
+ log(` 批 ${round + 1}/${batchCount}: 调用 ${book.cost.calls} 次 · 写 ${written.written} 条 · 已是 ${written.unchanged} 条 · 耗时 ${Date.now() - roundStart}ms · 当日 token ${dayTokens}`);
215
298
  }
216
299
  } finally {
217
- try { db.close(); } catch { /* 忽略 */ }
300
+ try {
301
+ db.close();
302
+ } catch {
303
+ /* 关不掉也不影响结果 */
304
+ }
218
305
  }
219
306
 
220
- // ── 汇总 + 日报一行 ───────────────────────────────────────────────────────
221
- for (const s of skippedOwner) state.skipped[String(s.id)] = `owner=${s.ownerUser}`;
222
- state.updatedAt = new Date().toISOString();
223
- D.writeState(stateFile, state);
224
- const usedTokens = Number(state.cost.promptTokens) + Number(state.cost.completionTokens);
307
+ // 台账里记一笔「这些为什么没送」,方便下次查账
308
+ for (const skip of foreign) book.skipped[String(skip.id)] = `owner=${skip.ownerUser}`;
309
+ book.updatedAt = new Date().toISOString();
310
+ D.writeState(stateFile, book);
311
+
312
+ const usedTokens = Number(book.cost.promptTokens) + Number(book.cost.completionTokens);
225
313
  const line = D.renderDailyLine({
226
- mode: 'apply', stats: state.stats, cost: state.cost, state: { processedCount: Object.keys(state.processed).length },
227
- limits, stoppedReason: result.stopping,
314
+ mode: 'apply',
315
+ stats: book.stats,
316
+ cost: book.cost,
317
+ state: { processedCount: Object.keys(book.processed).length },
318
+ limits,
319
+ stoppedReason: result.stopping,
228
320
  });
229
- result.cost = { ...state.cost, usedTokens };
230
- result.state = { file: stateFile, processed: Object.keys(state.processed).length };
231
- log(`\n完成:批次 ${state.stats.batches} · 本轮写 ${state.stats.tagged} 条 · 当日调用 ${state.cost.calls}/${limits.callsPerDay} · 当日 token ${usedTokens}/${limits.tokensPerDay} · 用时 ${Date.now() - runStart}ms`);
321
+ result.cost = { ...book.cost, usedTokens };
322
+ result.state = { file: stateFile, processed: Object.keys(book.processed).length };
323
+ log(`\n完成:批次 ${book.stats.batches} · 本轮写 ${book.stats.tagged} 条 · 当日调用 ${book.cost.calls}/${limits.callsPerDay} · 当日 token ${usedTokens}/${limits.tokensPerDay} · 用时 ${Date.now() - startedAt}ms`);
324
+
232
325
  if (!flags['no-daily-report']) {
233
326
  const reportFile = flags['daily-report'] ? path.resolve(String(flags['daily-report'])) : D.dailyReportPath({});
234
- const r = D.updateDailySection({ file: reportFile, line, dryRun: false });
235
- result.dailyReport = { file: reportFile, ...r, line };
236
- log(`${r.ok ? '✓' : '⚠️'} 日报:${reportFile}${r.ok ? `(${r.replaced ? '段替换' : '追加段'})` : `(${r.reason})`}`);
327
+ const done = D.updateDailySection({ file: reportFile, line, dryRun: false });
328
+ result.dailyReport = { file: reportFile, ...done, line };
329
+ const suffix = done.ok ? `(${done.replaced ? '段替换' : '追加段'})` : `(${done.reason})`;
330
+ log(`${done.ok ? '✓' : '⚠️'} 日报:${reportFile}${suffix}`);
237
331
  }
238
332
  result.dailyLine = line;
239
333
  result.ok = true;
240
- return finish(result, 0, json, log);
241
- }
242
-
243
- function finish(result, code, json, log) {
244
- result.exitCode = code;
245
- if (json) {
246
- process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
247
- } else {
248
- log(`\n[${code === 0 ? 'OK' : 'STOP'}] exit=${code}`);
249
- }
250
- process.exitCode = code;
334
+ return seal(result, EXIT.OK, { asJson, log });
251
335
  }