@kira905/dsh-memory-toolkit 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +92 -0
- package/README.en.md +225 -2
- package/README.md +180 -2
- package/lib/deploy-guard.mjs +463 -275
- package/lib/guard-http.mjs +59 -62
- package/lib/pitfall-form-map.mjs +94 -84
- package/lib/pitfall-score.mjs +262 -185
- package/lib/session-file-name.mjs +116 -100
- package/lib/zstd-frames.mjs +88 -67
- package/memory-db-tools/README.md +29 -1
- package/memory-db-tools/audit-analyze.mjs +1 -1
- package/memory-db-tools/audit-export.mjs +60 -16
- package/memory-db-tools/audit-struct-probe.mjs +29 -8
- package/memory-db-tools/lib/known-devices.mjs +137 -0
- package/memory-engine/README.md +58 -4
- package/memory-engine/bench/bench.mjs +1 -1
- package/memory-engine/cli/band-sentinel.mjs +164 -0
- package/memory-engine/cli/distill-rollback.mjs +112 -44
- package/memory-engine/cli/distill-sessions.mjs +237 -0
- package/memory-engine/cli/distill.mjs +196 -95
- package/memory-engine/cli/doc-index-rebuild.mjs +142 -108
- package/memory-engine/cli/doc-index.mjs +47 -38
- package/memory-engine/cli/domain-tags.mjs +246 -162
- package/memory-engine/cli/layer-migrate.mjs +176 -0
- package/memory-engine/cli/layer-move.mjs +189 -0
- package/memory-engine/cli/pitfall-hook.mjs +136 -70
- package/memory-engine/cli/recall.mjs +243 -110
- package/memory-engine/cli/selfcheck.mjs +93 -44
- package/memory-engine/cli/serve-start.mjs +221 -140
- package/memory-engine/cli/sweep.mjs +83 -44
- package/memory-engine/cli/sync.mjs +93 -57
- package/memory-engine/config/audit-devices.json +6 -0
- package/memory-engine/config/bands.json +7 -0
- package/memory-engine/config/layers.json +7 -0
- package/memory-engine/dict/LICENSE-/350/257/215/345/205/270.md +53 -0
- package/memory-engine/dict/lexicon-config.json +11 -0
- package/memory-engine/dict//350/257/215/345/205/270.txt +109750 -0
- package/memory-engine/dict//350/257/215/351/242/221.txt +20000 -0
- package/memory-engine/lib/band-sentinel.mjs +417 -0
- package/memory-engine/lib/cli-strict.mjs +128 -0
- package/memory-engine/lib/cli-util.mjs +147 -125
- package/memory-engine/lib/config.mjs +83 -96
- package/memory-engine/lib/dirty-ledger.mjs +127 -94
- package/memory-engine/lib/distill-rollback.mjs +173 -133
- package/memory-engine/lib/distill.mjs +878 -585
- package/memory-engine/lib/doc-index-store.mjs +208 -235
- package/memory-engine/lib/doc-index.mjs +351 -350
- package/memory-engine/lib/doc-scan.mjs +178 -156
- package/memory-engine/lib/doc-summarize.mjs +220 -205
- package/memory-engine/lib/domain-tags.mjs +358 -277
- package/memory-engine/lib/embedder.mjs +175 -194
- package/memory-engine/lib/hashes.mjs +22 -7
- package/memory-engine/lib/layer-migrate.mjs +330 -0
- package/memory-engine/lib/layer-move.mjs +307 -0
- package/memory-engine/lib/layers.mjs +191 -0
- package/memory-engine/lib/lexicon-recall.mjs +473 -0
- package/memory-engine/lib/memory-source.mjs +78 -80
- package/memory-engine/lib/pitfall-source.mjs +115 -114
- package/memory-engine/lib/policy-source.mjs +279 -261
- package/memory-engine/lib/recall-log.mjs +285 -0
- package/memory-engine/lib/recall.mjs +565 -240
- package/memory-engine/lib/selfcheck.mjs +142 -106
- package/memory-engine/lib/session-distill.mjs +603 -0
- package/memory-engine/lib/session-source.mjs +421 -0
- package/memory-engine/lib/sweep.mjs +107 -130
- package/memory-engine/lib/sync.mjs +110 -140
- package/memory-engine/lib/tick-hook.mjs +123 -148
- package/memory-engine/lib/vectors-store.mjs +213 -188
- package/memory-engine/test/12-layering.test.mjs +539 -0
- package/memory-engine/test/13-session-distill.test.mjs +616 -0
- package/memory-engine/tools/bridge-smoke.mjs +94 -55
- package/memory-engine/tools/fetch-model.mjs +167 -87
- package/memory-engine/tools/smoke-recall.mjs +94 -38
- package/memory-engine/tools/verify-recall-chain.mjs +413 -0
- package/package.json +1 -1
|
@@ -1,69 +1,156 @@
|
|
|
1
|
-
//
|
|
1
|
+
// 记忆域标签补齐 CLI
|
|
2
2
|
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
// node cli/domain-tags.mjs --db <库路径> --apply # 真跑(副本库;分批 + 可续跑)
|
|
6
|
-
// node cli/domain-tags.mjs --db <库路径> --apply --snapshot <dir> # 跑前 VACUUM INTO 快照
|
|
7
|
-
// node cli/domain-tags.mjs --db <库路径> --limit 25 --batch 25 # 只跑 1 批(试跑)
|
|
8
|
-
// node cli/domain-tags.mjs --db <库路径> --apply --json # 机器可读(stdout 只有 JSON)
|
|
3
|
+
// 干的事:把库里待处理的记忆条目按批送给分类模型,模型只准从固定的域标签集合里挑,
|
|
4
|
+
// 挑完写回库、并把「这批问过了」记进幂等台账(所以中断重跑不会重复花钱)。
|
|
9
5
|
//
|
|
10
|
-
//
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
//
|
|
6
|
+
// 命令行形态(对外契约,参数名 / 默认值 / 退出码一律照旧):
|
|
7
|
+
// --db <path> 必填。本工具没有默认库路径 —— 写死库路径是明令禁止的
|
|
8
|
+
// --apply 真跑(缺省是 dry-run:只规划、不调模型、不写库)
|
|
9
|
+
// --snapshot <dir> 真跑前先 VACUUM INTO 到 <dir>/memory.db
|
|
10
|
+
// --batch <n> 每批条数(25)
|
|
11
|
+
// --limit <n> 本次最多取多少条(0 = 不限)
|
|
12
|
+
// --max-chars-per-item <n> 送模型时单条正文截断长度(240)
|
|
13
|
+
// --max-calls / --max-tokens 每日成本闸(缺省取库里的 COST_LIMITS)
|
|
14
|
+
// --owner-user <id> 本机 owner 的 userid;缺省先查身份表,再不行用内置常量
|
|
15
|
+
// --state <file> 幂等台账落点(缺省 <home>/memory/domain-tags-state.json)
|
|
16
|
+
// --model / --base-url 分类模型档
|
|
17
|
+
// --no-daily-report 不写日报段
|
|
18
|
+
// --daily-report <file> 覆盖日报落点
|
|
19
|
+
// --json stdout 只留 JSON,人看的话全走 stderr
|
|
14
20
|
//
|
|
15
|
-
// 退出码:0 =
|
|
21
|
+
// 退出码:0 = 正常收尾(包含「成本闸到点、主动停」这种受控停止)
|
|
22
|
+
// 1 = 有异常 / 2 = 用法错 / 3 = 环境不可用(没 key、库不在)
|
|
16
23
|
import fs from 'node:fs';
|
|
17
|
-
import os from 'node:os';
|
|
18
24
|
import path from 'node:path';
|
|
19
|
-
import { DatabaseSync } from 'node:sqlite';
|
|
20
25
|
import { isMainEntry, parseArgs, num, makeLogger } from '../lib/cli-util.mjs';
|
|
21
26
|
import * as D from '../lib/domain-tags.mjs';
|
|
22
27
|
|
|
23
|
-
const
|
|
28
|
+
const EXIT = { OK: 0, FAIL: 1, USAGE: 2, ENV: 3 };
|
|
29
|
+
const OWNER_FALLBACK = 'LiJianXiao';
|
|
30
|
+
const OWNER_SOURCES = ['../../memory-db-tools/lib/memory-scope.mjs', '../dsh-memory-local/src/schema-migrate.mjs'];
|
|
31
|
+
|
|
32
|
+
/** 数值型开关:源长什么样、落到哪个字段、不给时取什么。 */
|
|
33
|
+
const NUMERIC_OPTIONS = [
|
|
34
|
+
{ from: 'batch', to: 'batchSize', fallback: 25 },
|
|
35
|
+
{ from: 'limit', to: 'limit', fallback: 0 },
|
|
36
|
+
{ from: 'max-chars-per-item', to: 'maxCharsPerItem', fallback: 240 },
|
|
37
|
+
{ from: 'max-calls', to: 'callsPerDay', fallback: D.COST_LIMITS.callsPerDay },
|
|
38
|
+
{ from: 'max-tokens', to: 'tokensPerDay', fallback: D.COST_LIMITS.tokensPerDay },
|
|
39
|
+
];
|
|
40
|
+
|
|
24
41
|
if (isMainEntry(import.meta.url)) {
|
|
25
42
|
main().catch((e) => {
|
|
26
|
-
console.error(`未捕获异常:${e
|
|
27
|
-
process.exitCode =
|
|
43
|
+
console.error(`未捕获异常:${e?.stack ?? e}`);
|
|
44
|
+
process.exitCode = EXIT.FAIL;
|
|
28
45
|
});
|
|
29
46
|
}
|
|
30
47
|
|
|
31
|
-
/**
|
|
48
|
+
/**
|
|
49
|
+
* 本机 owner 的 userid:先问权威实现,再问自检镜像,最后才用内置常量。
|
|
50
|
+
* 顺序不能反 —— 常量只是兜底,不是口径。
|
|
51
|
+
*/
|
|
32
52
|
async function resolveOwnerUser(explicit, home) {
|
|
33
53
|
if (explicit) return { user: explicit, source: 'cli' };
|
|
34
|
-
const
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
const mod = await import(new URL(rel, import.meta.url).href);
|
|
38
|
-
const identity = mod.readIdentityTable ? mod.readIdentityTable(home) : null;
|
|
39
|
-
const r = mod.resolveDefaultOwner ? mod.resolveDefaultOwner(identity) : null;
|
|
40
|
-
if (r && r.ownerUser) return { user: r.ownerUser, source: `${rel} (${r.source})`, notes: r.notes ?? [] };
|
|
41
|
-
} catch { /* 试下一个候选 */ }
|
|
54
|
+
for (const rel of OWNER_SOURCES) {
|
|
55
|
+
const hit = await askOwnerSource(rel, home);
|
|
56
|
+
if (hit) return hit;
|
|
42
57
|
}
|
|
43
|
-
return {
|
|
58
|
+
return {
|
|
59
|
+
user: OWNER_FALLBACK,
|
|
60
|
+
source: '内置回落(两条权威源都不可用)',
|
|
61
|
+
notes: ['⚠️ 身份表解析不可用 ⇒ 回落常量'],
|
|
62
|
+
};
|
|
44
63
|
}
|
|
45
64
|
|
|
46
|
-
async function
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
65
|
+
async function askOwnerSource(rel, home) {
|
|
66
|
+
try {
|
|
67
|
+
const mod = await import(new URL(rel, import.meta.url).href);
|
|
68
|
+
const table = mod.readIdentityTable ? mod.readIdentityTable(home) : null;
|
|
69
|
+
const resolved = mod.resolveDefaultOwner ? mod.resolveDefaultOwner(table) : null;
|
|
70
|
+
if (resolved?.ownerUser) {
|
|
71
|
+
return { user: resolved.ownerUser, source: `${rel} (${resolved.source})`, notes: resolved.notes ?? [] };
|
|
72
|
+
}
|
|
73
|
+
} catch {
|
|
74
|
+
/* 这一路不可用,换下一路 */
|
|
55
75
|
}
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
76
|
+
return null;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** 数值开关一次性取齐(表驱动,省得逐个 if)。 */
|
|
80
|
+
function takeNumbers(flags) {
|
|
81
|
+
const bag = {};
|
|
82
|
+
for (const opt of NUMERIC_OPTIONS) bag[opt.to] = num(flags[opt.from], opt.fallback);
|
|
83
|
+
return bag;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** 选工作集:非本机 owner 的先摘出去(归属从严,绝不出机),台账里盖过章的也不重复问。 */
|
|
87
|
+
function planWork(items, { ownerColumn, ownerUser, processed }) {
|
|
88
|
+
const queue = [];
|
|
89
|
+
const foreign = [];
|
|
90
|
+
for (const row of items) {
|
|
91
|
+
const holder = ownerColumn ? row.ownerUser : ownerUser;
|
|
92
|
+
if (holder !== ownerUser) {
|
|
93
|
+
foreign.push({ id: row.id, ownerUser: holder ?? '(空)' });
|
|
94
|
+
continue;
|
|
95
|
+
}
|
|
96
|
+
if (processed[String(row.id)]) continue;
|
|
97
|
+
queue.push(row);
|
|
60
98
|
}
|
|
99
|
+
return { queue, foreign };
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** 模型输出落库前的清洗:只认本批的 id、只认固定集合里的标签(防模型自由发挥)。 */
|
|
103
|
+
function sanitizeModelItems(raw, batch) {
|
|
104
|
+
const wanted = new Map(batch.map((row) => [Number(row.id), row]));
|
|
105
|
+
const keep = [];
|
|
106
|
+
let dropped = 0;
|
|
107
|
+
for (const item of raw) {
|
|
108
|
+
const id = Number(item?.id);
|
|
109
|
+
if (!Number.isFinite(id) || !wanted.has(id)) continue;
|
|
110
|
+
const labels = Array.isArray(item?.domains) ? item.domains : Array.isArray(item?.domain) ? item.domain : [];
|
|
111
|
+
const valid = labels.filter((label) => D.DOMAIN_TOKEN_BY_LABEL[String(label).trim()] !== undefined);
|
|
112
|
+
dropped += labels.length - valid.length;
|
|
113
|
+
keep.push({ id, domains: valid });
|
|
114
|
+
}
|
|
115
|
+
// 模型漏答的条目按「没有标签」记 —— 写库时就当空集合,别让它们卡住后面的批
|
|
116
|
+
for (const row of batch) {
|
|
117
|
+
const id = Number(row.id);
|
|
118
|
+
if (!keep.some((r) => r.id === id)) keep.push({ id, domains: [] });
|
|
119
|
+
}
|
|
120
|
+
return { rows: keep, dropped };
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function bump(state, field, delta) {
|
|
124
|
+
state.stats[field] = Number(state.stats[field] ?? 0) + delta;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** 收尾:把 JSON 或人话吐出来,再把退出码钉住。 */
|
|
128
|
+
function seal(result, code, opts) {
|
|
129
|
+
result.exitCode = code;
|
|
130
|
+
if (opts.asJson) process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
|
|
131
|
+
else opts.log(`\n[${code === EXIT.OK ? 'OK' : 'STOP'}] exit=${code}`);
|
|
132
|
+
process.exitCode = code;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
function refuse(code, message, asJson) {
|
|
136
|
+
process.stderr.write(`${message}\n`);
|
|
137
|
+
process.exitCode = code;
|
|
138
|
+
void asJson;
|
|
139
|
+
return code;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
async function main() {
|
|
143
|
+
const { flags } = parseArgs();
|
|
144
|
+
const asJson = Boolean(flags.json);
|
|
145
|
+
const log = makeLogger({ json: asJson });
|
|
61
146
|
|
|
147
|
+
const dbPath = flags.db ? path.resolve(String(flags.db)) : '';
|
|
148
|
+
if (!dbPath) return refuse(EXIT.USAGE, '用法错:必须给 --db <库路径>(本脚本没有默认库路径 —— 写死路径是明令禁止的)', asJson);
|
|
149
|
+
if (!fs.existsSync(dbPath)) return refuse(EXIT.ENV, `环境不可用:库不存在 ${dbPath}`, asJson);
|
|
150
|
+
|
|
151
|
+
const conf = takeNumbers(flags);
|
|
152
|
+
const limits = { callsPerDay: conf.callsPerDay, tokensPerDay: conf.tokensPerDay };
|
|
62
153
|
const apply = Boolean(flags.apply);
|
|
63
|
-
const batchSize = num(flags.batch, 25);
|
|
64
|
-
const limit = num(flags.limit, 0);
|
|
65
|
-
const maxCharsPerItem = num(flags['max-chars-per-item'], 240);
|
|
66
|
-
const limits = { callsPerDay: num(flags['max-calls'], D.COST_LIMITS.callsPerDay), tokensPerDay: num(flags['max-tokens'], D.COST_LIMITS.tokensPerDay) };
|
|
67
154
|
const home = path.dirname(path.dirname(dbPath)); // <home>/memory/memory.db → <home>
|
|
68
155
|
const stateFile = flags.state ? path.resolve(String(flags.state)) : path.join(home, 'memory', 'domain-tags-state.json');
|
|
69
156
|
const snapshotDir = flags.snapshot ? path.resolve(String(flags.snapshot)) : null;
|
|
@@ -73,179 +160,176 @@ async function main() {
|
|
|
73
160
|
|
|
74
161
|
const owner = await resolveOwnerUser(flags['owner-user'] ? String(flags['owner-user']) : null, home);
|
|
75
162
|
const result = {
|
|
76
|
-
at: nowIso,
|
|
77
|
-
|
|
78
|
-
|
|
163
|
+
at: nowIso,
|
|
164
|
+
mode: apply ? 'apply' : 'dry-run',
|
|
165
|
+
db: dbPath,
|
|
166
|
+
model,
|
|
167
|
+
baseUrl,
|
|
168
|
+
ownerUser: owner.user,
|
|
169
|
+
ownerSource: owner.source,
|
|
170
|
+
limits,
|
|
171
|
+
batchSize: conf.batchSize,
|
|
172
|
+
maxCharsPerItem: conf.maxCharsPerItem,
|
|
173
|
+
stats: {},
|
|
174
|
+
cost: null,
|
|
175
|
+
stopping: null,
|
|
176
|
+
ok: false,
|
|
177
|
+
exitCode: EXIT.OK,
|
|
79
178
|
};
|
|
80
179
|
|
|
81
|
-
log(
|
|
180
|
+
log(`目标库 : ${dbPath}`);
|
|
82
181
|
log(`本机 owner: ${owner.user}(来源 ${owner.source})`);
|
|
83
|
-
log(
|
|
84
|
-
for (const
|
|
182
|
+
log(`分类模型 : ${model} @ ${baseUrl}(成本闸 ≤${limits.callsPerDay} 次 / ≤${limits.tokensPerDay} token 每日)`);
|
|
183
|
+
for (const note of owner.notes ?? []) log(`⚠️ ${note}`);
|
|
85
184
|
|
|
86
|
-
|
|
87
|
-
const
|
|
88
|
-
const state = D.readState(stateFile, dbPath, nowIso);
|
|
185
|
+
const scanned = D.readCandidates(dbPath, { limit: conf.limit, maxCharsPerItem: conf.maxCharsPerItem });
|
|
186
|
+
const book = D.readState(stateFile, dbPath, nowIso);
|
|
89
187
|
if (!scanned.ownerColumn) log('⚠️ 库无 owner_user 列(未迁移的老库形态)⇒ 全部条目按「本机 owner」处理');
|
|
90
188
|
|
|
91
|
-
const
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
}
|
|
100
|
-
if (state.processed[String(it.id)]) continue; // 幂等台账:已处理过的不再送模型
|
|
101
|
-
pending.push(it);
|
|
102
|
-
}
|
|
103
|
-
const batches = Math.ceil(pending.length / batchSize);
|
|
104
|
-
const estimated = pending.slice(0, batchSize).length ? D.estimateTokens(pending.slice(0, batchSize)) : 0;
|
|
189
|
+
const { queue, foreign } = planWork(scanned.items, {
|
|
190
|
+
ownerColumn: scanned.ownerColumn,
|
|
191
|
+
ownerUser: owner.user,
|
|
192
|
+
processed: book.processed,
|
|
193
|
+
});
|
|
194
|
+
const batchCount = Math.ceil(queue.length / conf.batchSize);
|
|
195
|
+
const firstBatchTokens = queue.length ? D.estimateTokens(queue.slice(0, conf.batchSize)) : 0;
|
|
196
|
+
|
|
105
197
|
result.stats = {
|
|
106
|
-
scanned: scanned.items.length,
|
|
107
|
-
|
|
198
|
+
scanned: scanned.items.length,
|
|
199
|
+
skippedOwner: foreign.length,
|
|
200
|
+
alreadyProcessed: scanned.items.length - queue.length - foreign.length,
|
|
201
|
+
pending: queue.length,
|
|
202
|
+
plannedBatches: batchCount,
|
|
203
|
+
estimatedTokensPerBatch: firstBatchTokens,
|
|
108
204
|
};
|
|
109
|
-
log(`扫描 ${scanned.items.length} 条 · 跳过(非本机 owner) ${
|
|
110
|
-
if (
|
|
205
|
+
log(`扫描 ${scanned.items.length} 条 · 跳过(非本机 owner) ${foreign.length} · 已处理(台账) ${result.stats.alreadyProcessed} · 待处理 ${queue.length}(${batchCount} 批)`);
|
|
206
|
+
if (foreign.length) {
|
|
207
|
+
const preview = foreign.map((x) => x.id).slice(0, 20).join(',');
|
|
208
|
+
log(` ⚠️ 跳过条目(归属从严:不进第三方)id=[${preview}${foreign.length > 20 ? '…' : ''}]`);
|
|
209
|
+
}
|
|
111
210
|
|
|
112
211
|
if (!apply) {
|
|
113
212
|
log('\n[DRY-RUN] 未写任何内容(未调模型、未写 state);要真跑加 --apply');
|
|
114
213
|
result.ok = true;
|
|
115
|
-
result.state = { file: stateFile, cost:
|
|
116
|
-
return
|
|
214
|
+
result.state = { file: stateFile, cost: book.cost };
|
|
215
|
+
return seal(result, EXIT.OK, { asJson, log });
|
|
117
216
|
}
|
|
118
217
|
|
|
119
|
-
|
|
120
|
-
if (pending.length === 0) {
|
|
218
|
+
if (!queue.length) {
|
|
121
219
|
log('\n无待处理条目(幂等台账已覆盖)—— 零写入');
|
|
122
220
|
result.ok = true;
|
|
123
|
-
result.state = { file: stateFile, cost:
|
|
124
|
-
return
|
|
221
|
+
result.state = { file: stateFile, cost: book.cost, processed: Object.keys(book.processed).length };
|
|
222
|
+
return seal(result, EXIT.OK, { asJson, log });
|
|
125
223
|
}
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
224
|
+
|
|
225
|
+
// 动钱之前先算总账:首批都过不去就干脆一批都不跑
|
|
226
|
+
const preflight = D.costGate(book, firstBatchTokens, limits);
|
|
227
|
+
if (!preflight.ok) {
|
|
228
|
+
result.stopping = preflight.reason;
|
|
229
|
+
log(`\n[STOP] 成本闸:${preflight.reason}(超限即停,不重试)`);
|
|
130
230
|
result.ok = true;
|
|
131
|
-
return
|
|
231
|
+
return seal(result, EXIT.OK, { asJson, log });
|
|
132
232
|
}
|
|
133
233
|
|
|
134
|
-
// ── 快照(§3.6 前置) ──────────────────────────────────────────────────────
|
|
135
234
|
if (snapshotDir) {
|
|
136
|
-
const
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
result.snapshot = { file, bytes: st.size };
|
|
140
|
-
log(`✓ 快照:${file}(${st.size} B)`);
|
|
235
|
+
const file = D.vacuumInto(dbPath, path.join(snapshotDir, 'memory.db'));
|
|
236
|
+
result.snapshot = { file, bytes: fs.statSync(file).size };
|
|
237
|
+
log(`✓ 快照:${result.snapshot.file}(${result.snapshot.bytes} B)`);
|
|
141
238
|
} else {
|
|
142
239
|
log('⚠️ 未给 --snapshot:本次无快照(副本演练可接受,生产前置必须有)');
|
|
143
240
|
result.snapshot = null;
|
|
144
241
|
}
|
|
145
242
|
|
|
146
|
-
// ── 分类模型 key ──────────────────────────────────────────────────────────
|
|
147
243
|
const apiKey = process.env[D.DEFAULT_LLM.credentialKey] || D.readCredential(D.DEFAULT_LLM.credentialKey);
|
|
148
244
|
if (!apiKey) {
|
|
149
245
|
log(`环境不可用:找不到 ${D.DEFAULT_LLM.credentialKey}(env 与 .credentials.yaml 都没有)`);
|
|
150
|
-
result.exitCode =
|
|
151
|
-
return
|
|
246
|
+
result.exitCode = EXIT.ENV;
|
|
247
|
+
return seal(result, EXIT.ENV, { asJson, log });
|
|
152
248
|
}
|
|
153
249
|
|
|
154
|
-
|
|
250
|
+
const startedAt = Date.now();
|
|
155
251
|
const db = D.openDb(dbPath, { write: true });
|
|
156
|
-
|
|
157
|
-
const runStart = Date.now();
|
|
252
|
+
book.stats.runs = Number(book.stats.runs ?? 0) + 1;
|
|
158
253
|
try {
|
|
159
|
-
for (let
|
|
160
|
-
const
|
|
161
|
-
if (
|
|
162
|
-
const
|
|
163
|
-
const gate = D.costGate(state, est, limits);
|
|
254
|
+
for (let round = 0; round < batchCount; round += 1) {
|
|
255
|
+
const batch = queue.slice(round * conf.batchSize, (round + 1) * conf.batchSize);
|
|
256
|
+
if (!batch.length) break;
|
|
257
|
+
const gate = D.costGate(book, D.estimateTokens(batch), limits);
|
|
164
258
|
if (!gate.ok) {
|
|
165
259
|
result.stopping = gate.reason;
|
|
166
260
|
log(`\n[STOP] 成本闸:${gate.reason}(超限即停,不重试)`);
|
|
167
261
|
break;
|
|
168
262
|
}
|
|
169
|
-
|
|
170
|
-
const
|
|
171
|
-
|
|
172
|
-
|
|
263
|
+
|
|
264
|
+
const roundStart = Date.now();
|
|
265
|
+
const reply = await D.callClassifier({ items: batch, apiKey, baseUrl, model });
|
|
266
|
+
if (!reply.ok) {
|
|
267
|
+
result.stopping = `模型调用失败(第 ${round + 1} 批):${reply.error}`;
|
|
173
268
|
log(`\n[STOP] ${result.stopping}`);
|
|
174
269
|
break;
|
|
175
270
|
}
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
// 模型输出 → 只保留本批条目的 id(防模型编 id);域标签只从固定集合里认(appendDomainTags 里过滤)
|
|
182
|
-
const inBatch = new Map(slice.map((it) => [Number(it.id), it]));
|
|
183
|
-
const results = [];
|
|
184
|
-
let unknownDomainDropped = 0;
|
|
185
|
-
for (const item of resp.items) {
|
|
186
|
-
const id = Number(item?.id);
|
|
187
|
-
if (!Number.isFinite(id) || !inBatch.has(id)) continue;
|
|
188
|
-
const rawDomains = Array.isArray(item?.domains) ? item.domains : (Array.isArray(item?.domain) ? item.domain : []);
|
|
189
|
-
const valid = rawDomains.filter((d) => D.DOMAIN_TOKEN_BY_LABEL[String(d).trim()] !== undefined);
|
|
190
|
-
unknownDomainDropped += rawDomains.length - valid.length;
|
|
191
|
-
results.push({ id, domains: valid });
|
|
192
|
-
}
|
|
193
|
-
const missing = slice.filter((it) => !results.some((r) => r.id === Number(it.id)));
|
|
194
|
-
for (const m of missing) results.push({ id: Number(m.id), domains: [] }); // 模型漏答 ⇒ 记为"无标签",不写库
|
|
271
|
+
book.cost.calls = Number(book.cost.calls ?? 0) + 1;
|
|
272
|
+
book.cost.promptTokens = Number(book.cost.promptTokens ?? 0) + Number(reply.usage?.prompt_tokens ?? 0);
|
|
273
|
+
book.cost.completionTokens = Number(book.cost.completionTokens ?? 0) + Number(reply.usage?.completion_tokens ?? 0);
|
|
274
|
+
book.cost.date = nowIso.slice(0, 10);
|
|
195
275
|
|
|
196
|
-
const
|
|
197
|
-
|
|
198
|
-
|
|
276
|
+
const cleaned = sanitizeModelItems(reply.items, batch);
|
|
277
|
+
const written = await D.applyResults(db, cleaned.rows);
|
|
278
|
+
if (!written.ok) {
|
|
279
|
+
result.stopping = `写库失败(第 ${round + 1} 批):${written.error}`;
|
|
199
280
|
log(`\n[STOP] ${result.stopping}`);
|
|
200
281
|
break;
|
|
201
282
|
}
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
283
|
+
|
|
284
|
+
// 「问过了」入账:连「模型说没标签」的也算 —— 否则下一轮又会花钱问同一批
|
|
285
|
+
for (const row of batch) {
|
|
286
|
+
const hit = cleaned.rows.find((r) => r.id === Number(row.id));
|
|
287
|
+
book.processed[String(row.id)] = D.tagSetHash(hit ? hit.domains : []);
|
|
206
288
|
}
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
D.writeState(stateFile,
|
|
214
|
-
|
|
289
|
+
book.stats.batches = Number(book.stats.batches ?? 0) + 1;
|
|
290
|
+
bump(book, 'tagged', written.written);
|
|
291
|
+
bump(book, 'noTag', cleaned.rows.filter((r) => r.domains.length === 0).length);
|
|
292
|
+
book.stats.skippedOwner = foreign.length;
|
|
293
|
+
bump(book, 'skippedUnknownDomain', cleaned.dropped);
|
|
294
|
+
book.updatedAt = new Date().toISOString();
|
|
295
|
+
D.writeState(stateFile, book); // 每批落盘 ⇒ 断电也能接着跑
|
|
296
|
+
const dayTokens = Number(book.cost.promptTokens) + Number(book.cost.completionTokens);
|
|
297
|
+
log(` 批 ${round + 1}/${batchCount}: 调用 ${book.cost.calls} 次 · 写 ${written.written} 条 · 已是 ${written.unchanged} 条 · 耗时 ${Date.now() - roundStart}ms · 当日 token ${dayTokens}`);
|
|
215
298
|
}
|
|
216
299
|
} finally {
|
|
217
|
-
try {
|
|
300
|
+
try {
|
|
301
|
+
db.close();
|
|
302
|
+
} catch {
|
|
303
|
+
/* 关不掉也不影响结果 */
|
|
304
|
+
}
|
|
218
305
|
}
|
|
219
306
|
|
|
220
|
-
//
|
|
221
|
-
for (const
|
|
222
|
-
|
|
223
|
-
D.writeState(stateFile,
|
|
224
|
-
|
|
307
|
+
// 台账里记一笔「这些为什么没送」,方便下次查账
|
|
308
|
+
for (const skip of foreign) book.skipped[String(skip.id)] = `owner=${skip.ownerUser}`;
|
|
309
|
+
book.updatedAt = new Date().toISOString();
|
|
310
|
+
D.writeState(stateFile, book);
|
|
311
|
+
|
|
312
|
+
const usedTokens = Number(book.cost.promptTokens) + Number(book.cost.completionTokens);
|
|
225
313
|
const line = D.renderDailyLine({
|
|
226
|
-
mode: 'apply',
|
|
227
|
-
|
|
314
|
+
mode: 'apply',
|
|
315
|
+
stats: book.stats,
|
|
316
|
+
cost: book.cost,
|
|
317
|
+
state: { processedCount: Object.keys(book.processed).length },
|
|
318
|
+
limits,
|
|
319
|
+
stoppedReason: result.stopping,
|
|
228
320
|
});
|
|
229
|
-
result.cost = { ...
|
|
230
|
-
result.state = { file: stateFile, processed: Object.keys(
|
|
231
|
-
log(`\n完成:批次 ${
|
|
321
|
+
result.cost = { ...book.cost, usedTokens };
|
|
322
|
+
result.state = { file: stateFile, processed: Object.keys(book.processed).length };
|
|
323
|
+
log(`\n完成:批次 ${book.stats.batches} · 本轮写 ${book.stats.tagged} 条 · 当日调用 ${book.cost.calls}/${limits.callsPerDay} · 当日 token ${usedTokens}/${limits.tokensPerDay} · 用时 ${Date.now() - startedAt}ms`);
|
|
324
|
+
|
|
232
325
|
if (!flags['no-daily-report']) {
|
|
233
326
|
const reportFile = flags['daily-report'] ? path.resolve(String(flags['daily-report'])) : D.dailyReportPath({});
|
|
234
|
-
const
|
|
235
|
-
result.dailyReport = { file: reportFile, ...
|
|
236
|
-
|
|
327
|
+
const done = D.updateDailySection({ file: reportFile, line, dryRun: false });
|
|
328
|
+
result.dailyReport = { file: reportFile, ...done, line };
|
|
329
|
+
const suffix = done.ok ? `(${done.replaced ? '段替换' : '追加段'})` : `(${done.reason})`;
|
|
330
|
+
log(`${done.ok ? '✓' : '⚠️'} 日报:${reportFile}${suffix}`);
|
|
237
331
|
}
|
|
238
332
|
result.dailyLine = line;
|
|
239
333
|
result.ok = true;
|
|
240
|
-
return
|
|
241
|
-
}
|
|
242
|
-
|
|
243
|
-
function finish(result, code, json, log) {
|
|
244
|
-
result.exitCode = code;
|
|
245
|
-
if (json) {
|
|
246
|
-
process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
|
|
247
|
-
} else {
|
|
248
|
-
log(`\n[${code === 0 ? 'OK' : 'STOP'}] exit=${code}`);
|
|
249
|
-
}
|
|
250
|
-
process.exitCode = code;
|
|
334
|
+
return seal(result, EXIT.OK, { asJson, log });
|
|
251
335
|
}
|