@kira905/dsh-memory-toolkit 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/LICENSE +661 -0
- package/README.en.md +110 -0
- package/README.md +102 -2
- package/dsh-memory-local/FORK-BASE.json +24 -0
- package/dsh-memory-local/README.md +113 -0
- package/dsh-memory-local/build.mjs +101 -0
- package/dsh-memory-local/cordis.patch.yml +20 -0
- package/dsh-memory-local/deploy.mjs +294 -0
- package/dsh-memory-local/entity-status.mjs +75 -0
- package/dsh-memory-local/lib/bridge.mjs +108 -0
- package/dsh-memory-local/lib/host-adapter.js +1171 -0
- package/dsh-memory-local/lib/index.d.ts +128 -0
- package/dsh-memory-local/lib/index.js +944 -0
- package/dsh-memory-local/lib/injector.mjs +400 -0
- package/dsh-memory-local/lib/memory-read.mjs +116 -0
- package/dsh-memory-local/lib/module-gate.mjs +68 -0
- package/dsh-memory-local/lib/pointer.mjs +152 -0
- package/dsh-memory-local/lib/prod-gate.mjs +121 -0
- package/dsh-memory-local/lib/recall-log.mjs +118 -0
- package/dsh-memory-local/lib/recall.mjs +192 -0
- package/dsh-memory-local/lib/schema-migrate.mjs +365 -0
- package/dsh-memory-local/lib/user-message.mjs +32 -0
- package/dsh-memory-local/lib/vectors-read.mjs +110 -0
- package/dsh-memory-local/package.json +51 -0
- package/dsh-memory-local/restore.mjs +259 -0
- package/dsh-memory-local/rollback-injector-fix.mjs +166 -0
- package/dsh-memory-local/src/bridge.mjs +108 -0
- package/dsh-memory-local/src/host-adapter.js +1171 -0
- package/dsh-memory-local/src/index.d.ts +128 -0
- package/dsh-memory-local/src/index.js +944 -0
- package/dsh-memory-local/src/injector.mjs +400 -0
- package/dsh-memory-local/src/memory-read.mjs +116 -0
- package/dsh-memory-local/src/module-gate.mjs +68 -0
- package/dsh-memory-local/src/pointer.mjs +152 -0
- package/dsh-memory-local/src/recall-log.mjs +118 -0
- package/dsh-memory-local/src/recall.mjs +192 -0
- package/dsh-memory-local/src/schema-migrate.mjs +365 -0
- package/dsh-memory-local/src/user-message.mjs +32 -0
- package/dsh-memory-local/src/vectors-read.mjs +110 -0
- package/dsh-memory-local/test/fixtures-s2b.mjs +216 -0
- package/dsh-memory-local/test/probe-entity.mjs +170 -0
- package/dsh-memory-local/test/regression.mjs +270 -0
- package/dsh-memory-local/test/s2b.test.mjs +568 -0
- package/dsh-memory-local/test/schema-migrate.test.mjs +334 -0
- package/dsh-memory-local/test/test-adapter-converge.mjs +127 -0
- package/dsh-memory-local/test/visible-injections.test.mjs +239 -0
- package/dsh-memory-local/verify-after-restart.mjs +181 -0
- package/lib/deploy-guard.mjs +391 -0
- package/lib/guard-http.mjs +93 -0
- package/lib/pitfall-form-map.mjs +130 -0
- package/lib/pitfall-score.mjs +185 -0
- package/lib/session-file-name.mjs +160 -0
- package/lib/zstd-frames.mjs +78 -0
- package/memory-db-tools/README.md +68 -0
- package/memory-db-tools/_test/test-memory-scope.mjs +302 -0
- package/memory-db-tools/audit-analyze.mjs +352 -0
- package/memory-db-tools/audit-dump-ids.mjs +245 -0
- package/memory-db-tools/audit-export.mjs +270 -0
- package/memory-db-tools/audit-hit-probe.mjs +227 -0
- package/memory-db-tools/audit-ledger-probe.mjs +236 -0
- package/memory-db-tools/audit-persona-overlap.mjs +277 -0
- package/memory-db-tools/audit-struct-probe.mjs +213 -0
- package/memory-db-tools/audit-verify-unchanged.mjs +276 -0
- package/memory-db-tools/check-memory-db.mjs +8 -0
- package/memory-db-tools/dump-memory-db.mjs +12 -0
- package/memory-db-tools/lib/memory-scope.mjs +446 -0
- package/memory-db-tools/list-memory-summary.mjs +477 -0
- package/memory-db-tools/memory-identity-check.mjs +101 -0
- package/memory-db-tools/memory-schema-migrate.mjs +290 -0
- package/memory-db-tools/memory-scope-fixture.mjs +197 -0
- package/memory-db-tools/read-memories.mjs +26 -0
- package/memory-db-tools/read-memory-db.mjs +20 -0
- package/memory-db-tools/refine-memories.mjs +247 -0
- package/memory-db-tools/show-mem-times.mjs +22 -0
- package/memory-engine/README.md +70 -0
- package/memory-engine/bench/bench.mjs +221 -0
- package/memory-engine/cli/distill-rollback.mjs +86 -0
- package/memory-engine/cli/distill.mjs +156 -0
- package/memory-engine/cli/doc-index-rebuild.mjs +155 -0
- package/memory-engine/cli/doc-index.mjs +48 -0
- package/memory-engine/cli/domain-tags.mjs +251 -0
- package/memory-engine/cli/pitfall-hook.mjs +91 -0
- package/memory-engine/cli/recall.mjs +132 -0
- package/memory-engine/cli/selfcheck.mjs +57 -0
- package/memory-engine/cli/serve-start.mjs +247 -0
- package/memory-engine/cli/sweep.mjs +58 -0
- package/memory-engine/cli/sync.mjs +69 -0
- package/memory-engine/lib/cli-util.mjs +197 -0
- package/memory-engine/lib/config.mjs +135 -0
- package/memory-engine/lib/dirty-ledger.mjs +174 -0
- package/memory-engine/lib/distill-rollback.mjs +178 -0
- package/memory-engine/lib/distill.mjs +898 -0
- package/memory-engine/lib/doc-index-store.mjs +325 -0
- package/memory-engine/lib/doc-index.mjs +529 -0
- package/memory-engine/lib/doc-scan.mjs +271 -0
- package/memory-engine/lib/doc-summarize.mjs +305 -0
- package/memory-engine/lib/domain-tags.mjs +429 -0
- package/memory-engine/lib/embedder.mjs +242 -0
- package/memory-engine/lib/hashes.mjs +12 -0
- package/memory-engine/lib/memory-source.mjs +127 -0
- package/memory-engine/lib/pitfall-source.mjs +170 -0
- package/memory-engine/lib/policy-source.mjs +371 -0
- package/memory-engine/lib/recall.mjs +331 -0
- package/memory-engine/lib/selfcheck.mjs +131 -0
- package/memory-engine/lib/sweep.mjs +159 -0
- package/memory-engine/lib/sync.mjs +180 -0
- package/memory-engine/lib/tick-hook.mjs +218 -0
- package/memory-engine/lib/vectors-store.mjs +292 -0
- package/memory-engine/package-lock.json +1086 -0
- package/memory-engine/package.json +22 -0
- package/memory-engine/serve.mjs +171 -0
- package/memory-engine/test/01-store.test.mjs +127 -0
- package/memory-engine/test/02-sync.test.mjs +196 -0
- package/memory-engine/test/03-sweep.test.mjs +145 -0
- package/memory-engine/test/04-recall.test.mjs +214 -0
- package/memory-engine/test/05-selfcheck.test.mjs +175 -0
- package/memory-engine/test/06-doc-index.test.mjs +928 -0
- package/memory-engine/test/07-domain-tags.test.mjs +298 -0
- package/memory-engine/test/08-distill.test.mjs +686 -0
- package/memory-engine/test/09-pitfall-source.test.mjs +365 -0
- package/memory-engine/test/10-tick-hook.test.mjs +252 -0
- package/memory-engine/test/11-policy-source.test.mjs +224 -0
- package/memory-engine/test/_tickhook-worker.mjs +56 -0
- package/memory-engine/test/fixtures.mjs +205 -0
- package/memory-engine/test/harness.mjs +73 -0
- package/memory-engine/test/run.mjs +35 -0
- package/memory-engine/tools/bridge-smoke.mjs +80 -0
- package/memory-engine/tools/fetch-model.mjs +128 -0
- package/memory-engine/tools/smoke-recall.mjs +76 -0
- package/package.json +53 -4
- package/persona-memory-audit.mjs +217 -0
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// 记忆引擎 v2 · S3c —— 提炼 CLI(`cli/distill.mjs`)
|
|
3
|
+
//
|
|
4
|
+
// 用法:
|
|
5
|
+
// node cli/distill.mjs --db <memory.db> [选项] # 默认 **dry-run**(只打印不写库)
|
|
6
|
+
// node cli/distill.mjs --db <memory.db> --apply # 真跑:调模型 + 写库 + 写 state
|
|
7
|
+
//
|
|
8
|
+
// 选项:
|
|
9
|
+
// --db <path> 记忆库(**必填**,无默认 —— 写死生产路径是明令禁止的)
|
|
10
|
+
// --vectors-db <path> 向量库(默认 <home>/memory-vectors/vectors.db;home = db 的上上级)
|
|
11
|
+
// --state <path> 提炼 state(默认 <home>/butler-distill-state.json)
|
|
12
|
+
// --owner-user <id> 本机 owner 的 userid(缺省由身份表解析;解析不出 ⇒ **本轮 0 调用**)
|
|
13
|
+
// --window-hours <n> 增量窗口(默认 24)
|
|
14
|
+
// --limit <n> 输入条数上限(默认 0 = 不限)
|
|
15
|
+
// --max-calls / --max-tokens 成本闸(默认 20 次 / 150000 token 每日)
|
|
16
|
+
// --model / --base-url 模型档(默认 deepseek-flash @ 本鱼链;2026-09-21 主人拍板,不许改回第三方)
|
|
17
|
+
// --sim-threshold <f> 聚类余弦阈值(默认 0.60)
|
|
18
|
+
// --min-cluster-chars <n> 整组正文合计低于此值不调模型(默认 120)
|
|
19
|
+
// --min-cluster-size <n> 组内少于 N 条不提炼(默认 **2**:单条自成一组不算聚类,且少写几条以护住 memory:recall 段)
|
|
20
|
+
// --max-clusters <n> 单轮最多处理几组(默认 20)
|
|
21
|
+
// --now <iso> 以某个时刻为"现在"(演练/三天不自我指涉模拟用)
|
|
22
|
+
// --snapshot-dir <path> 写库前 `VACUUM INTO` 快照目录
|
|
23
|
+
// --no-dirty-probe 不跑全表 hash 比对(不读 memories_dirty 补充源)
|
|
24
|
+
// --daily-report <file> 追加/替换日报段(默认:今天的 <workspace>/docs/管家报告/<日期>-<设备>.md)
|
|
25
|
+
// --no-daily 不写日报段
|
|
26
|
+
// --json 机器可读(stdout 只出 JSON,日志走 stderr)
|
|
27
|
+
// --quiet / --help
|
|
28
|
+
//
|
|
29
|
+
// 退出码:0 = 正常(含"没活干")/ 1 = 失败 / 2 = 用法错 / 3 = 环境不可用(库不在)
|
|
30
|
+
import fs from 'node:fs';
|
|
31
|
+
import path from 'node:path';
|
|
32
|
+
import { isMainEntry, parseArgs, makeLogger, num } from '../lib/cli-util.mjs';
|
|
33
|
+
import {
|
|
34
|
+
runDistill, readState, renderDailyLine, updateDailySection, dailyReportPath,
|
|
35
|
+
COST_LIMITS, DEFAULT_LLM, DEFAULT_WINDOW_HOURS, CLUSTER_DEFAULTS, STATE_SCHEMA,
|
|
36
|
+
} from '../lib/distill.mjs';
|
|
37
|
+
|
|
38
|
+
const USAGE = 'node cli/distill.mjs --db <memory.db> [--apply] [--window-hours 24] [--limit N] '
|
|
39
|
+
+ '[--max-calls 20] [--max-tokens 150000] [--max-clusters 20] [--min-cluster-size 2] [--min-cluster-chars 120] '
|
|
40
|
+
+ '[--model deepseek-flash] [--now <iso>] [--snapshot-dir <dir>] [--daily-report <file>] [--no-dirty-probe] '
|
|
41
|
+
+ '[--no-daily] [--json] [--help]';
|
|
42
|
+
|
|
43
|
+
/** 解析「本机 owner 的 userid」:权威实现优先,其次 fork 自检镜像,最后内置回落(与 domain-tags 同口径)。 */
|
|
44
|
+
async function resolveOwnerUser(explicit, home) {
|
|
45
|
+
if (explicit) return { user: explicit, source: 'cli' };
|
|
46
|
+
const cands = ['../../memory-db-tools/lib/memory-scope.mjs', '../dsh-memory-local/src/schema-migrate.mjs'];
|
|
47
|
+
for (const rel of cands) {
|
|
48
|
+
try {
|
|
49
|
+
const mod = await import(new URL(rel, import.meta.url).href);
|
|
50
|
+
const identity = mod.readIdentityTable ? mod.readIdentityTable(home) : null;
|
|
51
|
+
const r = mod.resolveDefaultOwner ? mod.resolveDefaultOwner(identity) : null;
|
|
52
|
+
if (r && r.ownerUser) return { user: r.ownerUser, source: `${rel} (${r.source})`, notes: r.notes ?? [] };
|
|
53
|
+
} catch { /* 试下一个候选 */ }
|
|
54
|
+
}
|
|
55
|
+
return { user: null, source: '解析失败(两条权威源都不可用)', notes: ['⚠️ 本机 owner 解析不出来 ⇒ 按 D2 从严:本轮不调用任何第三方模型'] };
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
async function main() {
|
|
59
|
+
const { flags } = parseArgs();
|
|
60
|
+
const json = Boolean(flags.json);
|
|
61
|
+
const log = makeLogger({ json, quiet: Boolean(flags.quiet) });
|
|
62
|
+
if (flags.help) { process.stdout.write(`用法:${USAGE}\n`); return 0; }
|
|
63
|
+
|
|
64
|
+
const dbPath = flags.db ? path.resolve(String(flags.db)) : null;
|
|
65
|
+
if (!dbPath) {
|
|
66
|
+
process.stderr.write('用法错:必须给 --db <库路径>(本脚本没有默认库路径)\n');
|
|
67
|
+
return 2;
|
|
68
|
+
}
|
|
69
|
+
if (!fs.existsSync(dbPath)) {
|
|
70
|
+
process.stderr.write(`环境不可用:库不存在 ${dbPath}\n`);
|
|
71
|
+
return 3;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const home = path.resolve(flags.home ? String(flags.home) : path.dirname(path.dirname(dbPath)));
|
|
75
|
+
const vectorsDb = flags['vectors-db'] ? path.resolve(String(flags['vectors-db'])) : path.join(home, 'memory-vectors', 'vectors.db');
|
|
76
|
+
const stateFile = flags.state ? path.resolve(String(flags.state)) : path.join(home, 'butler-distill-state.json');
|
|
77
|
+
const apply = Boolean(flags.apply);
|
|
78
|
+
const now = flags.now ? Date.parse(String(flags.now)) : Date.now();
|
|
79
|
+
if (!Number.isFinite(now)) { process.stderr.write(`用法错:--now 不是合法时间:${flags.now}\n`); return 2; }
|
|
80
|
+
const limits = {
|
|
81
|
+
callsPerDay: num(flags['max-calls'], COST_LIMITS.callsPerDay),
|
|
82
|
+
tokensPerDay: num(flags['max-tokens'], COST_LIMITS.tokensPerDay),
|
|
83
|
+
};
|
|
84
|
+
const owner = await resolveOwnerUser(flags['owner-user'] ? String(flags['owner-user']) : null, home);
|
|
85
|
+
const snapshotDir = flags['snapshot-dir'] ? path.resolve(String(flags['snapshot-dir'])) : null;
|
|
86
|
+
|
|
87
|
+
log(`库 : ${dbPath}`);
|
|
88
|
+
log(`向量库 : ${vectorsDb}`);
|
|
89
|
+
log(`state : ${stateFile}`);
|
|
90
|
+
log(`本机 owner: ${owner.user ?? '(未解析)'}(来源 ${owner.source})`);
|
|
91
|
+
log(`模型 : ${flags.model ? String(flags.model) : DEFAULT_LLM.model}(成本闸 ≤${limits.callsPerDay} 次 / ≤${limits.tokensPerDay} token 每日)`);
|
|
92
|
+
log(`模式 : ${apply ? 'APPLY(写库)' : 'DRY-RUN(只打印不写库)'}`);
|
|
93
|
+
for (const n of owner.notes ?? []) log(`⚠️ ${n}`);
|
|
94
|
+
|
|
95
|
+
const r = await runDistill({
|
|
96
|
+
memoryDb: dbPath, vectorsDb, stateFile, now,
|
|
97
|
+
windowHours: num(flags['window-hours'], DEFAULT_WINDOW_HOURS),
|
|
98
|
+
apply,
|
|
99
|
+
limits,
|
|
100
|
+
model: flags.model ? String(flags.model) : DEFAULT_LLM.model,
|
|
101
|
+
baseUrl: flags['base-url'] ? String(flags['base-url']) : DEFAULT_LLM.baseUrl,
|
|
102
|
+
localOwner: owner.user,
|
|
103
|
+
limit: num(flags.limit, 0),
|
|
104
|
+
simThreshold: num(flags['sim-threshold'], CLUSTER_DEFAULTS.simThreshold),
|
|
105
|
+
minClusterChars: num(flags['min-cluster-chars'], CLUSTER_DEFAULTS.minClusterChars),
|
|
106
|
+
minClusterSize: num(flags['min-cluster-size'], CLUSTER_DEFAULTS.minClusterSize),
|
|
107
|
+
maxClustersPerRun: num(flags['max-clusters'], CLUSTER_DEFAULTS.maxClustersPerRun),
|
|
108
|
+
dirtyProbe: !flags['no-dirty-probe'],
|
|
109
|
+
snapshotFile: snapshotDir ? path.join(snapshotDir, `memory-before-distill-${new Date(now).toISOString().replace(/[:.]/g, '-')}.db`) : null,
|
|
110
|
+
log,
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
// 不调模型(dry-run / 环境问题)时也要留一行可查的"当日成本",方便日报段
|
|
114
|
+
const state = r.state ?? readState(stateFile, dbPath, new Date(now).toISOString());
|
|
115
|
+
const dailyLine = r.dailyLine ?? renderDailyLine({ mode: r.mode, run: r.run ?? {}, cost: state.cost, limits, stopping: r.stopping });
|
|
116
|
+
|
|
117
|
+
let daily = { ok: false, reason: '未写日报(--no-daily)' };
|
|
118
|
+
if (!flags['no-daily'] && apply) {
|
|
119
|
+
const file = flags['daily-report'] ? path.resolve(String(flags['daily-report'])) : dailyReportPath({ now });
|
|
120
|
+
daily = updateDailySection({ file, line: dailyLine });
|
|
121
|
+
if (daily.ok) log(`日报段已更新:${file}(${daily.replaced ? '替换既有段' : '追加新段'})`);
|
|
122
|
+
else log(`日报段未写:${daily.reason}(${file})`);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
const out = {
|
|
126
|
+
ok: r.ok, mode: r.mode, at: new Date(now).toISOString(),
|
|
127
|
+
db: dbPath, vectorsDb, stateFile, model: DEFAULT_LLM.model, ownerUser: owner.user, ownerSource: owner.source,
|
|
128
|
+
limits, inputs: r.inputs, clusterStats: r.clusterStats ?? null, dirty: r.dirty ?? null,
|
|
129
|
+
planned: r.planned, calls: r.calls, written: r.written, writtenIds: r.writtenIds,
|
|
130
|
+
cost: state.cost ?? null, stopping: r.stopping, warnings: r.warnings, dailyLine, daily,
|
|
131
|
+
snapshot: r.snapshot ?? null,
|
|
132
|
+
};
|
|
133
|
+
if (json) process.stdout.write(JSON.stringify(out, null, 2) + '\n');
|
|
134
|
+
else {
|
|
135
|
+
log('');
|
|
136
|
+
log(`输入 ${r.inputs?.accepted ?? 0} 条(窗口 ${r.inputs?.windowRows ?? 0} / 台账 ${r.inputs?.dirtyRows ?? 0})⇒ 接受 ${r.inputs?.accepted ?? 0}`);
|
|
137
|
+
log(`聚类 ${r.clusterStats?.clusters ?? 0} 组(单条组 ${r.clusterStats?.singletonClusters ?? 0} · 无向量 ${r.clusterStats?.noVector ?? 0})`);
|
|
138
|
+
for (const p of r.planned ?? []) log(` ${p.action === 'call' ? '→ 调模型' : '· 跳过 '} ${p.key} ids=[${p.ids.join(',')}] ${p.action}${p.chars ? ` (${p.chars} 字)` : ''}`);
|
|
139
|
+
for (const c of r.calls ?? []) log(` ✦ ${c.key} 调用 ${c.ok ? 'OK' : '失败'} · ${c.tokens} token · 结论 ${c.conclusionChars} 字${c.error ? ` · ${c.error}` : ''}`);
|
|
140
|
+
log(`提炼条数 ${r.written}${r.writtenIds?.length ? `(新 id ${r.writtenIds.join(',')})` : ''}`);
|
|
141
|
+
if (r.stopping) log(`停机:${r.stopping}`);
|
|
142
|
+
log(`当日调用 ${state.cost?.calls ?? 0}/${limits.callsPerDay} · 当日 token ${(Number(state.cost?.promptTokens ?? 0) + Number(state.cost?.completionTokens ?? 0))}/${limits.tokensPerDay}`);
|
|
143
|
+
for (const w of r.warnings ?? []) log(`⚠️ ${w}`);
|
|
144
|
+
log(`日报行:${dailyLine}`);
|
|
145
|
+
}
|
|
146
|
+
return r.ok ? 0 : (r.exitCode || 1);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
if (isMainEntry(import.meta.url)) {
|
|
150
|
+
main().then((code) => { process.exitCode = code; }).catch((e) => {
|
|
151
|
+
console.error(`未捕获异常:${e && e.stack ? e.stack : e}`);
|
|
152
|
+
process.exitCode = 1;
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
export { main, resolveOwnerUser, STATE_SCHEMA };
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
// 记忆引擎 v2 · 离线层 —— 文档索引**逃生舱**(方案 §7「doc-index.db 损坏 / 陈旧 → 重建器一键全量刷新」)
|
|
2
|
+
//
|
|
3
|
+
// 用法:
|
|
4
|
+
// node cli/doc-index-rebuild.mjs # 一键全量刷新(范围内按 path 重跑,幂等)
|
|
5
|
+
// node cli/doc-index-rebuild.mjs --purge # 先清空范围内全部条再重建(模拟库损坏/陈旧)
|
|
6
|
+
// node cli/doc-index-rebuild.mjs --no-snapshot # 不写重建前快照(默认会写)
|
|
7
|
+
// node cli/doc-index-rebuild.mjs --restore <快照文件> # 回滚:用快照覆盖当前库
|
|
8
|
+
// node cli/doc-index-rebuild.mjs --list-snapshots # 列出可用快照
|
|
9
|
+
// 路径可注入:--docs-root / --dsh-root / --doc-index-db / --state-file / --home / --snapshot-dir
|
|
10
|
+
//
|
|
11
|
+
// 退出码:0 成功 / 1 有失败 / 2 用法错
|
|
12
|
+
import fs from 'node:fs';
|
|
13
|
+
import path from 'node:path';
|
|
14
|
+
import os from 'node:os';
|
|
15
|
+
import { isMainEntry, parseArgs, makeLogger } from '../lib/cli-util.mjs';
|
|
16
|
+
import { runDocIndexOnce } from '../lib/doc-index.mjs';
|
|
17
|
+
import { openDocIndexStore, snapshotDb } from '../lib/doc-index-store.mjs';
|
|
18
|
+
|
|
19
|
+
const EXIT = { OK: 0, FAIL: 1, USAGE: 2 };
|
|
20
|
+
|
|
21
|
+
function resolvePaths(flags, env = process.env) {
|
|
22
|
+
const home = path.resolve(String(flags.home || env.DSH_HOME || path.join(os.homedir(), '.dsh')));
|
|
23
|
+
const vectorsDir = path.join(home, 'memory-vectors');
|
|
24
|
+
const dbPath = path.resolve(String(flags['doc-index-db'] || env.DOC_INDEX_DB || path.join(vectorsDir, 'doc-index.db')));
|
|
25
|
+
const snapshotDir = path.resolve(String(flags['snapshot-dir'] || path.join(vectorsDir, 'doc-index-snapshots')));
|
|
26
|
+
return { home, vectorsDir, dbPath, snapshotDir };
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function ts() {
|
|
30
|
+
const d = new Date();
|
|
31
|
+
const p = (n) => String(n).padStart(2, '0');
|
|
32
|
+
return `${d.getFullYear()}${p(d.getMonth() + 1)}${p(d.getDate())}-${p(d.getHours())}${p(d.getMinutes())}${p(d.getSeconds())}`;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** 重建前快照(VACUUM INTO;库不存在则跳过——没有可备份的东西)。 */
|
|
36
|
+
export function takeSnapshot({ dbPath, snapshotDir, log = () => {} }) {
|
|
37
|
+
if (!fs.existsSync(dbPath)) return { ok: true, skipped: true, reason: '库不存在(首次建库无需快照)' };
|
|
38
|
+
const dest = path.join(snapshotDir, `doc-index-${ts()}.db`);
|
|
39
|
+
const store = openDocIndexStore(dbPath, { create: false });
|
|
40
|
+
try {
|
|
41
|
+
snapshotDb(store.db, dest);
|
|
42
|
+
} finally {
|
|
43
|
+
try {
|
|
44
|
+
store.db.close();
|
|
45
|
+
} catch {
|
|
46
|
+
/* ignore */
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
const size = fs.statSync(dest).size;
|
|
50
|
+
log(`[rebuild] 重建前快照:${dest}(${size} B)`);
|
|
51
|
+
return { ok: true, dest, size };
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** 列出快照(按时间倒序)。 */
|
|
55
|
+
export function listSnapshots(snapshotDir) {
|
|
56
|
+
try {
|
|
57
|
+
return fs
|
|
58
|
+
.readdirSync(snapshotDir)
|
|
59
|
+
.filter((n) => n.startsWith('doc-index-') && n.endsWith('.db'))
|
|
60
|
+
.map((n) => ({ name: n, file: path.join(snapshotDir, n), size: fs.statSync(path.join(snapshotDir, n)).size, mtime: fs.statSync(path.join(snapshotDir, n)).mtimeMs }))
|
|
61
|
+
.sort((a, b) => b.mtime - a.mtime);
|
|
62
|
+
} catch {
|
|
63
|
+
return [];
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* 回滚:用快照覆盖当前库(先把当前库另存为 `.before-restore-<ts>`,绝不静默丢)。
|
|
69
|
+
* ⚠️ WAL 副作用:覆盖前必须清掉 `-wal` / `-shm`,否则新库会被旧 WAL 污染。
|
|
70
|
+
*/
|
|
71
|
+
export function restoreFromSnapshot({ dbPath, snapshotFile, log = () => {} }) {
|
|
72
|
+
if (!fs.existsSync(snapshotFile)) return { ok: false, error: `快照不存在:${snapshotFile}` };
|
|
73
|
+
if (!fs.existsSync(dbPath)) return { ok: false, error: `目标库不存在:${dbPath}` };
|
|
74
|
+
const backup = `${dbPath}.before-restore-${ts()}`;
|
|
75
|
+
fs.copyFileSync(dbPath, backup);
|
|
76
|
+
for (const suffix of ['-wal', '-shm']) {
|
|
77
|
+
const f = `${dbPath}${suffix}`;
|
|
78
|
+
if (fs.existsSync(f)) fs.rmSync(f, { force: true });
|
|
79
|
+
}
|
|
80
|
+
fs.copyFileSync(snapshotFile, dbPath);
|
|
81
|
+
log(`[rebuild] 已回滚:${snapshotFile} → ${dbPath}(回滚前副本:${backup})`);
|
|
82
|
+
return { ok: true, backup, from: snapshotFile };
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export async function main() {
|
|
86
|
+
const { flags } = parseArgs();
|
|
87
|
+
const json = Boolean(flags.json);
|
|
88
|
+
const log = makeLogger({ json, quiet: Boolean(flags.quiet) });
|
|
89
|
+
const { dbPath, snapshotDir, home } = resolvePaths(flags);
|
|
90
|
+
|
|
91
|
+
try {
|
|
92
|
+
if (flags['list-snapshots']) {
|
|
93
|
+
const snaps = listSnapshots(snapshotDir);
|
|
94
|
+
if (json) process.stdout.write(JSON.stringify({ ok: true, snapshotDir, snapshots: snaps }, null, 2) + '\n');
|
|
95
|
+
else {
|
|
96
|
+
log(`快照目录:${snapshotDir}`);
|
|
97
|
+
if (!snaps.length) log('(无快照)');
|
|
98
|
+
for (const s of snaps) log(` ${new Date(s.mtime).toLocaleString('zh-CN')} ${s.size} B ${s.file}`);
|
|
99
|
+
}
|
|
100
|
+
return EXIT.OK;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
if (typeof flags.restore === 'string') {
|
|
104
|
+
const r = restoreFromSnapshot({ dbPath, snapshotFile: path.resolve(flags.restore), log });
|
|
105
|
+
if (json) process.stdout.write(JSON.stringify({ ok: r.ok, ...r }, null, 2) + '\n');
|
|
106
|
+
return r.ok ? EXIT.OK : EXIT.FAIL;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
// ① 重建前快照(默认开)
|
|
110
|
+
let snapshot = { ok: true, skipped: true, reason: '--no-snapshot' };
|
|
111
|
+
if (!flags['no-snapshot'] && !flags['dry-run']) {
|
|
112
|
+
snapshot = takeSnapshot({ dbPath, snapshotDir, log });
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// ② --purge:先清空范围内全部条(模拟"库损坏 / 陈旧"后的干净重建)
|
|
116
|
+
if (flags.purge && !flags['dry-run'] && fs.existsSync(dbPath)) {
|
|
117
|
+
const store = openDocIndexStore(dbPath, { create: false });
|
|
118
|
+
try {
|
|
119
|
+
const n = store.db.prepare('DELETE FROM doc_index').run().changes;
|
|
120
|
+
log(`[rebuild] --purge:已清空 doc_index(删除 ${Number(n) || 0} 条)`);
|
|
121
|
+
} finally {
|
|
122
|
+
try {
|
|
123
|
+
store.db.close();
|
|
124
|
+
} catch {
|
|
125
|
+
/* ignore */
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
// ③ 一键全量刷新(force = 忽略差分,按 path 覆盖)
|
|
131
|
+
const r = await runDocIndexOnce({
|
|
132
|
+
flags: { ...flags, rebuild: true, home, 'doc-index-db': dbPath },
|
|
133
|
+
});
|
|
134
|
+
for (const l of r.lines) log(l);
|
|
135
|
+
|
|
136
|
+
const out = {
|
|
137
|
+
ok: r.status !== 'failed',
|
|
138
|
+
status: r.status,
|
|
139
|
+
dbPath,
|
|
140
|
+
snapshot,
|
|
141
|
+
stats: r.stats,
|
|
142
|
+
dailyLine: r.state.lastDailyLine,
|
|
143
|
+
};
|
|
144
|
+
if (json) process.stdout.write(JSON.stringify(out, null, 2) + '\n');
|
|
145
|
+
else log(`[rebuild] 完成:${r.state.lastDailyLine}`);
|
|
146
|
+
return r.status === 'failed' ? EXIT.FAIL : EXIT.OK;
|
|
147
|
+
} catch (e) {
|
|
148
|
+
const msg = String(e && e.stack ? e.stack : e);
|
|
149
|
+
if (json) process.stdout.write(JSON.stringify({ ok: false, error: msg }, null, 2) + '\n');
|
|
150
|
+
else log(`❌ ${msg}`);
|
|
151
|
+
return EXIT.USAGE;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
if (isMainEntry(import.meta.url)) process.exitCode = await main();
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
// 记忆引擎 v2 · 离线层 —— 文档索引 CLI(方案 §3.2)
|
|
2
|
+
//
|
|
3
|
+
// 用法:
|
|
4
|
+
// node cli/doc-index.mjs # 增量(默认范围 = 首期 10 篇)
|
|
5
|
+
// node cli/doc-index.mjs --all # 扫描面全量(⚠️ 全量是 §10 T2 触发器事项)
|
|
6
|
+
// node cli/doc-index.mjs --only docs/a.md,docs/b.md
|
|
7
|
+
// node cli/doc-index.mjs --rebuild # 忽略差分,范围内全部重跑(按 path 覆盖,幂等)
|
|
8
|
+
// node cli/doc-index.mjs --dry-run # 只算不写(不建库、不改 state、不写日报)
|
|
9
|
+
// node cli/doc-index.mjs --llm # 摘要走便宜模型(受 §3.5 成本闸)
|
|
10
|
+
// node cli/doc-index.mjs --json # 机器可读(日志走 stderr)
|
|
11
|
+
// 路径可注入:--docs-root / --dsh-root / --doc-index-db / --state-file / --home
|
|
12
|
+
import { isMainEntry, parseArgs, makeLogger } from '../lib/cli-util.mjs';
|
|
13
|
+
import { runDocIndexOnce } from '../lib/doc-index.mjs';
|
|
14
|
+
|
|
15
|
+
export async function main() {
|
|
16
|
+
const { flags } = parseArgs();
|
|
17
|
+
const json = Boolean(flags.json);
|
|
18
|
+
const log = makeLogger({ json, quiet: Boolean(flags.quiet) });
|
|
19
|
+
try {
|
|
20
|
+
const r = await runDocIndexOnce({ flags });
|
|
21
|
+
if (json) {
|
|
22
|
+
process.stdout.write(
|
|
23
|
+
JSON.stringify(
|
|
24
|
+
{
|
|
25
|
+
ok: r.ok,
|
|
26
|
+
status: r.status,
|
|
27
|
+
exitCode: r.exitCode,
|
|
28
|
+
stats: r.stats,
|
|
29
|
+
dailyLine: r.state.lastDailyLine,
|
|
30
|
+
paths: r.state.paths,
|
|
31
|
+
},
|
|
32
|
+
null,
|
|
33
|
+
2,
|
|
34
|
+
) + '\n',
|
|
35
|
+
);
|
|
36
|
+
} else {
|
|
37
|
+
for (const l of r.lines) log(l);
|
|
38
|
+
}
|
|
39
|
+
process.exitCode = r.exitCode;
|
|
40
|
+
} catch (e) {
|
|
41
|
+
const msg = String(e && e.stack ? e.stack : e);
|
|
42
|
+
if (json) process.stdout.write(JSON.stringify({ ok: false, error: msg }, null, 2) + '\n');
|
|
43
|
+
else log(`❌ ${msg}`);
|
|
44
|
+
process.exitCode = 2;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
if (isMainEntry(import.meta.url)) await main();
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
// 记忆引擎 v2 · S3b —— 域标签补齐 CLI(方案 §3.6 / §5.4 / §7 / §3.9 D2)
|
|
2
|
+
//
|
|
3
|
+
// 用法:
|
|
4
|
+
// node cli/domain-tags.mjs --db <库路径> # dry-run(默认;只读规划,零写入)
|
|
5
|
+
// node cli/domain-tags.mjs --db <库路径> --apply # 真跑(副本库;分批 + 可续跑)
|
|
6
|
+
// node cli/domain-tags.mjs --db <库路径> --apply --snapshot <dir> # 跑前 VACUUM INTO 快照
|
|
7
|
+
// node cli/domain-tags.mjs --db <库路径> --limit 25 --batch 25 # 只跑 1 批(试跑)
|
|
8
|
+
// node cli/domain-tags.mjs --db <库路径> --apply --json # 机器可读(stdout 只有 JSON)
|
|
9
|
+
//
|
|
10
|
+
// 关键开关:`--batch`(每批条数,默认 25)· `--max-chars-per-item`(送模型的单条截断,默认 240)
|
|
11
|
+
// `--owner-user`(本机 owner 的 userid;缺省由身份表解析、再回落常量)· `--state <file>`
|
|
12
|
+
// `--max-calls`(默认 20)· `--max-tokens`(默认 150000)· `--model` · `--base-url`
|
|
13
|
+
// `--no-daily-report`(不写日报段)· `--daily-report <path>`(覆盖日报路径)
|
|
14
|
+
//
|
|
15
|
+
// 退出码:0 = 完成/无待处理(含"成本闸停机"这种受控停止);1 = 有异常;2 = 用法错;3 = 环境不可用(无 key / 库不存在)
|
|
16
|
+
import fs from 'node:fs';
|
|
17
|
+
import os from 'node:os';
|
|
18
|
+
import path from 'node:path';
|
|
19
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
20
|
+
import { isMainEntry, parseArgs, num, makeLogger } from '../lib/cli-util.mjs';
|
|
21
|
+
import * as D from '../lib/domain-tags.mjs';
|
|
22
|
+
|
|
23
|
+
const argv = process.argv.slice(2);
|
|
24
|
+
if (isMainEntry(import.meta.url)) {
|
|
25
|
+
main().catch((e) => {
|
|
26
|
+
console.error(`未捕获异常:${e && e.stack ? e.stack : e}`);
|
|
27
|
+
process.exitCode = 1;
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** 解析「本机 owner 的 userid」:优先 ⑤-3 的权威实现,其次 fork 的自检镜像,最后内置回落。 */
|
|
32
|
+
async function resolveOwnerUser(explicit, home) {
|
|
33
|
+
if (explicit) return { user: explicit, source: 'cli' };
|
|
34
|
+
const cands = ['../../memory-db-tools/lib/memory-scope.mjs', '../dsh-memory-local/src/schema-migrate.mjs'];
|
|
35
|
+
for (const rel of cands) {
|
|
36
|
+
try {
|
|
37
|
+
const mod = await import(new URL(rel, import.meta.url).href);
|
|
38
|
+
const identity = mod.readIdentityTable ? mod.readIdentityTable(home) : null;
|
|
39
|
+
const r = mod.resolveDefaultOwner ? mod.resolveDefaultOwner(identity) : null;
|
|
40
|
+
if (r && r.ownerUser) return { user: r.ownerUser, source: `${rel} (${r.source})`, notes: r.notes ?? [] };
|
|
41
|
+
} catch { /* 试下一个候选 */ }
|
|
42
|
+
}
|
|
43
|
+
return { user: 'LiJianXiao', source: '内置回落(两条权威源都不可用)', notes: ['⚠️ 身份表解析不可用 ⇒ 回落常量'] };
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
async function main() {
|
|
47
|
+
const { flags } = parseArgs(argv);
|
|
48
|
+
const json = Boolean(flags.json);
|
|
49
|
+
const log = makeLogger({ json });
|
|
50
|
+
const dbPath = flags.db ? path.resolve(String(flags.db)) : null;
|
|
51
|
+
if (!dbPath) {
|
|
52
|
+
console.error('用法错:必须给 --db <库路径>(本脚本没有默认库路径 —— 写死路径是明令禁止的)');
|
|
53
|
+
process.exitCode = 2;
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
if (!fs.existsSync(dbPath)) {
|
|
57
|
+
console.error(`环境不可用:库不存在 ${dbPath}`);
|
|
58
|
+
process.exitCode = 3;
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const apply = Boolean(flags.apply);
|
|
63
|
+
const batchSize = num(flags.batch, 25);
|
|
64
|
+
const limit = num(flags.limit, 0);
|
|
65
|
+
const maxCharsPerItem = num(flags['max-chars-per-item'], 240);
|
|
66
|
+
const limits = { callsPerDay: num(flags['max-calls'], D.COST_LIMITS.callsPerDay), tokensPerDay: num(flags['max-tokens'], D.COST_LIMITS.tokensPerDay) };
|
|
67
|
+
const home = path.dirname(path.dirname(dbPath)); // <home>/memory/memory.db → <home>
|
|
68
|
+
const stateFile = flags.state ? path.resolve(String(flags.state)) : path.join(home, 'memory', 'domain-tags-state.json');
|
|
69
|
+
const snapshotDir = flags.snapshot ? path.resolve(String(flags.snapshot)) : null;
|
|
70
|
+
const model = flags.model ? String(flags.model) : D.DEFAULT_LLM.model;
|
|
71
|
+
const baseUrl = flags['base-url'] ? String(flags['base-url']) : D.DEFAULT_LLM.baseUrl;
|
|
72
|
+
const nowIso = new Date().toISOString();
|
|
73
|
+
|
|
74
|
+
const owner = await resolveOwnerUser(flags['owner-user'] ? String(flags['owner-user']) : null, home);
|
|
75
|
+
const result = {
|
|
76
|
+
at: nowIso, mode: apply ? 'apply' : 'dry-run', db: dbPath, model, baseUrl,
|
|
77
|
+
ownerUser: owner.user, ownerSource: owner.source, limits, batchSize, maxCharsPerItem,
|
|
78
|
+
stats: {}, cost: null, stopping: null, ok: false, exitCode: 0,
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
log(`库 : ${dbPath}`);
|
|
82
|
+
log(`本机 owner: ${owner.user}(来源 ${owner.source})`);
|
|
83
|
+
log(`模型 : ${model} @ ${baseUrl}(成本闸 ≤${limits.callsPerDay} 次 / ≤${limits.tokensPerDay} token 每日)`);
|
|
84
|
+
for (const n of owner.notes ?? []) log(`⚠️ ${n}`);
|
|
85
|
+
|
|
86
|
+
// ── 只读取样 + 过滤 ────────────────────────────────────────────────────────
|
|
87
|
+
const scanned = D.readCandidates(dbPath, { limit, maxCharsPerItem });
|
|
88
|
+
const state = D.readState(stateFile, dbPath, nowIso);
|
|
89
|
+
if (!scanned.ownerColumn) log('⚠️ 库无 owner_user 列(未迁移的老库形态)⇒ 全部条目按「本机 owner」处理');
|
|
90
|
+
|
|
91
|
+
const pending = [];
|
|
92
|
+
const skippedOwner = [];
|
|
93
|
+
for (const it of scanned.items) {
|
|
94
|
+
const ownerUser = scanned.ownerColumn ? it.ownerUser : owner.user;
|
|
95
|
+
if (ownerUser !== owner.user) {
|
|
96
|
+
// D2 从严:归属不明 / 非本机 owner ⇒ **一律不送第三方模型**,跳过并记账
|
|
97
|
+
skippedOwner.push({ id: it.id, ownerUser: ownerUser ?? '(空)' });
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
100
|
+
if (state.processed[String(it.id)]) continue; // 幂等台账:已处理过的不再送模型
|
|
101
|
+
pending.push(it);
|
|
102
|
+
}
|
|
103
|
+
const batches = Math.ceil(pending.length / batchSize);
|
|
104
|
+
const estimated = pending.slice(0, batchSize).length ? D.estimateTokens(pending.slice(0, batchSize)) : 0;
|
|
105
|
+
result.stats = {
|
|
106
|
+
scanned: scanned.items.length, skippedOwner: skippedOwner.length, alreadyProcessed: scanned.items.length - pending.length - skippedOwner.length,
|
|
107
|
+
pending: pending.length, plannedBatches: batches, estimatedTokensPerBatch: estimated,
|
|
108
|
+
};
|
|
109
|
+
log(`扫描 ${scanned.items.length} 条 · 跳过(非本机 owner) ${skippedOwner.length} · 已处理(台账) ${result.stats.alreadyProcessed} · 待处理 ${pending.length}(${batches} 批)`);
|
|
110
|
+
if (skippedOwner.length) log(` ⚠️ 跳过条目(D2 从严:不进第三方)id=[${skippedOwner.map((s) => s.id).slice(0, 20).join(',')}${skippedOwner.length > 20 ? '…' : ''}]`);
|
|
111
|
+
|
|
112
|
+
if (!apply) {
|
|
113
|
+
log('\n[DRY-RUN] 未写任何内容(未调模型、未写 state);要真跑加 --apply');
|
|
114
|
+
result.ok = true;
|
|
115
|
+
result.state = { file: stateFile, cost: state.cost };
|
|
116
|
+
return finish(result, 0, json, log);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// ── 成本闸(调用前总检查) ─────────────────────────────────────────────────
|
|
120
|
+
if (pending.length === 0) {
|
|
121
|
+
log('\n无待处理条目(幂等台账已覆盖)—— 零写入');
|
|
122
|
+
result.ok = true;
|
|
123
|
+
result.state = { file: stateFile, cost: state.cost, processed: Object.keys(state.processed).length };
|
|
124
|
+
return finish(result, 0, json, log);
|
|
125
|
+
}
|
|
126
|
+
const gate0 = D.costGate(state, estimated, limits);
|
|
127
|
+
if (!gate0.ok) {
|
|
128
|
+
result.stopping = gate0.reason;
|
|
129
|
+
log(`\n[STOP] 成本闸:${gate0.reason}(超限即停,不重试)`);
|
|
130
|
+
result.ok = true;
|
|
131
|
+
return finish(result, 0, json, log);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// ── 快照(§3.6 前置) ──────────────────────────────────────────────────────
|
|
135
|
+
if (snapshotDir) {
|
|
136
|
+
const dest = path.join(snapshotDir, 'memory.db');
|
|
137
|
+
const file = D.vacuumInto(dbPath, dest);
|
|
138
|
+
const st = fs.statSync(file);
|
|
139
|
+
result.snapshot = { file, bytes: st.size };
|
|
140
|
+
log(`✓ 快照:${file}(${st.size} B)`);
|
|
141
|
+
} else {
|
|
142
|
+
log('⚠️ 未给 --snapshot:本次无快照(副本演练可接受,生产前置必须有)');
|
|
143
|
+
result.snapshot = null;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// ── 分类模型 key ──────────────────────────────────────────────────────────
|
|
147
|
+
const apiKey = process.env[D.DEFAULT_LLM.credentialKey] || D.readCredential(D.DEFAULT_LLM.credentialKey);
|
|
148
|
+
if (!apiKey) {
|
|
149
|
+
log(`环境不可用:找不到 ${D.DEFAULT_LLM.credentialKey}(env 与 .credentials.yaml 都没有)`);
|
|
150
|
+
result.exitCode = 3;
|
|
151
|
+
return finish(result, 3, json, log);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
// ── 分批跑 ────────────────────────────────────────────────────────────────
|
|
155
|
+
const db = D.openDb(dbPath, { write: true });
|
|
156
|
+
state.stats.runs = Number(state.stats.runs ?? 0) + 1;
|
|
157
|
+
const runStart = Date.now();
|
|
158
|
+
try {
|
|
159
|
+
for (let b = 0; b < batches; b += 1) {
|
|
160
|
+
const slice = pending.slice(b * batchSize, (b + 1) * batchSize);
|
|
161
|
+
if (slice.length === 0) break;
|
|
162
|
+
const est = D.estimateTokens(slice);
|
|
163
|
+
const gate = D.costGate(state, est, limits);
|
|
164
|
+
if (!gate.ok) {
|
|
165
|
+
result.stopping = gate.reason;
|
|
166
|
+
log(`\n[STOP] 成本闸:${gate.reason}(超限即停,不重试)`);
|
|
167
|
+
break;
|
|
168
|
+
}
|
|
169
|
+
const callStart = Date.now();
|
|
170
|
+
const resp = await D.callClassifier({ items: slice, apiKey, baseUrl, model });
|
|
171
|
+
if (!resp.ok) {
|
|
172
|
+
result.stopping = `模型调用失败(第 ${b + 1} 批):${resp.error}`;
|
|
173
|
+
log(`\n[STOP] ${result.stopping}`);
|
|
174
|
+
break;
|
|
175
|
+
}
|
|
176
|
+
state.cost.calls = Number(state.cost.calls ?? 0) + 1;
|
|
177
|
+
state.cost.promptTokens = Number(state.cost.promptTokens ?? 0) + Number(resp.usage?.prompt_tokens ?? 0);
|
|
178
|
+
state.cost.completionTokens = Number(state.cost.completionTokens ?? 0) + Number(resp.usage?.completion_tokens ?? 0);
|
|
179
|
+
state.cost.date = nowIso.slice(0, 10);
|
|
180
|
+
|
|
181
|
+
// 模型输出 → 只保留本批条目的 id(防模型编 id);域标签只从固定集合里认(appendDomainTags 里过滤)
|
|
182
|
+
const inBatch = new Map(slice.map((it) => [Number(it.id), it]));
|
|
183
|
+
const results = [];
|
|
184
|
+
let unknownDomainDropped = 0;
|
|
185
|
+
for (const item of resp.items) {
|
|
186
|
+
const id = Number(item?.id);
|
|
187
|
+
if (!Number.isFinite(id) || !inBatch.has(id)) continue;
|
|
188
|
+
const rawDomains = Array.isArray(item?.domains) ? item.domains : (Array.isArray(item?.domain) ? item.domain : []);
|
|
189
|
+
const valid = rawDomains.filter((d) => D.DOMAIN_TOKEN_BY_LABEL[String(d).trim()] !== undefined);
|
|
190
|
+
unknownDomainDropped += rawDomains.length - valid.length;
|
|
191
|
+
results.push({ id, domains: valid });
|
|
192
|
+
}
|
|
193
|
+
const missing = slice.filter((it) => !results.some((r) => r.id === Number(it.id)));
|
|
194
|
+
for (const m of missing) results.push({ id: Number(m.id), domains: [] }); // 模型漏答 ⇒ 记为"无标签",不写库
|
|
195
|
+
|
|
196
|
+
const wr = await D.applyResults(db, results);
|
|
197
|
+
if (!wr.ok) {
|
|
198
|
+
result.stopping = `写库失败(第 ${b + 1} 批):${wr.error}`;
|
|
199
|
+
log(`\n[STOP] ${result.stopping}`);
|
|
200
|
+
break;
|
|
201
|
+
}
|
|
202
|
+
// 幂等台账:本批已处理(含"无标签"——它们也算处理过,避免每轮重复问模型)
|
|
203
|
+
for (const it of slice) {
|
|
204
|
+
const r = results.find((x) => x.id === Number(it.id));
|
|
205
|
+
state.processed[String(it.id)] = D.tagSetHash(r ? r.domains : []);
|
|
206
|
+
}
|
|
207
|
+
state.stats.batches = Number(state.stats.batches ?? 0) + 1;
|
|
208
|
+
state.stats.tagged = Number(state.stats.tagged ?? 0) + wr.written;
|
|
209
|
+
state.stats.noTag = Number(state.stats.noTag ?? 0) + results.filter((r) => r.domains.length === 0).length;
|
|
210
|
+
state.stats.skippedOwner = skippedOwner.length;
|
|
211
|
+
state.stats.skippedUnknownDomain = Number(state.stats.skippedUnknownDomain ?? 0) + unknownDomainDropped;
|
|
212
|
+
state.updatedAt = new Date().toISOString();
|
|
213
|
+
D.writeState(stateFile, state); // **每批落盘** ⇒ 可中断续跑
|
|
214
|
+
log(` 批 ${b + 1}/${batches}: 调用 ${state.cost.calls} 次 · 写 ${wr.written} 条 · 已是 ${wr.unchanged} 条 · 耗时 ${Date.now() - callStart}ms · 当日 token ${Number(state.cost.promptTokens) + Number(state.cost.completionTokens)}`);
|
|
215
|
+
}
|
|
216
|
+
} finally {
|
|
217
|
+
try { db.close(); } catch { /* 忽略 */ }
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// ── 汇总 + 日报一行 ───────────────────────────────────────────────────────
|
|
221
|
+
for (const s of skippedOwner) state.skipped[String(s.id)] = `owner=${s.ownerUser}`;
|
|
222
|
+
state.updatedAt = new Date().toISOString();
|
|
223
|
+
D.writeState(stateFile, state);
|
|
224
|
+
const usedTokens = Number(state.cost.promptTokens) + Number(state.cost.completionTokens);
|
|
225
|
+
const line = D.renderDailyLine({
|
|
226
|
+
mode: 'apply', stats: state.stats, cost: state.cost, state: { processedCount: Object.keys(state.processed).length },
|
|
227
|
+
limits, stoppedReason: result.stopping,
|
|
228
|
+
});
|
|
229
|
+
result.cost = { ...state.cost, usedTokens };
|
|
230
|
+
result.state = { file: stateFile, processed: Object.keys(state.processed).length };
|
|
231
|
+
log(`\n完成:批次 ${state.stats.batches} · 本轮写 ${state.stats.tagged} 条 · 当日调用 ${state.cost.calls}/${limits.callsPerDay} · 当日 token ${usedTokens}/${limits.tokensPerDay} · 用时 ${Date.now() - runStart}ms`);
|
|
232
|
+
if (!flags['no-daily-report']) {
|
|
233
|
+
const reportFile = flags['daily-report'] ? path.resolve(String(flags['daily-report'])) : D.dailyReportPath({});
|
|
234
|
+
const r = D.updateDailySection({ file: reportFile, line, dryRun: false });
|
|
235
|
+
result.dailyReport = { file: reportFile, ...r, line };
|
|
236
|
+
log(`${r.ok ? '✓' : '⚠️'} 日报:${reportFile}${r.ok ? `(${r.replaced ? '段替换' : '追加段'})` : `(${r.reason})`}`);
|
|
237
|
+
}
|
|
238
|
+
result.dailyLine = line;
|
|
239
|
+
result.ok = true;
|
|
240
|
+
return finish(result, 0, json, log);
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
function finish(result, code, json, log) {
|
|
244
|
+
result.exitCode = code;
|
|
245
|
+
if (json) {
|
|
246
|
+
process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
|
|
247
|
+
} else {
|
|
248
|
+
log(`\n[${code === 0 ? 'OK' : 'STOP'}] exit=${code}`);
|
|
249
|
+
}
|
|
250
|
+
process.exitCode = code;
|
|
251
|
+
}
|