@kira905/dsh-memory-toolkit 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/LICENSE +661 -0
- package/README.en.md +110 -0
- package/README.md +102 -2
- package/dsh-memory-local/FORK-BASE.json +24 -0
- package/dsh-memory-local/README.md +113 -0
- package/dsh-memory-local/build.mjs +101 -0
- package/dsh-memory-local/cordis.patch.yml +20 -0
- package/dsh-memory-local/deploy.mjs +306 -0
- package/dsh-memory-local/entity-status.mjs +75 -0
- package/dsh-memory-local/lib/bridge.mjs +108 -0
- package/dsh-memory-local/lib/host-adapter.js +1171 -0
- package/dsh-memory-local/lib/index.d.ts +128 -0
- package/dsh-memory-local/lib/index.js +944 -0
- package/dsh-memory-local/lib/injector.mjs +400 -0
- package/dsh-memory-local/lib/memory-read.mjs +116 -0
- package/dsh-memory-local/lib/module-gate.mjs +68 -0
- package/dsh-memory-local/lib/pointer.mjs +152 -0
- package/dsh-memory-local/lib/prod-gate.mjs +121 -0
- package/dsh-memory-local/lib/recall-log.mjs +118 -0
- package/dsh-memory-local/lib/recall.mjs +192 -0
- package/dsh-memory-local/lib/schema-migrate.mjs +365 -0
- package/dsh-memory-local/lib/user-message.mjs +32 -0
- package/dsh-memory-local/lib/vectors-read.mjs +110 -0
- package/dsh-memory-local/package.json +51 -0
- package/dsh-memory-local/restore.mjs +259 -0
- package/dsh-memory-local/rollback-injector-fix.mjs +166 -0
- package/dsh-memory-local/src/bridge.mjs +108 -0
- package/dsh-memory-local/src/host-adapter.js +1171 -0
- package/dsh-memory-local/src/index.d.ts +128 -0
- package/dsh-memory-local/src/index.js +944 -0
- package/dsh-memory-local/src/injector.mjs +400 -0
- package/dsh-memory-local/src/memory-read.mjs +116 -0
- package/dsh-memory-local/src/module-gate.mjs +68 -0
- package/dsh-memory-local/src/pointer.mjs +152 -0
- package/dsh-memory-local/src/recall-log.mjs +118 -0
- package/dsh-memory-local/src/recall.mjs +192 -0
- package/dsh-memory-local/src/schema-migrate.mjs +365 -0
- package/dsh-memory-local/src/user-message.mjs +32 -0
- package/dsh-memory-local/src/vectors-read.mjs +110 -0
- package/dsh-memory-local/test/fixtures-s2b.mjs +216 -0
- package/dsh-memory-local/test/probe-entity.mjs +170 -0
- package/dsh-memory-local/test/regression.mjs +270 -0
- package/dsh-memory-local/test/s2b.test.mjs +568 -0
- package/dsh-memory-local/test/schema-migrate.test.mjs +334 -0
- package/dsh-memory-local/test/test-adapter-converge.mjs +127 -0
- package/dsh-memory-local/test/visible-injections.test.mjs +239 -0
- package/dsh-memory-local/verify-after-restart.mjs +181 -0
- package/lib/deploy-guard.mjs +391 -0
- package/lib/guard-http.mjs +93 -0
- package/lib/pitfall-form-map.mjs +130 -0
- package/lib/pitfall-score.mjs +185 -0
- package/lib/session-file-name.mjs +160 -0
- package/lib/zstd-frames.mjs +78 -0
- package/memory-db-tools/README.md +68 -0
- package/memory-db-tools/_test/test-memory-scope.mjs +302 -0
- package/memory-db-tools/audit-analyze.mjs +352 -0
- package/memory-db-tools/audit-dump-ids.mjs +245 -0
- package/memory-db-tools/audit-export.mjs +270 -0
- package/memory-db-tools/audit-hit-probe.mjs +227 -0
- package/memory-db-tools/audit-ledger-probe.mjs +236 -0
- package/memory-db-tools/audit-persona-overlap.mjs +277 -0
- package/memory-db-tools/audit-struct-probe.mjs +213 -0
- package/memory-db-tools/audit-verify-unchanged.mjs +276 -0
- package/memory-db-tools/check-memory-db.mjs +8 -0
- package/memory-db-tools/dump-memory-db.mjs +12 -0
- package/memory-db-tools/lib/memory-scope.mjs +446 -0
- package/memory-db-tools/list-memory-summary.mjs +477 -0
- package/memory-db-tools/memory-identity-check.mjs +101 -0
- package/memory-db-tools/memory-schema-migrate.mjs +290 -0
- package/memory-db-tools/memory-scope-fixture.mjs +197 -0
- package/memory-db-tools/read-memories.mjs +26 -0
- package/memory-db-tools/read-memory-db.mjs +20 -0
- package/memory-db-tools/refine-memories.mjs +247 -0
- package/memory-db-tools/show-mem-times.mjs +22 -0
- package/memory-engine/README.md +70 -0
- package/memory-engine/bench/bench.mjs +221 -0
- package/memory-engine/cli/distill-rollback.mjs +86 -0
- package/memory-engine/cli/distill.mjs +156 -0
- package/memory-engine/cli/doc-index-rebuild.mjs +155 -0
- package/memory-engine/cli/doc-index.mjs +48 -0
- package/memory-engine/cli/domain-tags.mjs +251 -0
- package/memory-engine/cli/pitfall-hook.mjs +91 -0
- package/memory-engine/cli/recall.mjs +132 -0
- package/memory-engine/cli/selfcheck.mjs +57 -0
- package/memory-engine/cli/serve-start.mjs +247 -0
- package/memory-engine/cli/sweep.mjs +58 -0
- package/memory-engine/cli/sync.mjs +69 -0
- package/memory-engine/lib/cli-util.mjs +197 -0
- package/memory-engine/lib/config.mjs +135 -0
- package/memory-engine/lib/dirty-ledger.mjs +174 -0
- package/memory-engine/lib/distill-rollback.mjs +178 -0
- package/memory-engine/lib/distill.mjs +898 -0
- package/memory-engine/lib/doc-index-store.mjs +325 -0
- package/memory-engine/lib/doc-index.mjs +529 -0
- package/memory-engine/lib/doc-scan.mjs +271 -0
- package/memory-engine/lib/doc-summarize.mjs +305 -0
- package/memory-engine/lib/domain-tags.mjs +429 -0
- package/memory-engine/lib/embedder.mjs +242 -0
- package/memory-engine/lib/hashes.mjs +12 -0
- package/memory-engine/lib/memory-source.mjs +127 -0
- package/memory-engine/lib/pitfall-source.mjs +170 -0
- package/memory-engine/lib/policy-source.mjs +371 -0
- package/memory-engine/lib/recall.mjs +331 -0
- package/memory-engine/lib/selfcheck.mjs +131 -0
- package/memory-engine/lib/sweep.mjs +159 -0
- package/memory-engine/lib/sync.mjs +180 -0
- package/memory-engine/lib/tick-hook.mjs +218 -0
- package/memory-engine/lib/vectors-store.mjs +292 -0
- package/memory-engine/package-lock.json +1086 -0
- package/memory-engine/package.json +22 -0
- package/memory-engine/serve.mjs +171 -0
- package/memory-engine/test/01-store.test.mjs +127 -0
- package/memory-engine/test/02-sync.test.mjs +196 -0
- package/memory-engine/test/03-sweep.test.mjs +145 -0
- package/memory-engine/test/04-recall.test.mjs +214 -0
- package/memory-engine/test/05-selfcheck.test.mjs +175 -0
- package/memory-engine/test/06-doc-index.test.mjs +928 -0
- package/memory-engine/test/07-domain-tags.test.mjs +298 -0
- package/memory-engine/test/08-distill.test.mjs +686 -0
- package/memory-engine/test/09-pitfall-source.test.mjs +365 -0
- package/memory-engine/test/10-tick-hook.test.mjs +252 -0
- package/memory-engine/test/11-policy-source.test.mjs +224 -0
- package/memory-engine/test/_tickhook-worker.mjs +56 -0
- package/memory-engine/test/fixtures.mjs +205 -0
- package/memory-engine/test/harness.mjs +73 -0
- package/memory-engine/test/run.mjs +35 -0
- package/memory-engine/tools/bridge-smoke.mjs +80 -0
- package/memory-engine/tools/fetch-model.mjs +128 -0
- package/memory-engine/tools/smoke-recall.mjs +76 -0
- package/package.json +53 -4
- package/persona-memory-audit.mjs +217 -0
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* refine-memories.mjs —— 记忆库「按 payload 逐条改写」通用写入入口(先 dry-run,--apply 才写)
|
|
4
|
+
*
|
|
5
|
+
* 与既有一次性大扫除脚本的关系:
|
|
6
|
+
* · 后者是 **2026-09-15 长期库大扫除 L4 那次的一次性脚本** —— 它的 `validate()` 内置
|
|
7
|
+
* `IDS_EXPECTED = [17 个固定 id]` 覆盖度自检("载荷必须恰好是那 17 条")⇒ **新 payload 必被 fail-closed 拒**,
|
|
8
|
+
* 不可复用(2026-09-24 实测:新载荷报「载荷缺条目 / 含计划外条目」)。
|
|
9
|
+
* · 本脚本 = 同一套安全骨架(单事务 / 逐条校验 / 改前全文留档 / 可回滚),**去掉硬编码 id 清单**,
|
|
10
|
+
* 改为「payload 声明什么就改什么」;其余纪律一致。**未改动旧脚本一个字节。**
|
|
11
|
+
*
|
|
12
|
+
* 安全纪律(与旧脚本同):
|
|
13
|
+
* · 只 UPDATE memories.text(按 id 逐条),**不碰 pinned / tags / created_at**,不插入、不删除;
|
|
14
|
+
* · 单事务:任一条校验失败 ⇒ 整体不执行;
|
|
15
|
+
* · 校验:id 必须真实存在 · text 非空 · `must_contain` 逐条字面命中 · `forbid` 正则不得命中 · 载荷无重复 id;
|
|
16
|
+
* · 报告含**改前全文**(供回滚),默认落 `_archive`;`--rollback <报告>` 可逐条写回。
|
|
17
|
+
*
|
|
18
|
+
* 用法:
|
|
19
|
+
* node refine-memories.mjs --payload <f> # dry-run(默认)
|
|
20
|
+
* node refine-memories.mjs --payload <f> --apply # 真写(单事务)
|
|
21
|
+
* node refine-memories.mjs --payload <f> --self-test # 在临时库副本上跑完整通路 + 反校验
|
|
22
|
+
* node refine-memories.mjs --rollback <报告.json> # 按报告把改前全文写回
|
|
23
|
+
* (可选 --db <库> --report <报告>;默认见常量)
|
|
24
|
+
* 退出码:0 正常 / 1 校验或执行失败 / 2 用法错 / 3 环境不可用
|
|
25
|
+
*/
|
|
26
|
+
import fs from 'node:fs';
|
|
27
|
+
import os from 'node:os';
|
|
28
|
+
import path from 'node:path';
|
|
29
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
30
|
+
|
|
31
|
+
const HOME = (process.env.USERPROFILE ?? '').replaceAll('\\', '/');
|
|
32
|
+
const DEF_DB = `${HOME}/.dsh/memory/memory.db`;
|
|
33
|
+
const DEF_REPORT = path.join(process.cwd(), '_archive', 'memory-refine', 'refine-report.json');
|
|
34
|
+
|
|
35
|
+
function argOf(argv, name, def) {
|
|
36
|
+
const i = argv.indexOf(name);
|
|
37
|
+
return i >= 0 && argv[i + 1] ? argv[i + 1] : def;
|
|
38
|
+
}
|
|
39
|
+
function realOr(p) {
|
|
40
|
+
try { return fs.realpathSync.native(p); } catch { return path.resolve(p); }
|
|
41
|
+
}
|
|
42
|
+
function mainEntry() {
|
|
43
|
+
const a1 = process.argv[1] ? realOr(process.argv[1]) : '';
|
|
44
|
+
const a2 = realOr(new URL(import.meta.url).pathname.replace(/^\/([A-Za-z]:)/, '$1'));
|
|
45
|
+
return a1.toLowerCase() === a2.toLowerCase();
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** 逐条校验;返回 {ok, errors, rows} */
|
|
49
|
+
function validate(db, payload) {
|
|
50
|
+
const errors = [];
|
|
51
|
+
const rows = [];
|
|
52
|
+
const seen = new Set();
|
|
53
|
+
for (const e of payload.entries ?? []) {
|
|
54
|
+
if (!Number.isInteger(e.id) || typeof e.text !== 'string' || !e.text.trim()) {
|
|
55
|
+
errors.push(`#${e.id} 载荷形态非法(id 需整数、text 需非空字符串)`);
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
if (seen.has(e.id)) { errors.push(`#${e.id} 载荷内重复 id`); continue; }
|
|
59
|
+
seen.add(e.id);
|
|
60
|
+
const cur = db.prepare('SELECT text, pinned FROM memories WHERE id=?').get(e.id);
|
|
61
|
+
if (!cur) { errors.push(`#${e.id} 库中不存在(拒绝凭空插入)`); continue; }
|
|
62
|
+
for (const need of e.must_contain ?? []) {
|
|
63
|
+
if (!e.text.includes(need)) errors.push(`#${e.id} 新正文未命中必需关键词「${need}」`);
|
|
64
|
+
}
|
|
65
|
+
for (const pat of e.forbid ?? []) {
|
|
66
|
+
const hit = e.text.match(new RegExp(pat, 'g'));
|
|
67
|
+
if (hit) errors.push(`#${e.id} 新正文命中禁串正则 /${pat}/ → ${hit.slice(0, 2).join(' | ')}`);
|
|
68
|
+
}
|
|
69
|
+
rows.push({ id: e.id, oldLen: cur.text.length, newLen: e.text.length, pinned: cur.pinned });
|
|
70
|
+
}
|
|
71
|
+
if (!rows.length && !errors.length) errors.push('载荷为空');
|
|
72
|
+
return { ok: errors.length === 0, errors, rows };
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function selfTest(payload, srcDb) {
|
|
76
|
+
console.log('=== self-test:在临时库副本上跑完整 UPDATE 通路(不碰生产库)===');
|
|
77
|
+
const tmp = path.join(os.tmpdir(), `mem-refine-selftest-${Date.now()}.db`);
|
|
78
|
+
try {
|
|
79
|
+
const src = new DatabaseSync(srcDb, { readOnly: true });
|
|
80
|
+
src.exec(`VACUUM INTO '${tmp.replaceAll("'", "''")}'`);
|
|
81
|
+
src.close();
|
|
82
|
+
const t = new DatabaseSync(tmp);
|
|
83
|
+
const v = validate(t, payload);
|
|
84
|
+
if (!v.ok) { console.error(` ✗ 校验未过:\n ${v.errors.join('\n ')}`); t.close(); return false; }
|
|
85
|
+
t.exec('BEGIN');
|
|
86
|
+
const ts = Date.now();
|
|
87
|
+
const up = t.prepare('UPDATE memories SET text=?, updated_at=? WHERE id=?');
|
|
88
|
+
for (const e of payload.entries) up.run(e.text, ts, e.id);
|
|
89
|
+
t.exec('COMMIT');
|
|
90
|
+
let bad = 0;
|
|
91
|
+
for (const e of payload.entries) {
|
|
92
|
+
const after = t.prepare('SELECT text FROM memories WHERE id=?').get(e.id).text;
|
|
93
|
+
if (after !== e.text) { console.error(` ✗ #${e.id} 写回不一致`); bad++; }
|
|
94
|
+
}
|
|
95
|
+
const nAll = t.prepare('SELECT COUNT(*) n FROM memories').get().n;
|
|
96
|
+
const nSrc = (() => { const d = new DatabaseSync(srcDb, { readOnly: true }); const n = d.prepare('SELECT COUNT(*) n FROM memories').get().n; d.close(); return n; })();
|
|
97
|
+
t.close();
|
|
98
|
+
const ok = bad === 0 && nAll === nSrc;
|
|
99
|
+
console.log(` ${payload.entries.length} 条 UPDATE → 条数 ${nAll}(源 ${nSrc})· 逐条比对不一致 ${bad} 处 ${ok ? '✅ 通路正常' : '✗'}`);
|
|
100
|
+
return ok;
|
|
101
|
+
} catch (e) {
|
|
102
|
+
console.error(` ✗ self-test 失败:${e.message}`);
|
|
103
|
+
return false;
|
|
104
|
+
} finally {
|
|
105
|
+
try { fs.unlinkSync(tmp); } catch { /* 忽略 */ }
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function main() {
|
|
110
|
+
const argv = process.argv.slice(2);
|
|
111
|
+
if (argv.includes('--help')) {
|
|
112
|
+
console.log('用法: node refine-memories.mjs --payload <f> [--apply|--self-test] [--db <库>] [--report <报告>]');
|
|
113
|
+
console.log(' node refine-memories.mjs --rollback <报告.json>');
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
const dbPath = argOf(argv, '--db', DEF_DB);
|
|
117
|
+
const reportPath = argOf(argv, '--report', DEF_REPORT);
|
|
118
|
+
|
|
119
|
+
// ---- 回滚分支
|
|
120
|
+
const rbi = argv.indexOf('--rollback');
|
|
121
|
+
if (rbi >= 0) {
|
|
122
|
+
const rep = argv[rbi + 1];
|
|
123
|
+
if (!rep || !fs.existsSync(rep)) { console.error(`[rollback] 报告不存在:${rep}`); process.exitCode = 2; return; }
|
|
124
|
+
const r = JSON.parse(fs.readFileSync(rep, 'utf8'));
|
|
125
|
+
const before = r.before ?? {};
|
|
126
|
+
const beforePinned = r.beforePinned ?? {};
|
|
127
|
+
const ids = Object.keys(before).map(Number);
|
|
128
|
+
const pinIds = Object.keys(beforePinned).map(Number);
|
|
129
|
+
if (!ids.length && !pinIds.length) { console.error('[rollback] 报告既无 before 也无 beforePinned 段'); process.exitCode = 1; return; }
|
|
130
|
+
const db = new DatabaseSync(dbPath);
|
|
131
|
+
db.exec('BEGIN');
|
|
132
|
+
try {
|
|
133
|
+
let n = 0;
|
|
134
|
+
if (ids.length) {
|
|
135
|
+
const up = db.prepare('UPDATE memories SET text=?, updated_at=? WHERE id=?');
|
|
136
|
+
for (const id of ids) { const c = up.run(before[id], Date.now(), id); if (c.changes !== 1) throw new Error(`#${id} 回写影响行数 ${c.changes}`); n++; }
|
|
137
|
+
}
|
|
138
|
+
if (pinIds.length) {
|
|
139
|
+
const up = db.prepare('UPDATE memories SET pinned=?, updated_at=? WHERE id=?');
|
|
140
|
+
for (const id of pinIds) { const c = up.run(beforePinned[id], Date.now(), id); if (c.changes !== 1) throw new Error(`#${id} pinned 回写影响行数 ${c.changes}`); n++; }
|
|
141
|
+
}
|
|
142
|
+
db.exec('COMMIT');
|
|
143
|
+
console.log(`[rollback] 已写回 ${n} 条(text ${ids.length} · pinned ${pinIds.length})`);
|
|
144
|
+
} catch (e) { db.exec('ROLLBACK'); console.error(`[rollback] 失败已回滚:${e.message}`); process.exitCode = 1; }
|
|
145
|
+
db.close();
|
|
146
|
+
return;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// ---- unpin 分支:把指定 id 的 pinned 置 0(**内容保留**,只减注入量)
|
|
150
|
+
const upi = argv.indexOf('--unpin');
|
|
151
|
+
if (upi >= 0) {
|
|
152
|
+
const ids = String(argv[upi + 1] ?? '').split(',').map((s) => Number(s.trim())).filter(Number.isInteger);
|
|
153
|
+
if (!ids.length) { console.error('[unpin] 需要 --unpin <id,id,...>'); process.exitCode = 2; return; }
|
|
154
|
+
const db = new DatabaseSync(dbPath);
|
|
155
|
+
const rows = ids.map((id) => {
|
|
156
|
+
const r = db.prepare('SELECT id, pinned, LENGTH(text) AS len FROM memories WHERE id=?').get(id);
|
|
157
|
+
return { id, exists: !!r, pinned: r?.pinned ?? null, len: r?.len ?? null };
|
|
158
|
+
});
|
|
159
|
+
const missing = rows.filter((r) => !r.exists).map((r) => r.id);
|
|
160
|
+
const pinnedNow = db.prepare('SELECT COUNT(*) n FROM memories WHERE pinned=1').get().n;
|
|
161
|
+
console.log(`[unpin] 目标 ${ids.length} 条 · 缺失 ${missing.length} · 当前 pinned 总数 ${pinnedNow}`);
|
|
162
|
+
for (const r of rows) console.log(` #${r.id} pinned=${r.pinned} ${r.len} 字符`);
|
|
163
|
+
if (missing.length) { console.error(`[fail-closed] 以下 id 不存在:${missing.join(',')}`); db.close(); process.exitCode = 1; return; }
|
|
164
|
+
if (!argv.includes('--apply')) { db.close(); console.log('[unpin] dry-run 未写库;加 --apply 执行'); return; }
|
|
165
|
+
const beforePinned = {};
|
|
166
|
+
db.exec('BEGIN');
|
|
167
|
+
try {
|
|
168
|
+
const up = db.prepare('UPDATE memories SET pinned=0, updated_at=? WHERE id=?');
|
|
169
|
+
let n = 0;
|
|
170
|
+
for (const r of rows) {
|
|
171
|
+
beforePinned[r.id] = r.pinned;
|
|
172
|
+
const c = up.run(Date.now(), r.id);
|
|
173
|
+
if (c.changes !== 1) throw new Error(`#${r.id} 影响行数=${c.changes}`);
|
|
174
|
+
n++;
|
|
175
|
+
}
|
|
176
|
+
db.exec('COMMIT');
|
|
177
|
+
const np = db.prepare('SELECT COUNT(*) n FROM memories WHERE pinned=1').get().n;
|
|
178
|
+
console.log(`\n✅ 已降级 ${n} 条 → 现 pinned 条数 = ${np}`);
|
|
179
|
+
fs.mkdirSync(path.dirname(reportPath), { recursive: true });
|
|
180
|
+
fs.writeFileSync(reportPath, JSON.stringify({
|
|
181
|
+
at: new Date().toISOString(), session: process.env.DSH_SESSION_ID ?? null, dbPath,
|
|
182
|
+
unpinned: Object.keys(beforePinned).map(Number), beforePinned, pinnedAfter: np,
|
|
183
|
+
}, null, 2) + '\n', 'utf8');
|
|
184
|
+
console.log(`留档:${reportPath}\n回滚:node refine-memories.mjs --rollback "${reportPath}"`);
|
|
185
|
+
} catch (e) { db.exec('ROLLBACK'); console.error(`✗ 事务失败已回滚:${e.message}`); process.exitCode = 1; }
|
|
186
|
+
db.close();
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
const payloadPath = argOf(argv, '--payload', null);
|
|
191
|
+
if (!payloadPath || !fs.existsSync(payloadPath)) { console.error(`[fail-closed] 找不到载荷 ${payloadPath}`); process.exitCode = 3; return; }
|
|
192
|
+
if (!fs.existsSync(dbPath)) { console.error(`[fail-closed] 找不到记忆库 ${dbPath}`); process.exitCode = 3; return; }
|
|
193
|
+
|
|
194
|
+
const payload = JSON.parse(fs.readFileSync(payloadPath, 'utf8'));
|
|
195
|
+
console.log(`载荷:${payloadPath}(${(payload.entries ?? []).length} 条)\n目标库:${dbPath}`);
|
|
196
|
+
|
|
197
|
+
if (argv.includes('--self-test')) {
|
|
198
|
+
process.exitCode = selfTest(payload, dbPath) ? 0 : 1;
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
const db = new DatabaseSync(dbPath);
|
|
203
|
+
const v = validate(db, payload);
|
|
204
|
+
console.log(`\n校验:${v.ok ? '✅ 全通过' : `✗ ${v.errors.length} 项不通过`}`);
|
|
205
|
+
for (const e of v.errors) console.log(` ✗ ${e}`);
|
|
206
|
+
console.log('\n逐条明细(id · 改前→改后字符数 · pinned):');
|
|
207
|
+
for (const r of v.rows) console.log(` #${String(r.id).padStart(4)} ${r.oldLen} → ${r.newLen} 字符 pinned=${r.pinned}`);
|
|
208
|
+
|
|
209
|
+
if (!v.ok) { db.close(); process.exitCode = 1; return; }
|
|
210
|
+
if (!argv.includes('--apply')) {
|
|
211
|
+
db.close();
|
|
212
|
+
console.log('\n(dry-run:未写库;加 --apply 执行改写)');
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
const stamp = Date.now();
|
|
217
|
+
db.exec('BEGIN');
|
|
218
|
+
try {
|
|
219
|
+
const up = db.prepare('UPDATE memories SET text=?, updated_at=? WHERE id=?');
|
|
220
|
+
const rd = db.prepare('SELECT text FROM memories WHERE id=?');
|
|
221
|
+
const before = {};
|
|
222
|
+
for (const e of payload.entries) before[e.id] = rd.get(e.id).text;
|
|
223
|
+
let n = 0;
|
|
224
|
+
for (const e of payload.entries) {
|
|
225
|
+
const r = up.run(e.text, stamp, e.id);
|
|
226
|
+
if (r.changes !== 1) throw new Error(`#${e.id} UPDATE 影响行数=${r.changes}(期望 1)`);
|
|
227
|
+
n++;
|
|
228
|
+
}
|
|
229
|
+
db.exec('COMMIT');
|
|
230
|
+
const after = db.prepare('SELECT COUNT(*) n, SUM(LENGTH(text)) c FROM memories').get();
|
|
231
|
+
console.log(`\n✅ 已改写 ${n} 条 → 库内 ${after.n} 条 / ${after.c} 字符`);
|
|
232
|
+
fs.mkdirSync(path.dirname(reportPath), { recursive: true });
|
|
233
|
+
fs.writeFileSync(reportPath, JSON.stringify({
|
|
234
|
+
at: new Date().toISOString(), session: process.env.DSH_SESSION_ID ?? null,
|
|
235
|
+
payloadPath, dbPath, updated: n, totalAfter: after.n, charsAfter: after.c, before,
|
|
236
|
+
}, null, 2) + '\n', 'utf8');
|
|
237
|
+
console.log(`改前全文留档(供回滚):${reportPath}`);
|
|
238
|
+
console.log(`回滚:node refine-memories.mjs --rollback "${reportPath}"`);
|
|
239
|
+
} catch (err) {
|
|
240
|
+
db.exec('ROLLBACK');
|
|
241
|
+
console.error(`✗ 事务失败已回滚:${err.message}`);
|
|
242
|
+
process.exitCode = 1;
|
|
243
|
+
}
|
|
244
|
+
db.close();
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
if (mainEntry()) main();
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { DatabaseSync } from "node:sqlite";
|
|
2
|
+
|
|
3
|
+
// 只读列出全部记忆的 id / 创建时间 / 更新时间 / 标题(毫秒 epoch 已转本地时间)。
|
|
4
|
+
// 用法:node show-mem-times.mjs [id ...] (不给 id 则列全部;dsh 运行中也可安全执行)
|
|
5
|
+
const db = new DatabaseSync(`${process.env.USERPROFILE.replaceAll("\\", "/")}/.dsh/memory/memory.db`, { readOnly: true });
|
|
6
|
+
const fmt = ms => new Date(ms).toLocaleString("zh-CN", { hour12: false, timeZone: "Asia/Shanghai" });
|
|
7
|
+
|
|
8
|
+
const args = process.argv.slice(2);
|
|
9
|
+
let rows;
|
|
10
|
+
if (args.length === 0) {
|
|
11
|
+
rows = db.prepare("SELECT id, created_at, updated_at, text, pinned FROM memories ORDER BY id").all();
|
|
12
|
+
} else {
|
|
13
|
+
const ids = args.map(Number).filter(Number.isInteger);
|
|
14
|
+
if (!ids.length) { console.error("用法:node show-mem-times.mjs <id> [id ...](不给 id 列全部)"); process.exit(1); }
|
|
15
|
+
const ph = ids.map(() => "?").join(",");
|
|
16
|
+
rows = db.prepare(`SELECT id, created_at, updated_at, text, pinned FROM memories WHERE id IN (${ph}) ORDER BY id`).all(...ids);
|
|
17
|
+
}
|
|
18
|
+
for (const r of rows) {
|
|
19
|
+
const firstLine = r.text.split("\n")[0].slice(0, 60);
|
|
20
|
+
console.log(`#${String(r.id).padStart(3)} [${r.pinned === 1 ? "pin" : " -"}] 创建 ${fmt(r.created_at)} 更新 ${fmt(r.updated_at)} ${firstLine}`);
|
|
21
|
+
}
|
|
22
|
+
db.close();
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# 记忆引擎 v2 · 离线层(S1)
|
|
2
|
+
|
|
3
|
+
> 定位:DSH 生态的**离线记忆层**(向量库 / 增量同步 / 召回 / 自检 / 编码桥),可独立运行。
|
|
4
|
+
> 落点:向量库默认 `~/.dsh/memory-vectors/`;依赖自带于本目录,**绝不装进宿主的 `profiles/node_modules`**。
|
|
5
|
+
|
|
6
|
+
## 这是什么
|
|
7
|
+
|
|
8
|
+
记忆引擎 v2 的**离线层**(不含 S2 注入插件):
|
|
9
|
+
|
|
10
|
+
| 能力 | 实现 | 方案条款 |
|
|
11
|
+
|---|---|---|
|
|
12
|
+
| 独立向量库 | `vectors` 表 + `meta` 表(schemaVersion / hostId / modelId / dim / lastIncrementalSyncAt / lastFullHashSweepAt) | §3.1 |
|
|
13
|
+
| 增量同步 | 按 `updated_at ≥ 水位` 只读扫 `memory.db`,批 50,失败保留进度可续跑;换模型/维度自动全量重建 | §3.1 |
|
|
14
|
+
| 兜底校验 | 每周按 id 段轮转做全表 `text_hash` 校验(治「改 text 不动 updated_at」的外部写口,A7) | §3.1 / §6.2 |
|
|
15
|
+
| 召回 | 向量余弦 top-20 为主 + FTS5 仅补标签/英文/数字 + RRF(k=60) 融合 + **排除 pinned**;非空即触发,纯符号/单字跳过 | §3.3 |
|
|
16
|
+
| 启动自检 | 模型可用性 / 维度一致 / 表存在性(缺则重建)/ hostId 校验 | §8.3 #9 |
|
|
17
|
+
| 编码桥 | `serve.mjs`,**仅绑 127.0.0.1:8201**,`GET /health` + `POST /encode` | §3.1 编码通道契约 |
|
|
18
|
+
|
|
19
|
+
## 目录
|
|
20
|
+
|
|
21
|
+
```
|
|
22
|
+
memory-engine/
|
|
23
|
+
├─ serve.mjs 编码桥(回环 HTTP,模型常驻)
|
|
24
|
+
├─ cli/ sync.mjs · sweep.mjs · recall.mjs · selfcheck.mjs
|
|
25
|
+
├─ lib/ config / vectors-store / memory-source / embedder / sync / sweep / recall / selfcheck / cli-util / hashes
|
|
26
|
+
├─ tools/fetch-model.mjs 模型权重获取器(node fetch,绕开 Windows curl 的 schannel 坑)
|
|
27
|
+
├─ test/ 单测(夹具全部落系统临时目录)+ run.mjs
|
|
28
|
+
└─ bench/bench.mjs V1 / V2 基准(方案 §9)
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## 常用命令
|
|
32
|
+
|
|
33
|
+
```powershell
|
|
34
|
+
# 一次性:取权重(fp32 90MB / int8 约 25MB)
|
|
35
|
+
node tools\fetch-model.mjs --out <模型目录> --variant fp32
|
|
36
|
+
|
|
37
|
+
# 启动自检
|
|
38
|
+
node cli\selfcheck.mjs [--json]
|
|
39
|
+
|
|
40
|
+
# 编码桥幂等启动(S2c):探活优先(在跑就跳过),不在跑才 detached 拉起;
|
|
41
|
+
# 端口 8201 被占 +1 顺延,真实端口按子进程 pid 定位后落 bridge-port.json
|
|
42
|
+
node cli\serve-start.mjs [--port 8201] [--port-file <台账路径>] [--no-start] [--json]
|
|
43
|
+
# 退出码:0 已在跑 / 拉起就绪 · 1 拉起失败 · 3 --no-start 且桥不在(让路)· 2 用法错
|
|
44
|
+
|
|
45
|
+
# 增量同步 / 全量重建(--memory-db 必须指向记忆库副本,勿指生产库)
|
|
46
|
+
node cli\sync.mjs --memory-db <副本> [--full] [--dry-run]
|
|
47
|
+
|
|
48
|
+
# 兜底 hash 轮转:每次 1 段,--all 一轮完整覆盖
|
|
49
|
+
node cli\sweep.mjs --memory-db <副本> [--rounds N | --all]
|
|
50
|
+
|
|
51
|
+
# 召回(人工查验用)
|
|
52
|
+
node cli\recall.mjs --query "退下吧" --json
|
|
53
|
+
|
|
54
|
+
# 基准(写报告)
|
|
55
|
+
node bench\bench.mjs --memory-db <副本> --out bench-report.json
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
## 边界(硬约束)
|
|
59
|
+
|
|
60
|
+
1. **对 `memory.db` 只读**——所有打开方式都是 `new DatabaseSync(path, { readOnly: true })`,同步只写独立向量库。
|
|
61
|
+
2. **向量库不进任何同步仓**:默认落 `~/.dsh/memory-vectors/`,在仓1 采集白名单之外(方案 D3 的机制收益)。
|
|
62
|
+
3. **模型权重不进同步仓**:放中枢机本地目录(`MEMORY_ENGINE_MODEL_DIR` 可覆盖),两机各下一份。
|
|
63
|
+
4. **依赖不污染闭包**:`node_modules` 只在本目录,绝不装进 `profiles\node_modules`(生态铁律)。
|
|
64
|
+
5. 一切路径可注入:`--memory-db / --vectors-db / --model-dir`,或环境变量 `MEMORY_ENGINE_*`;**测试一律落临时目录**。
|
|
65
|
+
|
|
66
|
+
## 已知坑(实测)
|
|
67
|
+
|
|
68
|
+
- **Windows curl 取不到 HF 大文件**:HF 的 LFS 文件 302 到 Xet CAS 桥(`cas-bridge.xethub.hf.co`),部分 Windows 主机上 schannel 握手失败(`curl: (35) SEC_E_ILLEGAL_MESSAGE`)。用 `tools/fetch-model.mjs`(node 自带 TLS 栈)下载,别用 curl。
|
|
69
|
+
- **`memory-source.close()` 幂等**:调用方常在 `finally` 里再关一次,重复 `db.close()` 会抛 `database is not open`——已做幂等保护。
|
|
70
|
+
- **入口守卫必须 realpath**:工作区里常有目录 junction,`import.meta.url` 解析为物理路径而 `process.argv[1]` 是逻辑路径,直接字符串比较会让脚本静默退出 0(`lib/cli-util.mjs` 的 `isMainEntry`)。
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
// 记忆引擎 v2 · 离线层 —— 基准(方案 §9 V1 / V2,判据只认真机实测)
|
|
2
|
+
//
|
|
3
|
+
// V1:① 单句编码延迟 ② 全量记忆条编码耗时(首次全量 / 重建成本)
|
|
4
|
+
// V2:100 次召回(含 query 编码 + 余弦 + FTS 补位 + RRF 融合)对照 §3.4 的 300ms 硬预算
|
|
5
|
+
//
|
|
6
|
+
// 用法(数据源必须是记忆库副本,绝不直连在用库):
|
|
7
|
+
// node bench/bench.mjs --memory-db <记忆库副本路径>
|
|
8
|
+
// [--vectors-db <path>] [--model-dir <dir>] [--iterations 100] [--json] [--out report.json]
|
|
9
|
+
import fs from 'node:fs';
|
|
10
|
+
import os from 'node:os';
|
|
11
|
+
import path from 'node:path';
|
|
12
|
+
import { parseArgs, isMainEntry } from '../lib/cli-util.mjs';
|
|
13
|
+
import { resolveConfig } from '../lib/config.mjs';
|
|
14
|
+
import { openVectorsStore, countVectors } from '../lib/vectors-store.mjs';
|
|
15
|
+
import { openMemorySource } from '../lib/memory-source.mjs';
|
|
16
|
+
import { createLocalEmbedder } from '../lib/embedder.mjs';
|
|
17
|
+
import { runIncrementalSync } from '../lib/sync.mjs';
|
|
18
|
+
import { recall, buildVectorIndex, shouldTrigger } from '../lib/recall.mjs';
|
|
19
|
+
import { runSelfcheck } from '../lib/selfcheck.mjs';
|
|
20
|
+
|
|
21
|
+
const SHORT_QUERIES = ['退下吧', '整夜干活', '去深水区', '落卡', '回浅水区'];
|
|
22
|
+
|
|
23
|
+
function stats(samples) {
|
|
24
|
+
const a = [...samples].sort((x, y) => x - y);
|
|
25
|
+
const at = (p) => a[Math.min(a.length - 1, Math.floor((p / 100) * a.length))];
|
|
26
|
+
const mean = a.reduce((s, v) => s + v, 0) / (a.length || 1);
|
|
27
|
+
return {
|
|
28
|
+
n: a.length,
|
|
29
|
+
min: +a[0].toFixed(2),
|
|
30
|
+
p50: +at(50).toFixed(2),
|
|
31
|
+
p95: +at(95).toFixed(2),
|
|
32
|
+
max: +a[a.length - 1].toFixed(2),
|
|
33
|
+
mean: +mean.toFixed(2),
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export async function runBench({ config, iterations = 100, log = console.log, doSync = true }) {
|
|
38
|
+
const report = {
|
|
39
|
+
at: new Date().toISOString(),
|
|
40
|
+
host: os.hostname(),
|
|
41
|
+
platform: `${process.platform} ${os.arch()}`,
|
|
42
|
+
node: process.version,
|
|
43
|
+
cpu: (os.cpus()[0] || {}).model || 'unknown',
|
|
44
|
+
cpuCount: os.cpus().length,
|
|
45
|
+
totalMemGB: +(os.totalmem() / 1024 ** 3).toFixed(1),
|
|
46
|
+
config: {
|
|
47
|
+
memoryDb: config.memoryDb,
|
|
48
|
+
vectorsDb: config.vectorsDb,
|
|
49
|
+
modelId: config.modelId,
|
|
50
|
+
modelDir: config.modelDir,
|
|
51
|
+
},
|
|
52
|
+
v1: {},
|
|
53
|
+
v2: {},
|
|
54
|
+
selfcheck: null,
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
// 冷启动:模型加载计时
|
|
58
|
+
const tLoad0 = performance.now();
|
|
59
|
+
const embedder = await createLocalEmbedder({
|
|
60
|
+
modelDir: config.modelDir,
|
|
61
|
+
modelId: config.modelId,
|
|
62
|
+
dim: config.dim,
|
|
63
|
+
quantized: Boolean(config.quantized),
|
|
64
|
+
});
|
|
65
|
+
report.v1.modelLoadMs = +(performance.now() - tLoad0).toFixed(1);
|
|
66
|
+
report.v1.dim = embedder.dim;
|
|
67
|
+
report.v1.embedder = embedder.describe();
|
|
68
|
+
log(`[V1] 模型加载(冷启动):${report.v1.modelLoadMs} ms · dim=${embedder.dim} · ${JSON.stringify(embedder.describe())}`);
|
|
69
|
+
|
|
70
|
+
const store = openVectorsStore(config.vectorsDb);
|
|
71
|
+
store.hostId = config.hostId;
|
|
72
|
+
const source = openMemorySource(config.memoryDb);
|
|
73
|
+
const total = source.count();
|
|
74
|
+
report.v1.memoryRows = total;
|
|
75
|
+
|
|
76
|
+
// ① 全量编码计时(逐条,含批 50 的同步写库)
|
|
77
|
+
let syncStats = null;
|
|
78
|
+
if (doSync) {
|
|
79
|
+
const tFull0 = performance.now();
|
|
80
|
+
syncStats = await runIncrementalSync({
|
|
81
|
+
source,
|
|
82
|
+
store,
|
|
83
|
+
embedder,
|
|
84
|
+
modelId: config.modelId,
|
|
85
|
+
dim: config.dim,
|
|
86
|
+
batchSize: config.batchSize,
|
|
87
|
+
full: true,
|
|
88
|
+
log: () => {},
|
|
89
|
+
});
|
|
90
|
+
const fullMs = performance.now() - tFull0;
|
|
91
|
+
report.v1.fullEncode = {
|
|
92
|
+
rows: syncStats.encoded,
|
|
93
|
+
batches: syncStats.batches,
|
|
94
|
+
totalMs: +fullMs.toFixed(1),
|
|
95
|
+
perItemMs: +(fullMs / Math.max(1, syncStats.encoded)).toFixed(2),
|
|
96
|
+
error: syncStats.error || null,
|
|
97
|
+
};
|
|
98
|
+
log(
|
|
99
|
+
`[V1] 全量编码 ${syncStats.encoded} 条:${fullMs.toFixed(0)} ms(${(fullMs / Math.max(1, syncStats.encoded)).toFixed(2)} ms/条,${syncStats.batches} 批)`,
|
|
100
|
+
);
|
|
101
|
+
} else {
|
|
102
|
+
report.v1.fullEncode = { rows: countVectors(store.db, 'memory'), skipped: true };
|
|
103
|
+
log('[V1] 跳过全量编码(--no-sync)');
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// ② 单句编码计时
|
|
107
|
+
const singleSamples = [];
|
|
108
|
+
const probeText = '跨机同步的挡推裁决口径与三仓推送顺序';
|
|
109
|
+
for (let i = 0; i < iterations; i += 1) {
|
|
110
|
+
const t0 = performance.now();
|
|
111
|
+
await embedder.encode(probeText);
|
|
112
|
+
singleSamples.push(performance.now() - t0);
|
|
113
|
+
}
|
|
114
|
+
report.v1.singleEncode = stats(singleSamples);
|
|
115
|
+
log(`[V1] 单句编码 ${iterations} 次:P50 ${report.v1.singleEncode.p50} ms / P95 ${report.v1.singleEncode.p95} ms`);
|
|
116
|
+
|
|
117
|
+
// ③ 长句编码(记忆条均长 740 字)
|
|
118
|
+
const longRows = source.incrementalPage(0, 0, 20);
|
|
119
|
+
const longText = longRows.map((r) => r.text).join('\n').slice(0, 2000) || probeText.repeat(20);
|
|
120
|
+
const longSamples = [];
|
|
121
|
+
for (let i = 0; i < Math.min(30, iterations); i += 1) {
|
|
122
|
+
const t0 = performance.now();
|
|
123
|
+
await embedder.encode(longText);
|
|
124
|
+
longSamples.push(performance.now() - t0);
|
|
125
|
+
}
|
|
126
|
+
report.v1.longEncode = { chars: longText.length, ...stats(longSamples) };
|
|
127
|
+
log(
|
|
128
|
+
`[V1] 长文本(${longText.length} 字)编码 ${longSamples.length} 次:P50 ${report.v1.longEncode.p50} ms / P95 ${report.v1.longEncode.p95} ms`,
|
|
129
|
+
);
|
|
130
|
+
|
|
131
|
+
// V2:召回计时
|
|
132
|
+
const index = buildVectorIndex(store.db, { source: 'memory' });
|
|
133
|
+
report.v2.indexSize = index.count;
|
|
134
|
+
const idx = { source: 'memory', model: config.modelId };
|
|
135
|
+
void idx;
|
|
136
|
+
const queries = [];
|
|
137
|
+
const seed = source.incrementalPage(0, 0, Math.min(10, total)) || [];
|
|
138
|
+
for (const r of seed) queries.push(String(r.text).slice(0, 40));
|
|
139
|
+
queries.push(...SHORT_QUERIES);
|
|
140
|
+
const fireable = queries.filter((q) => shouldTrigger(q).trigger);
|
|
141
|
+
report.v2.querySet = { count: fireable.length, samples: fireable.slice(0, 5) };
|
|
142
|
+
|
|
143
|
+
const recallSamples = [];
|
|
144
|
+
const parts = { encodeMs: [], vectorMs: [], ftsMs: [], fuseMs: [] };
|
|
145
|
+
const qStats = new Map();
|
|
146
|
+
for (let i = 0; i < iterations; i += 1) {
|
|
147
|
+
const q = fireable[i % fireable.length];
|
|
148
|
+
const r = await recall({ query: q, store, source, embedder, index, resultTopK: config.resultTopK });
|
|
149
|
+
recallSamples.push(r.timings.totalMs);
|
|
150
|
+
parts.encodeMs.push(r.timings.encodeMs);
|
|
151
|
+
parts.vectorMs.push(r.timings.vectorMs);
|
|
152
|
+
parts.ftsMs.push(r.timings.ftsMs);
|
|
153
|
+
parts.fuseMs.push(r.timings.fuseMs);
|
|
154
|
+
const cur = qStats.get(q) || [];
|
|
155
|
+
cur.push(r.timings.totalMs);
|
|
156
|
+
qStats.set(q, cur);
|
|
157
|
+
}
|
|
158
|
+
report.v2.recall = stats(recallSamples);
|
|
159
|
+
report.v2.budgetMs = 300;
|
|
160
|
+
report.v2.verdict = report.v2.recall.p95 <= 300 ? 'PASS' : 'FAIL';
|
|
161
|
+
report.v2.breakdown = {
|
|
162
|
+
encode: stats(parts.encodeMs),
|
|
163
|
+
vector: stats(parts.vectorMs),
|
|
164
|
+
fts: stats(parts.ftsMs),
|
|
165
|
+
fuse: stats(parts.fuseMs),
|
|
166
|
+
};
|
|
167
|
+
report.v2.perQuery = [...qStats.entries()].map(([q, v]) => ({ query: q, ...stats(v) }));
|
|
168
|
+
log(
|
|
169
|
+
`[V2] 召回 ${iterations} 次:P50 ${report.v2.recall.p50} ms / P95 ${report.v2.recall.p95} ms / max ${report.v2.recall.max} ms(预算 ${report.v2.budgetMs} ms ⇒ ${report.v2.verdict})`,
|
|
170
|
+
);
|
|
171
|
+
log(
|
|
172
|
+
` 分解 P50:编码 ${report.v2.breakdown.encode.p50} · 向量 ${report.v2.breakdown.vector.p50} · FTS ${report.v2.breakdown.fts.p50} · 融合 ${report.v2.breakdown.fuse.p50} ms`,
|
|
173
|
+
);
|
|
174
|
+
|
|
175
|
+
// 顺带跑一次启动自检(交付物 #9 的现场证据)
|
|
176
|
+
report.selfcheck = await runSelfcheck({ store, source, embedder, config });
|
|
177
|
+
log(`[自检] ${report.selfcheck.ok ? '✅ 通过' : '❌ 失败'}(warn ${report.selfcheck.warned} / fail ${report.selfcheck.failed})`);
|
|
178
|
+
|
|
179
|
+
source.close();
|
|
180
|
+
store.db.close();
|
|
181
|
+
return report;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
async function main() {
|
|
185
|
+
const { flags } = parseArgs();
|
|
186
|
+
const config = resolveConfig({
|
|
187
|
+
memoryDb: flags['memory-db'] || undefined,
|
|
188
|
+
vectorsDb: flags['vectors-db'] || undefined,
|
|
189
|
+
vectorsDir: flags['vectors-dir'] || undefined,
|
|
190
|
+
modelDir: flags['model-dir'] || undefined,
|
|
191
|
+
modelId: flags.model || undefined,
|
|
192
|
+
dim: flags.dim ? Number(flags.dim) : undefined,
|
|
193
|
+
hostId: flags.host || undefined,
|
|
194
|
+
batchSize: flags.batch ? Number(flags.batch) : undefined,
|
|
195
|
+
});
|
|
196
|
+
if (!fs.existsSync(config.memoryDb)) {
|
|
197
|
+
throw new Error(`记忆库不存在:${config.memoryDb}(本卡须用 ROG 生产库的只读副本,勿指向生产库)`);
|
|
198
|
+
}
|
|
199
|
+
console.log('=== 记忆引擎 v2 · 离线层基准 ===');
|
|
200
|
+
console.log(`主机 : ${os.hostname()} · ${os.cpus().length} 核 ${(os.cpus()[0] || {}).model}`);
|
|
201
|
+
console.log(`内存 : ${(os.totalmem() / 1024 ** 3).toFixed(1)} GB · node ${process.version}`);
|
|
202
|
+
console.log(`记忆库 : ${config.memoryDb}`);
|
|
203
|
+
console.log(`向量库 : ${config.vectorsDb}`);
|
|
204
|
+
console.log(`模型目录 : ${config.modelDir}`);
|
|
205
|
+
console.log('');
|
|
206
|
+
const report = await runBench({
|
|
207
|
+
config,
|
|
208
|
+
iterations: Number(flags.iterations || 100),
|
|
209
|
+
log: (m) => console.log(m),
|
|
210
|
+
doSync: !flags['no-sync'],
|
|
211
|
+
});
|
|
212
|
+
if (flags.out) {
|
|
213
|
+
fs.mkdirSync(path.dirname(path.resolve(flags.out)), { recursive: true });
|
|
214
|
+
fs.writeFileSync(path.resolve(flags.out), JSON.stringify(report, null, 2), 'utf8');
|
|
215
|
+
console.log(`\n报告已写入:${path.resolve(flags.out)}`);
|
|
216
|
+
}
|
|
217
|
+
if (flags.json) process.stdout.write(JSON.stringify(report, null, 2) + '\n');
|
|
218
|
+
process.exitCode = report.v2.verdict === 'PASS' || flags['allow-slow'] ? 0 : 3;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
if (isMainEntry(import.meta.url)) await main();
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// 记忆引擎 v2 · S3c —— 提炼产物回滚 CLI(`cli/distill-rollback.mjs`)
|
|
3
|
+
//
|
|
4
|
+
// 用法:
|
|
5
|
+
// node cli/distill-rollback.mjs --db <memory.db> # **默认 dry-run**:只列出会被删的条
|
|
6
|
+
// node cli/distill-rollback.mjs --db <memory.db> --apply # 真删(先 VACUUM INTO 快照 + 生成一键恢复脚本)
|
|
7
|
+
// node cli/distill-rollback.mjs --db <memory.db> --key <srcKey> # 只回滚某一组
|
|
8
|
+
// node cli/distill-rollback.mjs --db <memory.db> --id 1001 --apply
|
|
9
|
+
// node cli/distill-rollback.mjs --db <memory.db> --digest # 只算内容摘要(演练对照用)
|
|
10
|
+
//
|
|
11
|
+
// 判据:`tags LIKE 'distilled%'`(回滚面就是这个前缀——提炼产物唯一性标记)
|
|
12
|
+
// 退出码:0 = 正常 / 1 = 失败 / 2 = 用法错 / 3 = 环境不可用
|
|
13
|
+
import fs from 'node:fs';
|
|
14
|
+
import path from 'node:path';
|
|
15
|
+
import { isMainEntry, parseArgs, makeLogger, num } from '../lib/cli-util.mjs';
|
|
16
|
+
import { listDistilled, rollbackDistilled, contentDigest, fileSha256 } from '../lib/distill-rollback.mjs';
|
|
17
|
+
|
|
18
|
+
const USAGE = 'node cli/distill-rollback.mjs --db <memory.db> [--apply] [--key <srcKey>]... [--id <id>]... '
|
|
19
|
+
+ '[--snapshot-dir <dir>] [--limit N] [--digest [--real-only]] [--show N] [--json] [--help]';
|
|
20
|
+
|
|
21
|
+
async function main() {
|
|
22
|
+
const { flags } = parseArgs();
|
|
23
|
+
const json = Boolean(flags.json);
|
|
24
|
+
const log = makeLogger({ json, quiet: Boolean(flags.quiet) });
|
|
25
|
+
if (flags.help) { process.stdout.write(`用法:${USAGE}\n`); return 0; }
|
|
26
|
+
|
|
27
|
+
const dbPath = flags.db ? path.resolve(String(flags.db)) : null;
|
|
28
|
+
if (!dbPath) { process.stderr.write('用法错:必须给 --db <库路径>\n'); return 2; }
|
|
29
|
+
if (!fs.existsSync(dbPath)) { process.stderr.write(`环境不可用:库不存在 ${dbPath}\n`); return 3; }
|
|
30
|
+
|
|
31
|
+
const collect = (v) => (v === undefined ? [] : String(v).split(',').map((x) => x.trim()).filter(Boolean));
|
|
32
|
+
const keys = collect(flags.key);
|
|
33
|
+
const ids = collect(flags.id).map((x) => Number(x)).filter((x) => Number.isFinite(x));
|
|
34
|
+
|
|
35
|
+
if (flags.digest) {
|
|
36
|
+
// `--real-only`:只算**真实(非派生)条**的内容摘要 —— 演练判据要的是「回滚后真实条一个字节没变」
|
|
37
|
+
// (整表摘要会把被删掉的提炼条也算进去 ⇒ 对比必然"不一致",那是判据写错、不是回滚失败)
|
|
38
|
+
const exclude = flags['real-only'] ? 'distilled%' : null;
|
|
39
|
+
const d = contentDigest(dbPath, { excludeTagsLike: exclude });
|
|
40
|
+
const out = { db: dbPath, realOnly: Boolean(exclude), ...d, fileSha256: fileSha256(dbPath) };
|
|
41
|
+
if (json) process.stdout.write(JSON.stringify(out, null, 2) + '\n');
|
|
42
|
+
else log(`${exclude ? '真实条' : '整表'}内容摘要:条数 ${d.count} · digest ${d.digest} · 全表条数 ${d.allCount} · 文件 sha ${out.fileSha256}`);
|
|
43
|
+
return 0;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const apply = Boolean(flags.apply);
|
|
47
|
+
const before = contentDigest(dbPath);
|
|
48
|
+
const items = listDistilled({ dbPath, keys: keys.length ? keys : null, ids: ids.length ? ids : null, limit: num(flags.limit, 0) });
|
|
49
|
+
|
|
50
|
+
log(`库 : ${dbPath}`);
|
|
51
|
+
log(`模式 : ${apply ? 'APPLY(真删)' : 'DRY-RUN(只列不删)'}`);
|
|
52
|
+
log(`库现状 : 全表 ${before.allCount} 条(其中提炼条 ${before.allCount - before.count} 条)· 内容摘要 ${before.digest}`);
|
|
53
|
+
log(`命中筛选: ${items.length} 条(--key ${keys.length ? keys.join(',') : '-'} / --id ${ids.length ? ids.join(',') : '-'})`);
|
|
54
|
+
for (const it of items.slice(0, Number(flags.show ?? 50))) log(` #${it.id} src=${it.srcKey} date=${it.date} ${it.chars} 字 · ${it.head.slice(0, 70)}`);
|
|
55
|
+
|
|
56
|
+
if (!items.length) {
|
|
57
|
+
if (json) process.stdout.write(JSON.stringify({ ok: true, matched: 0, deleted: 0, before }, null, 2) + '\n');
|
|
58
|
+
return 0;
|
|
59
|
+
}
|
|
60
|
+
if (!apply) {
|
|
61
|
+
log('(dry-run:未删除任何东西;加 --apply 才真删,删除前会自动 VACUUM INTO 快照)');
|
|
62
|
+
if (json) process.stdout.write(JSON.stringify({ ok: true, dryRun: true, matched: items.length, ids: items.map((x) => x.id), before }, null, 2) + '\n');
|
|
63
|
+
return 0;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const r = rollbackDistilled({
|
|
67
|
+
dbPath, apply: true, keys: keys.length ? keys : null, ids: ids.length ? ids : null,
|
|
68
|
+
snapshotDir: flags['snapshot-dir'] ? path.resolve(String(flags['snapshot-dir'])) : null,
|
|
69
|
+
});
|
|
70
|
+
log(`已删除 : ${r.deleted} 条(id ${r.deletedIds.join(',')})`);
|
|
71
|
+
log(`删除前快照: ${r.snapshot}${r.snapshotSha256 ? ` · sha ${r.snapshotSha256}` : ''}`);
|
|
72
|
+
log(`一键恢复: ${r.rollbackFile}`);
|
|
73
|
+
log(`回滚后 : 全表 ${r.after.allCount} 条 · 内容摘要 ${r.after.digest}${r.after.digest === before.digest ? ' ✅ 与删除前一致' : ' ⚠️ 与删除前不一致'}`);
|
|
74
|
+
for (const w of r.warnings ?? []) log(`⚠️ ${w}`);
|
|
75
|
+
if (json) process.stdout.write(JSON.stringify({ ok: true, ...r, fileSha256: fileSha256(dbPath) }, null, 2) + '\n');
|
|
76
|
+
return 0;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
if (isMainEntry(import.meta.url)) {
|
|
80
|
+
main().then((code) => { process.exitCode = code; }).catch((e) => {
|
|
81
|
+
console.error(`未捕获异常:${e && e.stack ? e.stack : e}`);
|
|
82
|
+
process.exitCode = 1;
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export { main };
|