@kira905/dsh-memory-toolkit 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +92 -0
- package/README.en.md +225 -2
- package/README.md +180 -2
- package/lib/deploy-guard.mjs +463 -275
- package/lib/guard-http.mjs +59 -62
- package/lib/pitfall-form-map.mjs +94 -84
- package/lib/pitfall-score.mjs +262 -185
- package/lib/session-file-name.mjs +116 -100
- package/lib/zstd-frames.mjs +88 -67
- package/memory-db-tools/README.md +29 -1
- package/memory-db-tools/audit-analyze.mjs +1 -1
- package/memory-db-tools/audit-export.mjs +60 -16
- package/memory-db-tools/audit-struct-probe.mjs +29 -8
- package/memory-db-tools/lib/known-devices.mjs +137 -0
- package/memory-engine/README.md +58 -4
- package/memory-engine/bench/bench.mjs +1 -1
- package/memory-engine/cli/band-sentinel.mjs +164 -0
- package/memory-engine/cli/distill-rollback.mjs +112 -44
- package/memory-engine/cli/distill-sessions.mjs +237 -0
- package/memory-engine/cli/distill.mjs +196 -95
- package/memory-engine/cli/doc-index-rebuild.mjs +142 -108
- package/memory-engine/cli/doc-index.mjs +47 -38
- package/memory-engine/cli/domain-tags.mjs +246 -162
- package/memory-engine/cli/layer-migrate.mjs +176 -0
- package/memory-engine/cli/layer-move.mjs +189 -0
- package/memory-engine/cli/pitfall-hook.mjs +136 -70
- package/memory-engine/cli/recall.mjs +243 -110
- package/memory-engine/cli/selfcheck.mjs +93 -44
- package/memory-engine/cli/serve-start.mjs +221 -140
- package/memory-engine/cli/sweep.mjs +83 -44
- package/memory-engine/cli/sync.mjs +93 -57
- package/memory-engine/config/audit-devices.json +6 -0
- package/memory-engine/config/bands.json +7 -0
- package/memory-engine/config/layers.json +7 -0
- package/memory-engine/dict/LICENSE-/350/257/215/345/205/270.md +53 -0
- package/memory-engine/dict/lexicon-config.json +11 -0
- package/memory-engine/dict//350/257/215/345/205/270.txt +109750 -0
- package/memory-engine/dict//350/257/215/351/242/221.txt +20000 -0
- package/memory-engine/lib/band-sentinel.mjs +417 -0
- package/memory-engine/lib/cli-strict.mjs +128 -0
- package/memory-engine/lib/cli-util.mjs +147 -125
- package/memory-engine/lib/config.mjs +83 -96
- package/memory-engine/lib/dirty-ledger.mjs +127 -94
- package/memory-engine/lib/distill-rollback.mjs +173 -133
- package/memory-engine/lib/distill.mjs +878 -585
- package/memory-engine/lib/doc-index-store.mjs +208 -235
- package/memory-engine/lib/doc-index.mjs +351 -350
- package/memory-engine/lib/doc-scan.mjs +178 -156
- package/memory-engine/lib/doc-summarize.mjs +220 -205
- package/memory-engine/lib/domain-tags.mjs +358 -277
- package/memory-engine/lib/embedder.mjs +175 -194
- package/memory-engine/lib/hashes.mjs +22 -7
- package/memory-engine/lib/layer-migrate.mjs +330 -0
- package/memory-engine/lib/layer-move.mjs +307 -0
- package/memory-engine/lib/layers.mjs +191 -0
- package/memory-engine/lib/lexicon-recall.mjs +473 -0
- package/memory-engine/lib/memory-source.mjs +78 -80
- package/memory-engine/lib/pitfall-source.mjs +115 -114
- package/memory-engine/lib/policy-source.mjs +279 -261
- package/memory-engine/lib/recall-log.mjs +285 -0
- package/memory-engine/lib/recall.mjs +565 -240
- package/memory-engine/lib/selfcheck.mjs +142 -106
- package/memory-engine/lib/session-distill.mjs +603 -0
- package/memory-engine/lib/session-source.mjs +421 -0
- package/memory-engine/lib/sweep.mjs +107 -130
- package/memory-engine/lib/sync.mjs +110 -140
- package/memory-engine/lib/tick-hook.mjs +123 -148
- package/memory-engine/lib/vectors-store.mjs +213 -188
- package/memory-engine/test/12-layering.test.mjs +539 -0
- package/memory-engine/test/13-session-distill.test.mjs +616 -0
- package/memory-engine/tools/bridge-smoke.mjs +94 -55
- package/memory-engine/tools/fetch-model.mjs +167 -87
- package/memory-engine/tools/smoke-recall.mjs +94 -38
- package/memory-engine/tools/verify-recall-chain.mjs +413 -0
- package/package.json +1 -1
|
@@ -0,0 +1,330 @@
|
|
|
1
|
+
// 加层字段的迁移引擎:探查 → 出计划 → 备份 → 事务内改 → 改完复核。
|
|
2
|
+
//
|
|
3
|
+
// 五条硬约束(每条都在代码里):
|
|
4
|
+
// ① **默认 dry-run**:只有显式 apply 才写;调用方没给 backup 就不许写(先备份再动 schema);
|
|
5
|
+
// ② **零删行**:本文件的 SQL 里没有 DELETE / DROP;行数在事务内与事务后各核一次;
|
|
6
|
+
// ③ **幂等**:列已在、空值已清零 ⇒ 一个字节都不写(第二次 apply 的 sha 与前一次相同);
|
|
7
|
+
// ④ **跨盘不 rename**:备份一律「写新文件 + sha 校验」;还原走「同目录临时文件 + rename」,
|
|
8
|
+
// 临时文件与目标同盘,因此不会踩 EXDEV;
|
|
9
|
+
// ⑤ **改完必复核**:行数、id 上界、sqlite_sequence、PRAGMA integrity_check 四项与改前对拍。
|
|
10
|
+
|
|
11
|
+
import fs from 'node:fs';
|
|
12
|
+
import path from 'node:path';
|
|
13
|
+
import crypto from 'node:crypto';
|
|
14
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
15
|
+
import { GuardError } from './cli-strict.mjs';
|
|
16
|
+
|
|
17
|
+
export const MEMORY_TABLE = 'memories';
|
|
18
|
+
const BUSY_TIMEOUT_MS = 8000;
|
|
19
|
+
|
|
20
|
+
const sha256 = (buf) => crypto.createHash('sha256').update(buf).digest('hex');
|
|
21
|
+
/** SQLite 字符串字面量转义:单引号翻倍(路径里可能出现)。 */
|
|
22
|
+
const quoteSql = (text) => `'${String(text).replace(/'/g, "''")}'`;
|
|
23
|
+
|
|
24
|
+
function openRead(dbPath) {
|
|
25
|
+
if (!fs.existsSync(dbPath)) throw new Error(`库文件不存在:${dbPath}`);
|
|
26
|
+
const db = new DatabaseSync(dbPath, { readOnly: true });
|
|
27
|
+
db.exec(`PRAGMA busy_timeout = ${BUSY_TIMEOUT_MS};`);
|
|
28
|
+
return db;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** 只读取数:表清单 / 列清单 / 行数 / 序列 / 完整性。 */
|
|
32
|
+
export function inspectDatabase(dbPath) {
|
|
33
|
+
const abs = path.resolve(dbPath);
|
|
34
|
+
const out = {
|
|
35
|
+
dbPath: abs,
|
|
36
|
+
exists: fs.existsSync(abs),
|
|
37
|
+
bytes: 0,
|
|
38
|
+
sha256: null,
|
|
39
|
+
tables: [],
|
|
40
|
+
hasMemoryTable: false,
|
|
41
|
+
columns: [],
|
|
42
|
+
rowCount: null,
|
|
43
|
+
seq: null,
|
|
44
|
+
integrity: null,
|
|
45
|
+
error: null,
|
|
46
|
+
};
|
|
47
|
+
if (!out.exists) {
|
|
48
|
+
out.error = `库文件不存在:${abs}`;
|
|
49
|
+
return out;
|
|
50
|
+
}
|
|
51
|
+
out.bytes = fs.statSync(abs).size;
|
|
52
|
+
out.sha256 = sha256(fs.readFileSync(abs));
|
|
53
|
+
|
|
54
|
+
let db = null;
|
|
55
|
+
try {
|
|
56
|
+
db = openRead(abs);
|
|
57
|
+
out.tables = db
|
|
58
|
+
.prepare("SELECT name FROM sqlite_master WHERE type IN ('table','view') ORDER BY name")
|
|
59
|
+
.all()
|
|
60
|
+
.map((r) => String(r.name));
|
|
61
|
+
out.hasMemoryTable = out.tables.includes(MEMORY_TABLE);
|
|
62
|
+
if (out.hasMemoryTable) {
|
|
63
|
+
out.columns = db.prepare(`PRAGMA table_info(${MEMORY_TABLE})`).all().map((r) => ({
|
|
64
|
+
name: String(r.name),
|
|
65
|
+
type: String(r.type ?? ''),
|
|
66
|
+
notnull: Number(r.notnull ?? 0),
|
|
67
|
+
dflt: r.dflt_value === null ? null : String(r.dflt_value),
|
|
68
|
+
}));
|
|
69
|
+
out.rowCount = Number(db.prepare(`SELECT COUNT(*) AS n FROM ${MEMORY_TABLE}`).get().n);
|
|
70
|
+
try {
|
|
71
|
+
const s = db.prepare("SELECT seq FROM sqlite_sequence WHERE name = ?").get(MEMORY_TABLE);
|
|
72
|
+
out.seq = s ? Number(s.seq) : null;
|
|
73
|
+
} catch {
|
|
74
|
+
out.seq = null; // 没有 AUTOINCREMENT 就不会有 sqlite_sequence,不算错
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
try {
|
|
78
|
+
const r = db.prepare('PRAGMA integrity_check').get();
|
|
79
|
+
out.integrity = r ? String(r.integrity_check ?? Object.values(r)[0]) : null;
|
|
80
|
+
} catch (e) {
|
|
81
|
+
out.integrity = `读不动:${e && e.message}`;
|
|
82
|
+
}
|
|
83
|
+
} catch (e) {
|
|
84
|
+
out.error = `打不开(只读):${e && e.message}`;
|
|
85
|
+
} finally {
|
|
86
|
+
try {
|
|
87
|
+
if (db) db.close();
|
|
88
|
+
} catch {
|
|
89
|
+
/* ignore */
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return out;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function countEmptyLayer(db, cfg) {
|
|
96
|
+
const col = cfg.column;
|
|
97
|
+
try {
|
|
98
|
+
const r = db.prepare(`SELECT COUNT(*) AS n FROM ${MEMORY_TABLE} WHERE ${col} IS NULL OR TRIM(${col}) = ''`).get();
|
|
99
|
+
return Number(r.n);
|
|
100
|
+
} catch {
|
|
101
|
+
return null;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function layerHistogram(db, cfg) {
|
|
106
|
+
const col = cfg.column;
|
|
107
|
+
const rows = db.prepare(`SELECT ${col} AS v, COUNT(*) AS n FROM ${MEMORY_TABLE} GROUP BY ${col} ORDER BY n DESC`).all();
|
|
108
|
+
const known = new Set(cfg.layers.map((n) => n.toLowerCase()));
|
|
109
|
+
const hist = {};
|
|
110
|
+
const outOfSet = {};
|
|
111
|
+
for (const row of rows) {
|
|
112
|
+
const key = row.v === null || String(row.v).trim() === '' ? '(空)' : String(row.v);
|
|
113
|
+
hist[key] = Number(row.n);
|
|
114
|
+
if (key !== '(空)' && !known.has(key.toLowerCase())) outOfSet[key] = Number(row.n);
|
|
115
|
+
}
|
|
116
|
+
return { hist, outOfSet };
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** 出计划:只读,绝不写。 */
|
|
120
|
+
export function buildPlan(dbPath, cfg) {
|
|
121
|
+
const info = inspectDatabase(dbPath);
|
|
122
|
+
const plan = {
|
|
123
|
+
dbPath: info.dbPath,
|
|
124
|
+
column: cfg.column,
|
|
125
|
+
defaultLayer: cfg.defaultLayer,
|
|
126
|
+
layerNames: cfg.layers,
|
|
127
|
+
info,
|
|
128
|
+
blockers: [],
|
|
129
|
+
hasColumn: false,
|
|
130
|
+
needsAlter: false,
|
|
131
|
+
needsBackfill: false,
|
|
132
|
+
rowCount: info.rowCount,
|
|
133
|
+
emptyLayerRows: null,
|
|
134
|
+
histogram: {},
|
|
135
|
+
outOfSet: {},
|
|
136
|
+
seq: info.seq,
|
|
137
|
+
integrity: info.integrity,
|
|
138
|
+
};
|
|
139
|
+
if (!info.exists) plan.blockers.push(info.error);
|
|
140
|
+
else if (info.error) plan.blockers.push(info.error);
|
|
141
|
+
else if (!info.hasMemoryTable) plan.blockers.push(`库里没有 ${MEMORY_TABLE} 表 —— 这不是一本记忆库`);
|
|
142
|
+
if (info.integrity && info.integrity !== 'ok') plan.blockers.push(`PRAGMA integrity_check 不是 ok:${info.integrity}`);
|
|
143
|
+
|
|
144
|
+
if (plan.blockers.length) return plan;
|
|
145
|
+
|
|
146
|
+
plan.hasColumn = info.columns.some((c) => c.name.toLowerCase() === cfg.column.toLowerCase());
|
|
147
|
+
let db = null;
|
|
148
|
+
try {
|
|
149
|
+
db = openRead(info.dbPath);
|
|
150
|
+
if (plan.hasColumn) {
|
|
151
|
+
plan.emptyLayerRows = countEmptyLayer(db, cfg);
|
|
152
|
+
const { hist, outOfSet } = layerHistogram(db, cfg);
|
|
153
|
+
plan.histogram = hist;
|
|
154
|
+
plan.outOfSet = outOfSet;
|
|
155
|
+
plan.needsBackfill = Number(plan.emptyLayerRows) > 0;
|
|
156
|
+
} else {
|
|
157
|
+
plan.needsAlter = true;
|
|
158
|
+
// 列还没有 ⇒ 加列即回填,条数就是全表
|
|
159
|
+
plan.emptyLayerRows = plan.rowCount;
|
|
160
|
+
plan.needsBackfill = plan.rowCount > 0;
|
|
161
|
+
}
|
|
162
|
+
} catch (e) {
|
|
163
|
+
plan.blockers.push(`取数失败:${e && e.message}`);
|
|
164
|
+
} finally {
|
|
165
|
+
try {
|
|
166
|
+
if (db) db.close();
|
|
167
|
+
} catch {
|
|
168
|
+
/* ignore */
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
return plan;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* 做备份:默认**逐字节复制**(无 WAL 时最直白,还原后与原件逐字节相同);
|
|
176
|
+
* 库旁有非空 `-wal` 时改用 `VACUUM INTO`(单文件、含 WAL 内容,避免半截复制)。
|
|
177
|
+
* 两种办法都在写完之后**打开副本核对**:行数 / 序列 / 完整性。
|
|
178
|
+
*/
|
|
179
|
+
export function createBackup({ dbPath, backupPath }) {
|
|
180
|
+
const src = path.resolve(dbPath);
|
|
181
|
+
const dst = path.resolve(backupPath);
|
|
182
|
+
if (src.toLowerCase() === dst.toLowerCase()) throw new Error('备份路径不能是库本身');
|
|
183
|
+
if (fs.existsSync(dst)) throw new GuardError(`备份文件已存在,不覆盖:${dst}(换个带时间戳的名字,或先挪走)`);
|
|
184
|
+
fs.mkdirSync(path.dirname(dst), { recursive: true });
|
|
185
|
+
|
|
186
|
+
const before = inspectDatabase(src);
|
|
187
|
+
if (!before.exists || before.error) throw new Error(`源库读不了,拒绝备份:${before.error ?? src}`);
|
|
188
|
+
|
|
189
|
+
const wal = `${src}-wal`;
|
|
190
|
+
const hasLiveWal = fs.existsSync(wal) && fs.statSync(wal).size > 0;
|
|
191
|
+
let method = hasLiveWal ? 'vacuum-into' : 'file-copy';
|
|
192
|
+
|
|
193
|
+
if (hasLiveWal) {
|
|
194
|
+
let db = null;
|
|
195
|
+
try {
|
|
196
|
+
db = openRead(src);
|
|
197
|
+
db.exec(`VACUUM INTO ${quoteSql(dst)}`);
|
|
198
|
+
} catch (e) {
|
|
199
|
+
throw new Error(`备份失败(VACUUM INTO 不行,而源库带活的 WAL 又不能直接复制):${e && e.message}`);
|
|
200
|
+
} finally {
|
|
201
|
+
try {
|
|
202
|
+
if (db) db.close();
|
|
203
|
+
} catch {
|
|
204
|
+
/* ignore */
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
} else {
|
|
208
|
+
method = 'file-copy';
|
|
209
|
+
fs.copyFileSync(src, dst);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
const after = inspectDatabase(dst);
|
|
213
|
+
if (!after.exists || after.error) throw new Error(`备份文件打不开:${after.error}`);
|
|
214
|
+
if (after.rowCount !== before.rowCount) {
|
|
215
|
+
throw new Error(`备份行数不符(源 ${before.rowCount} / 备份 ${after.rowCount})—— 库可能正在被写,重跑一次`);
|
|
216
|
+
}
|
|
217
|
+
if (after.integrity !== 'ok') throw new Error(`备份完整性检查不是 ok:${after.integrity}`);
|
|
218
|
+
return {
|
|
219
|
+
method,
|
|
220
|
+
path: dst,
|
|
221
|
+
bytes: after.bytes,
|
|
222
|
+
sha256: after.sha256,
|
|
223
|
+
rows: after.rowCount,
|
|
224
|
+
seq: after.seq,
|
|
225
|
+
integrity: after.integrity,
|
|
226
|
+
byteIdentical: after.sha256 === before.sha256,
|
|
227
|
+
source: { path: src, bytes: before.bytes, sha256: before.sha256, rows: before.rowCount, seq: before.seq },
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/** 事务内把计划做完,然后复核四项。 */
|
|
232
|
+
export function applyMigration({ dbPath, cfg, backupPath }) {
|
|
233
|
+
const plan = buildPlan(dbPath, cfg);
|
|
234
|
+
if (plan.blockers.length) throw new Error(`计划有阻断项,不动库:${plan.blockers.join(';')}`);
|
|
235
|
+
if (!plan.needsAlter && !plan.needsBackfill) {
|
|
236
|
+
return { changed: false, plan, backup: null, verify: { rowCountSame: true, seqSame: true, integrity: plan.integrity } };
|
|
237
|
+
}
|
|
238
|
+
if (!backupPath) throw new Error('动 schema 前必须先备份:用 --backup <path> 指定一份(拒绝无备份写入)');
|
|
239
|
+
|
|
240
|
+
const backup = createBackup({ dbPath, backupPath });
|
|
241
|
+
const abs = path.resolve(dbPath);
|
|
242
|
+
const db = new DatabaseSync(abs);
|
|
243
|
+
db.exec(`PRAGMA busy_timeout = ${BUSY_TIMEOUT_MS};`);
|
|
244
|
+
let touched = 0;
|
|
245
|
+
try {
|
|
246
|
+
db.exec('BEGIN IMMEDIATE');
|
|
247
|
+
if (plan.needsAlter) {
|
|
248
|
+
db.exec(`ALTER TABLE ${MEMORY_TABLE} ADD COLUMN ${cfg.column} TEXT NOT NULL DEFAULT ${quoteSql(cfg.defaultLayer)}`);
|
|
249
|
+
}
|
|
250
|
+
if (plan.needsBackfill) {
|
|
251
|
+
const stmt = db.prepare(
|
|
252
|
+
`UPDATE ${MEMORY_TABLE} SET ${cfg.column} = ? WHERE ${cfg.column} IS NULL OR TRIM(${cfg.column}) = ''`,
|
|
253
|
+
);
|
|
254
|
+
const res = stmt.run(cfg.defaultLayer);
|
|
255
|
+
touched = Number(res.changes ?? 0);
|
|
256
|
+
}
|
|
257
|
+
// 加列时既有行由列默认值直接填好(0 次 UPDATE)—— 记账按"被填了多少行"算,别报 0
|
|
258
|
+
if (plan.needsAlter) touched += Number(plan.rowCount ?? 0);
|
|
259
|
+
db.exec('COMMIT');
|
|
260
|
+
} catch (e) {
|
|
261
|
+
try {
|
|
262
|
+
db.exec('ROLLBACK');
|
|
263
|
+
} catch {
|
|
264
|
+
/* ignore */
|
|
265
|
+
}
|
|
266
|
+
db.close();
|
|
267
|
+
throw new Error(`迁移事务回滚(库未改):${e && e.message};备份仍在 ${backup.path}`);
|
|
268
|
+
}
|
|
269
|
+
db.close();
|
|
270
|
+
|
|
271
|
+
const after = buildPlan(dbPath, cfg);
|
|
272
|
+
const verify = {
|
|
273
|
+
rowCountSame: after.rowCount === plan.rowCount,
|
|
274
|
+
seqSame: after.seq === plan.seq,
|
|
275
|
+
integrity: after.integrity,
|
|
276
|
+
emptyAfter: after.emptyLayerRows,
|
|
277
|
+
histogram: after.histogram,
|
|
278
|
+
};
|
|
279
|
+
if (!verify.rowCountSame) throw new Error(`行数变了(${plan.rowCount} → ${after.rowCount})—— 立刻用备份还原:${backup.path}`);
|
|
280
|
+
if (!verify.seqSame) throw new Error(`sqlite_sequence 变了(${plan.seq} → ${after.seq})—— 立刻用备份还原:${backup.path}`);
|
|
281
|
+
if (Number(verify.emptyAfter) !== 0) throw new Error(`回填后仍有 ${verify.emptyAfter} 行空层 —— 立刻用备份还原:${backup.path}`);
|
|
282
|
+
if (verify.integrity !== 'ok') throw new Error(`迁移后完整性不是 ok:${verify.integrity} —— 用备份还原:${backup.path}`);
|
|
283
|
+
|
|
284
|
+
return { changed: true, plan, backup, touched, verify, after };
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
/** 还原:从备份写回库文件。默认 dry-run;真写走「同目录临时文件 + sha 校验 + rename」。 */
|
|
288
|
+
export function restoreDatabase({ backupPath, dbPath, apply = false }) {
|
|
289
|
+
const backup = inspectDatabase(backupPath);
|
|
290
|
+
if (!backup.exists) throw new Error(`备份不存在:${path.resolve(backupPath)}`);
|
|
291
|
+
if (backup.error) throw new Error(`备份打不开:${backup.error}`);
|
|
292
|
+
if (!backup.hasMemoryTable) throw new Error(`备份里没有 ${MEMORY_TABLE} 表 —— 不是一本记忆库的备份`);
|
|
293
|
+
if (backup.integrity !== 'ok') throw new Error(`备份完整性不是 ok:${backup.integrity}`);
|
|
294
|
+
|
|
295
|
+
const target = inspectDatabase(dbPath);
|
|
296
|
+
const plan = {
|
|
297
|
+
backup: { path: backup.dbPath, bytes: backup.bytes, sha256: backup.sha256, rows: backup.rowCount, seq: backup.seq },
|
|
298
|
+
target: target.exists
|
|
299
|
+
? { path: target.dbPath, bytes: target.bytes, sha256: target.sha256, rows: target.rowCount, seq: target.seq }
|
|
300
|
+
: { path: path.resolve(dbPath), bytes: null, sha256: null, rows: null, seq: null, missing: true },
|
|
301
|
+
sidecars: ['-wal', '-shm'].filter((s) => fs.existsSync(path.resolve(dbPath) + s)),
|
|
302
|
+
applied: false,
|
|
303
|
+
verify: null,
|
|
304
|
+
};
|
|
305
|
+
if (!apply) return plan;
|
|
306
|
+
|
|
307
|
+
const abs = path.resolve(dbPath);
|
|
308
|
+
fs.mkdirSync(path.dirname(abs), { recursive: true });
|
|
309
|
+
// 陈旧 WAL 与新库文件配在一起 = 直接损坏库,必须先摘掉(这是备份还原的一部分,不是删数据)
|
|
310
|
+
for (const suffix of ['-wal', '-shm']) {
|
|
311
|
+
const side = abs + suffix;
|
|
312
|
+
if (fs.existsSync(side)) fs.rmSync(side, { force: true });
|
|
313
|
+
}
|
|
314
|
+
const tmp = `${abs}.restore-${process.pid}-${Date.now()}.tmp`;
|
|
315
|
+
fs.copyFileSync(backup.dbPath, tmp);
|
|
316
|
+
const tmpSha = sha256(fs.readFileSync(tmp));
|
|
317
|
+
if (tmpSha !== backup.sha256) {
|
|
318
|
+
fs.rmSync(tmp, { force: true });
|
|
319
|
+
throw new Error(`临时副本 sha 与备份不符(${tmpSha.slice(0, 12)} ≠ ${backup.sha256.slice(0, 12)})—— 未替换`);
|
|
320
|
+
}
|
|
321
|
+
fs.renameSync(tmp, abs); // 同目录 rename:不跨盘,不会 EXDEV
|
|
322
|
+
|
|
323
|
+
const after = inspectDatabase(abs);
|
|
324
|
+
if (!after.exists || after.error || after.sha256 !== backup.sha256) {
|
|
325
|
+
throw new Error(`还原后核对失败(${after.error ?? `sha ${String(after.sha256).slice(0, 12)}`})—— 库文件已替换,请人工确认`);
|
|
326
|
+
}
|
|
327
|
+
plan.applied = true;
|
|
328
|
+
plan.verify = { sha256: after.sha256, rows: after.rowCount, seq: after.seq, integrity: after.integrity };
|
|
329
|
+
return plan;
|
|
330
|
+
}
|
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
// 软淘汰执行器:把一整层「搬走」而不是删掉 —— 搬到本库内的一张边表(memories_evicted),一条命令出去、一条命令回来。
|
|
2
|
+
//
|
|
3
|
+
// 为什么选「另一张表」而不是「另一个库文件」(代价写在这里,README 同款):
|
|
4
|
+
// · 好处:同一个事务里搬进搬出,守恒可以当场断言;不会留下一个可能被误删的旁路库文件;
|
|
5
|
+
// FTS5 触发器仍然在 主表 上,搬出即从索引删除、搬回即重建,索引不会漂。
|
|
6
|
+
// · 代价:搬走的行**仍然占这个库文件的体积**(软淘汰省的是上下文与召回面,不是磁盘);
|
|
7
|
+
// 边表没有 FTS 索引 ⇒ 搬走的行在引擎召回里不可见,只能用 SQL 或 `--back` 取回。
|
|
8
|
+
//
|
|
9
|
+
// 四条硬判据:
|
|
10
|
+
// ① 守恒:搬出层行数 + 剩下行数 == 搬前总行数(事务内断言,不成立就整批回滚);
|
|
11
|
+
// ② 逐行可比:每行落边表时记 content_sha(与 lib/layers.mjs 的行指纹同算法),搬回时先核后写;
|
|
12
|
+
// ③ 默认 dry-run:不带 apply 一个字节都不写;
|
|
13
|
+
// ④ 不许把主表搬空:会搬走全部行时要显式 `--force`(防"层名写错"把库清空)。
|
|
14
|
+
|
|
15
|
+
import fs from 'node:fs';
|
|
16
|
+
import path from 'node:path';
|
|
17
|
+
import crypto from 'node:crypto';
|
|
18
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
19
|
+
import { rowDigest, canonicalLayerName } from './layers.mjs';
|
|
20
|
+
import { GuardError } from './cli-strict.mjs';
|
|
21
|
+
|
|
22
|
+
export const MEMORY_TABLE = 'memories';
|
|
23
|
+
export const EVICTED_TABLE = 'memories_evicted';
|
|
24
|
+
const BUSY_TIMEOUT_MS = 8000;
|
|
25
|
+
|
|
26
|
+
const sha256 = (text) => crypto.createHash('sha256').update(String(text), 'utf8').digest('hex');
|
|
27
|
+
|
|
28
|
+
/** 边表:列与主表一一对应 + 三个记账列(层 / 指纹 / 批次)。 */
|
|
29
|
+
export const EVICTED_DDL = `
|
|
30
|
+
CREATE TABLE IF NOT EXISTS ${EVICTED_TABLE} (
|
|
31
|
+
id INTEGER PRIMARY KEY,
|
|
32
|
+
text TEXT NOT NULL,
|
|
33
|
+
tags TEXT NOT NULL DEFAULT '',
|
|
34
|
+
pinned INTEGER NOT NULL DEFAULT 0,
|
|
35
|
+
created_at INTEGER NOT NULL,
|
|
36
|
+
updated_at INTEGER NOT NULL,
|
|
37
|
+
layer TEXT NOT NULL,
|
|
38
|
+
content_sha TEXT NOT NULL,
|
|
39
|
+
batch TEXT NOT NULL,
|
|
40
|
+
moved_at INTEGER NOT NULL
|
|
41
|
+
)`;
|
|
42
|
+
|
|
43
|
+
const MEMORY_COLUMNS = 'id, text, tags, pinned, created_at, updated_at';
|
|
44
|
+
|
|
45
|
+
function open(dbPath, { readOnly = false } = {}) {
|
|
46
|
+
const abs = path.resolve(dbPath);
|
|
47
|
+
if (!fs.existsSync(abs)) throw new Error(`库文件不存在:${abs}`);
|
|
48
|
+
const db = new DatabaseSync(abs, readOnly ? { readOnly: true } : {});
|
|
49
|
+
db.exec(`PRAGMA busy_timeout = ${BUSY_TIMEOUT_MS};`);
|
|
50
|
+
return db;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function tableExists(db, name) {
|
|
54
|
+
return Boolean(db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = ?").get(name));
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function columnExists(db, table, column) {
|
|
58
|
+
return db.prepare(`PRAGMA table_info(${table})`).all().some((c) => String(c.name).toLowerCase() === column.toLowerCase());
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
const countAll = (db) => Number(db.prepare(`SELECT COUNT(*) AS n FROM ${MEMORY_TABLE}`).get().n);
|
|
62
|
+
const countLayerInMain = (db, cfg, layer) =>
|
|
63
|
+
Number(
|
|
64
|
+
db.prepare(`SELECT COUNT(*) AS n FROM ${MEMORY_TABLE} WHERE LOWER(${cfg.column}) = LOWER(?)`).get(String(layer)).n,
|
|
65
|
+
);
|
|
66
|
+
|
|
67
|
+
function countEvicted(db, layer) {
|
|
68
|
+
if (!tableExists(db, EVICTED_TABLE)) return 0;
|
|
69
|
+
return Number(db.prepare(`SELECT COUNT(*) AS n FROM ${EVICTED_TABLE} WHERE LOWER(layer) = LOWER(?)`).get(String(layer)).n);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function seqOf(db) {
|
|
73
|
+
try {
|
|
74
|
+
const r = db.prepare('SELECT seq FROM sqlite_sequence WHERE name = ?').get(MEMORY_TABLE);
|
|
75
|
+
return r ? Number(r.seq) : null;
|
|
76
|
+
} catch {
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function ensureEvictedTable(db) {
|
|
82
|
+
db.exec(EVICTED_DDL);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** 只说出计划:会不会搬、搬几行、有没有阻断项。 */
|
|
86
|
+
export function planMove({ dbPath, cfg, layer }) {
|
|
87
|
+
const db = open(dbPath, { readOnly: true });
|
|
88
|
+
try {
|
|
89
|
+
const plan = {
|
|
90
|
+
dbPath: path.resolve(dbPath),
|
|
91
|
+
layer,
|
|
92
|
+
totalRows: countAll(db),
|
|
93
|
+
hasColumn: columnExists(db, MEMORY_TABLE, cfg.column),
|
|
94
|
+
hasEvictedTable: tableExists(db, EVICTED_TABLE),
|
|
95
|
+
layerRows: 0,
|
|
96
|
+
evictedRows: 0,
|
|
97
|
+
seq: seqOf(db),
|
|
98
|
+
blockers: [],
|
|
99
|
+
};
|
|
100
|
+
if (!plan.hasColumn) {
|
|
101
|
+
plan.blockers.push(`主表没有 ${cfg.column} 列 —— 先跑 layer-migrate --apply 把列加上`);
|
|
102
|
+
return plan;
|
|
103
|
+
}
|
|
104
|
+
plan.layerRows = countLayerInMain(db, cfg, layer);
|
|
105
|
+
plan.evictedRows = countEvicted(db, layer);
|
|
106
|
+
plan.leavesEmpty = plan.layerRows > 0 && plan.layerRows === plan.totalRows;
|
|
107
|
+
return plan;
|
|
108
|
+
} finally {
|
|
109
|
+
db.close();
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* 搬出:主表 → 边表。事务内断言守恒;任何一步不符就整批回滚。
|
|
115
|
+
* @returns {{moved:number,totalBefore:number,remaining:number,evicted:number,batch:string,digests:number}}
|
|
116
|
+
*/
|
|
117
|
+
export function moveOut({ dbPath, cfg, layer, batch, force = false }) {
|
|
118
|
+
const name = canonicalLayerName(cfg, layer);
|
|
119
|
+
if (!name) throw new Error(`「${layer}」不在层名集合 [${cfg.layers.join(', ')}] 里`);
|
|
120
|
+
const plan = planMove({ dbPath, cfg, layer: name });
|
|
121
|
+
if (plan.blockers.length) throw new Error(plan.blockers.join(';'));
|
|
122
|
+
if (plan.layerRows === 0) {
|
|
123
|
+
return { moved: 0, totalBefore: plan.totalRows, remaining: plan.totalRows, evicted: plan.evictedRows, batch, digests: 0 };
|
|
124
|
+
}
|
|
125
|
+
if (plan.leavesEmpty && !force) {
|
|
126
|
+
throw new GuardError(
|
|
127
|
+
`搬走「${name}」会把主表搬空(${plan.layerRows}/${plan.totalRows} 行)—— 这通常是层名填错。确实要搬请加 --force`,
|
|
128
|
+
);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
const db = open(dbPath);
|
|
132
|
+
const before = { rows: countAll(db), seq: seqOf(db) };
|
|
133
|
+
let movedRows = 0;
|
|
134
|
+
try {
|
|
135
|
+
ensureEvictedTable(db);
|
|
136
|
+
const rows = db
|
|
137
|
+
.prepare(`SELECT ${MEMORY_COLUMNS}, ${cfg.column} AS layer_value FROM ${MEMORY_TABLE} WHERE LOWER(${cfg.column}) = LOWER(?) ORDER BY id`)
|
|
138
|
+
.all(name);
|
|
139
|
+
db.exec('BEGIN IMMEDIATE');
|
|
140
|
+
const insert = db.prepare(
|
|
141
|
+
`INSERT INTO ${EVICTED_TABLE} (id, text, tags, pinned, created_at, updated_at, layer, content_sha, batch, moved_at)
|
|
142
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
143
|
+
);
|
|
144
|
+
const remove = db.prepare(`DELETE FROM ${MEMORY_TABLE} WHERE id = ? AND LOWER(${cfg.column}) = LOWER(?)`);
|
|
145
|
+
const clash = db.prepare(`SELECT id FROM ${EVICTED_TABLE} WHERE id = ?`);
|
|
146
|
+
const now = Date.now();
|
|
147
|
+
for (const row of rows) {
|
|
148
|
+
const layerValue = String(row.layer_value ?? name);
|
|
149
|
+
const digest = rowDigest({ ...row, [cfg.column]: layerValue }, cfg);
|
|
150
|
+
if (clash.get(Number(row.id))) {
|
|
151
|
+
throw new GuardError(`id ${row.id} 已经在边表里(上次搬出没搬回)—— 先 layer-move --back,别覆盖`);
|
|
152
|
+
}
|
|
153
|
+
insert.run(
|
|
154
|
+
Number(row.id), String(row.text ?? ''), String(row.tags ?? ''), Number(row.pinned ?? 0),
|
|
155
|
+
Number(row.created_at ?? 0), Number(row.updated_at ?? 0), layerValue, digest, batch, now,
|
|
156
|
+
);
|
|
157
|
+
const res = remove.run(Number(row.id), name);
|
|
158
|
+
if (Number(res.changes ?? 0) !== 1) throw new Error(`删除主表 id ${row.id} 失败(影响 ${res.changes} 行)`);
|
|
159
|
+
movedRows += 1;
|
|
160
|
+
}
|
|
161
|
+
const remaining = countAll(db);
|
|
162
|
+
const evicted = countEvicted(db, name);
|
|
163
|
+
if (evicted + remaining !== before.rows) {
|
|
164
|
+
throw new Error(`守恒不成立:搬出 ${evicted} + 剩下 ${remaining} ≠ 搬前 ${before.rows}`);
|
|
165
|
+
}
|
|
166
|
+
db.exec('COMMIT');
|
|
167
|
+
return { moved: movedRows, totalBefore: before.rows, remaining, evicted, batch, digests: movedRows, seqBefore: before.seq, seqAfter: seqOf(db) };
|
|
168
|
+
} catch (e) {
|
|
169
|
+
try {
|
|
170
|
+
db.exec('ROLLBACK');
|
|
171
|
+
} catch {
|
|
172
|
+
/* ignore */
|
|
173
|
+
}
|
|
174
|
+
throw e;
|
|
175
|
+
} finally {
|
|
176
|
+
db.close();
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* 搬回:边表 → 主表。先逐行核指纹(不符整批拒动),再插回并删边表。
|
|
182
|
+
*/
|
|
183
|
+
export function moveBack({ dbPath, cfg, layer }) {
|
|
184
|
+
const name = canonicalLayerName(cfg, layer);
|
|
185
|
+
if (!name) throw new Error(`「${layer}」不在层名集合 [${cfg.layers.join(', ')}] 里`);
|
|
186
|
+
const db = open(dbPath);
|
|
187
|
+
const before = { rows: countAll(db), seq: seqOf(db) };
|
|
188
|
+
try {
|
|
189
|
+
if (!tableExists(db, EVICTED_TABLE)) {
|
|
190
|
+
return { restored: 0, totalBefore: before.rows, totalAfter: before.rows, mismatched: [], reason: '边表不存在(没搬出过)' };
|
|
191
|
+
}
|
|
192
|
+
ensureEvictedTable(db);
|
|
193
|
+
const rows = db.prepare(`SELECT * FROM ${EVICTED_TABLE} WHERE LOWER(layer) = LOWER(?) ORDER BY id`).all(name);
|
|
194
|
+
if (rows.length === 0) {
|
|
195
|
+
return { restored: 0, totalBefore: before.rows, totalAfter: before.rows, mismatched: [], reason: `边表里没有「${name}」的行` };
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
// 先验后写:任一指纹不符 / 主表已存在同 id ⇒ 整批不动
|
|
199
|
+
const mismatched = [];
|
|
200
|
+
const collisions = [];
|
|
201
|
+
for (const row of rows) {
|
|
202
|
+
const digest = rowDigest({ ...row, [cfg.column]: String(row.layer ?? name) }, cfg);
|
|
203
|
+
if (digest !== String(row.content_sha)) mismatched.push({ id: Number(row.id), stored: String(row.content_sha), now: digest });
|
|
204
|
+
const hit = db.prepare(`SELECT id FROM ${MEMORY_TABLE} WHERE id = ?`).get(Number(row.id));
|
|
205
|
+
if (hit) collisions.push(Number(row.id));
|
|
206
|
+
}
|
|
207
|
+
if (mismatched.length) {
|
|
208
|
+
const ids = mismatched.slice(0, 5).map((m) => m.id).join(', ');
|
|
209
|
+
throw new GuardError(`边表里有 ${mismatched.length} 行内容被改过(指纹不符,例:${ids})—— 整批拒动`);
|
|
210
|
+
}
|
|
211
|
+
if (collisions.length) {
|
|
212
|
+
throw new GuardError(`主表里已存在同 id 的行(例:${collisions.slice(0, 5).join(', ')})—— 整批拒动,先查清哪一份该留`);
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
db.exec('BEGIN IMMEDIATE');
|
|
216
|
+
const insert = db.prepare(
|
|
217
|
+
`INSERT INTO ${MEMORY_TABLE} (id, text, tags, pinned, created_at, updated_at, ${cfg.column})
|
|
218
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
|
219
|
+
);
|
|
220
|
+
const remove = db.prepare(`DELETE FROM ${EVICTED_TABLE} WHERE id = ? AND LOWER(layer) = LOWER(?)`);
|
|
221
|
+
let restored = 0;
|
|
222
|
+
for (const row of rows) {
|
|
223
|
+
insert.run(
|
|
224
|
+
Number(row.id), String(row.text ?? ''), String(row.tags ?? ''), Number(row.pinned ?? 0),
|
|
225
|
+
Number(row.created_at ?? 0), Number(row.updated_at ?? 0), String(row.layer ?? name),
|
|
226
|
+
);
|
|
227
|
+
const res = remove.run(Number(row.id), name);
|
|
228
|
+
if (Number(res.changes ?? 0) !== 1) throw new Error(`从边表删 id ${row.id} 失败(影响 ${res.changes} 行)`);
|
|
229
|
+
restored += 1;
|
|
230
|
+
}
|
|
231
|
+
const after = countAll(db);
|
|
232
|
+
if (after !== before.rows + restored) {
|
|
233
|
+
throw new Error(`守恒不成立:搬前 ${before.rows} + 搬回 ${restored} ≠ 搬后 ${after}`);
|
|
234
|
+
}
|
|
235
|
+
db.exec('COMMIT');
|
|
236
|
+
return { restored, totalBefore: before.rows, totalAfter: after, mismatched: [], seqBefore: before.seq, seqAfter: seqOf(db) };
|
|
237
|
+
} catch (e) {
|
|
238
|
+
try {
|
|
239
|
+
db.exec('ROLLBACK');
|
|
240
|
+
} catch {
|
|
241
|
+
/* ignore */
|
|
242
|
+
}
|
|
243
|
+
throw e;
|
|
244
|
+
} finally {
|
|
245
|
+
db.close();
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/** 导出主表(或某一层)的逐行指纹,当作「搬前长什么样」的凭据。 */
|
|
250
|
+
export function dumpDigests({ dbPath, cfg, layer = null }) {
|
|
251
|
+
const db = open(dbPath, { readOnly: true });
|
|
252
|
+
try {
|
|
253
|
+
if (!columnExists(db, MEMORY_TABLE, cfg.column)) throw new Error(`主表没有 ${cfg.column} 列 —— 先跑 layer-migrate`);
|
|
254
|
+
const rows = layer
|
|
255
|
+
? db
|
|
256
|
+
.prepare(`SELECT ${MEMORY_COLUMNS}, ${cfg.column} AS layer_value FROM ${MEMORY_TABLE} WHERE LOWER(${cfg.column}) = LOWER(?) ORDER BY id`)
|
|
257
|
+
.all(String(layer))
|
|
258
|
+
: db.prepare(`SELECT ${MEMORY_COLUMNS}, ${cfg.column} AS layer_value FROM ${MEMORY_TABLE} ORDER BY id`).all();
|
|
259
|
+
const items = rows.map((row) => ({
|
|
260
|
+
id: Number(row.id),
|
|
261
|
+
digest: rowDigest({ ...row, [cfg.column]: String(row.layer_value ?? '') }, cfg),
|
|
262
|
+
}));
|
|
263
|
+
return { at: new Date().toISOString(), dbPath: path.resolve(dbPath), layer, count: items.length, rows: items };
|
|
264
|
+
} finally {
|
|
265
|
+
db.close();
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/** 拿导出件对拍当前主表:缺行 / 多行 / 内容变了 三类分开报。 */
|
|
270
|
+
export function verifyAgainstDump({ dbPath, cfg, dump }) {
|
|
271
|
+
const now = dumpDigests({ dbPath, cfg, layer: dump.layer ?? null });
|
|
272
|
+
const before = new Map((dump.rows ?? []).map((r) => [Number(r.id), String(r.digest)]));
|
|
273
|
+
const after = new Map(now.rows.map((r) => [Number(r.id), String(r.digest)]));
|
|
274
|
+
const missing = [];
|
|
275
|
+
const extra = [];
|
|
276
|
+
const changed = [];
|
|
277
|
+
for (const [id, digest] of before) {
|
|
278
|
+
if (!after.has(id)) missing.push(id);
|
|
279
|
+
else if (after.get(id) !== digest) changed.push(id);
|
|
280
|
+
}
|
|
281
|
+
for (const id of after.keys()) if (!before.has(id)) extra.push(id);
|
|
282
|
+
const byId = (a, b) => a - b;
|
|
283
|
+
return {
|
|
284
|
+
ok: missing.length === 0 && extra.length === 0 && changed.length === 0,
|
|
285
|
+
checked: before.size,
|
|
286
|
+
currentCount: after.size,
|
|
287
|
+
missing: missing.sort(byId),
|
|
288
|
+
extra: extra.sort(byId),
|
|
289
|
+
changed: changed.sort(byId),
|
|
290
|
+
};
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/** 写导出件(JSON 只落文件)。 */
|
|
294
|
+
export function writeDump(file, dump) {
|
|
295
|
+
const abs = path.resolve(file);
|
|
296
|
+
fs.mkdirSync(path.dirname(abs), { recursive: true });
|
|
297
|
+
fs.writeFileSync(abs, `${JSON.stringify(dump, null, 2)}\n`, 'utf8');
|
|
298
|
+
return abs;
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
export function readDump(file) {
|
|
302
|
+
const abs = path.resolve(file);
|
|
303
|
+
if (!fs.existsSync(abs)) throw new Error(`指纹导出件不在:${abs}`);
|
|
304
|
+
return JSON.parse(fs.readFileSync(abs, 'utf8'));
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
export { sha256 as sha256OfText };
|