@kira905/dsh-memory-toolkit 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/CHANGELOG.md +130 -1
  2. package/README.en.md +225 -2
  3. package/README.md +208 -2
  4. package/cordis.patch.yml +32 -0
  5. package/lib/deploy-guard.mjs +463 -275
  6. package/lib/guard-http.mjs +59 -62
  7. package/lib/pitfall-form-map.mjs +94 -84
  8. package/lib/pitfall-score.mjs +262 -185
  9. package/lib/session-file-name.mjs +116 -100
  10. package/lib/zstd-frames.mjs +88 -67
  11. package/memory-db-tools/README.md +29 -1
  12. package/memory-db-tools/audit-analyze.mjs +1 -1
  13. package/memory-db-tools/audit-export.mjs +60 -16
  14. package/memory-db-tools/audit-struct-probe.mjs +29 -8
  15. package/memory-db-tools/lib/known-devices.mjs +137 -0
  16. package/memory-engine/README.md +58 -4
  17. package/memory-engine/bench/bench.mjs +1 -1
  18. package/memory-engine/cli/band-sentinel.mjs +164 -0
  19. package/memory-engine/cli/distill-rollback.mjs +112 -44
  20. package/memory-engine/cli/distill-sessions.mjs +237 -0
  21. package/memory-engine/cli/distill.mjs +196 -95
  22. package/memory-engine/cli/doc-index-rebuild.mjs +142 -108
  23. package/memory-engine/cli/doc-index.mjs +47 -38
  24. package/memory-engine/cli/domain-tags.mjs +246 -162
  25. package/memory-engine/cli/layer-migrate.mjs +176 -0
  26. package/memory-engine/cli/layer-move.mjs +189 -0
  27. package/memory-engine/cli/pitfall-hook.mjs +136 -70
  28. package/memory-engine/cli/recall.mjs +243 -110
  29. package/memory-engine/cli/selfcheck.mjs +93 -44
  30. package/memory-engine/cli/serve-start.mjs +221 -140
  31. package/memory-engine/cli/sweep.mjs +83 -44
  32. package/memory-engine/cli/sync.mjs +93 -57
  33. package/memory-engine/config/audit-devices.json +6 -0
  34. package/memory-engine/config/bands.json +7 -0
  35. package/memory-engine/config/layers.json +7 -0
  36. package/memory-engine/dict/LICENSE-/350/257/215/345/205/270.md +53 -0
  37. package/memory-engine/dict/lexicon-config.json +11 -0
  38. package/memory-engine/dict//350/257/215/345/205/270.txt +109750 -0
  39. package/memory-engine/dict//350/257/215/351/242/221.txt +20000 -0
  40. package/memory-engine/lib/band-sentinel.mjs +417 -0
  41. package/memory-engine/lib/cli-strict.mjs +128 -0
  42. package/memory-engine/lib/cli-util.mjs +147 -125
  43. package/memory-engine/lib/config.mjs +83 -96
  44. package/memory-engine/lib/dirty-ledger.mjs +127 -94
  45. package/memory-engine/lib/distill-rollback.mjs +173 -133
  46. package/memory-engine/lib/distill.mjs +878 -585
  47. package/memory-engine/lib/doc-index-store.mjs +208 -235
  48. package/memory-engine/lib/doc-index.mjs +351 -350
  49. package/memory-engine/lib/doc-scan.mjs +178 -156
  50. package/memory-engine/lib/doc-summarize.mjs +220 -205
  51. package/memory-engine/lib/domain-tags.mjs +358 -277
  52. package/memory-engine/lib/embedder.mjs +175 -194
  53. package/memory-engine/lib/hashes.mjs +22 -7
  54. package/memory-engine/lib/layer-migrate.mjs +330 -0
  55. package/memory-engine/lib/layer-move.mjs +307 -0
  56. package/memory-engine/lib/layers.mjs +191 -0
  57. package/memory-engine/lib/lexicon-recall.mjs +473 -0
  58. package/memory-engine/lib/memory-source.mjs +78 -80
  59. package/memory-engine/lib/pitfall-source.mjs +115 -114
  60. package/memory-engine/lib/policy-source.mjs +279 -261
  61. package/memory-engine/lib/recall-log.mjs +285 -0
  62. package/memory-engine/lib/recall.mjs +565 -240
  63. package/memory-engine/lib/selfcheck.mjs +142 -106
  64. package/memory-engine/lib/session-distill.mjs +603 -0
  65. package/memory-engine/lib/session-source.mjs +421 -0
  66. package/memory-engine/lib/sweep.mjs +107 -130
  67. package/memory-engine/lib/sync.mjs +110 -140
  68. package/memory-engine/lib/tick-hook.mjs +123 -148
  69. package/memory-engine/lib/vectors-store.mjs +213 -188
  70. package/memory-engine/test/12-layering.test.mjs +539 -0
  71. package/memory-engine/test/13-session-distill.test.mjs +616 -0
  72. package/memory-engine/tools/bridge-smoke.mjs +94 -55
  73. package/memory-engine/tools/fetch-model.mjs +167 -87
  74. package/memory-engine/tools/smoke-recall.mjs +94 -38
  75. package/memory-engine/tools/verify-recall-chain.mjs +413 -0
  76. package/package.json +11 -1
@@ -1,35 +1,54 @@
1
- // 记忆引擎 v2 · 离线层 —— 文档索引独立库(方案 §3.2 / D1 已拍:出主库)
1
+ // 文档索引独立库 —— 建表 / 键值元数据 / 并发保护 / 批量读写与清理
2
2
  //
3
- // 为什么独立库(B3/B5 的正解,三条硬理由,勿重开):
4
- // 1. 主库 `memories` **无 path 列、无外部唯一索引** ⇒ 幂等键无处可放;
5
- // 且 `forPrompt` 的 LIMIT 名额会被 `updated_at=now` 的索引条整批占掉;
6
- // 2. `memory.db` 是 DELETE 模式 + `busy_timeout=0` ⇒ 生成器写主库 = 与 `memory_write` 抢锁;
7
- // 3. 索引条描述**本机 docs 的 mtime**,两机 mtime 不同 ⇒ 跨机双写会互相覆盖。
8
- // 独立库同时继承 §3.1 的好处:不进任何同步仓、可重建、无锁竞争。
3
+ // 为什么单开一个库文件(三条理由,别重新讨论):
4
+ // 1. 索引条要有天然的幂等键(一条文档一个主键),主记忆库没有这样的唯一列可用;
5
+ // 2. 索引条上记的规模数字跟着本机文件 mtime 走,两台机器各算各的 ⇒ 塞进同一个库会互相覆盖;
6
+ // 3. 独立库不进任何同步链路,删掉整份可重建,也就不会跟别的写入抢锁。
7
+ //
8
+ // 对外只承诺「打开 / 读 / 写 / 清理」四件事,SQL 细节不出这个文件。
9
9
  import fs from 'node:fs';
10
10
  import path from 'node:path';
11
11
  import { DatabaseSync } from 'node:sqlite';
12
12
  import { DOC_INDEX_SCHEMA_VERSION } from './config.mjs';
13
13
 
14
- /** 表结构 —— 逐列照方案 §3.2(验收要贴 PRAGMA table_info 逐列比对,勿自行增删列)。 */
15
- const DDL = `
16
- CREATE TABLE IF NOT EXISTS doc_index (
17
- path TEXT PRIMARY KEY,
18
- mtime INTEGER NOT NULL,
19
- size INTEGER NOT NULL,
20
- text_hash TEXT NOT NULL,
21
- summary TEXT NOT NULL,
22
- owner_user TEXT NOT NULL DEFAULT 'LiJianXiao',
23
- visibility TEXT NOT NULL DEFAULT 'private_owner',
24
- updated_at INTEGER NOT NULL
25
- );
26
- CREATE TABLE IF NOT EXISTS meta (
27
- k TEXT PRIMARY KEY,
28
- v TEXT NOT NULL
29
- );
30
- `;
31
-
32
- /** meta 键名(卡要求:schemaVersion / hostId / lastRunAt;后两项为运行状态,照 S1 的 lastSyncStats 先例)。 */
14
+ // ── 表形状(验收按 PRAGMA table_info 逐列核对,列序/类型/NOT NULL/默认值一个都不许漂) ──
15
+
16
+ const T_DOC = 'doc_index';
17
+ const T_META = 'meta';
18
+ const ALL_TABLES = [T_DOC, T_META];
19
+
20
+ const DDL = [
21
+ `CREATE TABLE IF NOT EXISTS ${T_DOC} (`,
22
+ ' path TEXT PRIMARY KEY,',
23
+ ' mtime INTEGER NOT NULL,',
24
+ ' size INTEGER NOT NULL,',
25
+ ' text_hash TEXT NOT NULL,',
26
+ ' summary TEXT NOT NULL,',
27
+ " owner_user TEXT NOT NULL DEFAULT 'LiJianXiao',",
28
+ " visibility TEXT NOT NULL DEFAULT 'private_owner',",
29
+ ' updated_at INTEGER NOT NULL',
30
+ ');',
31
+ `CREATE TABLE IF NOT EXISTS ${T_META} (`,
32
+ ' k TEXT PRIMARY KEY,',
33
+ ' v TEXT NOT NULL',
34
+ ');',
35
+ ].join('\n');
36
+
37
+ // 列定义写成元组表,再由元组生成对外对象 —— 与上面的 DDL 一一对应,改一处就得同步另一处
38
+ const COLUMN_SPEC = [
39
+ ['path', 'TEXT', 0, null, 1],
40
+ ['mtime', 'INTEGER', 1, null, 0],
41
+ ['size', 'INTEGER', 1, null, 0],
42
+ ['text_hash', 'TEXT', 1, null, 0],
43
+ ['summary', 'TEXT', 1, null, 0],
44
+ ['owner_user', 'TEXT', 1, "'LiJianXiao'", 0],
45
+ ['visibility', 'TEXT', 1, "'private_owner'", 0],
46
+ ['updated_at', 'INTEGER', 1, null, 0],
47
+ ];
48
+
49
+ export const COLUMNS = COLUMN_SPEC.map(([name, type, notnull, dflt, pk]) => ({ name, type, notnull, dflt, pk }));
50
+
51
+ // 元数据键:schemaVersion 是结构版本;hostId / lastRunAt 是运行状态;lastRunStats 存最近一轮计数
33
52
  export const META_KEYS = {
34
53
  schemaVersion: 'schemaVersion',
35
54
  hostId: 'hostId',
@@ -37,287 +56,241 @@ export const META_KEYS = {
37
56
  lastRunStats: 'lastRunStats',
38
57
  };
39
58
 
40
- /** 列序(验收用:与 PRAGMA table_info 逐列比对) */
41
- export const COLUMNS = [
42
- { name: 'path', type: 'TEXT', notnull: 0, dflt: null, pk: 1 },
43
- { name: 'mtime', type: 'INTEGER', notnull: 1, dflt: null, pk: 0 },
44
- { name: 'size', type: 'INTEGER', notnull: 1, dflt: null, pk: 0 },
45
- { name: 'text_hash', type: 'TEXT', notnull: 1, dflt: null, pk: 0 },
46
- { name: 'summary', type: 'TEXT', notnull: 1, dflt: null, pk: 0 },
47
- { name: 'owner_user', type: 'TEXT', notnull: 1, dflt: "'LiJianXiao'", pk: 0 },
48
- { name: 'visibility', type: 'TEXT', notnull: 1, dflt: "'private_owner'", pk: 0 },
49
- { name: 'updated_at', type: 'INTEGER', notnull: 1, dflt: null, pk: 0 },
50
- ];
51
-
52
- /** 并发保护参数(方案 §3.2 / §5.2):busy_timeout + 重试 3 次、总退避上限 60 s。 */
59
+ // 并发保护参数:每次等锁 2s × 最多 4 次尝试,退避 1s / 4s / 16s,总预算 60s
53
60
  export const RETRY = {
54
61
  retries: 3,
55
- // 单次等锁上限。独立库的写者只有本模块自己(罕见的手工操作才会占锁)⇒ 2s 足够;
56
- // 取 5s 会让"4 次尝试"的墙钟被等锁拖到 ~87s(2026-09-21 实测),与 §5.2「总退避上限 60s」的口径读起来不像同一件事。
57
62
  busyTimeoutMs: 2000,
58
- // 指数退避:1s / 4s / 16s(累计 21s)
59
63
  delaysMs: [1000, 4000, 16000],
60
- // 总预算(含等锁):重试前判据 —— 已耗时 + 下一次退避 > 预算就直接放弃
61
64
  totalBudgetMs: 60000,
62
65
  };
63
66
 
64
- function tableExists(db, name) {
65
- return Boolean(
66
- db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?").get(name),
67
- );
67
+ const DEFAULT_OWNER = 'LiJianXiao';
68
+ const DEFAULT_VISIBILITY = 'private_owner';
69
+
70
+ // 长 SQL 集中在这里,读代码时不用在一堆模板串里找
71
+ const UPSERT_SQL = [
72
+ `INSERT INTO ${T_DOC} (path, mtime, size, text_hash, summary, owner_user, visibility, updated_at)`,
73
+ 'VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
74
+ 'ON CONFLICT(path) DO UPDATE SET',
75
+ ' mtime = excluded.mtime, size = excluded.size, text_hash = excluded.text_hash,',
76
+ ' summary = excluded.summary, owner_user = excluded.owner_user,',
77
+ ' visibility = excluded.visibility, updated_at = excluded.updated_at',
78
+ ].join('\n');
79
+
80
+ const SQL_HAS_ROW = `SELECT 1 AS hit FROM ${T_DOC} WHERE path = ? OR path LIKE ? ESCAPE '\\' LIMIT 1`;
81
+ const SQL_LIST_ALL = `SELECT * FROM ${T_DOC} ORDER BY path`;
82
+ const SQL_LIST_TREE = `SELECT * FROM ${T_DOC} WHERE path = ? OR path LIKE ? ESCAPE '\\' ORDER BY path`;
83
+ const SQL_COUNT = `SELECT COUNT(*) AS total FROM ${T_DOC}`;
84
+ const SQL_DELETE_TREE = `DELETE FROM ${T_DOC} WHERE path = ? OR path LIKE ? ESCAPE '\\'`;
85
+
86
+ // ── 内部小件 ────────────────────────────────────────────────────────────────
87
+
88
+ const asInt = (v) => Math.trunc(Number(v) || 0);
89
+
90
+ // LIKE 元字符转义(path 里可能有 % _ \,不转义会误匹配到别的篇)
91
+ export const escapeLike = (s) => String(s).replace(/[\\%_]/g, (m) => `\\${m}`);
92
+
93
+ // 子条键前缀:长文档切成 path#§1..N 那一批时用
94
+ const subKeyPattern = (p) => `${escapeLike(p)}#§%`;
95
+
96
+ const tableExists = (db, name) =>
97
+ Boolean(db.prepare('SELECT 1 AS hit FROM sqlite_master WHERE type = ? AND name = ?').get('table', name));
98
+
99
+ // 事务壳:体里抛错就回滚;回滚本身再炸也只把原始错误往外抛
100
+ function inTransaction(db, body) {
101
+ db.exec('BEGIN');
102
+ try {
103
+ const outcome = body();
104
+ db.exec('COMMIT');
105
+ return outcome;
106
+ } catch (err) {
107
+ try {
108
+ db.exec('ROLLBACK');
109
+ } catch {
110
+ // 连接已经坏了才会走到这儿 —— 原始错误更有诊断价值
111
+ }
112
+ throw err;
113
+ }
68
114
  }
69
115
 
116
+ // ── 建库 / 打开 ─────────────────────────────────────────────────────────────
117
+
70
118
  export function ensureSchema(db) {
71
- const created = [];
72
- if (!tableExists(db, 'doc_index')) created.push('doc_index');
73
- if (!tableExists(db, 'meta')) created.push('meta');
119
+ const created = ALL_TABLES.filter((name) => !tableExists(db, name));
74
120
  db.exec(DDL);
75
- return {
76
- created,
77
- tables: { doc_index: tableExists(db, 'doc_index'), meta: tableExists(db, 'meta') },
78
- };
121
+ const tables = {};
122
+ for (const name of ALL_TABLES) tables[name] = tableExists(db, name);
123
+ return { created, tables };
79
124
  }
80
125
 
81
- /**
82
- * 打开(必要时创建)文档索引库。
83
- * @param {string} dbPath
84
- * @param {{create?: boolean, hostId?: string}} [opts]
85
- */
126
+ // 打开(必要时建)文档索引库:建目录 → 设等锁时长 → 开 WAL → 补 schema / hostId 两个元数据键
86
127
  export function openDocIndexStore(dbPath, opts = {}) {
87
- const create = opts.create !== false;
88
- if (create) fs.mkdirSync(path.dirname(dbPath), { recursive: true });
89
- if (create === false && !fs.existsSync(dbPath)) {
90
- throw new Error(`文档索引库不存在: ${dbPath}`);
91
- }
128
+ const canCreate = opts.create !== false;
129
+ if (canCreate) fs.mkdirSync(path.dirname(dbPath), { recursive: true });
130
+ else if (!fs.existsSync(dbPath)) throw new Error(`文档索引库不存在: ${dbPath}`);
131
+
92
132
  const db = new DatabaseSync(dbPath);
93
133
  db.exec(`PRAGMA busy_timeout = ${Number(opts.busyTimeoutMs) || RETRY.busyTimeoutMs};`);
94
134
  try {
95
135
  db.exec('PRAGMA journal_mode = WAL;');
96
136
  } catch {
97
- /* 只读介质上 WAL 可能不可用;本机不属此情况 */
137
+ // 只读介质上 WAL 可能开不起来,不影响主流程
98
138
  }
139
+
99
140
  const schema = ensureSchema(db);
100
- if (getMeta(db, META_KEYS.schemaVersion) == null) {
101
- setMeta(db, META_KEYS.schemaVersion, DOC_INDEX_SCHEMA_VERSION);
102
- }
103
- if (opts.hostId && getMeta(db, META_KEYS.hostId) == null) {
104
- setMeta(db, META_KEYS.hostId, String(opts.hostId));
105
- }
141
+ if (getMeta(db, META_KEYS.schemaVersion) == null) setMeta(db, META_KEYS.schemaVersion, DOC_INDEX_SCHEMA_VERSION);
142
+ if (opts.hostId && getMeta(db, META_KEYS.hostId) == null) setMeta(db, META_KEYS.hostId, String(opts.hostId));
106
143
  return { db, dbPath, schema };
107
144
  }
108
145
 
109
- export function getMeta(db, k) {
110
- const r = db.prepare('SELECT v FROM meta WHERE k = ?').get(k);
111
- return r ? r.v : null;
112
- }
146
+ // ── 元数据键值 ──────────────────────────────────────────────────────────────
147
+
148
+ export const getMeta = (db, k) => db.prepare(`SELECT v FROM ${T_META} WHERE k = ?`).get(k)?.v ?? null;
113
149
 
114
- export function setMeta(db, k, v) {
115
- db.prepare('INSERT INTO meta (k, v) VALUES (?, ?) ON CONFLICT(k) DO UPDATE SET v = excluded.v')
150
+ export const setMeta = (db, k, v) =>
151
+ db
152
+ .prepare(`INSERT INTO ${T_META} (k, v) VALUES (?, ?) ON CONFLICT(k) DO UPDATE SET v = excluded.v`)
116
153
  .run(String(k), String(v));
117
- }
118
154
 
119
- export function getMetaAll(db) {
155
+ export const getMetaAll = (db) => {
120
156
  const out = {};
121
- for (const r of db.prepare('SELECT k, v FROM meta ORDER BY k').all()) out[r.k] = r.v;
157
+ for (const row of db.prepare(`SELECT k, v FROM ${T_META} ORDER BY k`).all()) out[row.k] = row.v;
122
158
  return out;
123
- }
159
+ };
124
160
 
125
- // ---------------------------------------------------------------- 并发保护
161
+ // ── 并发保护 ────────────────────────────────────────────────────────────────
126
162
 
127
- /** 是否为"库被锁"类错误(可重试);其它错误一律立即上抛(不掩盖真 bug)。 */
128
- export function isBusyError(e) {
129
- const msg = String((e && e.message) || e || '').toLowerCase();
130
- return (
131
- msg.includes('database is locked') ||
132
- msg.includes('database table is locked') ||
133
- msg.includes('sqlite_busy') ||
134
- msg.includes('busy')
135
- );
136
- }
163
+ // 只有「库被占着」类错误值得等;其它错误立刻收手,免得把真 bug 掩盖成重试
164
+ export const isBusyError = (e) => /busy|locked/i.test(String((e && e.message) || e || ''));
137
165
 
138
- const defaultSleep = (ms) => new Promise((r) => setTimeout(r, ms));
166
+ const realSleep = (ms) => new Promise((done) => setTimeout(done, ms));
139
167
 
140
- /**
141
- * 带重试地执行写操作(方案 §5.2:重试 3 次 / 退避上限 60 s)。
142
- * 超限不抛错给调用方判死——返回 {ok:false, error},由上层记 failed(卡要求"不报错给主人")。
143
- * @template T
144
- * @param {() => T} fn
145
- * @param {{retries?:number, delaysMs?:number[], sleep?:(ms:number)=>Promise<void>, onRetry?:(i:number, e:Error)=>void}} [opts]
146
- * @returns {Promise<{ok:boolean, value?:T, error?:string, attempts:number, busyWaits:number}>}
147
- */
168
+ // 带退避地执行一次写操作。超限**不抛**:返回 {ok:false, error} 交上层记失败
169
+ // (调用方约定:库里出问题只记账,不把异常甩给用户)
148
170
  export async function withBusyRetry(fn, opts = {}) {
149
- const retries = opts.retries ?? RETRY.retries;
171
+ const maxRetries = opts.retries ?? RETRY.retries;
150
172
  const delays = opts.delaysMs ?? RETRY.delaysMs;
151
- const sleep = opts.sleep ?? defaultSleep;
152
- const started = Date.now();
153
- let attempts = 0;
154
- for (let i = 0; ; i += 1) {
155
- attempts += 1;
173
+ const nap = opts.sleep ?? realSleep;
174
+ const budget = opts.totalBudgetMs ?? RETRY.totalBudgetMs;
175
+ const t0 = Date.now();
176
+ let attempt = 0;
177
+
178
+ for (;;) {
179
+ attempt += 1;
156
180
  try {
157
- return { ok: true, value: fn(), attempts, busyWaits: i };
158
- } catch (e) {
159
- if (!isBusyError(e) || i >= retries) {
160
- return { ok: false, error: String((e && e.message) || e), attempts, busyWaits: i };
181
+ return { ok: true, value: fn(), attempts: attempt, busyWaits: attempt - 1 };
182
+ } catch (err) {
183
+ const text = String((err && err.message) || err);
184
+ const waited = attempt - 1;
185
+ if (!isBusyError(err) || attempt > maxRetries) {
186
+ return { ok: false, error: text, attempts: attempt, busyWaits: waited };
161
187
  }
162
- const delay = delays[Math.min(i, delays.length - 1)] ?? 1000;
163
- if (Date.now() - started + delay > (opts.totalBudgetMs ?? RETRY.totalBudgetMs)) {
164
- return { ok: false, error: `退避预算耗尽(${opts.totalBudgetMs ?? RETRY.totalBudgetMs}ms):${String((e && e.message) || e)}`, attempts, busyWaits: i + 1 };
188
+ const pause = delays[Math.min(waited, delays.length - 1)] ?? 1000;
189
+ if (Date.now() - t0 + pause > budget) {
190
+ return { ok: false, error: `退避预算耗尽(${budget}ms):${text}`, attempts: attempt, busyWaits: waited + 1 };
165
191
  }
166
- if (opts.onRetry) opts.onRetry(i + 1, e);
167
- await sleep(delay);
192
+ if (typeof opts.onRetry === 'function') opts.onRetry(attempt, err);
193
+ await nap(pause);
168
194
  }
169
195
  }
170
196
  }
171
197
 
172
- // ---------------------------------------------------------------- 读写
198
+ // ── 读写 ────────────────────────────────────────────────────────────────────
173
199
 
174
- /**
175
- * 批量 UPSERT(单事务)。幂等键 = `path` 主键:重复执行不产生孤条。
176
- * @param {import('node:sqlite').DatabaseSync} db
177
- * @param {Array<{path:string, mtime:number, size:number, textHash:string, summary:string,
178
- * ownerUser?:string, visibility?:string, updatedAt:number}>} rows
179
- * @returns {{inserted:number, updated:number, unchanged:number}}
180
- */
200
+ // 批量 UPSERT(一个事务)。幂等键 = path:同一批重复执行不会长出第二条。
201
+ // 返回 {inserted, updated, unchanged};unchanged 是「摘要哈希没变、整行没重写」的那些。
181
202
  export function upsertDocs(db, rows) {
182
- const out = { inserted: 0, updated: 0, unchanged: 0 };
183
- if (!rows.length) return out;
184
- const existing = new Map();
185
- const selStmt = db.prepare('SELECT text_hash FROM doc_index WHERE path = ?');
186
- for (const r of rows) {
187
- const cur = selStmt.get(r.path);
188
- existing.set(r.path, cur ? cur.text_hash : null);
203
+ const tally = { inserted: 0, updated: 0, unchanged: 0 };
204
+ if (!rows?.length) return tally;
205
+
206
+ const readHash = db.prepare(`SELECT text_hash FROM ${T_DOC} WHERE path = ?`);
207
+ const pending = [];
208
+ for (const row of rows) {
209
+ const seen = readHash.get(row.path);
210
+ const before = seen ? seen.text_hash : null;
211
+ if (before === row.textHash) {
212
+ // 摘要没变 ⇒ 整行不重写(mtime / updated_at 都不许动)
213
+ tally.unchanged += 1;
214
+ continue;
215
+ }
216
+ pending.push({ row, before });
189
217
  }
190
- const stmt = db.prepare(`
191
- INSERT INTO doc_index (path, mtime, size, text_hash, summary, owner_user, visibility, updated_at)
192
- VALUES (?, ?, ?, ?, ?, ?, ?, ?)
193
- ON CONFLICT(path) DO UPDATE SET
194
- mtime = excluded.mtime,
195
- size = excluded.size,
196
- text_hash = excluded.text_hash,
197
- summary = excluded.summary,
198
- owner_user = excluded.owner_user,
199
- visibility = excluded.visibility,
200
- updated_at = excluded.updated_at
201
- `);
202
- const now = Date.now();
203
- db.exec('BEGIN');
204
- try {
205
- for (const r of rows) {
206
- const prev = existing.get(r.path);
207
- if (prev === r.textHash) {
208
- // 摘要内容未变 ⇒ 不重写该条(验收项「text_hash 未变的条目未重建」)
209
- out.unchanged += 1;
210
- continue;
211
- }
212
- if (prev == null) out.inserted += 1;
213
- else out.updated += 1;
214
- stmt.run(
215
- String(r.path),
216
- Math.trunc(Number(r.mtime) || 0),
217
- Math.trunc(Number(r.size) || 0),
218
- String(r.textHash),
219
- String(r.summary),
220
- String(r.ownerUser || 'LiJianXiao'),
221
- String(r.visibility || 'private_owner'),
222
- Math.trunc(Number(r.updatedAt) || now),
218
+ if (!pending.length) return tally;
219
+
220
+ const write = db.prepare(UPSERT_SQL);
221
+ const stamp = Date.now();
222
+ inTransaction(db, () => {
223
+ for (const { row, before } of pending) {
224
+ write.run(
225
+ String(row.path),
226
+ asInt(row.mtime),
227
+ asInt(row.size),
228
+ String(row.textHash),
229
+ String(row.summary),
230
+ String(row.ownerUser || DEFAULT_OWNER),
231
+ String(row.visibility || DEFAULT_VISIBILITY),
232
+ asInt(row.updatedAt) || stamp,
223
233
  );
234
+ if (before == null) tally.inserted += 1;
235
+ else tally.updated += 1;
224
236
  }
225
- db.exec('COMMIT');
226
- } catch (e) {
227
- try {
228
- db.exec('ROLLBACK');
229
- } catch {
230
- /* ignore */
231
- }
232
- throw e;
233
- }
234
- return out;
237
+ });
238
+ return tally;
235
239
  }
236
240
 
237
- /** 取指定 path 的已有 text_hash(跳过判据)。 */
241
+ // 一批 path 已有的 text_hash(调用方据此判「要不要重算」)
238
242
  export function getHashesForPaths(db, paths) {
239
- const out = new Map();
240
- if (!paths.length) return out;
241
- const stmt = db.prepare('SELECT text_hash FROM doc_index WHERE path = ?');
243
+ const known = new Map();
244
+ if (!paths?.length) return known;
245
+ const stmt = db.prepare(`SELECT text_hash FROM ${T_DOC} WHERE path = ?`);
242
246
  for (const p of paths) {
243
- const r = stmt.get(String(p));
244
- if (r) out.set(String(p), r.text_hash);
247
+ const key = String(p);
248
+ const hit = stmt.get(key);
249
+ if (hit) known.set(key, hit.text_hash);
245
250
  }
246
- return out;
251
+ return known;
247
252
  }
248
253
 
249
- export function getRow(db, p) {
250
- const r = db.prepare('SELECT * FROM doc_index WHERE path = ?').get(String(p));
251
- return r ?? null;
252
- }
254
+ export const getRow = (db, p) => db.prepare(`SELECT * FROM ${T_DOC} WHERE path = ?`).get(String(p)) ?? null;
253
255
 
254
- /**
255
- * 该篇在库里**是否有条**(主条 `path` **或** 任一分条 `path#§N`)。
256
- * ⚠️ 必须两条都查:长文档按 §3.2 切成 2~5 条时**只有分条、没有主条**
257
- * (实测踩过:只查主条会让长文档每轮都判 `missing-in-db` 反复重建,"第二轮全部跳过"永远不成立)。
258
- */
256
+ // 这一篇在库里有没有条:主条 path **或**任一分条 path#§N 都算。
257
+ // 分条必须一起查 —— 长文档切成 2~5 条时只存在分条,只查主条会判成「库里没有」而每轮重建。
259
258
  export function hasDocRows(db, p) {
260
- const r = db
261
- .prepare("SELECT 1 AS n FROM doc_index WHERE path = ? OR path LIKE ? ESCAPE '\\' LIMIT 1")
262
- .get(String(p), `${escapeLike(p)}#§%`);
263
- return Boolean(r);
259
+ const hit = db.prepare(SQL_HAS_ROW).get(String(p), subKeyPattern(p));
260
+ return Boolean(hit);
264
261
  }
265
262
 
266
- /** 全量列出(抽检/比对用;按 path 排序保证可复现)。 */
263
+ // 全量列出(按 path 排序,读数可复现);给 pathPrefix 就只管这一篇及其分条
267
264
  export function listRows(db, { pathPrefix = null } = {}) {
268
- if (pathPrefix) {
269
- return db
270
- .prepare("SELECT * FROM doc_index WHERE path = ? OR path LIKE ? ESCAPE '\\' ORDER BY path")
271
- .all(pathPrefix, `${escapeLike(pathPrefix)}#§%`);
272
- }
273
- return db.prepare('SELECT * FROM doc_index ORDER BY path').all();
265
+ if (!pathPrefix) return db.prepare(SQL_LIST_ALL).all();
266
+ return db.prepare(SQL_LIST_TREE).all(String(pathPrefix), subKeyPattern(pathPrefix));
274
267
  }
275
268
 
276
- export function countRows(db) {
277
- return db.prepare('SELECT COUNT(*) AS n FROM doc_index').get().n;
278
- }
279
-
280
- /** LIKE 元字符转义(path 里可能出现 % _ \,不转义会误匹配)。 */
281
- export function escapeLike(s) {
282
- return String(s).replace(/[\\%_]/g, (m) => `\\${m}`);
283
- }
269
+ export const countRows = (db) => db.prepare(SQL_COUNT).get().total;
284
270
 
285
- /**
286
- * 删除某篇文档的全部索引条(含 `path#§N` 长文档分条)——回滚口径:
287
- * `DELETE FROM doc_index WHERE path=?`(方案 §5.4)+ 分条一并清。
288
- */
271
+ // 删掉某篇的全部条(主条 + path#§N 分条)—— 回滚口径就这一条
289
272
  export function deleteDoc(db, p) {
290
- const r = db
291
- .prepare("DELETE FROM doc_index WHERE path = ? OR path LIKE ? ESCAPE '\\'")
292
- .run(String(p), `${escapeLike(p)}#§%`);
293
- return Number(r.changes || 0);
273
+ const res = db.prepare(SQL_DELETE_TREE).run(String(p), subKeyPattern(p));
274
+ return Number(res.changes || 0);
294
275
  }
295
276
 
296
- /** 删除本次未再生成的分条(治"文档变短后残留孤条")。 */
277
+ // 删本次没再生成的分条(文档从长变短时的残留),保留 keepKeys 里点名的那些
297
278
  export function deleteStaleRows(db, p, keepKeys) {
298
279
  const keep = new Set(keepKeys.map(String));
299
- const rows = db
300
- .prepare("SELECT path FROM doc_index WHERE path = ? OR path LIKE ? ESCAPE '\\'")
301
- .all(String(p), `${escapeLike(p)}#§%`);
280
+ const rows = db.prepare(`SELECT path FROM ${T_DOC} WHERE path = ? OR path LIKE ? ESCAPE '\\'`).all(
281
+ String(p),
282
+ subKeyPattern(p),
283
+ );
302
284
  const stale = rows.map((r) => r.path).filter((v) => !keep.has(v));
303
285
  if (!stale.length) return 0;
304
- const stmt = db.prepare('DELETE FROM doc_index WHERE path = ?');
305
- db.exec('BEGIN');
306
- try {
307
- for (const s of stale) stmt.run(s);
308
- db.exec('COMMIT');
309
- } catch (e) {
310
- try {
311
- db.exec('ROLLBACK');
312
- } catch {
313
- /* ignore */
314
- }
315
- throw e;
316
- }
286
+ const drop = db.prepare(`DELETE FROM ${T_DOC} WHERE path = ?`);
287
+ inTransaction(db, () => {
288
+ for (const key of stale) drop.run(key);
289
+ });
317
290
  return stale.length;
318
291
  }
319
292
 
320
- /** 库文件级快照(改动前备份;方案 §7「回滚点」)。 */
293
+ // 库文件级快照(改动前的回滚点)
321
294
  export function snapshotDb(db, destFile) {
322
295
  fs.mkdirSync(path.dirname(destFile), { recursive: true });
323
296
  db.prepare('VACUUM INTO ?').run(destFile);