@kira905/dsh-memory-toolkit 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +92 -0
- package/README.en.md +225 -2
- package/README.md +180 -2
- package/lib/deploy-guard.mjs +463 -275
- package/lib/guard-http.mjs +59 -62
- package/lib/pitfall-form-map.mjs +94 -84
- package/lib/pitfall-score.mjs +262 -185
- package/lib/session-file-name.mjs +116 -100
- package/lib/zstd-frames.mjs +88 -67
- package/memory-db-tools/README.md +29 -1
- package/memory-db-tools/audit-analyze.mjs +1 -1
- package/memory-db-tools/audit-export.mjs +60 -16
- package/memory-db-tools/audit-struct-probe.mjs +29 -8
- package/memory-db-tools/lib/known-devices.mjs +137 -0
- package/memory-engine/README.md +58 -4
- package/memory-engine/bench/bench.mjs +1 -1
- package/memory-engine/cli/band-sentinel.mjs +164 -0
- package/memory-engine/cli/distill-rollback.mjs +112 -44
- package/memory-engine/cli/distill-sessions.mjs +237 -0
- package/memory-engine/cli/distill.mjs +196 -95
- package/memory-engine/cli/doc-index-rebuild.mjs +142 -108
- package/memory-engine/cli/doc-index.mjs +47 -38
- package/memory-engine/cli/domain-tags.mjs +246 -162
- package/memory-engine/cli/layer-migrate.mjs +176 -0
- package/memory-engine/cli/layer-move.mjs +189 -0
- package/memory-engine/cli/pitfall-hook.mjs +136 -70
- package/memory-engine/cli/recall.mjs +243 -110
- package/memory-engine/cli/selfcheck.mjs +93 -44
- package/memory-engine/cli/serve-start.mjs +221 -140
- package/memory-engine/cli/sweep.mjs +83 -44
- package/memory-engine/cli/sync.mjs +93 -57
- package/memory-engine/config/audit-devices.json +6 -0
- package/memory-engine/config/bands.json +7 -0
- package/memory-engine/config/layers.json +7 -0
- package/memory-engine/dict/LICENSE-/350/257/215/345/205/270.md +53 -0
- package/memory-engine/dict/lexicon-config.json +11 -0
- package/memory-engine/dict//350/257/215/345/205/270.txt +109750 -0
- package/memory-engine/dict//350/257/215/351/242/221.txt +20000 -0
- package/memory-engine/lib/band-sentinel.mjs +417 -0
- package/memory-engine/lib/cli-strict.mjs +128 -0
- package/memory-engine/lib/cli-util.mjs +147 -125
- package/memory-engine/lib/config.mjs +83 -96
- package/memory-engine/lib/dirty-ledger.mjs +127 -94
- package/memory-engine/lib/distill-rollback.mjs +173 -133
- package/memory-engine/lib/distill.mjs +878 -585
- package/memory-engine/lib/doc-index-store.mjs +208 -235
- package/memory-engine/lib/doc-index.mjs +351 -350
- package/memory-engine/lib/doc-scan.mjs +178 -156
- package/memory-engine/lib/doc-summarize.mjs +220 -205
- package/memory-engine/lib/domain-tags.mjs +358 -277
- package/memory-engine/lib/embedder.mjs +175 -194
- package/memory-engine/lib/hashes.mjs +22 -7
- package/memory-engine/lib/layer-migrate.mjs +330 -0
- package/memory-engine/lib/layer-move.mjs +307 -0
- package/memory-engine/lib/layers.mjs +191 -0
- package/memory-engine/lib/lexicon-recall.mjs +473 -0
- package/memory-engine/lib/memory-source.mjs +78 -80
- package/memory-engine/lib/pitfall-source.mjs +115 -114
- package/memory-engine/lib/policy-source.mjs +279 -261
- package/memory-engine/lib/recall-log.mjs +285 -0
- package/memory-engine/lib/recall.mjs +565 -240
- package/memory-engine/lib/selfcheck.mjs +142 -106
- package/memory-engine/lib/session-distill.mjs +603 -0
- package/memory-engine/lib/session-source.mjs +421 -0
- package/memory-engine/lib/sweep.mjs +107 -130
- package/memory-engine/lib/sync.mjs +110 -140
- package/memory-engine/lib/tick-hook.mjs +123 -148
- package/memory-engine/lib/vectors-store.mjs +213 -188
- package/memory-engine/test/12-layering.test.mjs +539 -0
- package/memory-engine/test/13-session-distill.test.mjs +616 -0
- package/memory-engine/tools/bridge-smoke.mjs +94 -55
- package/memory-engine/tools/fetch-model.mjs +167 -87
- package/memory-engine/tools/smoke-recall.mjs +94 -38
- package/memory-engine/tools/verify-recall-chain.mjs +413 -0
- package/package.json +1 -1
|
@@ -1,35 +1,54 @@
|
|
|
1
|
-
//
|
|
1
|
+
// 文档索引独立库 —— 建表 / 键值元数据 / 并发保护 / 批量读写与清理
|
|
2
2
|
//
|
|
3
|
-
//
|
|
4
|
-
// 1.
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
//
|
|
3
|
+
// 为什么单开一个库文件(三条理由,别重新讨论):
|
|
4
|
+
// 1. 索引条要有天然的幂等键(一条文档一个主键),主记忆库没有这样的唯一列可用;
|
|
5
|
+
// 2. 索引条上记的规模数字跟着本机文件 mtime 走,两台机器各算各的 ⇒ 塞进同一个库会互相覆盖;
|
|
6
|
+
// 3. 独立库不进任何同步链路,删掉整份可重建,也就不会跟别的写入抢锁。
|
|
7
|
+
//
|
|
8
|
+
// 对外只承诺「打开 / 读 / 写 / 清理」四件事,SQL 细节不出这个文件。
|
|
9
9
|
import fs from 'node:fs';
|
|
10
10
|
import path from 'node:path';
|
|
11
11
|
import { DatabaseSync } from 'node:sqlite';
|
|
12
12
|
import { DOC_INDEX_SCHEMA_VERSION } from './config.mjs';
|
|
13
13
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
14
|
+
// ── 表形状(验收按 PRAGMA table_info 逐列核对,列序/类型/NOT NULL/默认值一个都不许漂) ──
|
|
15
|
+
|
|
16
|
+
const T_DOC = 'doc_index';
|
|
17
|
+
const T_META = 'meta';
|
|
18
|
+
const ALL_TABLES = [T_DOC, T_META];
|
|
19
|
+
|
|
20
|
+
const DDL = [
|
|
21
|
+
`CREATE TABLE IF NOT EXISTS ${T_DOC} (`,
|
|
22
|
+
' path TEXT PRIMARY KEY,',
|
|
23
|
+
' mtime INTEGER NOT NULL,',
|
|
24
|
+
' size INTEGER NOT NULL,',
|
|
25
|
+
' text_hash TEXT NOT NULL,',
|
|
26
|
+
' summary TEXT NOT NULL,',
|
|
27
|
+
" owner_user TEXT NOT NULL DEFAULT 'LiJianXiao',",
|
|
28
|
+
" visibility TEXT NOT NULL DEFAULT 'private_owner',",
|
|
29
|
+
' updated_at INTEGER NOT NULL',
|
|
30
|
+
');',
|
|
31
|
+
`CREATE TABLE IF NOT EXISTS ${T_META} (`,
|
|
32
|
+
' k TEXT PRIMARY KEY,',
|
|
33
|
+
' v TEXT NOT NULL',
|
|
34
|
+
');',
|
|
35
|
+
].join('\n');
|
|
36
|
+
|
|
37
|
+
// 列定义写成元组表,再由元组生成对外对象 —— 与上面的 DDL 一一对应,改一处就得同步另一处
|
|
38
|
+
const COLUMN_SPEC = [
|
|
39
|
+
['path', 'TEXT', 0, null, 1],
|
|
40
|
+
['mtime', 'INTEGER', 1, null, 0],
|
|
41
|
+
['size', 'INTEGER', 1, null, 0],
|
|
42
|
+
['text_hash', 'TEXT', 1, null, 0],
|
|
43
|
+
['summary', 'TEXT', 1, null, 0],
|
|
44
|
+
['owner_user', 'TEXT', 1, "'LiJianXiao'", 0],
|
|
45
|
+
['visibility', 'TEXT', 1, "'private_owner'", 0],
|
|
46
|
+
['updated_at', 'INTEGER', 1, null, 0],
|
|
47
|
+
];
|
|
48
|
+
|
|
49
|
+
export const COLUMNS = COLUMN_SPEC.map(([name, type, notnull, dflt, pk]) => ({ name, type, notnull, dflt, pk }));
|
|
50
|
+
|
|
51
|
+
// 元数据键:schemaVersion 是结构版本;hostId / lastRunAt 是运行状态;lastRunStats 存最近一轮计数
|
|
33
52
|
export const META_KEYS = {
|
|
34
53
|
schemaVersion: 'schemaVersion',
|
|
35
54
|
hostId: 'hostId',
|
|
@@ -37,287 +56,241 @@ export const META_KEYS = {
|
|
|
37
56
|
lastRunStats: 'lastRunStats',
|
|
38
57
|
};
|
|
39
58
|
|
|
40
|
-
|
|
41
|
-
export const COLUMNS = [
|
|
42
|
-
{ name: 'path', type: 'TEXT', notnull: 0, dflt: null, pk: 1 },
|
|
43
|
-
{ name: 'mtime', type: 'INTEGER', notnull: 1, dflt: null, pk: 0 },
|
|
44
|
-
{ name: 'size', type: 'INTEGER', notnull: 1, dflt: null, pk: 0 },
|
|
45
|
-
{ name: 'text_hash', type: 'TEXT', notnull: 1, dflt: null, pk: 0 },
|
|
46
|
-
{ name: 'summary', type: 'TEXT', notnull: 1, dflt: null, pk: 0 },
|
|
47
|
-
{ name: 'owner_user', type: 'TEXT', notnull: 1, dflt: "'LiJianXiao'", pk: 0 },
|
|
48
|
-
{ name: 'visibility', type: 'TEXT', notnull: 1, dflt: "'private_owner'", pk: 0 },
|
|
49
|
-
{ name: 'updated_at', type: 'INTEGER', notnull: 1, dflt: null, pk: 0 },
|
|
50
|
-
];
|
|
51
|
-
|
|
52
|
-
/** 并发保护参数(方案 §3.2 / §5.2):busy_timeout + 重试 3 次、总退避上限 60 s。 */
|
|
59
|
+
// 并发保护参数:每次等锁 2s × 最多 4 次尝试,退避 1s / 4s / 16s,总预算 60s
|
|
53
60
|
export const RETRY = {
|
|
54
61
|
retries: 3,
|
|
55
|
-
// 单次等锁上限。独立库的写者只有本模块自己(罕见的手工操作才会占锁)⇒ 2s 足够;
|
|
56
|
-
// 取 5s 会让"4 次尝试"的墙钟被等锁拖到 ~87s(2026-09-21 实测),与 §5.2「总退避上限 60s」的口径读起来不像同一件事。
|
|
57
62
|
busyTimeoutMs: 2000,
|
|
58
|
-
// 指数退避:1s / 4s / 16s(累计 21s)
|
|
59
63
|
delaysMs: [1000, 4000, 16000],
|
|
60
|
-
// 总预算(含等锁):重试前判据 —— 已耗时 + 下一次退避 > 预算就直接放弃
|
|
61
64
|
totalBudgetMs: 60000,
|
|
62
65
|
};
|
|
63
66
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
67
|
+
const DEFAULT_OWNER = 'LiJianXiao';
|
|
68
|
+
const DEFAULT_VISIBILITY = 'private_owner';
|
|
69
|
+
|
|
70
|
+
// 长 SQL 集中在这里,读代码时不用在一堆模板串里找
|
|
71
|
+
const UPSERT_SQL = [
|
|
72
|
+
`INSERT INTO ${T_DOC} (path, mtime, size, text_hash, summary, owner_user, visibility, updated_at)`,
|
|
73
|
+
'VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
|
|
74
|
+
'ON CONFLICT(path) DO UPDATE SET',
|
|
75
|
+
' mtime = excluded.mtime, size = excluded.size, text_hash = excluded.text_hash,',
|
|
76
|
+
' summary = excluded.summary, owner_user = excluded.owner_user,',
|
|
77
|
+
' visibility = excluded.visibility, updated_at = excluded.updated_at',
|
|
78
|
+
].join('\n');
|
|
79
|
+
|
|
80
|
+
const SQL_HAS_ROW = `SELECT 1 AS hit FROM ${T_DOC} WHERE path = ? OR path LIKE ? ESCAPE '\\' LIMIT 1`;
|
|
81
|
+
const SQL_LIST_ALL = `SELECT * FROM ${T_DOC} ORDER BY path`;
|
|
82
|
+
const SQL_LIST_TREE = `SELECT * FROM ${T_DOC} WHERE path = ? OR path LIKE ? ESCAPE '\\' ORDER BY path`;
|
|
83
|
+
const SQL_COUNT = `SELECT COUNT(*) AS total FROM ${T_DOC}`;
|
|
84
|
+
const SQL_DELETE_TREE = `DELETE FROM ${T_DOC} WHERE path = ? OR path LIKE ? ESCAPE '\\'`;
|
|
85
|
+
|
|
86
|
+
// ── 内部小件 ────────────────────────────────────────────────────────────────
|
|
87
|
+
|
|
88
|
+
const asInt = (v) => Math.trunc(Number(v) || 0);
|
|
89
|
+
|
|
90
|
+
// LIKE 元字符转义(path 里可能有 % _ \,不转义会误匹配到别的篇)
|
|
91
|
+
export const escapeLike = (s) => String(s).replace(/[\\%_]/g, (m) => `\\${m}`);
|
|
92
|
+
|
|
93
|
+
// 子条键前缀:长文档切成 path#§1..N 那一批时用
|
|
94
|
+
const subKeyPattern = (p) => `${escapeLike(p)}#§%`;
|
|
95
|
+
|
|
96
|
+
const tableExists = (db, name) =>
|
|
97
|
+
Boolean(db.prepare('SELECT 1 AS hit FROM sqlite_master WHERE type = ? AND name = ?').get('table', name));
|
|
98
|
+
|
|
99
|
+
// 事务壳:体里抛错就回滚;回滚本身再炸也只把原始错误往外抛
|
|
100
|
+
function inTransaction(db, body) {
|
|
101
|
+
db.exec('BEGIN');
|
|
102
|
+
try {
|
|
103
|
+
const outcome = body();
|
|
104
|
+
db.exec('COMMIT');
|
|
105
|
+
return outcome;
|
|
106
|
+
} catch (err) {
|
|
107
|
+
try {
|
|
108
|
+
db.exec('ROLLBACK');
|
|
109
|
+
} catch {
|
|
110
|
+
// 连接已经坏了才会走到这儿 —— 原始错误更有诊断价值
|
|
111
|
+
}
|
|
112
|
+
throw err;
|
|
113
|
+
}
|
|
68
114
|
}
|
|
69
115
|
|
|
116
|
+
// ── 建库 / 打开 ─────────────────────────────────────────────────────────────
|
|
117
|
+
|
|
70
118
|
export function ensureSchema(db) {
|
|
71
|
-
const created =
|
|
72
|
-
if (!tableExists(db, 'doc_index')) created.push('doc_index');
|
|
73
|
-
if (!tableExists(db, 'meta')) created.push('meta');
|
|
119
|
+
const created = ALL_TABLES.filter((name) => !tableExists(db, name));
|
|
74
120
|
db.exec(DDL);
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
};
|
|
121
|
+
const tables = {};
|
|
122
|
+
for (const name of ALL_TABLES) tables[name] = tableExists(db, name);
|
|
123
|
+
return { created, tables };
|
|
79
124
|
}
|
|
80
125
|
|
|
81
|
-
|
|
82
|
-
* 打开(必要时创建)文档索引库。
|
|
83
|
-
* @param {string} dbPath
|
|
84
|
-
* @param {{create?: boolean, hostId?: string}} [opts]
|
|
85
|
-
*/
|
|
126
|
+
// 打开(必要时建)文档索引库:建目录 → 设等锁时长 → 开 WAL → 补 schema / hostId 两个元数据键
|
|
86
127
|
export function openDocIndexStore(dbPath, opts = {}) {
|
|
87
|
-
const
|
|
88
|
-
if (
|
|
89
|
-
if (
|
|
90
|
-
|
|
91
|
-
}
|
|
128
|
+
const canCreate = opts.create !== false;
|
|
129
|
+
if (canCreate) fs.mkdirSync(path.dirname(dbPath), { recursive: true });
|
|
130
|
+
else if (!fs.existsSync(dbPath)) throw new Error(`文档索引库不存在: ${dbPath}`);
|
|
131
|
+
|
|
92
132
|
const db = new DatabaseSync(dbPath);
|
|
93
133
|
db.exec(`PRAGMA busy_timeout = ${Number(opts.busyTimeoutMs) || RETRY.busyTimeoutMs};`);
|
|
94
134
|
try {
|
|
95
135
|
db.exec('PRAGMA journal_mode = WAL;');
|
|
96
136
|
} catch {
|
|
97
|
-
|
|
137
|
+
// 只读介质上 WAL 可能开不起来,不影响主流程
|
|
98
138
|
}
|
|
139
|
+
|
|
99
140
|
const schema = ensureSchema(db);
|
|
100
|
-
if (getMeta(db, META_KEYS.schemaVersion) == null)
|
|
101
|
-
|
|
102
|
-
}
|
|
103
|
-
if (opts.hostId && getMeta(db, META_KEYS.hostId) == null) {
|
|
104
|
-
setMeta(db, META_KEYS.hostId, String(opts.hostId));
|
|
105
|
-
}
|
|
141
|
+
if (getMeta(db, META_KEYS.schemaVersion) == null) setMeta(db, META_KEYS.schemaVersion, DOC_INDEX_SCHEMA_VERSION);
|
|
142
|
+
if (opts.hostId && getMeta(db, META_KEYS.hostId) == null) setMeta(db, META_KEYS.hostId, String(opts.hostId));
|
|
106
143
|
return { db, dbPath, schema };
|
|
107
144
|
}
|
|
108
145
|
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
}
|
|
146
|
+
// ── 元数据键值 ──────────────────────────────────────────────────────────────
|
|
147
|
+
|
|
148
|
+
export const getMeta = (db, k) => db.prepare(`SELECT v FROM ${T_META} WHERE k = ?`).get(k)?.v ?? null;
|
|
113
149
|
|
|
114
|
-
export
|
|
115
|
-
db
|
|
150
|
+
export const setMeta = (db, k, v) =>
|
|
151
|
+
db
|
|
152
|
+
.prepare(`INSERT INTO ${T_META} (k, v) VALUES (?, ?) ON CONFLICT(k) DO UPDATE SET v = excluded.v`)
|
|
116
153
|
.run(String(k), String(v));
|
|
117
|
-
}
|
|
118
154
|
|
|
119
|
-
export
|
|
155
|
+
export const getMetaAll = (db) => {
|
|
120
156
|
const out = {};
|
|
121
|
-
for (const
|
|
157
|
+
for (const row of db.prepare(`SELECT k, v FROM ${T_META} ORDER BY k`).all()) out[row.k] = row.v;
|
|
122
158
|
return out;
|
|
123
|
-
}
|
|
159
|
+
};
|
|
124
160
|
|
|
125
|
-
//
|
|
161
|
+
// ── 并发保护 ────────────────────────────────────────────────────────────────
|
|
126
162
|
|
|
127
|
-
|
|
128
|
-
export
|
|
129
|
-
const msg = String((e && e.message) || e || '').toLowerCase();
|
|
130
|
-
return (
|
|
131
|
-
msg.includes('database is locked') ||
|
|
132
|
-
msg.includes('database table is locked') ||
|
|
133
|
-
msg.includes('sqlite_busy') ||
|
|
134
|
-
msg.includes('busy')
|
|
135
|
-
);
|
|
136
|
-
}
|
|
163
|
+
// 只有「库被占着」类错误值得等;其它错误立刻收手,免得把真 bug 掩盖成重试
|
|
164
|
+
export const isBusyError = (e) => /busy|locked/i.test(String((e && e.message) || e || ''));
|
|
137
165
|
|
|
138
|
-
const
|
|
166
|
+
const realSleep = (ms) => new Promise((done) => setTimeout(done, ms));
|
|
139
167
|
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
* 超限不抛错给调用方判死——返回 {ok:false, error},由上层记 failed(卡要求"不报错给主人")。
|
|
143
|
-
* @template T
|
|
144
|
-
* @param {() => T} fn
|
|
145
|
-
* @param {{retries?:number, delaysMs?:number[], sleep?:(ms:number)=>Promise<void>, onRetry?:(i:number, e:Error)=>void}} [opts]
|
|
146
|
-
* @returns {Promise<{ok:boolean, value?:T, error?:string, attempts:number, busyWaits:number}>}
|
|
147
|
-
*/
|
|
168
|
+
// 带退避地执行一次写操作。超限**不抛**:返回 {ok:false, error} 交上层记失败
|
|
169
|
+
// (调用方约定:库里出问题只记账,不把异常甩给用户)
|
|
148
170
|
export async function withBusyRetry(fn, opts = {}) {
|
|
149
|
-
const
|
|
171
|
+
const maxRetries = opts.retries ?? RETRY.retries;
|
|
150
172
|
const delays = opts.delaysMs ?? RETRY.delaysMs;
|
|
151
|
-
const
|
|
152
|
-
const
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
173
|
+
const nap = opts.sleep ?? realSleep;
|
|
174
|
+
const budget = opts.totalBudgetMs ?? RETRY.totalBudgetMs;
|
|
175
|
+
const t0 = Date.now();
|
|
176
|
+
let attempt = 0;
|
|
177
|
+
|
|
178
|
+
for (;;) {
|
|
179
|
+
attempt += 1;
|
|
156
180
|
try {
|
|
157
|
-
return { ok: true, value: fn(), attempts, busyWaits:
|
|
158
|
-
} catch (
|
|
159
|
-
|
|
160
|
-
|
|
181
|
+
return { ok: true, value: fn(), attempts: attempt, busyWaits: attempt - 1 };
|
|
182
|
+
} catch (err) {
|
|
183
|
+
const text = String((err && err.message) || err);
|
|
184
|
+
const waited = attempt - 1;
|
|
185
|
+
if (!isBusyError(err) || attempt > maxRetries) {
|
|
186
|
+
return { ok: false, error: text, attempts: attempt, busyWaits: waited };
|
|
161
187
|
}
|
|
162
|
-
const
|
|
163
|
-
if (Date.now() -
|
|
164
|
-
return { ok: false, error: `退避预算耗尽(${
|
|
188
|
+
const pause = delays[Math.min(waited, delays.length - 1)] ?? 1000;
|
|
189
|
+
if (Date.now() - t0 + pause > budget) {
|
|
190
|
+
return { ok: false, error: `退避预算耗尽(${budget}ms):${text}`, attempts: attempt, busyWaits: waited + 1 };
|
|
165
191
|
}
|
|
166
|
-
if (opts.onRetry) opts.onRetry(
|
|
167
|
-
await
|
|
192
|
+
if (typeof opts.onRetry === 'function') opts.onRetry(attempt, err);
|
|
193
|
+
await nap(pause);
|
|
168
194
|
}
|
|
169
195
|
}
|
|
170
196
|
}
|
|
171
197
|
|
|
172
|
-
//
|
|
198
|
+
// ── 读写 ────────────────────────────────────────────────────────────────────
|
|
173
199
|
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
* @param {import('node:sqlite').DatabaseSync} db
|
|
177
|
-
* @param {Array<{path:string, mtime:number, size:number, textHash:string, summary:string,
|
|
178
|
-
* ownerUser?:string, visibility?:string, updatedAt:number}>} rows
|
|
179
|
-
* @returns {{inserted:number, updated:number, unchanged:number}}
|
|
180
|
-
*/
|
|
200
|
+
// 批量 UPSERT(一个事务)。幂等键 = path:同一批重复执行不会长出第二条。
|
|
201
|
+
// 返回 {inserted, updated, unchanged};unchanged 是「摘要哈希没变、整行没重写」的那些。
|
|
181
202
|
export function upsertDocs(db, rows) {
|
|
182
|
-
const
|
|
183
|
-
if (!rows
|
|
184
|
-
|
|
185
|
-
const
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
203
|
+
const tally = { inserted: 0, updated: 0, unchanged: 0 };
|
|
204
|
+
if (!rows?.length) return tally;
|
|
205
|
+
|
|
206
|
+
const readHash = db.prepare(`SELECT text_hash FROM ${T_DOC} WHERE path = ?`);
|
|
207
|
+
const pending = [];
|
|
208
|
+
for (const row of rows) {
|
|
209
|
+
const seen = readHash.get(row.path);
|
|
210
|
+
const before = seen ? seen.text_hash : null;
|
|
211
|
+
if (before === row.textHash) {
|
|
212
|
+
// 摘要没变 ⇒ 整行不重写(mtime / updated_at 都不许动)
|
|
213
|
+
tally.unchanged += 1;
|
|
214
|
+
continue;
|
|
215
|
+
}
|
|
216
|
+
pending.push({ row, before });
|
|
189
217
|
}
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
for (const r of rows) {
|
|
206
|
-
const prev = existing.get(r.path);
|
|
207
|
-
if (prev === r.textHash) {
|
|
208
|
-
// 摘要内容未变 ⇒ 不重写该条(验收项「text_hash 未变的条目未重建」)
|
|
209
|
-
out.unchanged += 1;
|
|
210
|
-
continue;
|
|
211
|
-
}
|
|
212
|
-
if (prev == null) out.inserted += 1;
|
|
213
|
-
else out.updated += 1;
|
|
214
|
-
stmt.run(
|
|
215
|
-
String(r.path),
|
|
216
|
-
Math.trunc(Number(r.mtime) || 0),
|
|
217
|
-
Math.trunc(Number(r.size) || 0),
|
|
218
|
-
String(r.textHash),
|
|
219
|
-
String(r.summary),
|
|
220
|
-
String(r.ownerUser || 'LiJianXiao'),
|
|
221
|
-
String(r.visibility || 'private_owner'),
|
|
222
|
-
Math.trunc(Number(r.updatedAt) || now),
|
|
218
|
+
if (!pending.length) return tally;
|
|
219
|
+
|
|
220
|
+
const write = db.prepare(UPSERT_SQL);
|
|
221
|
+
const stamp = Date.now();
|
|
222
|
+
inTransaction(db, () => {
|
|
223
|
+
for (const { row, before } of pending) {
|
|
224
|
+
write.run(
|
|
225
|
+
String(row.path),
|
|
226
|
+
asInt(row.mtime),
|
|
227
|
+
asInt(row.size),
|
|
228
|
+
String(row.textHash),
|
|
229
|
+
String(row.summary),
|
|
230
|
+
String(row.ownerUser || DEFAULT_OWNER),
|
|
231
|
+
String(row.visibility || DEFAULT_VISIBILITY),
|
|
232
|
+
asInt(row.updatedAt) || stamp,
|
|
223
233
|
);
|
|
234
|
+
if (before == null) tally.inserted += 1;
|
|
235
|
+
else tally.updated += 1;
|
|
224
236
|
}
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
try {
|
|
228
|
-
db.exec('ROLLBACK');
|
|
229
|
-
} catch {
|
|
230
|
-
/* ignore */
|
|
231
|
-
}
|
|
232
|
-
throw e;
|
|
233
|
-
}
|
|
234
|
-
return out;
|
|
237
|
+
});
|
|
238
|
+
return tally;
|
|
235
239
|
}
|
|
236
240
|
|
|
237
|
-
|
|
241
|
+
// 一批 path 已有的 text_hash(调用方据此判「要不要重算」)
|
|
238
242
|
export function getHashesForPaths(db, paths) {
|
|
239
|
-
const
|
|
240
|
-
if (!paths
|
|
241
|
-
const stmt = db.prepare(
|
|
243
|
+
const known = new Map();
|
|
244
|
+
if (!paths?.length) return known;
|
|
245
|
+
const stmt = db.prepare(`SELECT text_hash FROM ${T_DOC} WHERE path = ?`);
|
|
242
246
|
for (const p of paths) {
|
|
243
|
-
const
|
|
244
|
-
|
|
247
|
+
const key = String(p);
|
|
248
|
+
const hit = stmt.get(key);
|
|
249
|
+
if (hit) known.set(key, hit.text_hash);
|
|
245
250
|
}
|
|
246
|
-
return
|
|
251
|
+
return known;
|
|
247
252
|
}
|
|
248
253
|
|
|
249
|
-
export
|
|
250
|
-
const r = db.prepare('SELECT * FROM doc_index WHERE path = ?').get(String(p));
|
|
251
|
-
return r ?? null;
|
|
252
|
-
}
|
|
254
|
+
export const getRow = (db, p) => db.prepare(`SELECT * FROM ${T_DOC} WHERE path = ?`).get(String(p)) ?? null;
|
|
253
255
|
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
* ⚠️ 必须两条都查:长文档按 §3.2 切成 2~5 条时**只有分条、没有主条**
|
|
257
|
-
* (实测踩过:只查主条会让长文档每轮都判 `missing-in-db` 反复重建,"第二轮全部跳过"永远不成立)。
|
|
258
|
-
*/
|
|
256
|
+
// 这一篇在库里有没有条:主条 path **或**任一分条 path#§N 都算。
|
|
257
|
+
// 分条必须一起查 —— 长文档切成 2~5 条时只存在分条,只查主条会判成「库里没有」而每轮重建。
|
|
259
258
|
export function hasDocRows(db, p) {
|
|
260
|
-
const
|
|
261
|
-
|
|
262
|
-
.get(String(p), `${escapeLike(p)}#§%`);
|
|
263
|
-
return Boolean(r);
|
|
259
|
+
const hit = db.prepare(SQL_HAS_ROW).get(String(p), subKeyPattern(p));
|
|
260
|
+
return Boolean(hit);
|
|
264
261
|
}
|
|
265
262
|
|
|
266
|
-
|
|
263
|
+
// 全量列出(按 path 排序,读数可复现);给 pathPrefix 就只管这一篇及其分条
|
|
267
264
|
export function listRows(db, { pathPrefix = null } = {}) {
|
|
268
|
-
if (pathPrefix)
|
|
269
|
-
|
|
270
|
-
.prepare("SELECT * FROM doc_index WHERE path = ? OR path LIKE ? ESCAPE '\\' ORDER BY path")
|
|
271
|
-
.all(pathPrefix, `${escapeLike(pathPrefix)}#§%`);
|
|
272
|
-
}
|
|
273
|
-
return db.prepare('SELECT * FROM doc_index ORDER BY path').all();
|
|
265
|
+
if (!pathPrefix) return db.prepare(SQL_LIST_ALL).all();
|
|
266
|
+
return db.prepare(SQL_LIST_TREE).all(String(pathPrefix), subKeyPattern(pathPrefix));
|
|
274
267
|
}
|
|
275
268
|
|
|
276
|
-
export
|
|
277
|
-
return db.prepare('SELECT COUNT(*) AS n FROM doc_index').get().n;
|
|
278
|
-
}
|
|
279
|
-
|
|
280
|
-
/** LIKE 元字符转义(path 里可能出现 % _ \,不转义会误匹配)。 */
|
|
281
|
-
export function escapeLike(s) {
|
|
282
|
-
return String(s).replace(/[\\%_]/g, (m) => `\\${m}`);
|
|
283
|
-
}
|
|
269
|
+
export const countRows = (db) => db.prepare(SQL_COUNT).get().total;
|
|
284
270
|
|
|
285
|
-
|
|
286
|
-
* 删除某篇文档的全部索引条(含 `path#§N` 长文档分条)——回滚口径:
|
|
287
|
-
* `DELETE FROM doc_index WHERE path=?`(方案 §5.4)+ 分条一并清。
|
|
288
|
-
*/
|
|
271
|
+
// 删掉某篇的全部条(主条 + path#§N 分条)—— 回滚口径就这一条
|
|
289
272
|
export function deleteDoc(db, p) {
|
|
290
|
-
const
|
|
291
|
-
|
|
292
|
-
.run(String(p), `${escapeLike(p)}#§%`);
|
|
293
|
-
return Number(r.changes || 0);
|
|
273
|
+
const res = db.prepare(SQL_DELETE_TREE).run(String(p), subKeyPattern(p));
|
|
274
|
+
return Number(res.changes || 0);
|
|
294
275
|
}
|
|
295
276
|
|
|
296
|
-
|
|
277
|
+
// 删本次没再生成的分条(文档从长变短时的残留),保留 keepKeys 里点名的那些
|
|
297
278
|
export function deleteStaleRows(db, p, keepKeys) {
|
|
298
279
|
const keep = new Set(keepKeys.map(String));
|
|
299
|
-
const rows = db
|
|
300
|
-
|
|
301
|
-
|
|
280
|
+
const rows = db.prepare(`SELECT path FROM ${T_DOC} WHERE path = ? OR path LIKE ? ESCAPE '\\'`).all(
|
|
281
|
+
String(p),
|
|
282
|
+
subKeyPattern(p),
|
|
283
|
+
);
|
|
302
284
|
const stale = rows.map((r) => r.path).filter((v) => !keep.has(v));
|
|
303
285
|
if (!stale.length) return 0;
|
|
304
|
-
const
|
|
305
|
-
db
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
db.exec('COMMIT');
|
|
309
|
-
} catch (e) {
|
|
310
|
-
try {
|
|
311
|
-
db.exec('ROLLBACK');
|
|
312
|
-
} catch {
|
|
313
|
-
/* ignore */
|
|
314
|
-
}
|
|
315
|
-
throw e;
|
|
316
|
-
}
|
|
286
|
+
const drop = db.prepare(`DELETE FROM ${T_DOC} WHERE path = ?`);
|
|
287
|
+
inTransaction(db, () => {
|
|
288
|
+
for (const key of stale) drop.run(key);
|
|
289
|
+
});
|
|
317
290
|
return stale.length;
|
|
318
291
|
}
|
|
319
292
|
|
|
320
|
-
|
|
293
|
+
// 库文件级快照(改动前的回滚点)
|
|
321
294
|
export function snapshotDb(db, destFile) {
|
|
322
295
|
fs.mkdirSync(path.dirname(destFile), { recursive: true });
|
|
323
296
|
db.prepare('VACUUM INTO ?').run(destFile);
|