dsh-cooper-memory 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/core.mjs ADDED
@@ -0,0 +1,439 @@
1
+ // lib/core.mjs —— 入库、检索、分块版本、记忆状态机
2
+ //
3
+ // 检索融合用加权 RRF(WeKnora 04-rag-pipeline.md:496)+ MMR 去冗余(:231,λ=0.7)。
4
+ // 因为没有向量库,RRF 只在"多个查询变体"之间融合;单变体时退化为单路排序。
5
+
6
+ import fs from 'node:fs';
7
+ import path from 'node:path';
8
+ import crypto from 'node:crypto';
9
+ import { openDb, nowIso, getMeta, setMeta } from './db.mjs';
10
+ import { splitDoc } from './chunk.mjs';
11
+
12
+ const TEXT_EXT = new Set(['.md', '.markdown', '.txt', '.log', '.csv', '.json', '.yaml', '.yml']);
13
+
14
+ const sha256 = (buf) => crypto.createHash('sha256').update(buf).digest('hex');
15
+ const norm = (s) => s.replace(/\r\n/g, '\n');
16
+
17
+ // ---------------- 入库 ----------------
18
+ export function ingestFolder(db, rootAbs, opts = {}) {
19
+ const cfg = {
20
+ chunk_size: opts.chunk_size ?? 512,
21
+ chunk_overlap: opts.chunk_overlap ?? 80,
22
+ strategy: opts.strategy ?? 'auto',
23
+ };
24
+ const exts = opts.exts ? new Set(opts.exts) : TEXT_EXT;
25
+ const maxFiles = opts.max_files ?? 5000;
26
+
27
+ const stats = { scanned: 0, imported: 0, skipped: 0, failed: 0, chunks: 0, tiers: {}, errors: [] };
28
+ const walk = (dir) => {
29
+ let entries = [];
30
+ try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }
31
+ for (const e of entries) {
32
+ if (stats.scanned >= maxFiles) return;
33
+ const abs = path.join(dir, e.name);
34
+ if (e.isDirectory()) {
35
+ if (/^(node_modules|\.git|dist|build|__pycache__|\.venv)$/i.test(e.name)) continue;
36
+ walk(abs);
37
+ continue;
38
+ }
39
+ if (!e.isFile()) continue;
40
+ if (!exts.has(path.extname(e.name).toLowerCase())) continue;
41
+ stats.scanned++;
42
+ try {
43
+ const rel = path.relative(rootAbs, abs).split(path.sep).join('/');
44
+ const buf = fs.readFileSync(abs);
45
+ const hash = sha256(buf);
46
+ const st = fs.statSync(abs);
47
+ const prev = db.prepare('SELECT id, sha256 FROM documents WHERE rel_path=?').get(rel);
48
+ if (prev && prev.sha256 === hash) { stats.skipped++; continue; }
49
+
50
+ const text = norm(buf.toString('utf8'));
51
+ const { chunks, tier } = splitDoc(text, cfg);
52
+ if (!chunks.length) { stats.failed++; stats.errors.push({ rel, error: 'no chunks' }); continue; }
53
+
54
+ db.exec('BEGIN');
55
+ try {
56
+ let docId;
57
+ if (prev) {
58
+ docId = prev.id;
59
+ db.prepare('DELETE FROM chunks WHERE doc_id=?').run(docId);
60
+ db.prepare('UPDATE documents SET size=?,mtime=?,sha256=?,imported_at=? WHERE id=?')
61
+ .run(st.size, st.mtime.toISOString(), hash, nowIso(), docId);
62
+ } else {
63
+ const r = db.prepare('INSERT INTO documents(rel_path,abs_path,size,mtime,sha256,imported_at) VALUES(?,?,?,?,?,?)')
64
+ .run(rel, abs, st.size, st.mtime.toISOString(), hash, nowIso());
65
+ docId = Number(r.lastInsertRowid);
66
+ }
67
+ const insChunk = db.prepare('INSERT INTO chunks(doc_id,ordinal,heading,context_header,text,tier,version) VALUES(?,?,?,?,?,?,1)');
68
+ const insVer = db.prepare('INSERT INTO chunk_versions(chunk_id,version,text,heading,editor,note,created_at) VALUES(?,1,?,?,?,?,?)');
69
+ for (const c of chunks) {
70
+ const r = insChunk.run(docId, c.ordinal, c.heading, c.context_header, c.text, c.tier);
71
+ insVer.run(Number(r.lastInsertRowid), c.text, c.heading, 'ingest', 'initial', nowIso());
72
+ }
73
+ db.exec('COMMIT');
74
+ stats.imported++;
75
+ stats.chunks += chunks.length;
76
+ stats.tiers[tier] = (stats.tiers[tier] || 0) + 1;
77
+ } catch (err) {
78
+ db.exec('ROLLBACK');
79
+ throw err;
80
+ }
81
+ } catch (err) {
82
+ stats.failed++;
83
+ stats.errors.push({ rel: path.relative(rootAbs, abs), error: String(err.message || err) });
84
+ }
85
+ }
86
+ };
87
+ walk(rootAbs);
88
+ setMeta(db, 'last_ingest_at', nowIso());
89
+ return stats;
90
+ }
91
+
92
+ // ---------------- 检索 ----------------
93
+ // 中文没有空格,trigram 只吃"精确子串",召回过窄。
94
+ // 所以拆词:ASCII 词按词切,中文按二元组切,两条路一起召回再融合。
95
+
96
+ function queryTerms(q) {
97
+ const s = q.trim();
98
+ const out = new Set();
99
+ for (const m of s.matchAll(/[A-Za-z0-9_.-]{2,}/g)) out.add(m[0].toLowerCase());
100
+ for (const run of s.match(/[\u4e00-\u9fff]+/g) || []) {
101
+ if (run.length === 1) out.add(run);
102
+ else for (let i = 0; i < run.length - 1; i++) out.add(run.slice(i, i + 2));
103
+ }
104
+ return [...out];
105
+ }
106
+
107
+ function ftsQuery(db, q, limit) {
108
+ const s = q.trim();
109
+ const terms = queryTerms(s).filter((t) => [...t].length >= 3 || /[a-z]/i.test(t));
110
+ if (!terms.length) return [];
111
+ const expr = terms.map((t) => `"${t.replace(/"/g, '""')}"`).join(' OR ');
112
+ try {
113
+ return db.prepare(`
114
+ SELECT c.id, c.doc_id, c.ordinal, c.text, c.heading, c.context_header, d.rel_path
115
+ FROM chunks_fts f JOIN chunks c ON c.id=f.rowid JOIN documents d ON d.id=c.doc_id
116
+ WHERE chunks_fts MATCH ? ORDER BY rank LIMIT ?`).all(expr, limit);
117
+ } catch {
118
+ return [];
119
+ }
120
+ }
121
+
122
+ function likeQuery(db, q, limit) {
123
+ const terms = queryTerms(q).slice(0, 12);
124
+ if (!terms.length) return [];
125
+ const where = terms.map(() => '(c.text LIKE ? OR c.heading LIKE ?)').join(' OR ');
126
+ const args = [];
127
+ for (const t of terms) { args.push(`%${t}%`, `%${t}%`); }
128
+ const rows = db.prepare(`
129
+ SELECT c.id, c.doc_id, c.ordinal, c.text, c.heading, c.context_header, d.rel_path
130
+ FROM chunks c JOIN documents d ON d.id=c.doc_id
131
+ WHERE ${where} LIMIT ?`).all(...args, limit * 4);
132
+ // 打分:命中词数,标题权重更高
133
+ const scored = rows.map((r) => {
134
+ let hit = 0;
135
+ const text = r.text || '';
136
+ const head = r.heading || '';
137
+ for (const t of terms) {
138
+ if (head.includes(t)) hit += 2;
139
+ else if (text.includes(t)) hit += 1;
140
+ }
141
+ return { ...r, hits: hit };
142
+ }).filter((r) => r.hits > 0).sort((a, b) => b.hits - a.hits);
143
+ return scored.slice(0, limit);
144
+ }
145
+
146
+ const toTrigrams = (s) => {
147
+ const set = new Set();
148
+ const t = s.toLowerCase().replace(/\s+/g, ' ');
149
+ for (let i = 0; i < t.length - 2; i++) set.add(t.slice(i, i + 3));
150
+ return set;
151
+ };
152
+ const jaccard = (a, b) => {
153
+ if (!a.size || !b.size) return 0;
154
+ let inter = 0;
155
+ for (const x of a) if (b.has(x)) inter++;
156
+ return inter / (a.size + b.size - inter);
157
+ };
158
+
159
+ /** 双路召回 + 加权 RRF 融合 + MMR 多样性 */
160
+ export function searchChunks(db, { query, variants = [], limit = 8, k = 60, mmrLambda = 0.7, pool = 60 }) {
161
+ const all = [query, ...variants.filter(Boolean)];
162
+ const lists = [];
163
+ for (const q of all) {
164
+ const f = ftsQuery(db, q, pool);
165
+ if (f.length) lists.push(f);
166
+ const l = likeQuery(db, q, pool);
167
+ if (l.length) lists.push(l);
168
+ }
169
+ if (!lists.length) return { results: [], usedQueries: all, fused: 0 };
170
+
171
+ const scores = new Map();
172
+ const meta = new Map();
173
+ for (const list of lists) {
174
+ list.forEach((row, idx) => {
175
+ const s = 1 / (k + idx + 1);
176
+ scores.set(row.id, (scores.get(row.id) || 0) + s);
177
+ if (!meta.has(row.id)) meta.set(row.id, row);
178
+ });
179
+ }
180
+ // 归一到"单路第一名"的理论最大值
181
+ const maxScore = 1 / (k + 1);
182
+ let cands = [...scores.entries()]
183
+ .map(([id, s]) => ({ ...meta.get(id), score: Math.min(1, s / maxScore) }))
184
+ .sort((a, b) => b.score - a.score);
185
+
186
+ // MMR
187
+ const picked = [];
188
+ const pool_ = cands.slice(0, pool);
189
+ while (picked.length < limit && pool_.length) {
190
+ let best = -1;
191
+ let bestVal = -Infinity;
192
+ for (let i = 0; i < pool_.length; i++) {
193
+ const c = pool_[i];
194
+ let red = 0;
195
+ for (const p of picked) red = Math.max(red, jaccard(toTrigrams(c.text), toTrigrams(p.text)));
196
+ const val = mmrLambda * c.score - (1 - mmrLambda) * red;
197
+ if (val > bestVal) { bestVal = val; best = i; }
198
+ }
199
+ picked.push(pool_.splice(best, 1)[0]);
200
+ }
201
+ return { results: picked, usedQueries: all, fused: cands.length };
202
+ }
203
+
204
+ export function getChunk(db, id) {
205
+ return db.prepare(`
206
+ SELECT c.*, d.rel_path FROM chunks c JOIN documents d ON d.id=c.doc_id WHERE c.id=?`).get(id);
207
+ }
208
+
209
+ // ---------------- 分块版本 ----------------
210
+ export function editChunk(db, chunkId, newText, { editor = 'user', note = '', baseVersion = null } = {}) {
211
+ const c = db.prepare('SELECT * FROM chunks WHERE id=?').get(chunkId);
212
+ if (!c) throw new Error(`chunk ${chunkId} not found`);
213
+ if (baseVersion != null && c.version !== baseVersion) {
214
+ throw new Error(`version conflict: current=${c.version}, base=${baseVersion}(请重新读取后再提交)`);
215
+ }
216
+ const v = c.version + 1;
217
+ db.exec('BEGIN');
218
+ try {
219
+ db.prepare('INSERT INTO chunk_versions(chunk_id,version,text,heading,editor,note,created_at) VALUES(?,?,?,?,?,?,?)')
220
+ .run(chunkId, v, newText, c.heading, editor, note, nowIso());
221
+ db.prepare('UPDATE chunks SET text=?,version=?,edited_at=?,edited_by=? WHERE id=?')
222
+ .run(newText, v, nowIso(), editor, chunkId);
223
+ db.exec('COMMIT');
224
+ } catch (e) { db.exec('ROLLBACK'); throw e; }
225
+ return { chunkId, version: v };
226
+ }
227
+
228
+ export function chunkHistory(db, chunkId) {
229
+ return db.prepare('SELECT version,editor,note,created_at,length(text) AS chars FROM chunk_versions WHERE chunk_id=? ORDER BY version DESC')
230
+ .all(chunkId);
231
+ }
232
+
233
+ export function chunkVersionText(db, chunkId, version) {
234
+ const r = db.prepare('SELECT text,heading,created_at FROM chunk_versions WHERE chunk_id=? AND version=?').get(chunkId, version);
235
+ return r || null;
236
+ }
237
+
238
+ export function rollbackChunk(db, chunkId, toVersion, { editor = 'user' } = {}) {
239
+ const old = chunkVersionText(db, chunkId, toVersion);
240
+ if (!old) throw new Error(`version ${toVersion} not found`);
241
+ return editChunk(db, chunkId, old.text, { editor, note: `rollback to v${toVersion}` });
242
+ }
243
+
244
+ /** 逐字符 diff(LCS 简化版:行级 + 字符级摘要) */
245
+ export function diffTexts(a, b) {
246
+ const al = a.split('\n');
247
+ const bl = b.split('\n');
248
+ const n = al.length, m = bl.length;
249
+ if (n * m > 4_000_000) return { mode: 'too-large', note: '文本过大,跳过逐行对齐', added: m, removed: n };
250
+ const dp = Array.from({ length: n + 1 }, () => new Uint16Array(m + 1));
251
+ for (let i = n - 1; i >= 0; i--) {
252
+ for (let j = m - 1; j >= 0; j--) {
253
+ dp[i][j] = al[i] === bl[j] ? dp[i + 1][j + 1] + 1 : Math.max(dp[i + 1][j], dp[i][j + 1]);
254
+ }
255
+ }
256
+ const out = [];
257
+ let i = 0, j = 0;
258
+ while (i < n && j < m) {
259
+ if (al[i] === bl[j]) { out.push({ op: ' ', line: al[i] }); i++; j++; }
260
+ else if (dp[i + 1][j] >= dp[i][j + 1]) { out.push({ op: '-', line: al[i] }); i++; }
261
+ else { out.push({ op: '+', line: bl[j] }); j++; }
262
+ }
263
+ while (i < n) out.push({ op: '-', line: al[i++] });
264
+ while (j < m) out.push({ op: '+', line: bl[j++] });
265
+ return { mode: 'line', hunks: out };
266
+ }
267
+
268
+ // ---------------- 目录树 ----------------
269
+ export function tree(db) {
270
+ const rows = db.prepare(`
271
+ SELECT d.rel_path, COUNT(c.id) AS chunks, d.id
272
+ FROM documents d LEFT JOIN chunks c ON c.doc_id=d.id GROUP BY d.id ORDER BY d.rel_path`).all();
273
+ const root = { name: '', dirs: new Map(), files: [] };
274
+ for (const r of rows) {
275
+ const parts = r.rel_path.split('/');
276
+ let node = root;
277
+ for (let i = 0; i < parts.length - 1; i++) {
278
+ if (!node.dirs.has(parts[i])) node.dirs.set(parts[i], { name: parts[i], dirs: new Map(), files: [] });
279
+ node = node.dirs.get(parts[i]);
280
+ }
281
+ node.files.push({ name: parts[parts.length - 1], path: r.rel_path, chunks: r.chunks, docId: r.id });
282
+ }
283
+ const render = (n, depth) => {
284
+ const lines = [];
285
+ for (const d of [...n.dirs.values()].sort((a, b) => a.name.localeCompare(b.name))) {
286
+ lines.push(`${' '.repeat(depth)}${d.name}/`);
287
+ lines.push(...render(d, depth + 1));
288
+ }
289
+ for (const f of n.files.sort((a, b) => a.name.localeCompare(b.name))) {
290
+ lines.push(`${' '.repeat(depth)}${f.name} (${f.chunks} chunks)`);
291
+ }
292
+ return lines;
293
+ };
294
+ return render(root, 0).join('\n');
295
+ }
296
+
297
+ // ---------------- 记忆 ----------------
298
+ const KINDS = new Set(['profile', 'preference', 'fact', 'todo', 'interest']);
299
+ const STANDING = ['profile', 'preference', 'interest'];
300
+
301
+ // 判定"同一主题"的阈值:subject 相同即视为同题;否则看文本相似度
302
+ const SUPERSEDE_THRESHOLD = 0.5;
303
+
304
+ /**
305
+ * 找同主题的旧记忆。
306
+ * 对齐 WeKnora 的语义:新推断**不立刻**顶掉旧条目,只在确认时替换
307
+ * (23-memory.md:25「确认前旧条目继续生效,确认时一并替换」)。
308
+ */
309
+ export function findConflict(db, { kind, subject = '', text, excludeId = null }) {
310
+ const rows = db.prepare(`SELECT * FROM memories WHERE kind=? AND status IN ('pending','active')`).all(kind);
311
+ const tri = toTrigrams(text || '');
312
+ let best = null;
313
+ for (const r of rows) {
314
+ if (excludeId && r.id === excludeId) continue;
315
+ const sameSubject = !!(subject && r.subject && subject.trim() === r.subject.trim());
316
+ const sim = jaccard(tri, toTrigrams(r.text));
317
+ if (sameSubject || sim >= SUPERSEDE_THRESHOLD) {
318
+ if (!best || sim > best.sim) best = { row: r, sim: Number(sim.toFixed(3)), sameSubject };
319
+ }
320
+ }
321
+ return best;
322
+ }
323
+
324
+ export function remember(db, { kind, text, subject = '', status = 'pending', confidence = 0.6, source = '', tags = '', supersedes = null, detail = null }) {
325
+ if (!KINDS.has(kind)) throw new Error(`kind 必须是 ${[...KINDS].join('/')}`);
326
+ if (!text || !text.trim()) throw new Error('text 不能为空');
327
+ const body = text.trim();
328
+ // 显式指定优先;否则自动探测同主题旧条目(只挂链接,不改旧条目状态)
329
+ let sup = supersedes ? Number(supersedes) : null;
330
+ let autoSim = null;
331
+ if (!sup) {
332
+ const c = findConflict(db, { kind, subject, text: body });
333
+ if (c) { sup = c.row.id; autoSim = c.sim; }
334
+ }
335
+ const r = db.prepare(`INSERT INTO memories(kind,subject,text,status,confidence,source,tags,created_at,confirmed_at,supersedes_id,detail)
336
+ VALUES(?,?,?,?,?,?,?,?,?,?,?)`).run(kind, subject, body, status, confidence, source, tags, nowIso(),
337
+ status === 'active' ? nowIso() : null, sup, detail);
338
+ const id = Number(r.lastInsertRowid);
339
+ db.prepare('INSERT INTO memory_versions(memory_id,version,text,editor,note,created_at) VALUES(?,1,?,?,?,?)')
340
+ .run(id, body, 'agent', 'created', nowIso());
341
+ return { id, status, supersedes: sup, similarity: autoSim };
342
+ }
343
+
344
+ export function confirmMemory(db, id, { editor = 'user' } = {}) {
345
+ const m = db.prepare('SELECT * FROM memories WHERE id=?').get(id);
346
+ if (!m) throw new Error(`memory ${id} not found`);
347
+ let replaced = null;
348
+ db.exec('BEGIN');
349
+ try {
350
+ if (m.supersedes_id) {
351
+ const old = db.prepare('SELECT id,status FROM memories WHERE id=?').get(m.supersedes_id);
352
+ if (old && old.status !== 'rejected' && old.status !== 'superseded') {
353
+ db.prepare("UPDATE memories SET status='superseded', superseded_by=? WHERE id=?").run(id, m.supersedes_id);
354
+ replaced = m.supersedes_id;
355
+ }
356
+ }
357
+ db.prepare('UPDATE memories SET status=?, confirmed_at=? WHERE id=?').run('active', nowIso(), id);
358
+ db.exec('COMMIT');
359
+ } catch (e) { db.exec('ROLLBACK'); throw e; }
360
+ return { id, status: 'active', replaced };
361
+ }
362
+
363
+ export function rejectMemory(db, id, { note = '' } = {}) {
364
+ const m = db.prepare('SELECT * FROM memories WHERE id=?').get(id);
365
+ if (!m) throw new Error(`memory ${id} not found`);
366
+ // 拒绝留抑制记录:status=rejected,保留正文供后续去重
367
+ db.prepare('UPDATE memories SET status=? WHERE id=?').run('rejected', id);
368
+ db.prepare('INSERT INTO memory_versions(memory_id,version,text,editor,note,created_at) VALUES(?,?,?,?,?,?)')
369
+ .run(id, (m.version || 1) + 1, m.text, 'user', note || 'rejected', nowIso());
370
+ return { id, status: 'rejected' };
371
+ }
372
+
373
+ export function editMemory(db, id, newText, { editor = 'user', note = '' } = {}) {
374
+ const m = db.prepare('SELECT * FROM memories WHERE id=?').get(id);
375
+ if (!m) throw new Error(`memory ${id} not found`);
376
+ const v = (m.version || 1) + 1;
377
+ db.exec('BEGIN');
378
+ try {
379
+ db.prepare('UPDATE memories SET text=?, version=?, status=? WHERE id=?').run(newText, v, m.status === 'pending' ? 'active' : m.status, id);
380
+ db.prepare('INSERT INTO memory_versions(memory_id,version,text,editor,note,created_at) VALUES(?,?,?,?,?,?)')
381
+ .run(id, v, newText, editor, note || 'edited', nowIso());
382
+ db.exec('COMMIT');
383
+ } catch (e) { db.exec('ROLLBACK'); throw e; }
384
+ return { id, version: v };
385
+ }
386
+
387
+ export function listMemories(db, { status = null, kind = null, limit = 100 } = {}) {
388
+ let sql = 'SELECT * FROM memories WHERE 1=1';
389
+ const args = [];
390
+ if (status) { sql += ' AND status=?'; args.push(status); }
391
+ if (kind) { sql += ' AND kind=?'; args.push(kind); }
392
+ sql += ' ORDER BY id DESC LIMIT ?';
393
+ args.push(limit);
394
+ return db.prepare(sql).all(...args);
395
+ }
396
+
397
+ /**
398
+ * 召回:常驻类(profile/preference/interest)+ 按问匹配(fact/todo)
399
+ * pending 一律不注入(WeKnora 23-memory.md:25)
400
+ */
401
+ export function recall(db, query = '', { budgetChars = 1200 } = {}) {
402
+ const out = [];
403
+ let used = 0;
404
+ const standing = db.prepare(`SELECT * FROM memories WHERE status='active' AND kind IN ('profile','preference','interest') ORDER BY confirmed_at DESC LIMIT 50`).all();
405
+ for (const m of standing) {
406
+ if (used + m.text.length > budgetChars * 0.6) break;
407
+ out.push({ ...m, reason: 'standing' });
408
+ used += m.text.length;
409
+ }
410
+ if (query.trim()) {
411
+ const terms = queryTerms(query).slice(0, 12);
412
+ if (terms.length) {
413
+ // 在 SQL 侧按词过滤,避免"只扫前 500 条"把老记忆永远挡住
414
+ const where = terms.map(() => '(text LIKE ? OR subject LIKE ? OR detail LIKE ?)').join(' OR ');
415
+ const args = [];
416
+ for (const t of terms) args.push(`%${t}%`, `%${t}%`, `%${t}%`);
417
+ const poolRows = db.prepare(`SELECT * FROM memories WHERE status='active' AND kind IN ('fact','todo') AND (${where}) LIMIT 600`).all(...args);
418
+ const scored = poolRows.map((m) => {
419
+ const head = (m.text + ' ' + (m.subject || '')).toLowerCase();
420
+ const full = head + ' ' + String(m.detail || '').toLowerCase();
421
+ let hit = 0;
422
+ for (const term of terms) {
423
+ const t = term.toLowerCase();
424
+ if (head.includes(t)) hit += 2; // 标题命中权重更高
425
+ else if (full.includes(t)) hit += 1; // 正文命中
426
+ }
427
+ return { m, hit };
428
+ }).filter((x) => x.hit > 0).sort((a, b) => b.hit - a.hit || b.m.confidence - a.m.confidence);
429
+ for (const { m } of scored) {
430
+ if (used + m.text.length > budgetChars) break;
431
+ out.push({ ...m, reason: 'matched' });
432
+ used += m.text.length;
433
+ }
434
+ }
435
+ }
436
+ return { items: out, chars: used };
437
+ }
438
+
439
+ export { KINDS, openDb };
package/lib/db.mjs ADDED
@@ -0,0 +1,138 @@
1
+ // lib/db.mjs —— 唯一的库结构定义与连接入口
2
+ //
3
+ // 设计依据(来自 WeKnora 的实践,见 docs):
4
+ // - 分块被手工编辑后,字符偏移不可靠 → 主键语义用 (doc_id, ordinal) + version
5
+ // - 分块编辑每次生成新版本,可回滚(02-knowledge-base.md:91)
6
+ // - 记忆 pending 不注入,确认后 active;新事实替代旧事实(23-memory.md:13-32)
7
+ // - 目录结构由 rel_path 派生,不建目录表(02-knowledge-base.md:395)
8
+
9
+ import { DatabaseSync } from 'node:sqlite';
10
+ import fs from 'node:fs';
11
+ import os from 'node:os';
12
+ import path from 'node:path';
13
+
14
+ export const ROOT = process.env.MEM_ROOT || path.join(os.homedir(), '.dsh-memory');
15
+ export const DB_PATH = process.env.MEM_DB || path.join(ROOT, 'mem.db');
16
+
17
+ const SCHEMA = `
18
+ PRAGMA journal_mode = WAL;
19
+ PRAGMA foreign_keys = ON;
20
+
21
+ CREATE TABLE IF NOT EXISTS documents (
22
+ id INTEGER PRIMARY KEY,
23
+ rel_path TEXT NOT NULL UNIQUE,
24
+ abs_path TEXT NOT NULL,
25
+ size INTEGER NOT NULL,
26
+ mtime TEXT NOT NULL,
27
+ sha256 TEXT NOT NULL,
28
+ imported_at TEXT NOT NULL
29
+ );
30
+
31
+ CREATE TABLE IF NOT EXISTS chunks (
32
+ id INTEGER PRIMARY KEY,
33
+ doc_id INTEGER NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
34
+ ordinal INTEGER NOT NULL,
35
+ heading TEXT NOT NULL DEFAULT '',
36
+ context_header TEXT NOT NULL DEFAULT '',
37
+ text TEXT NOT NULL,
38
+ tier TEXT NOT NULL DEFAULT '',
39
+ version INTEGER NOT NULL DEFAULT 1,
40
+ edited_at TEXT,
41
+ edited_by TEXT,
42
+ UNIQUE (doc_id, ordinal)
43
+ );
44
+ CREATE INDEX IF NOT EXISTS idx_chunks_doc ON chunks(doc_id, ordinal);
45
+
46
+ -- 每次编辑 / 回滚都追加一行,chunks.version 指向当前生效版本
47
+ CREATE TABLE IF NOT EXISTS chunk_versions (
48
+ id INTEGER PRIMARY KEY,
49
+ chunk_id INTEGER NOT NULL REFERENCES chunks(id) ON DELETE CASCADE,
50
+ version INTEGER NOT NULL,
51
+ text TEXT NOT NULL,
52
+ heading TEXT NOT NULL DEFAULT '',
53
+ editor TEXT,
54
+ note TEXT,
55
+ created_at TEXT NOT NULL,
56
+ UNIQUE (chunk_id, version)
57
+ );
58
+ CREATE INDEX IF NOT EXISTS idx_cv_chunk ON chunk_versions(chunk_id, version);
59
+
60
+ -- trigram 分词:中文无需词典即可子串召回
61
+ CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5 (text, heading, tokenize='trigram');
62
+
63
+ CREATE TRIGGER IF NOT EXISTS chunks_ai AFTER INSERT ON chunks BEGIN
64
+ INSERT INTO chunks_fts(rowid, text, heading) VALUES (new.id, new.text, new.heading);
65
+ END;
66
+ CREATE TRIGGER IF NOT EXISTS chunks_ad AFTER DELETE ON chunks BEGIN
67
+ DELETE FROM chunks_fts WHERE rowid = old.id;
68
+ END;
69
+ CREATE TRIGGER IF NOT EXISTS chunks_au AFTER UPDATE OF text, heading ON chunks BEGIN
70
+ DELETE FROM chunks_fts WHERE rowid = old.id;
71
+ INSERT INTO chunks_fts(rowid, text, heading) VALUES (new.id, new.text, new.heading);
72
+ END;
73
+
74
+ -- 记忆:profile/profile 偏好/fact/事项/兴趣 五类
75
+ CREATE TABLE IF NOT EXISTS memories (
76
+ id INTEGER PRIMARY KEY,
77
+ kind TEXT NOT NULL,
78
+ subject TEXT NOT NULL DEFAULT '',
79
+ text TEXT NOT NULL,
80
+ status TEXT NOT NULL DEFAULT 'pending',
81
+ confidence REAL NOT NULL DEFAULT 0.6,
82
+ source TEXT NOT NULL DEFAULT '',
83
+ tags TEXT NOT NULL DEFAULT '',
84
+ created_at TEXT NOT NULL,
85
+ confirmed_at TEXT,
86
+ superseded_by INTEGER REFERENCES memories(id),
87
+ version INTEGER NOT NULL DEFAULT 1
88
+ );
89
+ CREATE INDEX IF NOT EXISTS idx_mem_status ON memories(status, kind);
90
+
91
+ CREATE TABLE IF NOT EXISTS memory_versions (
92
+ id INTEGER PRIMARY KEY,
93
+ memory_id INTEGER NOT NULL REFERENCES memories(id) ON DELETE CASCADE,
94
+ version INTEGER NOT NULL,
95
+ text TEXT NOT NULL,
96
+ editor TEXT,
97
+ note TEXT,
98
+ created_at TEXT NOT NULL,
99
+ UNIQUE (memory_id, version)
100
+ );
101
+
102
+ CREATE TABLE IF NOT EXISTS meta (
103
+ key TEXT PRIMARY KEY,
104
+ value TEXT NOT NULL
105
+ );
106
+ `;
107
+
108
+ export function openDb(dbPath = DB_PATH) {
109
+ if (dbPath !== ':memory:') fs.mkdirSync(path.dirname(dbPath), { recursive: true });
110
+ const db = new DatabaseSync(dbPath);
111
+ db.exec(SCHEMA);
112
+ migrate(db);
113
+ return db;
114
+ }
115
+
116
+ /** 增量迁移:老库补列(CREATE TABLE IF NOT EXISTS 不会给已存在的表加列) */
117
+ function migrate(db) {
118
+ const cols = db.prepare('PRAGMA table_info(memories)').all().map((c) => c.name);
119
+ if (!cols.includes('supersedes_id')) {
120
+ db.exec('ALTER TABLE memories ADD COLUMN supersedes_id INTEGER REFERENCES memories(id)');
121
+ }
122
+ // 导入型记忆需要完整正文:text 只存一行结论,detail 存原文
123
+ if (!cols.includes('detail')) {
124
+ db.exec('ALTER TABLE memories ADD COLUMN detail TEXT');
125
+ }
126
+ }
127
+
128
+ export const nowIso = () => new Date().toISOString();
129
+
130
+ export function setMeta(db, key, value) {
131
+ db.prepare('INSERT INTO meta(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value')
132
+ .run(key, String(value));
133
+ }
134
+
135
+ export function getMeta(db, key) {
136
+ const r = db.prepare('SELECT value FROM meta WHERE key=?').get(key);
137
+ return r ? r.value : null;
138
+ }