dsh-cooper-memory 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/LICENSE +21 -0
- package/README.md +107 -2
- package/cli.mjs +177 -0
- package/cordis.patch.yml +12 -0
- package/docs/DETAIL.md +83 -0
- package/icon.svg +10 -0
- package/lib/chunk.mjs +283 -0
- package/lib/core.mjs +439 -0
- package/lib/db.mjs +138 -0
- package/lib/extract.mjs +383 -0
- package/lib/import-lingshu.mjs +164 -0
- package/lib/llm.mjs +280 -0
- package/locale/en.json +7 -0
- package/locale/zh.json +7 -0
- package/mcp.mjs +298 -0
- package/package.json +37 -3
- package/web.mjs +230 -0
package/lib/extract.mjs
ADDED
|
@@ -0,0 +1,383 @@
|
|
|
1
|
+
// lib/extract.mjs —— 自动提取记忆(无需 LLM,服务进程出不了网也能跑)
|
|
2
|
+
//
|
|
3
|
+
// 设计参照 WeKnora 的 auto 模式(23-memory.md:13-18):
|
|
4
|
+
// - 后台从会话提取,不阻塞对话
|
|
5
|
+
// - 提取结果一律 pending,确认后才注入
|
|
6
|
+
// - 拒绝过的内容留抑制记录,不再重复提取
|
|
7
|
+
// - 有延迟与最短间隔(这里用"文件水位线 + 每轮上限"近似)
|
|
8
|
+
//
|
|
9
|
+
// 与 WeKnora 的差异:它用 LLM 蒸馏,这里用规则打分(本机服务访问不到 API)。
|
|
10
|
+
|
|
11
|
+
import fs from 'node:fs';
|
|
12
|
+
import os from 'node:os';
|
|
13
|
+
import path from 'node:path';
|
|
14
|
+
import zlib from 'node:zlib';
|
|
15
|
+
import { nowIso, getMeta, setMeta } from './db.mjs';
|
|
16
|
+
import { findConflict } from './core.mjs';
|
|
17
|
+
|
|
18
|
+
const ZSTD_MAGIC = Buffer.from([0x28, 0xb5, 0x2f, 0xfd]);
|
|
19
|
+
|
|
20
|
+
/** 会话文件是追加式多 zstd frame,按魔数切帧逐个解 */
|
|
21
|
+
export function readSessionLines(file) {
|
|
22
|
+
const buf = fs.readFileSync(file);
|
|
23
|
+
if (!file.endsWith('.zstd')) return buf.toString('utf8').split('\n').filter(Boolean);
|
|
24
|
+
const offs = [];
|
|
25
|
+
let i = -1;
|
|
26
|
+
while ((i = buf.indexOf(ZSTD_MAGIC, i + 1)) !== -1) offs.push(i);
|
|
27
|
+
let text = '';
|
|
28
|
+
for (let k = 0; k < offs.length; k++) {
|
|
29
|
+
const s = offs[k];
|
|
30
|
+
const e = k + 1 < offs.length ? offs[k + 1] : buf.length;
|
|
31
|
+
try { text += zlib.zstdDecompressSync(buf.subarray(s, e)).toString('utf8'); } catch { /* 跳过坏帧 */ }
|
|
32
|
+
}
|
|
33
|
+
return text.split('\n').filter(Boolean);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const NOISE_PREFIX = [
|
|
37
|
+
'Current runtime context', 'The following information describes',
|
|
38
|
+
'System reminder', '【灵枢', 'Workspace instructions',
|
|
39
|
+
'<system-reminder', '<', 'Time sampled', 'Elapsed since', '(Omitted',
|
|
40
|
+
'background job ', 'pwsh-', '--- [', 'BUILD ', 'node.exe :', 'started background job',
|
|
41
|
+
];
|
|
42
|
+
|
|
43
|
+
/** 抽出真实的用户发言(排除注入的运行时上下文/提醒) */
|
|
44
|
+
export function collectUserTurns(file) {
|
|
45
|
+
const turns = [];
|
|
46
|
+
for (const line of readSessionLines(file)) {
|
|
47
|
+
let ev;
|
|
48
|
+
try { ev = JSON.parse(line); } catch { continue; }
|
|
49
|
+
if (ev?.type !== 'user/message') continue;
|
|
50
|
+
const parts = ev.data?.content;
|
|
51
|
+
if (!Array.isArray(parts)) continue;
|
|
52
|
+
const text = parts.filter((p) => p.type === 'text').map((p) => p.text).join('\n').trim();
|
|
53
|
+
if (!text) continue;
|
|
54
|
+
if (NOISE_PREFIX.some((p) => text.startsWith(p))) continue;
|
|
55
|
+
if (text.length < 4) continue;
|
|
56
|
+
turns.push({ seq: ev.seq, time: ev.time, text });
|
|
57
|
+
}
|
|
58
|
+
return turns;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// ---------- 规则 ----------
|
|
62
|
+
const FIRST_PERSON = /(我|我们|我的|咱|本人)/;
|
|
63
|
+
const MEMORY_VERB = /(记住|记一下|记下来|别忘|以后都|以后不要|以后别|从现在开始|下次要|下次别|请你记)/;
|
|
64
|
+
|
|
65
|
+
const RULES = [
|
|
66
|
+
// 显式要求记住 —— 最强信号
|
|
67
|
+
{ kind: 'preference', weight: 0.9, re: MEMORY_VERB },
|
|
68
|
+
// 明确的偏好/意图陈述
|
|
69
|
+
{ kind: 'preference', weight: 0.8, re: /(我(更|比较|最)?(喜欢|偏好|习惯|倾向)|我不(喜欢|想要|希望)|不要用|别用|一律|必须是|千万(别|不要)|我希望你|我要求你|我只要|我就想要|我需要的?是|别一开始|别一上来|不要一开始)/ },
|
|
70
|
+
// 身份 / 能力画像
|
|
71
|
+
{ kind: 'profile', weight: 0.82, re: /(我是(一|个|做|搞|写)?|我叫|我在.{0,15}(工作|上班)|我负责|我的(角色|岗位|职责)|我不懂|我不会|我是新手|我熟悉|我做过|我搞过)/ },
|
|
72
|
+
// 项目 / 仓库事实
|
|
73
|
+
{ kind: 'fact', weight: 0.76, re: /(我的?(项目|仓库|配置|目录).{0,8}(是|叫|在|用)|这个项目(用|是|只)|我们的(项目|团队|环境)|原因是|根源是|导致)/ },
|
|
74
|
+
// 本机环境事实(联网/读写/路径这类硬约束)
|
|
75
|
+
{ kind: 'fact', weight: 0.74, re: /((我|我的).{0,10}(环境|机器|电脑|系统|工作区))|(我(这边|这里)?).{0,8}(无法|不能|只能|可以|不行)|([A-Za-z甲乙丙丁])?\s?盘.{0,12}(不能|无法|可以|能|不行)/ },
|
|
76
|
+
{ kind: 'todo', weight: 0.8, re: /(待办|提醒我|记得做|回头要|后续要|等下要|稍后要)/ },
|
|
77
|
+
];
|
|
78
|
+
|
|
79
|
+
// 提问、抱怨、指令性噪音:除非显式要求记住,否则一律不收
|
|
80
|
+
const NOISE = /(你是不是|你搞错|你确定|为什么|怎么|什么|吗[??]?$|呢[??]?$|吧$|能不能|可不可以|有没有|帮我|检查一下|试试|看看$|你看下|是否可行)/;
|
|
81
|
+
|
|
82
|
+
const splitSentences = (text) =>
|
|
83
|
+
text.split(/(?<=[。!?!?;;\n,,])/).map((s) => s.trim()).filter(Boolean);
|
|
84
|
+
|
|
85
|
+
function scoreSentence(sentence) {
|
|
86
|
+
if (sentence.length < 8 || sentence.length > 240) return null;
|
|
87
|
+
if (/[??]\s*$/.test(sentence)) return null; // 问句不收
|
|
88
|
+
const explicit = MEMORY_VERB.test(sentence);
|
|
89
|
+
if (!FIRST_PERSON.test(sentence) && !explicit) return null;
|
|
90
|
+
if (/^\s*你/.test(sentence) && !explicit) return null; // 对助手的指令,不是关于我的
|
|
91
|
+
if (!explicit && NOISE.test(sentence)) return null; // 噪音句只放行"记住…"
|
|
92
|
+
let best = null;
|
|
93
|
+
for (const r of RULES) {
|
|
94
|
+
if (!r.re.test(sentence)) continue;
|
|
95
|
+
if (!best || r.weight > best.weight) best = { kind: r.kind, weight: r.weight };
|
|
96
|
+
}
|
|
97
|
+
return best;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const normalize = (s) => s
|
|
101
|
+
.replace(/^[\s\-*•\d.、))]+/, '')
|
|
102
|
+
.replace(/\s+/g, ' ')
|
|
103
|
+
.replace(/[。!?!?;;,,、]+$/, '')
|
|
104
|
+
.trim();
|
|
105
|
+
|
|
106
|
+
export const dedupeKey = (s) => normalize(s).toLowerCase().replace(/[^\p{L}\p{N}]/gu, '');
|
|
107
|
+
|
|
108
|
+
const toTrigrams = (s) => {
|
|
109
|
+
const set = new Set();
|
|
110
|
+
const t = s.toLowerCase().replace(/\s+/g, ' ');
|
|
111
|
+
for (let i = 0; i < t.length - 2; i++) set.add(t.slice(i, i + 3));
|
|
112
|
+
return set;
|
|
113
|
+
};
|
|
114
|
+
const jaccard = (a, b) => {
|
|
115
|
+
if (!a.size || !b.size) return 0;
|
|
116
|
+
let inter = 0;
|
|
117
|
+
for (const x of a) if (b.has(x)) inter++;
|
|
118
|
+
return inter / (a.size + b.size - inter);
|
|
119
|
+
};
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* 判重阈值按长度分档。
|
|
123
|
+
* 短句 trigram 只有十几个,改一个词就能让 jaccard 掉到 0.6——阈值定高了会把
|
|
124
|
+
* 同一件事判成两条,定低了又会把两件不同的事合并。所以:
|
|
125
|
+
* 短句要更严(接近逐字相同才算重复)
|
|
126
|
+
* 长句可以更松(差异被长度稀释,90% 重合基本就是同一句)
|
|
127
|
+
* 长度差超过 2.5 倍的,不判重(长的很可能是短句的补充版本,交给 supersede)。
|
|
128
|
+
*/
|
|
129
|
+
export function dedupeThreshold(a, b) {
|
|
130
|
+
const min = Math.min(a.length, b.length);
|
|
131
|
+
const max = Math.max(a.length, b.length);
|
|
132
|
+
if (min === 0) return 1.01;
|
|
133
|
+
if (max / min > 2.5) return 1.01;
|
|
134
|
+
if (max < 30) return 0.90; // 短句:几乎要逐字相同,措辞变了交给 supersede
|
|
135
|
+
if (max <= 100) return 0.80; // 实测"同一句改一个词"约 0.86
|
|
136
|
+
return 0.78; // 长句:差异被长度稀释
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** 两个文本是否算同一条记忆(长度感知) */
|
|
140
|
+
export function isDuplicateText(a, b, triA = null, triB = null) {
|
|
141
|
+
const ta = triA || toTrigrams(a);
|
|
142
|
+
const tb = triB || toTrigrams(b);
|
|
143
|
+
return jaccard(ta, tb) >= dedupeThreshold(a, b);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function walkSessions(dir) {
|
|
147
|
+
const out = [];
|
|
148
|
+
const walk = (d) => {
|
|
149
|
+
let es = [];
|
|
150
|
+
try { es = fs.readdirSync(d, { withFileTypes: true }); } catch { return; }
|
|
151
|
+
for (const e of es) {
|
|
152
|
+
const p = path.join(d, e.name);
|
|
153
|
+
if (e.isDirectory()) walk(p);
|
|
154
|
+
else if (e.name.endsWith('.jsonl') || e.name.endsWith('.jsonl.zstd')) out.push(p);
|
|
155
|
+
}
|
|
156
|
+
};
|
|
157
|
+
walk(dir);
|
|
158
|
+
return out.sort();
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* 扫描会话日志,把候选记忆写成 pending
|
|
163
|
+
* @param {object} db
|
|
164
|
+
* @param {{sessionsDir?:string, limit?:number, dryRun?:boolean, maxFiles?:number, minAgeMs?:number}} opts
|
|
165
|
+
*/
|
|
166
|
+
export function runAutoExtract(db, opts = {}) {
|
|
167
|
+
const sessionsDir = opts.sessionsDir || path.join(os.homedir(), '.dsh', 'sessions');
|
|
168
|
+
const limit = opts.limit ?? 30;
|
|
169
|
+
const dryRun = !!opts.dryRun;
|
|
170
|
+
const maxFiles = opts.maxFiles ?? 50;
|
|
171
|
+
|
|
172
|
+
const files = walkSessions(sessionsDir).slice(-maxFiles);
|
|
173
|
+
const existing = db.prepare("SELECT text FROM memories WHERE status IN ('pending','active','rejected')").all();
|
|
174
|
+
const existingKeys = new Set(existing.map((m) => dedupeKey(m.text)));
|
|
175
|
+
const existingTri = existing.map((m) => ({ text: m.text, tri: toTrigrams(m.text) }));
|
|
176
|
+
|
|
177
|
+
const result = {
|
|
178
|
+
scannedFiles: 0, newFiles: 0, turns: 0, candidates: 0,
|
|
179
|
+
added: [], skipped: { dup: 0, short: 0, weak: 0 }, dryRun,
|
|
180
|
+
};
|
|
181
|
+
const seen = new Set();
|
|
182
|
+
|
|
183
|
+
for (const file of files) {
|
|
184
|
+
const st = fs.statSync(file);
|
|
185
|
+
const rel = path.relative(sessionsDir, file);
|
|
186
|
+
const stamp = `${st.size}:${st.mtimeMs}`;
|
|
187
|
+
result.scannedFiles++;
|
|
188
|
+
if (!dryRun && getMeta(db, 'extract:' + rel) === stamp) continue;
|
|
189
|
+
result.newFiles++;
|
|
190
|
+
|
|
191
|
+
const turns = collectUserTurns(file);
|
|
192
|
+
result.turns += turns.length;
|
|
193
|
+
for (const t of turns) {
|
|
194
|
+
for (const raw of splitSentences(t.text)) {
|
|
195
|
+
const scored = scoreSentence(raw);
|
|
196
|
+
if (!scored) continue;
|
|
197
|
+
const text = normalize(raw);
|
|
198
|
+
if (text.length < 8) { result.skipped.short++; continue; }
|
|
199
|
+
const key = dedupeKey(text);
|
|
200
|
+
if (existingKeys.has(key) || seen.has(key)) { result.skipped.dup++; continue; }
|
|
201
|
+
const tri = toTrigrams(text);
|
|
202
|
+
const nearDup = existingTri.some((e) => jaccard(tri, e.tri) > 0.85);
|
|
203
|
+
if (nearDup) { result.skipped.dup++; continue; }
|
|
204
|
+
seen.add(key);
|
|
205
|
+
const cand = {
|
|
206
|
+
kind: scored.kind, text, confidence: scored.weight,
|
|
207
|
+
source: 'auto:' + rel.replace(/\\/g, '/'),
|
|
208
|
+
};
|
|
209
|
+
result.candidates++;
|
|
210
|
+
if (dryRun) { if (result.added.length < limit) result.added.push(cand); continue; }
|
|
211
|
+
if (result.added.length >= limit) continue;
|
|
212
|
+
const r = db.prepare(`INSERT INTO memories(kind,subject,text,status,confidence,source,tags,created_at)
|
|
213
|
+
VALUES(?,?,?,'pending',?,?,'auto',?)`).run(cand.kind, '', cand.text, cand.confidence, cand.source, nowIso());
|
|
214
|
+
db.prepare('INSERT INTO memory_versions(memory_id,version,text,editor,note,created_at) VALUES(?,1,?,?,?,?)')
|
|
215
|
+
.run(Number(r.lastInsertRowid), cand.text, 'auto-extract', 'pending', nowIso());
|
|
216
|
+
result.added.push({ id: Number(r.lastInsertRowid), ...cand });
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
if (!dryRun) setMeta(db, 'extract:' + rel, stamp);
|
|
220
|
+
}
|
|
221
|
+
if (!dryRun) setMeta(db, 'last_extract_at', nowIso());
|
|
222
|
+
return result;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
// ---------------- LLM 蒸馏(可选,取决于服务进程能否出网) ----------------
|
|
226
|
+
import { chat, chatWithRetry } from './llm.mjs';
|
|
227
|
+
|
|
228
|
+
const DISTILL_SYSTEM = `你是长期记忆提取器。输入是用户在一段对话里的原话(可能多条)。
|
|
229
|
+
请只提取**关于用户本人**、且**长期有效**的信息,输出 JSON 数组,不要输出任何解释。
|
|
230
|
+
|
|
231
|
+
可提取的类型:
|
|
232
|
+
- profile:身份/职业/技能水平(例:用户不懂 Docker)
|
|
233
|
+
- preference:稳定的偏好与要求(例:回答用中文、先给结论)
|
|
234
|
+
- fact:环境中长期成立的事实(例:C 盘无法联网,D 盘可以)
|
|
235
|
+
- todo:用户明确要求以后要做的事
|
|
236
|
+
|
|
237
|
+
不要提取:
|
|
238
|
+
- 一次性的任务请求("帮我改这个文件")
|
|
239
|
+
- 临时的运行状态("现在启动不了")
|
|
240
|
+
- 疑问句、抱怨、对助手的指令
|
|
241
|
+
- 对话里没有依据的推测
|
|
242
|
+
|
|
243
|
+
必须严格输出这个形状的 JSON 数组,字段名只能是 kind、text、confidence:
|
|
244
|
+
[{"kind":"profile","text":"一句话陈述","confidence":0.8}]
|
|
245
|
+
|
|
246
|
+
字段规则:
|
|
247
|
+
- kind 只能取 profile / preference / fact / todo 之一
|
|
248
|
+
- text 是完整的中文陈述句,脱离上下文也能读懂
|
|
249
|
+
- confidence 是 0 到 1 的小数
|
|
250
|
+
- 不要使用 content、category、title 等其他字段名
|
|
251
|
+
|
|
252
|
+
注意:同一批里常混着大量无用内容(构建日志、工具输出、一次性任务)。请**逐条判断**——只要有一条含可用信息就提取它,不要因为其他条无关而返回空数组。
|
|
253
|
+
|
|
254
|
+
示例输入:
|
|
255
|
+
[1] 我不用 Docker,机器上装不了
|
|
256
|
+
[2] 帮我看看这个报错
|
|
257
|
+
示例输出:
|
|
258
|
+
[{"kind":"profile","text":"用户不使用 Docker,且本机无法安装","confidence":0.85}]
|
|
259
|
+
|
|
260
|
+
没有可提取内容时,只输出:[]`;
|
|
261
|
+
|
|
262
|
+
const VALID_KINDS = new Set(['profile', 'preference', 'fact', 'todo', 'interest']);
|
|
263
|
+
|
|
264
|
+
const KIND_BY_CATEGORY = {
|
|
265
|
+
工作环境: 'fact', 技术栈: 'fact', 部署环境: 'fact', 项目: 'fact', 环境: 'fact', 事实: 'fact',
|
|
266
|
+
工作偏好: 'preference', 协作习惯: 'preference', 偏好: 'preference', 沟通偏好: 'preference', 习惯: 'preference',
|
|
267
|
+
身份: 'profile', 职业: 'profile', 能力: 'profile', 画像: 'profile',
|
|
268
|
+
待办: 'todo', 计划: 'todo', 事项: 'todo',
|
|
269
|
+
};
|
|
270
|
+
|
|
271
|
+
/** 模型有时会换成 content/category,这里统一收敛成 kind/text */
|
|
272
|
+
function normalizeCandidate(c) {
|
|
273
|
+
if (!c || typeof c !== 'object') return null;
|
|
274
|
+
let kind = String(c.kind || '').toLowerCase().trim();
|
|
275
|
+
if (!VALID_KINDS.has(kind)) kind = KIND_BY_CATEGORY[String(c.category || '').trim()] || '';
|
|
276
|
+
const text = String(c.text || c.content || c.statement || c.value || '').trim();
|
|
277
|
+
if (!VALID_KINDS.has(kind)) {
|
|
278
|
+
if (/偏好|习惯|希望|不要|喜欢|倾向|要求/.test(text)) kind = 'preference';
|
|
279
|
+
else if (/待办|以后要|提醒|计划/.test(text)) kind = 'todo';
|
|
280
|
+
else if (/我是|我在|职业|从事|技能|不懂|不会/.test(text)) kind = 'profile';
|
|
281
|
+
else kind = 'fact';
|
|
282
|
+
}
|
|
283
|
+
const conf = typeof c.confidence === 'number' ? c.confidence
|
|
284
|
+
: c.confidence === 'high' ? 0.85 : c.confidence === 'medium' ? 0.7 : c.confidence === 'low' ? 0.5 : 0.7;
|
|
285
|
+
return { kind, text, confidence: conf };
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
function parseJsonArray(s) {
|
|
289
|
+
const m = s.match(/\[[\s\S]*\]/);
|
|
290
|
+
if (!m) return [];
|
|
291
|
+
try {
|
|
292
|
+
const arr = JSON.parse(m[0]);
|
|
293
|
+
return Array.isArray(arr) ? arr : [];
|
|
294
|
+
} catch { return []; }
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
/** 取最近 N 条用户发言(按文件修改时间取最近的会话,而不是按路径名) */
|
|
298
|
+
export function recentTurns(sessionsDir, n = 60) {
|
|
299
|
+
const dir = sessionsDir || path.join(os.homedir(), '.dsh', 'sessions');
|
|
300
|
+
const files = walkSessions(dir)
|
|
301
|
+
.map((f) => { try { return { f, m: fs.statSync(f).mtimeMs }; } catch { return null; } })
|
|
302
|
+
.filter(Boolean)
|
|
303
|
+
.sort((a, b) => a.m - b.m)
|
|
304
|
+
.slice(-12)
|
|
305
|
+
.map((x) => x.f);
|
|
306
|
+
const all = [];
|
|
307
|
+
for (const f of files) for (const t of collectUserTurns(f)) all.push(t);
|
|
308
|
+
all.sort((a, b) => (a.time || 0) - (b.time || 0));
|
|
309
|
+
return all.slice(-n);
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
/** 写入前统一去重(精确 + 近似 + 已拒绝抑制) */
|
|
313
|
+
export function saveCandidates(db, cands, { limit = 30, source = 'llm' } = {}) {
|
|
314
|
+
const existing = db.prepare("SELECT text FROM memories WHERE status IN ('pending','active','rejected')").all();
|
|
315
|
+
const keys = new Set(existing.map((m) => dedupeKey(m.text)));
|
|
316
|
+
const triList = existing.map((m) => ({ text: m.text, tri: toTrigrams(m.text) }));
|
|
317
|
+
const added = [];
|
|
318
|
+
for (const c of cands) {
|
|
319
|
+
const text = normalize(String(c.text || ''));
|
|
320
|
+
if (text.length < 8) continue;
|
|
321
|
+
if (!['profile', 'preference', 'fact', 'todo', 'interest'].includes(c.kind)) continue;
|
|
322
|
+
const k = dedupeKey(text);
|
|
323
|
+
if (keys.has(k)) continue;
|
|
324
|
+
const tri = toTrigrams(text);
|
|
325
|
+
// 长度感知判重:短句更严、长句更松,长度差过大直接不判重
|
|
326
|
+
if (triList.some((e) => isDuplicateText(text, e.text, tri, e.tri))) continue;
|
|
327
|
+
keys.add(k); triList.push({ text, tri });
|
|
328
|
+
const conf = Math.max(0, Math.min(1, Number(c.confidence) || 0.7));
|
|
329
|
+
// 同主题但措辞不同的旧条目:只挂"拟替代"链接,确认时才真正替换
|
|
330
|
+
const conflict = findConflict(db, { kind: c.kind, subject: c.subject || '', text });
|
|
331
|
+
const sup = conflict ? conflict.row.id : null;
|
|
332
|
+
const r = db.prepare(`INSERT INTO memories(kind,subject,text,status,confidence,source,tags,created_at,supersedes_id)
|
|
333
|
+
VALUES(?,?,?,'pending',?,?,?,?,?)`).run(c.kind, c.subject || '', text, conf, source, 'auto', nowIso(), sup);
|
|
334
|
+
const id = Number(r.lastInsertRowid);
|
|
335
|
+
db.prepare('INSERT INTO memory_versions(memory_id,version,text,editor,note,created_at) VALUES(?,1,?,?,?,?)')
|
|
336
|
+
.run(id, text, 'llm-distill', 'pending', nowIso());
|
|
337
|
+
added.push({ id, kind: c.kind, text, confidence: conf, supersedes: sup, similarity: conflict ? conflict.sim : null });
|
|
338
|
+
if (added.length >= limit) break;
|
|
339
|
+
}
|
|
340
|
+
return added;
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
/** 用 LLM 蒸馏一批发言 → 写 pending(分批调用,避免整批被判为噪声) */
|
|
344
|
+
export async function llmDistill(db, { turns, limit = 30, dryRun = false, batchSize = 15 } = {}) {
|
|
345
|
+
const clean = (turns || []).filter((t) => {
|
|
346
|
+
const s = String(t.text || '').trim();
|
|
347
|
+
if (s.length < 6) return false;
|
|
348
|
+
if (s.length > 1200) return false;
|
|
349
|
+
return !/^(background job|pwsh-|node\.exe|--- \[|BUILD |started background job)/.test(s);
|
|
350
|
+
});
|
|
351
|
+
if (!clean.length) return { candidates: [], added: [], note: '过滤后没有可用发言' };
|
|
352
|
+
|
|
353
|
+
const all = [];
|
|
354
|
+
const seen = new Set();
|
|
355
|
+
for (let i = 0; i < clean.length; i += batchSize) {
|
|
356
|
+
const batch = clean.slice(i, i + batchSize);
|
|
357
|
+
const payload = batch.map((t, k) => `[${k + 1}] ${t.text}`).join('\n');
|
|
358
|
+
let out = '';
|
|
359
|
+
try {
|
|
360
|
+
out = await chatWithRetry(
|
|
361
|
+
[{ role: 'system', content: DISTILL_SYSTEM }, { role: 'user', content: payload }],
|
|
362
|
+
{ maxTokens: 4000 },
|
|
363
|
+
{ retries: 2, baseDelayMs: 800 },
|
|
364
|
+
);
|
|
365
|
+
} catch (e) {
|
|
366
|
+
// 单批失败不影响其他批
|
|
367
|
+
if (process.env.DEBUG_DISTILL) process.stderr.write('BATCH_ERR>>>' + String(e.message || e) + '\n');
|
|
368
|
+
continue;
|
|
369
|
+
}
|
|
370
|
+
if (process.env.DEBUG_DISTILL) {
|
|
371
|
+
process.stderr.write(`RAW_LLM_OUTPUT[batch ${i / batchSize + 1}]>>>` + String(out).slice(0, 400) + '\n');
|
|
372
|
+
}
|
|
373
|
+
for (const c of parseJsonArray(out).map(normalizeCandidate)) {
|
|
374
|
+
if (!c || c.text.length < 8) continue;
|
|
375
|
+
const k = dedupeKey(c.text);
|
|
376
|
+
if (seen.has(k)) continue;
|
|
377
|
+
seen.add(k);
|
|
378
|
+
all.push(c);
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
if (dryRun) return { candidates: all, added: [], batches: Math.ceil(clean.length / batchSize) };
|
|
382
|
+
return { candidates: all, added: saveCandidates(db, all, { limit, source: 'llm' }), batches: Math.ceil(clean.length / batchSize) };
|
|
383
|
+
}
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
// lib/import-lingshu.mjs —— 把灵枢(dsh-memory)已存记忆导入本地记忆库
|
|
2
|
+
//
|
|
3
|
+
// 灵枢节点是 Markdown + YAML frontmatter,正文是 CCG 六要素:
|
|
4
|
+
// # 功能名:xxx ← 一行结论,作为我们的 text
|
|
5
|
+
// # 生效条件 / # 执行 ← 细节,作为 detail
|
|
6
|
+
// 分类靠 tags 里的 `cat:*`,项目靠 `proj:*`。
|
|
7
|
+
//
|
|
8
|
+
// 映射规则(宁可归到 fact,也不要乱猜成 preference,因为 preference 是常驻注入的):
|
|
9
|
+
// user_behavior / user_communication / 偏好类 → preference
|
|
10
|
+
// 项目介绍 / 架构 / 技术栈 / 环境 / 构建 / IDE / SCM / 依赖 → fact
|
|
11
|
+
// 经验类(task_summary / pitfalls / decision / skill / tool / mcp) → fact
|
|
12
|
+
// structural(审核裁决记录) → 跳过,那是审计不是记忆
|
|
13
|
+
|
|
14
|
+
import fs from 'node:fs';
|
|
15
|
+
import os from 'node:os';
|
|
16
|
+
import path from 'node:path';
|
|
17
|
+
import { nowIso } from './db.mjs';
|
|
18
|
+
import { dedupeKey, isDuplicateText } from './extract.mjs';
|
|
19
|
+
|
|
20
|
+
const DEFAULT_DIR = path.join(os.homedir(), '.dsh', '.dsh-memory', 'data', 'mdcg');
|
|
21
|
+
|
|
22
|
+
const PREF_CATS = new Set([
|
|
23
|
+
'cat:user_behavior', 'cat:user_communication', 'cat:preference', 'cat:user_preference',
|
|
24
|
+
]);
|
|
25
|
+
const SKIP_CATS = new Set(['cat:audit', 'cat:review-record', 'cat:audit_trace']);
|
|
26
|
+
|
|
27
|
+
/** 极简 frontmatter 解析:只取需要的字段 */
|
|
28
|
+
function parseFrontmatter(text) {
|
|
29
|
+
const m = text.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?([\s\S]*)$/);
|
|
30
|
+
if (!m) return { meta: {}, body: text };
|
|
31
|
+
const meta = {};
|
|
32
|
+
for (const line of m[1].split(/\r?\n/)) {
|
|
33
|
+
const mm = line.match(/^([a-z_]+):\s*(.*)$/);
|
|
34
|
+
if (!mm) continue;
|
|
35
|
+
let v = mm[2].trim();
|
|
36
|
+
if (/^".*"$/.test(v)) v = v.slice(1, -1);
|
|
37
|
+
meta[mm[1]] = v;
|
|
38
|
+
}
|
|
39
|
+
return { meta, body: m[2] };
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function parseTags(raw) {
|
|
43
|
+
return (raw || '')
|
|
44
|
+
.replace(/^\[|\]$/g, '')
|
|
45
|
+
.split(',')
|
|
46
|
+
.map((s) => s.trim().replace(/^["']|["']$/g, ''))
|
|
47
|
+
.filter(Boolean);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function walk(dir) {
|
|
51
|
+
const out = [];
|
|
52
|
+
if (!fs.existsSync(dir)) return out;
|
|
53
|
+
for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
54
|
+
const p = path.join(dir, e.name);
|
|
55
|
+
if (e.isDirectory()) out.push(...walk(p));
|
|
56
|
+
else if (e.name.endsWith('.md')) out.push(p);
|
|
57
|
+
}
|
|
58
|
+
return out;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** 从 CCG 正文里取一行结论 */
|
|
62
|
+
function extractTitle(body, fallback) {
|
|
63
|
+
const m = body.match(/^#\s*功能名[::]\s*(.+)$/m);
|
|
64
|
+
if (m) return m[1].trim();
|
|
65
|
+
const h1 = body.match(/^#\s+(.+)$/m);
|
|
66
|
+
if (h1) return h1[1].trim();
|
|
67
|
+
return fallback;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export function parseLingshuNode(file) {
|
|
71
|
+
const raw = fs.readFileSync(file, 'utf8');
|
|
72
|
+
const { meta, body } = parseFrontmatter(raw);
|
|
73
|
+
const tags = parseTags(meta.tags);
|
|
74
|
+
const cats = tags.filter((t) => t.startsWith('cat:'));
|
|
75
|
+
const proj = tags.find((t) => t.startsWith('proj:'));
|
|
76
|
+
const id = meta.id || path.basename(file, '.md');
|
|
77
|
+
const title = extractTitle(body, id);
|
|
78
|
+
let kind = 'fact';
|
|
79
|
+
if (cats.some((c) => PREF_CATS.has(c))) kind = 'preference';
|
|
80
|
+
return {
|
|
81
|
+
id,
|
|
82
|
+
layer: meta.layer || '',
|
|
83
|
+
lifecycle: (meta.lifecycle_state || 'active').trim(),
|
|
84
|
+
importance: Number(meta.importance) || 0.6,
|
|
85
|
+
confidence: Number(meta.confidence) || 0.6,
|
|
86
|
+
sensitivity: meta.sensitivity || 'internal',
|
|
87
|
+
createdAt: Number(meta.created_at) || 0,
|
|
88
|
+
tags,
|
|
89
|
+
cats,
|
|
90
|
+
subject: proj ? proj.slice(5) : (meta.layer === 'contextual' ? '' : ''),
|
|
91
|
+
kind,
|
|
92
|
+
title,
|
|
93
|
+
body: body.trim(),
|
|
94
|
+
skip: cats.some((c) => SKIP_CATS.has(c)) || tags.some((t) => ['audit', 'review-record', 'audit_trace'].includes(t)),
|
|
95
|
+
file,
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* @param {object} db
|
|
101
|
+
* @param {{dir?:string, dryRun?:boolean, limit?:number, layers?:string[]}} opts
|
|
102
|
+
*/
|
|
103
|
+
export function importLingshu(db, opts = {}) {
|
|
104
|
+
const dir = opts.dir || DEFAULT_DIR;
|
|
105
|
+
const layers = opts.layers || ['knowledge', 'contextual'];
|
|
106
|
+
const dryRun = !!opts.dryRun;
|
|
107
|
+
const limit = opts.limit ?? 5000;
|
|
108
|
+
|
|
109
|
+
const existing = db.prepare("SELECT text FROM memories WHERE status IN ('pending','active','rejected')").all();
|
|
110
|
+
const keys = new Set(existing.map((m) => dedupeKey(m.text)));
|
|
111
|
+
const tris = existing.map((m) => ({ text: m.text, tri: toTri(m.text) }));
|
|
112
|
+
|
|
113
|
+
const stats = { files: 0, imported: 0, skipped: 0, dup: 0, byKind: {}, byLayer: {}, errors: [] };
|
|
114
|
+
const samples = [];
|
|
115
|
+
const insert = db.prepare(`INSERT INTO memories(kind,subject,text,status,confidence,source,tags,created_at,confirmed_at,detail)
|
|
116
|
+
VALUES(?,?,?,'active',?,?,?,?,?,?)`);
|
|
117
|
+
|
|
118
|
+
for (const layer of layers) {
|
|
119
|
+
const layerDir = path.join(dir, layer);
|
|
120
|
+
for (const file of walk(layerDir)) {
|
|
121
|
+
if (stats.imported + stats.dup >= limit) break;
|
|
122
|
+
stats.files++;
|
|
123
|
+
try {
|
|
124
|
+
const node = parseLingshuNode(file);
|
|
125
|
+
if (node.skip) { stats.skipped++; continue; }
|
|
126
|
+
if (node.lifecycle !== 'active') { stats.skipped++; continue; }
|
|
127
|
+
const text = node.title.slice(0, 240);
|
|
128
|
+
if (text.length < 4) { stats.skipped++; continue; }
|
|
129
|
+
const key = dedupeKey(text);
|
|
130
|
+
const tri = toTri(text);
|
|
131
|
+
if (keys.has(key) || tris.some((e) => isDuplicateText(text, e.text, tri, e.tri))) {
|
|
132
|
+
stats.dup++; continue;
|
|
133
|
+
}
|
|
134
|
+
keys.add(key); tris.push({ text, tri });
|
|
135
|
+
stats.byKind[node.kind] = (stats.byKind[node.kind] || 0) + 1;
|
|
136
|
+
stats.byLayer[layer] = (stats.byLayer[layer] || 0) + 1;
|
|
137
|
+
if (samples.length < 8) samples.push({ kind: node.kind, subject: node.subject, text: text.slice(0, 60), tags: node.tags.slice(0, 3) });
|
|
138
|
+
if (dryRun) { stats.imported++; continue; }
|
|
139
|
+
insert.run(
|
|
140
|
+
node.kind,
|
|
141
|
+
node.subject,
|
|
142
|
+
text,
|
|
143
|
+
Math.max(0, Math.min(1, node.importance || node.confidence)),
|
|
144
|
+
`lingshu:${layer}:${node.id}`,
|
|
145
|
+
['lingshu', ...node.tags].join(','),
|
|
146
|
+
nowIso(),
|
|
147
|
+
nowIso(),
|
|
148
|
+
node.body,
|
|
149
|
+
);
|
|
150
|
+
stats.imported++;
|
|
151
|
+
} catch (e) {
|
|
152
|
+
stats.errors.push({ file: path.basename(file), error: String(e.message || e) });
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
return { ...stats, samples, dryRun };
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
const toTri = (s) => {
|
|
160
|
+
const set = new Set();
|
|
161
|
+
const t = s.toLowerCase().replace(/\s+/g, ' ');
|
|
162
|
+
for (let i = 0; i < t.length - 2; i++) set.add(t.slice(i, i + 3));
|
|
163
|
+
return set;
|
|
164
|
+
};
|