memory-eternal 0.10.23 → 0.10.24

package/lib/refine.js

@@ -130,6 +130,132 @@ export async function refineScan(root, { llm, limit = 12, provider, model, minSi
   return { ok: true, candidates: cands.length, useModel: route.provider + '/' + route.model, findings, skipped }
 }
 
+// -- 「同目标、不同做法」聚类（issue #36 第三块）----------------------------------
+//
+// 为什么不能用正文相似度：同一件事的不同做法（例如「用 7z 压缩」「用 tar.gz 分发」「npm 包瘦身」）
+// 字面差异大、相似度低，按正文召回根本抓不到；而它们**目标**是一致的。
+// 所以先让模型给每张卡抽一句「这张卡解决什么问题」，再按**目标**聚类。抽取结果缓存在
+// refine-goals.json（同一张卡只抽一次），避免每次扫描都烧一遍 token。
+
+const GOALS_FILE = 'refine-goals.json'
+
+/** 已抽取的「卡片目标」缓存：{ [path]: { goal, at } }。 */
+export function readGoals(root) {
+  try { return JSON.parse(fs.readFileSync(path.join(root, GOALS_FILE), 'utf8')) || {} } catch { return {} }
+}
+
+function writeGoals(root, data) {
+  const file = path.join(root, GOALS_FILE)
+  const tmp = file + '.tmp'
+  fs.writeFileSync(tmp, JSON.stringify(data, null, 2), 'utf8')
+  fs.renameSync(tmp, file)
+}
+
+/** 让模型用一句话说清「这张卡解决什么问题」。 */
+export async function extractGoal(llm, route, card) {
+  const system = [
+    '你在给本地知识库的卡片做「目标索引」。只回答一个问题：**这张卡解决什么问题 / 给什么场景用**。',
+    '只输出严格 JSON（不要 markdown 代码块、不要多余文字）：',
+    '{"goal":"一句话（不超过 40 字，动词开头，写清要解决的问题）","topic":"一个主题词"}',
+    '不要复述卡里的做法与细节，只写目标。同一件事的不同做法，goal 应当几乎一样。',
+  ].join('\n')
+  const user = '标题：' + card.title + '\n\n正文：\n' + String(card.body || '').slice(0, 1800)
+  const res = await callLlmJson(llm, route, { system, user, maxTokens: 300, purpose: 'memory-refine-goal' })
+  if (res.failure) return { failure: res.failure }
+  let s = String(res.text || '').trim().replace(/^\x60\x60\x60(?:json)?/i, '').replace(/\x60\x60\x60$/, '').trim()
+  const a = s.indexOf('{'); const b = s.lastIndexOf('}')
+  if (a >= 0 && b > a) s = s.slice(a, b + 1)
+  try {
+    const obj = JSON.parse(s)
+    const goal = String(obj.goal || '').trim().slice(0, 120)
+    if (!goal) return { failure: { code: 'UNPARSEABLE_OUTPUT', message: '模型没给出 goal' } }
+    return { goal, topic: String(obj.topic || '').trim().slice(0, 40) }
+  } catch { return { failure: { code: 'UNPARSEABLE_OUTPUT', message: '目标抽取输出不是合法 JSON：' + String(res.text || '').slice(0, 120) } } }
+}
+
+/**
+ * 扫描「同目标、不同做法」的卡片群：抽目标（带缓存）→ 按目标相似度聚类 → 过滤已裁决的对。
+ * **只读**：不写卡片，只更新目标缓存（refine-goals.json）。
+ */
+export async function goalScan(root, { llm, limit = 30, threshold = 0.55, maxCards = 400, provider, model } = {}) {
+  if (!llm) return { ok: false, error: '当前进程拿不到 llm 服务（请在 DSH 面板里跑，或由宿主触发）' }
+  const rows = rowsOf(root).slice(0, Math.max(2, Number(maxCards) || 400))
+  let routes = []
+  try { routes = await routeCandidates(llm, { provider, model }) } catch { routes = [] }
+  if (!routes.length) return { ok: false, error: '取不到模型路由（llm.listProviders 为空）' }
+  const route = routes[0]
+  const cache = readGoals(root)
+  const cards = rows.map((r) => ({ path: r.path, title: r.title, kind: r.kind, body: String(r.body || '') }))
+  const pending = cards.filter((c) => !(cache[c.path] && cache[c.path].goal))
+  let calls = 0
+  const skipped = []
+  for (const c of pending.slice(0, Math.max(1, Number(limit) || 30))) {
+    let r
+    try { r = await extractGoal(llm, route, c) } catch (e) { r = { failure: { code: 'EXCEPTION', message: String((e && e.message) || e) } } }
+    calls += 1
+    if (r.failure) { skipped.push({ path: c.path, error: r.failure.code + ': ' + r.failure.message }); continue }
+    cache[c.path] = { goal: r.goal, topic: r.topic || '', at: Date.now() }
+  }
+  try { writeGoals(root, cache) } catch (e) { skipped.push({ path: '(cache)', error: '目标缓存写入失败：' + String((e && e.message) || e) }) }
+
+  // 并查集聚类：目标相似度 >= threshold 视为同一目标
+  const withGoal = cards.filter((c) => cache[c.path] && cache[c.path].goal)
+  const parent = new Map(withGoal.map((c) => [c.path, c.path]))
+  const find = (x) => { while (parent.get(x) !== x) { parent.set(x, parent.get(parent.get(x))); x = parent.get(x) } return x }
+  const union = (x, y) => { const a = find(x), b = find(y); if (a !== b) parent.set(a, b) }
+  const sim = (x, y) => textSimilarity(String(cache[x.path].goal), String(cache[y.path].goal))
+  for (let i = 0; i < withGoal.length; i++) {
+    for (let j = i + 1; j < withGoal.length; j++) {
+      if (sim(withGoal[i], withGoal[j]) >= Number(threshold) || cache[withGoal[i].path].goal === cache[withGoal[j].path].goal) union(withGoal[i].path, withGoal[j].path)
+    }
+  }
+  const groups = new Map()
+  for (const c of withGoal) {
+    const k = find(c.path)
+    if (!groups.has(k)) groups.set(k, [])
+    groups.get(k).push(c)
+  }
+  const decided = readDecisions(root)
+  const clusters = []
+  for (const members of groups.values()) {
+    if (members.length < 2) continue
+    // 组内任意一对已被裁决（都保留 / 保留其一）→ 整组不再提示：人已经看过了
+    const already = members.some((a, i) => members.slice(i + 1).some((b) => decided[pairKey(a.path, b.path)]))
+    clusters.push({
+      goal: cache[members[0].path].goal,
+      members: members.map((m) => ({ path: m.path, title: m.title, kind: m.kind, goal: cache[m.path].goal })),
+      decided: already,
+    })
+  }
+  clusters.sort((a, b) => b.members.length - a.members.length)
+  return {
+    ok: true,
+    useModel: route.provider + '/' + route.model,
+    cached: withGoal.length,
+    calls,
+    candidates: pending.length,
+    clusters: clusters.filter((c) => !c.decided),
+    decidedClusters: clusters.filter((c) => c.decided).length,
+    skipped,
+  }
+}
+
+/** 把一组「同目标」的卡标成「都保留」（记两两决策，避免下次再提示）。 */
+export async function markClusterKept(root, { paths = [], note = '同目标不同做法，都保留' } = {}) {
+  const list = (Array.isArray(paths) ? paths : []).map((x) => String(x || '').trim()).filter(Boolean)
+  if (list.length < 2) return { ok: false, error: '至少给两张卡的 path' }
+  const data = readDecisions(root)
+  let n = 0
+  for (let i = 0; i < list.length; i++) {
+    for (let j = i + 1; j < list.length; j++) {
+      data[pairKey(list[i], list[j])] = { action: 'keepBoth', relation: 'complement', note: String(note).slice(0, 500), at: Date.now() }
+      n += 1
+    }
+  }
+  try { writeDecisions(root, data) } catch (e) { return { ok: false, error: '决策写入失败：' + String((e && e.message) || e) } }
+  return { ok: true, recorded: n }
+}
+
 /** 技能目录条目行的形态（与 capture.js 的噪声闸门同一口径）。 */
 const CATALOG_LINE_RE = /^\s*[-*]\s*\x60[a-z0-9][a-z0-9-]{2,40}\x60:\s/
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "memory-eternal",
-  "version": "0.10.23",
+  "version": "0.10.24",
   "description": "记忆核心（Memory Eternal）：自研的 DeepSeek Harness 记忆插件，不移植任何既有记忆框架——对话结束后自动沉淀知识卡到本地 Markdown Vault（自研去重、自研 CJK 检索、可 git 管理），设置页提供图形化知识库（统计 / 搜索 / 知识图谱 + 侧边栏一键弹窗），Agent 通过 memory_recall 工具按需召回历史上下文。零人工干预。",
   "license": "MIT",
   "repository": {