memory-eternal 0.10.22 → 0.10.24

package/lib/refine.js

@@ -14,7 +14,7 @@ import fs from 'node:fs'
 import path from 'node:path'
 import { getDb } from './db.js'
 import { deleteCard, mergeCards, textSimilarity } from './vault.js'
-import { callLlmJson, routeCandidates } from './capture.js'
+import { callLlmJson, routeCandidates, stripRuntimeNoise, looksTruncated } from './capture.js'
 
 const DECISIONS_FILE = 'refine-decisions.json'
 const RELATIONS = new Set(['duplicate', 'complement', 'conflict', 'unrelated'])
@@ -130,6 +130,182 @@ export async function refineScan(root, { llm, limit = 12, provider, model, minSi
   return { ok: true, candidates: cands.length, useModel: route.provider + '/' + route.model, findings, skipped }
 }
 
+// -- 「同目标、不同做法」聚类（issue #36 第三块）----------------------------------
+//
+// 为什么不能用正文相似度：同一件事的不同做法（例如「用 7z 压缩」「用 tar.gz 分发」「npm 包瘦身」）
+// 字面差异大、相似度低，按正文召回根本抓不到；而它们**目标**是一致的。
+// 所以先让模型给每张卡抽一句「这张卡解决什么问题」，再按**目标**聚类。抽取结果缓存在
+// refine-goals.json（同一张卡只抽一次），避免每次扫描都烧一遍 token。
+
+const GOALS_FILE = 'refine-goals.json'
+
+/** 已抽取的「卡片目标」缓存：{ [path]: { goal, at } }。 */
+export function readGoals(root) {
+  try { return JSON.parse(fs.readFileSync(path.join(root, GOALS_FILE), 'utf8')) || {} } catch { return {} }
+}
+
+function writeGoals(root, data) {
+  const file = path.join(root, GOALS_FILE)
+  const tmp = file + '.tmp'
+  fs.writeFileSync(tmp, JSON.stringify(data, null, 2), 'utf8')
+  fs.renameSync(tmp, file)
+}
+
+/** 让模型用一句话说清「这张卡解决什么问题」。 */
+export async function extractGoal(llm, route, card) {
+  const system = [
+    '你在给本地知识库的卡片做「目标索引」。只回答一个问题：**这张卡解决什么问题 / 给什么场景用**。',
+    '只输出严格 JSON（不要 markdown 代码块、不要多余文字）：',
+    '{"goal":"一句话（不超过 40 字，动词开头，写清要解决的问题）","topic":"一个主题词"}',
+    '不要复述卡里的做法与细节，只写目标。同一件事的不同做法，goal 应当几乎一样。',
+  ].join('\n')
+  const user = '标题：' + card.title + '\n\n正文：\n' + String(card.body || '').slice(0, 1800)
+  const res = await callLlmJson(llm, route, { system, user, maxTokens: 300, purpose: 'memory-refine-goal' })
+  if (res.failure) return { failure: res.failure }
+  let s = String(res.text || '').trim().replace(/^\x60\x60\x60(?:json)?/i, '').replace(/\x60\x60\x60$/, '').trim()
+  const a = s.indexOf('{'); const b = s.lastIndexOf('}')
+  if (a >= 0 && b > a) s = s.slice(a, b + 1)
+  try {
+    const obj = JSON.parse(s)
+    const goal = String(obj.goal || '').trim().slice(0, 120)
+    if (!goal) return { failure: { code: 'UNPARSEABLE_OUTPUT', message: '模型没给出 goal' } }
+    return { goal, topic: String(obj.topic || '').trim().slice(0, 40) }
+  } catch { return { failure: { code: 'UNPARSEABLE_OUTPUT', message: '目标抽取输出不是合法 JSON：' + String(res.text || '').slice(0, 120) } } }
+}
+
+/**
+ * 扫描「同目标、不同做法」的卡片群：抽目标（带缓存）→ 按目标相似度聚类 → 过滤已裁决的对。
+ * **只读**：不写卡片，只更新目标缓存（refine-goals.json）。
+ */
+export async function goalScan(root, { llm, limit = 30, threshold = 0.55, maxCards = 400, provider, model } = {}) {
+  if (!llm) return { ok: false, error: '当前进程拿不到 llm 服务（请在 DSH 面板里跑，或由宿主触发）' }
+  const rows = rowsOf(root).slice(0, Math.max(2, Number(maxCards) || 400))
+  let routes = []
+  try { routes = await routeCandidates(llm, { provider, model }) } catch { routes = [] }
+  if (!routes.length) return { ok: false, error: '取不到模型路由（llm.listProviders 为空）' }
+  const route = routes[0]
+  const cache = readGoals(root)
+  const cards = rows.map((r) => ({ path: r.path, title: r.title, kind: r.kind, body: String(r.body || '') }))
+  const pending = cards.filter((c) => !(cache[c.path] && cache[c.path].goal))
+  let calls = 0
+  const skipped = []
+  for (const c of pending.slice(0, Math.max(1, Number(limit) || 30))) {
+    let r
+    try { r = await extractGoal(llm, route, c) } catch (e) { r = { failure: { code: 'EXCEPTION', message: String((e && e.message) || e) } } }
+    calls += 1
+    if (r.failure) { skipped.push({ path: c.path, error: r.failure.code + ': ' + r.failure.message }); continue }
+    cache[c.path] = { goal: r.goal, topic: r.topic || '', at: Date.now() }
+  }
+  try { writeGoals(root, cache) } catch (e) { skipped.push({ path: '(cache)', error: '目标缓存写入失败：' + String((e && e.message) || e) }) }
+
+  // 并查集聚类：目标相似度 >= threshold 视为同一目标
+  const withGoal = cards.filter((c) => cache[c.path] && cache[c.path].goal)
+  const parent = new Map(withGoal.map((c) => [c.path, c.path]))
+  const find = (x) => { while (parent.get(x) !== x) { parent.set(x, parent.get(parent.get(x))); x = parent.get(x) } return x }
+  const union = (x, y) => { const a = find(x), b = find(y); if (a !== b) parent.set(a, b) }
+  const sim = (x, y) => textSimilarity(String(cache[x.path].goal), String(cache[y.path].goal))
+  for (let i = 0; i < withGoal.length; i++) {
+    for (let j = i + 1; j < withGoal.length; j++) {
+      if (sim(withGoal[i], withGoal[j]) >= Number(threshold) || cache[withGoal[i].path].goal === cache[withGoal[j].path].goal) union(withGoal[i].path, withGoal[j].path)
+    }
+  }
+  const groups = new Map()
+  for (const c of withGoal) {
+    const k = find(c.path)
+    if (!groups.has(k)) groups.set(k, [])
+    groups.get(k).push(c)
+  }
+  const decided = readDecisions(root)
+  const clusters = []
+  for (const members of groups.values()) {
+    if (members.length < 2) continue
+    // 组内任意一对已被裁决（都保留 / 保留其一）→ 整组不再提示：人已经看过了
+    const already = members.some((a, i) => members.slice(i + 1).some((b) => decided[pairKey(a.path, b.path)]))
+    clusters.push({
+      goal: cache[members[0].path].goal,
+      members: members.map((m) => ({ path: m.path, title: m.title, kind: m.kind, goal: cache[m.path].goal })),
+      decided: already,
+    })
+  }
+  clusters.sort((a, b) => b.members.length - a.members.length)
+  return {
+    ok: true,
+    useModel: route.provider + '/' + route.model,
+    cached: withGoal.length,
+    calls,
+    candidates: pending.length,
+    clusters: clusters.filter((c) => !c.decided),
+    decidedClusters: clusters.filter((c) => c.decided).length,
+    skipped,
+  }
+}
+
+/** 把一组「同目标」的卡标成「都保留」（记两两决策，避免下次再提示）。 */
+export async function markClusterKept(root, { paths = [], note = '同目标不同做法，都保留' } = {}) {
+  const list = (Array.isArray(paths) ? paths : []).map((x) => String(x || '').trim()).filter(Boolean)
+  if (list.length < 2) return { ok: false, error: '至少给两张卡的 path' }
+  const data = readDecisions(root)
+  let n = 0
+  for (let i = 0; i < list.length; i++) {
+    for (let j = i + 1; j < list.length; j++) {
+      data[pairKey(list[i], list[j])] = { action: 'keepBoth', relation: 'complement', note: String(note).slice(0, 500), at: Date.now() }
+      n += 1
+    }
+  }
+  try { writeDecisions(root, data) } catch (e) { return { ok: false, error: '决策写入失败：' + String((e && e.message) || e) } }
+  return { ok: true, recorded: n }
+}
+
+/** 技能目录条目行的形态（与 capture.js 的噪声闸门同一口径）。 */
+const CATALOG_LINE_RE = /^\s*[-*]\s*\x60[a-z0-9][a-z0-9-]{2,40}\x60:\s/
+
+/**
+ * 找出「注入垃圾卡」：正文其实是被注入的运行时文本（技能目录、system-reminder、环境快照…），
+ * 剥掉噪声后几乎不剩人话。
+ *
+ * 为什么需要（实测）：库里已经有一批「- \`code-review\`: Review the changes…」这种**和对话毫无关系**的卡
+ * —— DSH 的 available-skills 目录被当 user 消息注入，旧噪声闸门没挡住，于是被当成知识沉淀下来。
+ * 新版本已不再生成，但**存量还在**。这里只做**提议**（返回清单，不动数据），由人决定回收。
+ */
+export async function findInjectedCards(root, { limit = 500 } = {}) {
+  const rows = rowsOf(root).slice(0, Math.max(1, Number(limit) || 500))
+  const out = []
+  for (const r of rows) {
+    const body = String(r.body || '')
+    if (!body.trim()) continue
+    const cleaned = stripRuntimeNoise(body)
+    const ratio = body.length ? cleaned.length / body.length : 0
+    const allLines = body.split('\n').filter((l) => l.trim())
+    const catalogLines = allLines.filter((l) => CATALOG_LINE_RE.test(l)).length
+    const reasons = []
+    if (/<available_skills>|<system-reminder>/i.test(body)) reasons.push('含运行时注入标签')
+    if (catalogLines >= 3 && catalogLines / Math.max(1, allLines.length) >= 0.3) reasons.push('技能目录条目 ' + catalogLines + ' 行')
+    if (ratio < 0.35 && cleaned.length < 200) reasons.push('剥离注入后只剩 ' + cleaned.length + ' 字')
+    if (looksTruncated(String(r.title || ''))) reasons.push('标题是碎片')
+    if (reasons.length) out.push({ path: r.path, title: r.title, kind: r.kind, reasons, ratio: Math.round(ratio * 100) / 100, cleanedLen: cleaned.length })
+  }
+  return out
+}
+
+/**
+ * 回收这些卡 —— **走回收站（可还原）**，不是永久删除；逐个报告失败项。
+ * @param {{paths?:string[]}} opts
+ */
+export async function purgeInjected(root, { paths = [] } = {}) {
+  const list = (Array.isArray(paths) ? paths : []).map((x) => String(x || '').trim()).filter(Boolean)
+  if (!list.length) return { ok: false, error: '没有给出要回收的 path' }
+  const removed = []
+  const failed = []
+  for (const p of list) {
+    try {
+      const r = await deleteCard(root, p)     // 默认非永久 → 进回收站
+      if (r && r.ok === false) failed.push({ path: p, error: r.error || '删除失败' })
+      else removed.push(p)
+    } catch (e) { failed.push({ path: p, error: String((e && e.message) || e) }) }
+  }
+  return { ok: failed.length === 0, removed, failed, recoverable: true }
+}
+
 /**
  * 执行人的决定。**只做可恢复动作**：
  *   keepA / keepB → 另一张进**回收站**（deleteCard 默认非永久，可从回收中心还原）

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "memory-eternal",
-  "version": "0.10.22",
+  "version": "0.10.24",
   "description": "记忆核心（Memory Eternal）：自研的 DeepSeek Harness 记忆插件，不移植任何既有记忆框架——对话结束后自动沉淀知识卡到本地 Markdown Vault（自研去重、自研 CJK 检索、可 git 管理），设置页提供图形化知识库（统计 / 搜索 / 知识图谱 + 侧边栏一键弹窗），Agent 通过 memory_recall 工具按需召回历史上下文。零人工干预。",
   "license": "MIT",
   "repository": {

package/scripts/release.mjs

@@ -62,7 +62,8 @@ const npmHasVersion = async (v) => {
     return r.status === 200
   } catch { return false }
 }
-const waitNpmVersion = async (v, { tries = 12, delayMs = 5000 } = {}) => {
+// 实测：这个 registry 的 version 端点偶尔要 2 分钟才可见（0.10.22 就因为 60s 超时中断过一次发布）→ 给足 3 分钟
+const waitNpmVersion = async (v, { tries = 36, delayMs = 5000 } = {}) => {
   for (let i = 0; i < tries; i++) {
     if (await npmHasVersion(v)) return true
     await new Promise((r) => setTimeout(r, delayMs))
@@ -196,6 +197,25 @@ for (const remote of remotes) {
   if (!b.ok) problems.push(`${remote} tag ${tag} 未推送成功（${b.out.slice(0, 120)}）→ 重试：git push ${remote} ${tag}`)
 }
 
+// 推完再**实证**每个远端的 tag 指向：Gitee / GitHub 的 Release API 会在 main 还没推上去时
+// 把 tag 建在"当时的 main"上（实测踩过三次：gitee v0.10.19/20/21、origin v0.10.22 —— 都是
+// "Release 创建成功"但其实指向旧提交）。这里逐个核对 peeled 提交，不一致就强制修正并如实报告。
+const headSha = run('git', ['rev-parse', 'HEAD']).out
+for (const remote of remotes) {
+  if (DRY) continue
+  const ls = run('git', ['ls-remote', remote, `refs/tags/${tag}*`], { allowFail: true })
+  if (!ls.ok) { problems.push(`${remote} tag ${tag} 无法核对（${ls.out.slice(0, 120)}）`); continue }
+  const peeled = (ls.out.split(/\r?\n/).find((l) => l.includes('^{}')) || '').split(/\s+/)[0] || ''
+  if (!peeled) { problems.push(`${remote} 上找不到 tag ${tag} → 重试：git push ${remote} ${tag}`); continue }
+  if (peeled !== headSha) {
+    const fix = run('git', ['push', '--force', remote, tag], { allowFail: true })
+    say(`${remote}: tag ${tag} 原指向 ${peeled.slice(0, 7)}（≠ HEAD ${headSha.slice(0, 7)}）→ ${fix.ok ? '已强制修正' : '修正失败：' + fix.out.slice(0, 120)}`)
+    if (!fix.ok) problems.push(`${remote} tag ${tag} 指向错误且修正失败 → 重试：git push --force ${remote} ${tag}`)
+  } else {
+    say(`${remote}: tag ${tag} → ${headSha.slice(0, 7)} ✓`)
+  }
+}
+
 // ---- 5. GitHub Release ---------------------------------------------------
 step(5, 'GitHub Release')
 if (DRY) say(`[dry-run] 将执行：gh release create ${tag} --title … --notes-file …`)