memory-eternal 0.9.0 → 0.9.1

package/lib/vault.js

@@ -64,25 +64,38 @@ export function safeSlug(name) {
   return base || 'card'
 }
 
-/** Jaccard-like 相似度：字符 bigram 集合。 */
-export function textSimilarity(a, b) {
-  const bigrams = (text) => {
-    const cleaned = String(text)
-      .replace(/[\s#*_`|[\]()（）\-—・]+/g, '')
-      .toLowerCase()
-    const out = new Set()
-    for (let i = 0; i < cleaned.length - 1; i++) out.add(cleaned.slice(i, i + 2))
-    if (cleaned.length === 1) out.add(cleaned)
-    return out
-  }
-  const ba = bigrams(a)
-  const bb = bigrams(b)
-  if (ba.size === 0 || bb.size === 0) return 0
+/** 字符 bigram 集合（textSimilarity 的预处理部分，单独暴露供 O(n²) 场景复用）。 */
+export function bigramSet(text) {
+  const cleaned = String(text)
+    .replace(/[\s#*_`|[\]()（）\-—・]+/g, '')
+    .toLowerCase()
+  const out = new Set()
+  for (let i = 0; i < cleaned.length - 1; i++) out.add(cleaned.slice(i, i + 2))
+  if (cleaned.length === 1) out.add(cleaned)
+  return out
+}
+
+/**
+ * 用预算好的 bigram 集合算 Jaccard（结果与 textSimilarity 完全一致）。
+ *
+ * `minRatio` 是数学上界剪枝：J ≤ min(|A|,|B|)/max(|A|,|B|)，所以长度比低于阈值时
+ * 必然 `J < 阈值`，可直接返回 0 **不迭代**（对「J ≥ 阈值」的判定无损，不是近似）。
+ * 图谱的 O(n²) 配对就靠这一步从「每次重新分词两段正文」变成「集合求交」。
+ */
+export function jaccardSets(ba, bb, minRatio = 0) {
+  if (!ba || !bb || ba.size === 0 || bb.size === 0) return 0
+  const [small, large] = ba.size <= bb.size ? [ba, bb] : [bb, ba]
+  if (minRatio > 0 && small.size / large.size < minRatio) return 0
   let inter = 0
-  for (const g of ba) if (bb.has(g)) inter++
+  for (const g of small) if (large.has(g)) inter++
   return inter / (ba.size + bb.size - inter)
 }
 
+/** Jaccard-like 相似度：字符 bigram 集合。 */
+export function textSimilarity(a, b) {
+  return jaccardSets(bigramSet(a), bigramSet(b))
+}
+
 /** CJK 感知查询词：整词 + 中文字符 bigram。 */
 export function queryTerms(query) {
   const tokens = new Set(String(query).split(/[^\w\u4e00-\u9fff]+/).filter(Boolean))
@@ -165,15 +178,74 @@ function rowToMarkdown(row) {
   return { text: lines.join('\n'), tags }
 }
 
-/** 列出卡片（从 SQL）。 */
-export async function listCards(root, { status } = {}) {
+/**
+ * 智能体署名归一（与客户端 normAgent 同一套规则，供 SQL 过滤用）：
+ * 空 / agent / unknown / session:* / DeepSeek Harness → deepseek-harness；claude-code* → claude；其余精确匹配。
+ */
+function agentWhere(agent, args) {
+  const a = String(agent || '').toLowerCase()
+  if (a === 'deepseek-harness') {
+    args.push('', 'agent', 'unknown', 'deepseek-harness', 'deepseek harness')
+    return "(LOWER(COALESCE(submitted_by,'')) IN (?,?,?,?,?) OR submitted_by LIKE 'session:%')"
+  }
+  if (a === 'claude') { args.push('claude', 'claude-code%'); return "(LOWER(COALESCE(submitted_by,'')) = ? OR submitted_by LIKE ?)" }
+  args.push(a)
+  return "LOWER(COALESCE(submitted_by,'')) = ?"
+}
+
+/** WHERE 组装：状态白名单 + 可选 kind / agent。 */
+function cardsWhere({ want, kind, agent }) {
+  const args = [...want]
+  let where = `status IN (${want.map(() => '?').join(',')})`
+  if (kind) { where += ' AND kind = ?'; args.push(kind) }
+  if (agent) where += ' AND ' + agentWhere(agent, args)
+  return { where, args }
+}
+
+/** 标题排序用的中文排序器（拼音 + 数字感知）；固定 zh-Hans-CN，避免随进程 locale 漂移。 */
+export const TITLE_COLLATOR = new Intl.Collator('zh-Hans-CN', { numeric: true, sensitivity: 'base' })
+
+/**
+ * 列出卡片（从 SQL）。
+ * @param opts.status  状态白名单（默认 approved）
+ * @param opts.kind    类型过滤（SQL）
+ * @param opts.agent   署名过滤（服务端归一，如 deepseek-harness / claude）
+ * @param opts.limit   返回条数上限（缺省=全部，供内部调用保持原语义）
+ * @param opts.offset  偏移（配合 limit 做真分页）
+ * @param opts.sort    'recent'（updated_at DESC，默认）| 'title'（拼音升序，跨页全局有序）
+ */
+export async function listCards(root, { status, kind, agent, limit, offset = 0, sort = 'recent' } = {}) {
   const want = status ?? ['approved']
   const db = getDb(root)
-  const placeholders = want.map(() => '?').join(',')
-  const rows = db.prepare(`SELECT * FROM cards WHERE status IN (${placeholders}) ORDER BY updated_at DESC`).all(...want)
+  const { where, args } = cardsWhere({ want, kind, agent })
+  if (sort === 'title') {
+    // SQLite 的 ORDER BY 是 UTF-8 字节序，中文标题看着就是乱的 → 只取 path/title 用拼音排好，
+    // 再回表取这一页的正文（避免为了排序把所有正文读进内存）。
+    const light = db.prepare(`SELECT path, title FROM cards WHERE ${where}`).all(...args)
+    light.sort((a, b) => TITLE_COLLATOR.compare(a.title || '', b.title || ''))
+    const start = Math.max(0, offset | 0)
+    const page = Number.isInteger(limit) && limit > 0 ? light.slice(start, start + limit) : light.slice(start)
+    if (page.length === 0) return []
+    const ph = page.map(() => '?').join(',')
+    const rows = db.prepare(`SELECT * FROM cards WHERE path IN (${ph})`).all(...page.map((r) => r.path))
+    const byPath = new Map(rows.map((r) => [r.path, rowToCard(r)]))
+    return page.map((r) => byPath.get(r.path)).filter(Boolean)
+  }
+  if (Number.isInteger(limit) && limit > 0) {
+    const rows = db.prepare(`SELECT * FROM cards WHERE ${where} ORDER BY updated_at DESC LIMIT ? OFFSET ?`).all(...args, limit, Math.max(0, offset | 0))
+    return rows.map(rowToCard)
+  }
+  const rows = db.prepare(`SELECT * FROM cards WHERE ${where} ORDER BY updated_at DESC`).all(...args)
   return rows.map(rowToCard)
 }
 
+/** 计数（分页接口要用 total 判断还有没有下一页）。 */
+export async function countCards(root, { status, kind, agent } = {}) {
+  const db = getDb(root)
+  const { where, args } = cardsWhere({ want: status ?? ['approved'], kind, agent })
+  return db.prepare(`SELECT COUNT(*) n FROM cards WHERE ${where}`).get(...args).n
+}
+
 /** 审核队列：status=pending 或 rejected。 */
 export async function auditQueue(root) {
   const cards = await listCards(root, { status: ['pending', 'rejected'] })
@@ -404,9 +476,28 @@ export async function exportCards(root) {
   })
 }
 
+// 图谱缓存：一次生成约 300ms（301 卡），重复请求（多客户端/切页重开）直接命中。
+// 失效用「指纹」而不是 TTL 兜底：卡数 + 更新时间 + 状态直方图 + 审计日志/更新记录水位，
+// 任何写卡、审核、删卡、追加更新都会改变指纹 → 不存在脏读。
+const GRAPH_CACHE_MAX = 4
+const GRAPH_CACHE_TTL_MS = 5 * 60 * 1000
+const graphCache = new Map() // root -> { fp, value, at }
+
+function graphFingerprint(db) {
+  const c = db.prepare('SELECT COUNT(*) n, MAX(updated_at) m, SUM(id) s FROM cards').get()
+  const st = db.prepare('SELECT status, COUNT(*) n FROM cards GROUP BY status').all()
+  const log = db.prepare('SELECT MAX(id) id FROM audit_log').get()
+  let upd = 0
+  try { upd = db.prepare('SELECT MAX(id) id FROM card_updates').get().id || 0 } catch { /* 老库无此表 */ }
+  return [c.n, c.m || '', c.s || 0, st.map((r) => `${r.status}:${r.n}`).join(','), log.id || 0, upd].join('|')
+}
+
 /** 图谱：节点 = 卡片；边 = [[wikilink]] 或共享标签。 */
 export async function graph(root) {
   const db = getDb(root)
+  const fp = graphFingerprint(db)
+  const hit = graphCache.get(root)
+  if (hit && hit.fp === fp && Date.now() - hit.at < GRAPH_CACHE_TTL_MS) return hit.value
   // 只选图谱需要的列，不读 body（大幅减少内存和传输量）
   const rows = db.prepare("SELECT id, path, kind, title, tags, summary, status, updated_at FROM cards WHERE status = 'approved' ORDER BY updated_at DESC").all()
   // body 单独按需读取（只在需要 wikilink 匹配时）
@@ -462,23 +553,39 @@ export async function graph(root) {
         }
       }
     }
+  }
+  // 标签倒排（tag → paths，按 cards 顺序）：原先每个标签都要全表扫一遍，是 O(卡数²)
+  const tagIndex = new Map()
+  for (const c of cards) for (const t of c.tags) {
+    const arr = tagIndex.get(t)
+    if (arr) arr.push(c.path)
+    else tagIndex.set(t, [c.path])
+  }
+  for (const card of cards) {
     for (const tag of card.tags) {
-      for (const other of cards) {
-        if (other.path !== card.path && other.tags.includes(tag)) addEdge(card.path, other.path, `tag:${tag}`)
+      for (const other of tagIndex.get(tag) || []) {
+        if (other !== card.path) addEdge(card.path, other, `tag:${tag}`)
       }
     }
   }
 
-  // 相似度关联
+  // 相似度关联：bigram 集合只算一次（原先每对都重新分词两段正文＝指数级白烧），
+  // 再用数学上界剪枝（J ≤ min/max）跳过长度比过小的配对。
+  const SIM_THRESHOLD = 0.42
+  const bigramsByPath = new Map()
+  for (const c of cards) bigramsByPath.set(c.path, bigramSet(bodyMap.get(c.path) || ''))
   const similarCount = {}
   for (let i = 0; i < cards.length; i++) for (let j = i + 1; j < cards.length; j++) {
     const a = cards[i], b = cards[j]
     if (a.kind !== b.kind && !a.tags.some((t) => b.tags.includes(t))) continue
     if ((similarCount[a.path] || 0) >= 1 || (similarCount[b.path] || 0) >= 1) continue
-    const sim = textSimilarity(bodyMap.get(a.path) || '', bodyMap.get(b.path) || '')
-    if (sim >= 0.42) { addEdge(a.path, b.path, 'similar'); addDegree(a.path); addDegree(b.path); similarCount[a.path] = (similarCount[a.path] || 0) + 1; similarCount[b.path] = (similarCount[b.path] || 0) + 1 }
+    const sim = jaccardSets(bigramsByPath.get(a.path), bigramsByPath.get(b.path), SIM_THRESHOLD)
+    if (sim >= SIM_THRESHOLD) { addEdge(a.path, b.path, 'similar'); addDegree(a.path); addDegree(b.path); similarCount[a.path] = (similarCount[a.path] || 0) + 1; similarCount[b.path] = (similarCount[b.path] || 0) + 1 }
   }
-  return { nodes, edges }
+  const value = { nodes, edges }
+  graphCache.set(root, { fp, value, at: Date.now() })
+  if (graphCache.size > GRAPH_CACHE_MAX) graphCache.delete(graphCache.keys().next().value)
+  return value
 }
 
 /** 统计。 */

package/lib/watchdog.js

@@ -71,7 +71,8 @@ export async function probeTcp(port, timeoutMs = 600) {
  * @param {boolean} [opts.autoStart=true]  // 首次探测到死时是否自动拉起
  */
 export function startWatchdog({ port = 7999, vaultRoot, interval = 5000, maxRestart = 10, autoStart = true } = {}) {
-  let child = null   // 我们 spawn 的 web 进程
+  let child = null   // 我们 spawn 的 web 进程（最近一次）
+  const spawned = new Map() // child -> 它监听的端口（用于回收多余实例）
   let ownedByUs = false
   let restartCount = 0
   let stopped = false
@@ -100,14 +101,33 @@ export function startWatchdog({ port = 7999, vaultRoot, interval = 5000, maxRest
       windowsHide: true,
     })
     c.on('exit', (code, sig) => {
+      spawned.delete(c)
       if (!stopped && code !== 0) log(`web exited code=${code} sig=${sig}`)
     })
     c.unref()
+    spawned.set(c, targetPort)
     child = c
     ownedByUs = true
     return targetPort
   }
 
+  /**
+   * 回收「跑到备用端口上的多余实例」。
+   *
+   * findFreePort 会在目标端口被占时退到 port+1…（曾经因此留下一个 8000 的重复 web
+   * 进程常驻 66MB，而 tick 只探目标端口，永远发现不了）。目标端口确认可用后，
+   * 把不在目标端口上的自家子进程收掉即可（只杀我们自己 spawn 的，不动用户手起的）。
+   */
+  const reapExtraChildren = () => {
+    for (const [c, p] of spawned) {
+      if (p === port) continue
+      if (c.exitCode !== null || c.killed) { spawned.delete(c); continue }
+      try { c.kill('SIGTERM') } catch {}
+      spawned.delete(c)
+      log(`目标端口 ${port} 已可用 → 回收多余实例（端口 ${p}，pid ${c.pid}）`)
+    }
+  }
+
   const tick = async () => {
     if (stopped) return
     const alive = await probe(port)
@@ -117,6 +137,7 @@ export function startWatchdog({ port = 7999, vaultRoot, interval = 5000, maxRest
         log(`port ${port} 已存活（外部实例）→ 切换为保活模式（不接管）`)
         ownedByUs = true // 至少逻辑上"知道"它活了
       }
+      reapExtraChildren()
       return
     }
     // 端口死
@@ -153,10 +174,13 @@ export function startWatchdog({ port = 7999, vaultRoot, interval = 5000, maxRest
     if (stopped) return
     stopped = true
     if (timer) { clearInterval(timer); timer = null }
-    if (child && !child.killed) {
-      try { child.kill('SIGTERM') } catch {}
-      setTimeout(() => { try { child && child.kill('SIGKILL') } catch {} }, 1000).unref()
+    for (const c of spawned.keys()) {
+      if (c.exitCode === null && !c.killed) {
+        try { c.kill('SIGTERM') } catch {}
+        setTimeout(() => { try { c.kill('SIGKILL') } catch {} }, 1000).unref()
+      }
     }
+    spawned.clear()
     log('看门狗退出')
     process.exit(0)
   }

package/lib/web.js

@@ -14,7 +14,7 @@ import http from 'node:http'
 import { promises as fs, readFileSync } from 'node:fs'
 import path from 'node:path'
 import { fileURLToPath } from 'node:url'
-import { createApi, API_PREFIX } from './api.js'
+import { createApi, API_PREFIX, encodeBody } from './api.js'
 import { defaultVaultDir, configFilePath } from './capture-run.js'
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url))
@@ -50,8 +50,13 @@ export function startWebServer({ port = DEFAULT_WEB_PORT, vaultRoot = defaultVau
       }
       if (p === '/app.js') {
         const buf = await fs.readFile(path.join(WEB_ROOT, 'app.js'))
-        res.writeHead(200, { 'Content-Type': MIME['.js'], 'Cache-Control': 'no-store' })
-        return res.end(buf)
+        // 自包含 bundle 约 270KB，每次打开独立 Web 页都要重下 —— 协商 gzip 时压缩传输
+        const headers = { 'Content-Type': MIME['.js'], 'Cache-Control': 'no-store', 'Vary': 'Accept-Encoding' }
+        const { body: out, encoding } = encodeBody(res, buf)
+        if (encoding) headers['Content-Encoding'] = encoding
+        headers['Content-Length'] = out.length
+        res.writeHead(200, headers)
+        return res.end(out)
       }
       res.writeHead(404, { 'Content-Type': 'application/json' })
       res.end(JSON.stringify({ ok: false, error: 'not found' }))

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "memory-eternal",
-  "version": "0.9.0",
+  "version": "0.9.1",
   "description": "记忆核心（Memory Eternal）：自研的 DeepSeek Harness 记忆插件，不移植任何既有记忆框架——对话结束后自动沉淀知识卡到本地 Markdown Vault（自研去重、自研 CJK 检索、可 git 管理），设置页提供图形化知识库（统计 / 搜索 / 知识图谱 + 侧边栏一键弹窗），Agent 通过 memory_recall 工具按需召回历史上下文。零人工干预。",
   "license": "MIT",
   "repository": {