memory-eternal 0.9.12 → 0.9.14

package/lib/minhash.js

@@ -0,0 +1,99 @@
+// MinHash + LSH：相似卡候选生成（把 O(n²) 精确比对换成「签名分桶 → 少量精确校验」）。
+//
+// 用法：先给每张卡的 bigram 哈希集合算一个 k 维最小哈希签名；再按 band 分桶，
+// 同一 band 撞在一起的卡互为候选；最后只对候选做**精确** Jaccard（保证判定语义不变）。
+// 纯函数，方便单测。
+
+const M1 = 0x85ebca6b
+const M2 = 0xc2b2ae35
+
+/** 32 位整数混合（murmur3 finalizer 变体）—— 用作第 seed 号哈希函数。 */
+export function mix32(x, seed = 0) {
+  let h = (x ^ Math.imul(seed + 1, 0x9e3779b1)) >>> 0
+  h ^= h >>> 16
+  h = Math.imul(h, M1) >>> 0
+  h ^= h >>> 13
+  h = Math.imul(h, M2) >>> 0
+  h ^= h >>> 16
+  return h >>> 0
+}
+
+/**
+ * k 维最小哈希签名。
+ * @param {Iterable<number>} grams 哈希后的 bigram 集合
+ * @param {number} k 签名长度
+ * @returns {Uint32Array}
+ */
+export function minhashSignature(grams, k = 64) {
+  const sig = new Uint32Array(k).fill(0xffffffff)
+  for (const g of grams) {
+    for (let i = 0; i < k; i++) {
+      const h = mix32(g, i)
+      if (h < sig[i]) sig[i] = h
+    }
+  }
+  return sig
+}
+
+/** 两个签名的一致位比例 = Jaccard 的无偏估计。 */
+export function jaccardEstimate(a, b) {
+  if (!a || !b || a.length === 0 || a.length !== b.length) return 0
+  let same = 0
+  for (let i = 0; i < a.length; i++) if (a[i] === b[i]) same++
+  return same / a.length
+}
+
+/**
+ * LSH 分桶：把签名切成 bands 段，每段 rows 行，拼成桶 key。
+ * @returns {string[]} 长度 = bands（每段一个 key）
+ */
+export function lshBandKeys(sig, bands = 32, rows = 2) {
+  const keys = []
+  for (let b = 0; b < bands; b++) {
+    let key = ''
+    for (let r = 0; r < rows; r++) key += sig[b * rows + r].toString(36) + '.'
+    keys.push(key)
+  }
+  return keys
+}
+
+/**
+ * 生成候选：同一 band 撞在一起的卡互为候选，按「撞上的 band 数」降序截断。
+ * @param {Map<string, Uint32Array>} signatures id -> 签名
+ * @param {{bands?:number, rows?:number, maxPerCard?:number, minHits?:number}} [opts]
+ * @returns {Map<string, string[]>} id -> 候选 id 列表（已按命中数排序、已截断）
+ */
+export function lshCandidates(signatures, opts = {}) {
+  const bands = opts.bands || 32
+  const rows = opts.rows || 2
+  const maxPerCard = opts.maxPerCard || 40
+  const minHits = opts.minHits || 1
+  const hits = new Map() // id -> Map(otherId -> count)
+  const buckets = new Map() // bandKey -> id[]
+  for (const [id, sig] of signatures) {
+    const keys = lshBandKeys(sig, bands, rows)
+    for (const key of keys) {
+      const arr = buckets.get(key)
+      if (arr) arr.push(id)
+      else buckets.set(key, [id])
+    }
+  }
+  for (const arr of buckets.values()) {
+    if (arr.length < 2 || arr.length > 400) continue // 超大桶基本是「全库同质」，跳过以免退化
+    for (let i = 0; i < arr.length; i++) {
+      for (let j = i + 1; j < arr.length; j++) {
+        const a = arr[i], b = arr[j]
+        let ma = hits.get(a); if (!ma) { ma = new Map(); hits.set(a, ma) }
+        ma.set(b, (ma.get(b) || 0) + 1)
+        let mb = hits.get(b); if (!mb) { mb = new Map(); hits.set(b, mb) }
+        mb.set(a, (mb.get(a) || 0) + 1)
+      }
+    }
+  }
+  const out = new Map()
+  for (const [id, map] of hits) {
+    const ranked = [...map.entries()].filter(([, n]) => n >= minHits).sort((x, y) => y[1] - x[1])
+    out.set(id, (ranked.length <= maxPerCard ? ranked : ranked.slice(0, maxPerCard)).map(([other]) => other))
+  }
+  return out
+}

package/lib/vault.js

@@ -4,7 +4,9 @@
 // 底层依赖 lib/db.js 的 getDb(root) → DatabaseSync 实例。
 
 import path from 'node:path'
+import fs from 'node:fs'
 import { getDb, enforceAudit } from './db.js'
+import { minhashSignature, lshCandidates } from './minhash.js'
 
 /** 主题目录根。 */
 export const KIND_ROOTS = {
@@ -75,6 +77,31 @@ export function bigramSet(text) {
   return out
 }
 
+/**
+ * 与 bigramSet 同规则（同一套清洗），但把每个 bigram 压成 32 位整数再入集合。
+ * 为什么单独一份：Set<number> 的构建与求交都比 Set<string> 快数倍，
+ * 而 **去重判定** 仍必须用字符串版（语义不能变）—— 所以图谱专用、互不干扰。
+ */
+export function bigramHashSet(text) {
+  const cleaned = String(text)
+    .replace(/[\s#*_`|[\]()（）\-—・]+/g, '')
+    .toLowerCase()
+  const out = new Set()
+  for (let i = 0; i < cleaned.length - 1; i++) out.add(hash32(cleaned.slice(i, i + 2)))
+  if (cleaned.length === 1) out.add(hash32(cleaned))
+  return out
+}
+
+/** 32 位 FNV-1a：把 bigram 压成整数键。 */
+function hash32(s) {
+  let h = 0x811c9dc5
+  for (let i = 0; i < s.length; i++) {
+    h ^= s.charCodeAt(i)
+    h = Math.imul(h, 0x01000193) >>> 0
+  }
+  return h >>> 0
+}
+
 /**
  * 用预算好的 bigram 集合算 Jaccard（结果与 textSimilarity 完全一致）。
  *
@@ -483,6 +510,34 @@ const GRAPH_CACHE_MAX = 4
 const GRAPH_CACHE_TTL_MS = 5 * 60 * 1000
 const graphCache = new Map() // root -> { fp, value, at }
 
+// ---- P0-1：建图结果落盘 ------------------------------------------------------
+// 指纹一变就失效，命中直接读文件 —— 免掉整条建图管线（596 卡实测 ~600ms → ~25ms）。
+// 文件放 vault 根目录、原子写（tmp + rename），写失败/超限都只是「下次重建」，不影响功能。
+const GRAPH_CACHE_FILE = 'memory-eternal-graph-cache.json'
+const GRAPH_CACHE_MAX_BYTES = 8 * 1024 * 1024
+const graphCachePath = (root) => path.join(root, GRAPH_CACHE_FILE)
+
+function readGraphCache(root, fp) {
+  try {
+    const data = JSON.parse(fs.readFileSync(graphCachePath(root), 'utf8'))
+    if (!data || data.fp !== fp) return null
+    const v = data.value
+    if (!v || !Array.isArray(v.nodes) || !Array.isArray(v.edges)) return null
+    return v
+  } catch { return null }
+}
+
+function writeGraphCache(root, fp, value) {
+  try {
+    const payload = JSON.stringify({ fp, at: Date.now(), value })
+    if (payload.length > GRAPH_CACHE_MAX_BYTES) return
+    const file = graphCachePath(root)
+    const tmp = file + '.' + process.pid + '.tmp'
+    fs.writeFileSync(tmp, payload)
+    fs.renameSync(tmp, file)
+  } catch { /* 缓存写失败不影响功能 */ }
+}
+
 function graphFingerprint(db) {
   const c = db.prepare('SELECT COUNT(*) n, MAX(updated_at) m, SUM(id) s FROM cards').get()
   const st = db.prepare('SELECT status, COUNT(*) n FROM cards GROUP BY status').all()
@@ -498,6 +553,13 @@ export async function graph(root) {
   const fp = graphFingerprint(db)
   const hit = graphCache.get(root)
   if (hit && hit.fp === fp && Date.now() - hit.at < GRAPH_CACHE_TTL_MS) return hit.value
+  // P0-1：内存未命中就查磁盘缓存（指纹一致即视为有效）
+  const onDisk = readGraphCache(root, fp)
+  if (onDisk) {
+    graphCache.set(root, { fp, value: onDisk, at: Date.now() })
+    if (process.env.MEMORY_GRAPH_PROFILE === '1') console.error('[memory-eternal] graph: 命中磁盘缓存')
+    return onDisk
+  }
   // 只选图谱需要的列，不读 body（大幅减少内存和传输量）
   const rows = db.prepare("SELECT id, path, kind, title, tags, summary, status, updated_at FROM cards WHERE status = 'approved' ORDER BY updated_at DESC").all()
   // body 单独按需读取（只在需要 wikilink 匹配时）
@@ -589,69 +651,102 @@ export async function graph(root) {
     }
   }
 
-  // 相似度关联：bigram 集合只算一次（原先每对都重新分词两段正文＝指数级白烧），
-  // 再用数学上界剪枝（J ≤ min/max）跳过长度比过小的配对。
+  // 相似度关联（P0-3 + P1-1）：
+  //   · bigram 一律用 32 位哈希入集合（构建与求交都比字符串快数倍，碰撞率可忽略）；
+  //   · 候选来自两个互补来源：① 稀有 gram 倒排（按 df 计数排序，免每卡 sort 比较器）
+  //     ② MinHash + LSH 分桶（补足「共享 gram 都很常见」的近重复对）；
+  //   · 只对候选做**精确** Jaccard —— 判定语义与旧实现一致，只是不再全量配对。
   const SIM_THRESHOLD = 0.42
-  const bigramsByPath = new Map()
-  // df = 有多少张卡含这个 bigram。常见 bigram（df 很大）几乎均匀铺满全库，
-  // 撑不起 0.42 的 Jaccard；只用稀有 bigram 建倒排即可生成候选，避免 5.8 万对全量比对。
   const DF_CAP = 24
   const RARE_CAP = 120
+  const CAND_CAP = 40
+  const MH_K = 48
+  const MH_BANDS = 24
+  const MH_ROWS = 2
+  const prof = process.env.MEMORY_GRAPH_PROFILE === '1'
+  let tStage = Date.now()
+  const hashesByPath = new Map()
   const gramDf = new Map()
   for (const c of cards) {
-    const s = bigramSet(bodyMap.get(c.path) || '')
-    bigramsByPath.set(c.path, s)
+    const s = bigramHashSet(bodyMap.get(c.path) || '')
+    hashesByPath.set(c.path, s)
     for (const g of s) gramDf.set(g, (gramDf.get(g) || 0) + 1)
   }
-  const rareIndex = new Map()
+  const tHash = Date.now() - tStage
+  // ① 稀有 gram 倒排：df ∈ [1, DF_CAP] 直接进桶（计数排序），再按桶取最稀的 RARE_CAP 个
+  tStage = Date.now()
   const rareByPath = new Map()
   for (const c of cards) {
+    const buckets = []
+    for (let df = 0; df <= DF_CAP; df++) buckets.push(null)
+    for (const g of hashesByPath.get(c.path)) {
+      const df = gramDf.get(g) || 0
+      if (df > DF_CAP) continue
+      if (!buckets[df]) buckets[df] = []
+      buckets[df].push(g)
+    }
     const rare = []
-    for (const g of bigramsByPath.get(c.path)) if ((gramDf.get(g) || 0) <= DF_CAP) rare.push(g)
-    // 只取「最稀有」的 RARE_CAP 个参与候选生成：近重复卡一定共享大量稀有 gram，
-    // 而这一步是冷启动的真正热点（每张卡 × 每个稀有 gram 的倒排表）。
-    rare.sort((x, y) => (gramDf.get(x) || 0) - (gramDf.get(y) || 0))
-    rareByPath.set(c.path, rare.length <= RARE_CAP ? rare : rare.slice(0, RARE_CAP))
-    for (const g of rare) {
+    for (let df = 1; df <= DF_CAP && rare.length < RARE_CAP; df++) {
+      const b = buckets[df]
+      if (!b) continue
+      for (const g of b) { rare.push(g); if (rare.length >= RARE_CAP) break }
+    }
+    rareByPath.set(c.path, rare)
+  }
+  const rareIndex = new Map()
+  for (const c of cards) {
+    for (const g of rareByPath.get(c.path)) {
       const arr = rareIndex.get(g)
       if (arr) arr.push(c.path)
       else rareIndex.set(g, [c.path])
     }
   }
+  const tRare = Date.now() - tStage
+  // ② MinHash + LSH：签名只跑「稀有 gram」（≤120/卡），成本可控且正好覆盖近重复场景
+  tStage = Date.now()
+  const signatures = new Map()
+  for (const c of cards) signatures.set(c.path, minhashSignature(rareByPath.get(c.path), MH_K))
+  const lshCands = lshCandidates(signatures, { bands: MH_BANDS, rows: MH_ROWS, maxPerCard: CAND_CAP })
+  const tSig = Date.now() - tStage
+  tStage = Date.now()
   const similarCount = {}
-  // 标签集合预先建成 Set：原先每个配对都做 tags.some(includes) 是 O(标签数²)，配对 5.8 万次很可观。
   const tagSetByPath = new Map(cards.map((c) => [c.path, new Set(c.tags)]))
-  // 候选生成：按共享稀有 bigram 数降序，只对候选做精确 Jaccard。
-  // 每张卡最多保留 1 条 similar 边（与旧行为一致）。
-  const sharedRare = new Map()
+  const merged = new Map()
+  let jaccardCalls = 0
   for (const card of cards) {
+    merged.clear()
     for (const gram of rareByPath.get(card.path)) {
       for (const other of rareIndex.get(gram) || []) {
-        if (other !== card.path) sharedRare.set(other, (sharedRare.get(other) || 0) + 1)
+        if (other !== card.path) merged.set(other, (merged.get(other) || 0) + 1)
       }
     }
-    if (!sharedRare.size) continue
-    // 候选上限：共享稀有 gram 最多的 40 张（近重复卡一定排在最前面，召回不受影响）
-    const ranked = [...sharedRare.entries()].sort((x, y) => y[1] - x[1])
-    const cands = ranked.length <= 40 ? ranked : ranked.slice(0, 40)
-    sharedRare.clear()
-    const A = bigramsByPath.get(card.path)
-    for (const [other, shared] of cands) {
+    for (const other of lshCands.get(card.path) || []) {
+      if (other !== card.path) merged.set(other, (merged.get(other) || 0) + 2)
+    }
+    if (!merged.size) continue
+    const ranked = [...merged.entries()].sort((x, y) => y[1] - x[1])
+    const cands = ranked.length <= CAND_CAP ? ranked : ranked.slice(0, CAND_CAP)
+    const A = hashesByPath.get(card.path)
+    for (const [other] of cands) {
       if ((similarCount[card.path] || 0) >= 1) break
       if ((similarCount[other] || 0) >= 1) continue
       const b = byPath.get(other)
       if (!b) continue
       if (b.kind !== card.kind && !card.tags.some((t) => tagSetByPath.get(other).has(t))) continue
-      const B = bigramsByPath.get(other)
+      const B = hashesByPath.get(other)
       const lo = Math.min(A.size, B.size)
       const hi = Math.max(A.size, B.size)
       if (!hi || lo / hi < SIM_THRESHOLD) continue // 无损上界：J ≤ min/max
+      jaccardCalls++
       const sim = jaccardSets(A, B, SIM_THRESHOLD)
       if (sim >= SIM_THRESHOLD) { addEdge(card.path, other, 'similar'); addDegree(card.path); addDegree(other); similarCount[card.path] = 1; similarCount[other] = 1; break }
     }
   }
+  const tCand = Date.now() - tStage
+  if (prof) console.error('[memory-eternal] graph profile: hash=' + tHash + 'ms rare=' + tRare + 'ms sig=' + tSig + 'ms cand=' + tCand + 'ms jaccardCalls=' + jaccardCalls)
   const value = { nodes, edges }
   graphCache.set(root, { fp, value, at: Date.now() })
+  writeGraphCache(root, fp, value)
   if (graphCache.size > GRAPH_CACHE_MAX) graphCache.delete(graphCache.keys().next().value)
   return value
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "memory-eternal",
-  "version": "0.9.12",
+  "version": "0.9.14",
   "description": "记忆核心（Memory Eternal）：自研的 DeepSeek Harness 记忆插件，不移植任何既有记忆框架——对话结束后自动沉淀知识卡到本地 Markdown Vault（自研去重、自研 CJK 检索、可 git 管理），设置页提供图形化知识库（统计 / 搜索 / 知识图谱 + 侧边栏一键弹窗），Agent 通过 memory_recall 工具按需召回历史上下文。零人工干预。",
   "license": "MIT",
   "repository": {
@@ -60,7 +60,7 @@
     "build": "node build.mjs",
     "build:client": "node build.mjs",
     "prepublishOnly": "node build.mjs",
-    "test": "node build.mjs && node tests/vault.test.mjs && node tests/capture.test.mjs && node tests/capture-route.test.mjs && node tests/stall.test.mjs && node tests/vault-path.test.mjs && node tests/vault-resolve.test.mjs && node tests/api.test.mjs && node tests/graph-perf.test.mjs && node tests/settings-compat.test.mjs && node tests/node-bin.test.mjs && node tests/watchdog-lock.test.mjs && node tests/markdown.test.mjs && node tests/feedback.test.mjs && node tests/ssr.test.mjs"
+    "test": "node build.mjs && node tests/vault.test.mjs && node tests/capture.test.mjs && node tests/capture-route.test.mjs && node tests/stall.test.mjs && node tests/vault-path.test.mjs && node tests/vault-resolve.test.mjs && node tests/minhash.test.mjs && node tests/api.test.mjs && node tests/graph-perf.test.mjs && node tests/settings-compat.test.mjs && node tests/node-bin.test.mjs && node tests/watchdog-lock.test.mjs && node tests/markdown.test.mjs && node tests/feedback.test.mjs && node tests/graph-lod.test.mjs && node tests/graph-sprites.test.mjs && node tests/ssr.test.mjs"
   },
   "dependencies": {
     "@deepseek-ai/schemastery": "^3.18.1"