dsh-memu-memory 1.2.0 → 1.2.1

package/lib/curator.js

@@ -1,11 +1,11 @@
 import {
   globalDbPath, loadGlobalIndex, updateGlobalIndex, newMemoryId, getSettings, effectiveVectorRoute,
 } from './state.js'
-import { commit } from './memu.js'
+import { commit, forget } from './memu.js'
 import { loadPrompt } from './prompts.js'
 import { retryStoreWrite, redactSecrets, warn } from './paths.js'
 import { record } from './activity-log.js'
-import { assertVectorWriteAllowed, hasActiveVectorGeneration, readVectorState, withVectorWrite } from './vector-profile.js'
+import { activeEntryStamp, assertVectorWriteAllowed, hasActiveVectorGeneration, readVectorState, withVectorWrite } from './vector-profile.js'
 
 const MAX_SESSIONS = 10
 const MAX_TOTAL_CHARS = 90_000
@@ -214,6 +214,43 @@ export function cancelCurationJob(id) {
   return true
 }
 
+// 孤儿窗口补偿（这是补偿设计，不是事务）：curator 是 commit-before-index —— 批量提交
+// 成功而全局索引写失败时，引擎库里会留下索引不认识的孤儿行。这里对刚提交的每个 name
+// 逐个 forget，尽力撤销；这是尽力而为，不是回滚保证：删不掉的必须可见。补偿结果（包括
+// 哪些补上了、哪些没补上）逐条记录进活动日志 kind:'curate'，绝不吞掉。
+async function compensateCuratorCommit(originalError, { files, route }) {
+  const db = globalDbPath()
+  const compensated = []
+  const failed = []
+  for (const file of files) {
+    try {
+      const result = await forget({ db, name: file.name })
+      if (!result || typeof result !== 'object' || result.ok === false || result.error || result.parseWarning
+        || !Number.isSafeInteger(result.deleted) || result.deleted < 0) {
+        throw new Error('backend compensation delete was not acknowledged')
+      }
+      compensated.push(file.name)
+    } catch (compensationError) {
+      failed.push({ name: file.name, error: String(compensationError?.message || compensationError) })
+    }
+  }
+  if (failed.length) {
+    record({
+      kind: 'curate', scope: 'global', ok: false, level: 'error',
+      message: `全局提炼索引写失败后的补偿不完整：${compensated.length}/${files.length} 条已从向量库撤销，${failed.length} 条撤销失败（引擎库残留孤儿行，需手动清理）`,
+      detail: { route, compensated, failed: failed.slice(0, 20), originalError: String(originalError?.message || originalError) },
+      refs: files.map((file) => file.name).slice(0, 20),
+    })
+  } else {
+    record({
+      kind: 'curate', scope: 'global', ok: true, level: 'warn',
+      message: `全局提炼索引写失败，已从向量库回滚刚提交的 ${compensated.length} 条记忆（补偿成功；索引未变，重跑提炼即可）`,
+      detail: { route, compensated, failed: [], originalError: String(originalError?.message || originalError) },
+      refs: files.map((file) => file.name).slice(0, 20),
+    })
+  }
+}
+
 // Exported for the audit regression suite: merging the commit decision logic
 // into startGlobalCuration would require standing up subagents to test it.
 export async function commitCuratorOutput(output, { job }) {
@@ -302,36 +339,54 @@ export async function commitCuratorOutput(output, { job }) {
     // Managed agent mode still commits the exact same files to the active local
     // vector database; only pending_review entries remain index-only.
     const route = engineRoute()
-    if (route !== 'lexical') {
-      await commit({
-        payload: { recall_files: files, resource: [], user: {} },
-        route, db: globalDbPath(),
-      })
-    }
-
-    if (job?.abort?.signal?.aborted) throw new Error('cancelled by user')
-    await retryStoreWrite(() => updateGlobalIndex(fresh => {
-      for (const { id, entry } of pending) fresh.entries[id] = entry
-      for (const { id, entry, op, targetId, previous } of updates) {
-      if (previous && JSON.stringify(fresh.entries[targetId]) !== JSON.stringify(previous)) {
-        const heldId = newMemoryId('gmem')
-        fresh.entries[heldId] = { ...entry, status: 'pending_review', conflictingWith: targetId, reviewReason: 'target changed while curator was running' }
-        continue
-      }
-      if (op === 'merge' && previous) {
-        entry.content = entry.content || previous.content
-        entry.title = entry.title || previous.title
-        entry.evidence = [...(previous.evidence || []), ...(entry.evidence || [])].slice(-40)
-      }
-      fresh.entries[id] = entry
-      if ((op === 'supersede' || op === 'merge') && targetId && fresh.entries[targetId]) {
-        fresh.entries[targetId].status = 'superseded'
-        fresh.entries[targetId].supersededBy = id
-        fresh.entries[targetId].validTo = now
-        fresh.entries[targetId].updatedAt = now
-      }
+    let committedToBackend = false
+    try {
+      if (route !== 'lexical') {
+        await commit({
+          payload: { recall_files: files, resource: [], user: {} },
+          route, db: globalDbPath(),
+        })
+        committedToBackend = true
+        // 整理写入与单条写入同口径：提交成功就在条目上落"已入向量库"戳，否则这批
+        // 全局记忆在向量计数（membership 的戳腿）里永远缺失。pending 不提交，不戳。
+        const vector = activeEntryStamp(route)
+        if (vector) for (const update of updates) update.entry.vector = vector
       }
-    }))
+
+      if (job?.abort?.signal?.aborted) throw new Error('cancelled by user')
+      await retryStoreWrite(() => updateGlobalIndex(fresh => {
+        for (const { id, entry } of pending) fresh.entries[id] = entry
+        for (const { id, entry, op, targetId, previous } of updates) {
+        if (previous && JSON.stringify(fresh.entries[targetId]) !== JSON.stringify(previous)) {
+          const heldId = newMemoryId('gmem')
+          // 提交成功的只有 `id` 这一行，held 克隆从未进过向量库——它绝不能继承
+          // "提交成功即落戳"的 entry.vector：向量库里的行名是 `id` 而不是 heldId，
+          // 带戳会让 membership 把一条不存在的行计进来，而真实存在的 `id` 行反而
+          // 无处归属（它根本不会按这个克隆写进索引）。
+          const held = { ...entry, status: 'pending_review', conflictingWith: targetId, reviewReason: 'target changed while curator was running' }
+          delete held.vector
+          fresh.entries[heldId] = held
+          continue
+        }
+        if (op === 'merge' && previous) {
+          entry.content = entry.content || previous.content
+          entry.title = entry.title || previous.title
+          entry.evidence = [...(previous.evidence || []), ...(entry.evidence || [])].slice(-40)
+        }
+        fresh.entries[id] = entry
+        if ((op === 'supersede' || op === 'merge') && targetId && fresh.entries[targetId]) {
+          fresh.entries[targetId].status = 'superseded'
+          fresh.entries[targetId].supersededBy = id
+          fresh.entries[targetId].validTo = now
+          fresh.entries[targetId].updatedAt = now
+        }
+        }
+      }))
+    } catch (error) {
+      // 提交成功之后（abort 检查 / 索引写）的任何失败都会留下孤儿行：尽力撤销。
+      if (committedToBackend) await compensateCuratorCommit(error, { files, route })
+      throw error
+    }
   })
   const summary = {
     files,

package/lib/graph.js

@@ -1,6 +1,6 @@
 import path from 'node:path'
-import { indexDir, readJsonStrict, writeJson, withFileLock, stamp } from './paths.js'
-import { loadPrivateIndex, updatePrivateIndex, newMemoryId } from './state.js'
+import { indexDir, readJsonStrict, writeJson, withFileLock, stamp, warn } from './paths.js'
+import { loadPrivateIndex, updatePrivateIndex, newMemoryId, listLibraries } from './state.js'
 import { resolveLibrary, sessionIdOf, visibleToSession, isMemoryEligible, decayStrength } from './memory.js'
 
 export const LINK_KINDS = ['relation', 'commitment', 'event', 'person', 'conflict']
@@ -60,6 +60,16 @@ export function restoreGraphData(libraryId, data, memoryIdMap) {
     if (!evidenceIds.length) continue
     updateRelationship({ libraryId, fromId: entityMap[relationship.fromId], toId: entityMap[relationship.toId], state: relationship.state, evidenceIds, reason: relationship.reason || 'Imported relationship with explicit evidence.' })
   }
+  // A backup carries the whole entity store, but only the memories that made it
+  // through the export/import mapping come back, so entities nothing points at
+  // any more would sit in the file forever as invisible weight. Collect them
+  // once the remap is done; a prune failure must not fail a restore that
+  // already succeeded, and the startup sweep would catch it on the next launch.
+  try {
+    pruneOrphanEntities(scope.libraryId)
+  } catch (error) {
+    warn(`orphan entity prune after graph restore skipped: ${error?.message || error}`)
+  }
 }
 
 export function createEntity({ exec, libraryId, name, kind = 'person', aliases = [] }) {
@@ -102,6 +112,119 @@ export function renameEntity({ libraryId, entityId, name, aliases = [] }) {
   })
 }
 
+// What keeps an entity alive. Edges only live in one direction: an entry's
+// links[] names the entity, the entity stores nothing back. On top of that,
+// relationship snapshots (same file, keyed by the endpoint pair) describe the
+// pair with history and evidence that outlive the individual entries — deleting
+// an endpoint would leave a snapshot whose endpoints no longer resolve, so a
+// snapshot counts as a reference too. An entity referenced by neither is an
+// *orphan*: unreachable except as a graph placeholder, and safe to collect.
+// Orphans come from old builds whose permanent delete dropped the memory row
+// without cascading, and from partial backups.
+
+/**
+ * Remove every entity in one library that no entry links to and no relationship
+ * snapshot references. Idempotent: on healthy data it writes nothing. Returns
+ * `{ removed, total }` with the removed entity ids.
+ */
+export function pruneOrphanEntities(libraryId) {
+  const scope = scopeFor({ libraryId })
+  const file = entitiesPath(scope.libraryId)
+  return withFileLock(file, () => {
+    const store = readEntities(scope.libraryId)
+    const ids = Object.keys(store.entities)
+    if (!ids.length) return { removed: [], total: 0 }
+    const referenced = new Set()
+    for (const entry of Object.values(loadPrivateIndex(scope.libraryId).entries || {})) {
+      for (const link of entry.links || []) referenced.add(link.to)
+    }
+    for (const relationship of Object.values(store.relationships || {})) {
+      referenced.add(relationship.fromId)
+      referenced.add(relationship.toId)
+    }
+    const removed = ids.filter(id => !referenced.has(id))
+    // Write only when something actually goes, so the common case never touches
+    // the file (mtime-based caches, backups and concurrent readers all benefit).
+    if (removed.length) {
+      for (const id of removed) delete store.entities[id]
+      writeJson(file, store)
+    }
+    return { removed, total: ids.length }
+  })
+}
+
+/**
+ * The startup sweep: run pruneOrphanEntities over every active library. One
+ * corrupt store must not stop the rest, so per-library failures are collected
+ * instead of thrown; the caller decides how to log them. Archived libraries are
+ * deliberately left alone (frozen user data — restoring one runs the prune via
+ * restoreGraphData anyway). Returns `{ scanned, removed, failed }`.
+ */
+export function pruneOrphanEntitiesAll() {
+  const libraries = listLibraries()
+  const result = { scanned: libraries.length, removed: 0, failed: [] }
+  for (const library of libraries) {
+    try {
+      result.removed += pruneOrphanEntities(library.id).removed.length
+    } catch (error) {
+      result.failed.push({ libraryId: library.id, error: String(error?.message || error) })
+    }
+  }
+  return result
+}
+
+/**
+ * Delete an entity. This is graph bookkeeping, not a memory write: the entity
+ * row goes, no memory text is touched. An entity still referenced by links or
+ * relationship snapshots is refused by default — silently deleting it would
+ * turn every one of those edges into a `[deleted]` placeholder and rewrite what
+ * those memories are *about*. `force` deletes through that: inbound links are
+ * removed from every entry (not session-filtered — a link left behind in an
+ * entry this conversation cannot see would dangle exactly the way this guard
+ * exists to prevent, and the entry text itself is untouched) and relationship
+ * snapshots on the endpoints are dropped with their history, which is the loss
+ * `force` is asking for. Index links are cleaned before the entity row so a
+ * crash between the two writes leaves at worst a collectable orphan, never a
+ * link to an entity that is already gone.
+ */
+export function deleteEntity({ exec, libraryId, entityId, force = false }) {
+  const scope = scopeFor({ exec, libraryId })
+  if (!entityId || typeof entityId !== 'string') throw new Error('entityId is required')
+  const file = entitiesPath(scope.libraryId)
+  return withFileLock(file, () => {
+    const store = readEntities(scope.libraryId)
+    if (!store.entities[entityId]) throw new Error('unknown entity')
+    const index = loadPrivateIndex(scope.libraryId)
+    let inbound = 0
+    for (const entry of Object.values(index.entries || {})) {
+      for (const link of entry.links || []) if (link.to === entityId) inbound += 1
+    }
+    const relationshipKeys = Object.keys(store.relationships || {})
+      .filter(key => { const [from, to] = JSON.parse(key); return from === entityId || to === entityId })
+    if ((inbound || relationshipKeys.length) && !force) {
+      const parts = []
+      if (inbound) parts.push(`${inbound} link(s) point at it`)
+      if (relationshipKeys.length) parts.push(`${relationshipKeys.length} relationship snapshot(s) reference it`)
+      throw new Error(`entity is still referenced: ${parts.join(' and ')}; remove the links first or pass force to delete the entity with them`)
+    }
+    let removedLinks = 0
+    if (inbound) {
+      updatePrivateIndex(scope.libraryId, fresh => {
+        for (const entry of Object.values(fresh.entries)) {
+          if (!Array.isArray(entry.links) || !entry.links.some(link => link.to === entityId)) continue
+          const kept = entry.links.filter(link => link.to !== entityId)
+          removedLinks += entry.links.length - kept.length
+          entry.links = kept
+        }
+      })
+    }
+    for (const key of relationshipKeys) delete store.relationships[key]
+    delete store.entities[entityId]
+    writeJson(file, store)
+    return { entityId, deleted: true, removedLinks, removedRelationships: relationshipKeys.length }
+  })
+}
+
 export function linkEntries({ exec, libraryId, fromId, toId, kind, note, confidence = 0.7 }) {
   const scope = scopeFor({ exec, libraryId })
   if (!LINK_KINDS.includes(kind)) throw new Error(`link kind must be ${LINK_KINDS.join(', ')}`)
@@ -145,7 +268,7 @@ export function unlinkEntry({ libraryId, fromId, toId, kind }) {
   return { removed }
 }
 
-export function getGraph({ exec, libraryId, focusId = null, depth = 1, limit = 200 }) {
+export function getGraph({ exec, libraryId, focusId = null, depth = 1, limit = 200, includeOrphans = false }) {
   const scope = scopeFor({ exec, libraryId })
   const index = loadPrivateIndex(scope.libraryId)
   const entities = readEntities(scope.libraryId).entities
@@ -181,9 +304,16 @@ export function getGraph({ exec, libraryId, focusId = null, depth = 1, limit = 2
       frontier = next
     }
   } else {
+    // Orphan entities — no edge from any visible entry — are hidden by default.
+    // After the permanent-delete cascade and pruneOrphanEntities they should not
+    // exist on disk at all; a leftover from old data is invisible weight, and a
+    // session-scoped viewer already never saw them (the check below). Pass
+    // includeOrphans to see what an old entities.json still holds. An explicit
+    // focusId always wins: asking for a node by id is intent, not browsing.
     for (const id of [...visible, ...Object.keys(entities), ...adjacency.keys()]) {
       if (selected.size >= cap) break
       if (sessionIdOf(exec) && !visible.has(id) && !adjacency.has(id)) continue
+      if (!includeOrphans && entities[id] && !adjacency.has(id)) continue
       selected.add(id)
     }
   }
@@ -207,8 +337,12 @@ export function recallRelations(libraryId, index, id, sessionId) {
       const to = from === id ? link.to : from
       const target = index.entries[to]
       if (target && (!isMemoryEligible(target, { automatic: true }) || !visibleToSession(target, sessionId))) continue
-      if (!target && !entities[to]) continue
-      out.push({ id: to, title: target?.title || entities[to]?.name, kind: link.kind, note: String(link.note || '').slice(0, 80), confidence: link.confidence ?? 0.7 })
+      // A dangling edge (entry and entity both gone) used to be dropped here
+      // while getGraph rendered the same disk state as a [deleted] placeholder —
+      // the two views disagreed. Both now report the edge and mark it `missing`;
+      // the caller filters if it wants the old silence back.
+      const missing = !target && !entities[to]
+      out.push({ id: to, title: target?.title || entities[to]?.name || '[deleted]', kind: link.kind, note: String(link.note || '').slice(0, 80), confidence: link.confidence ?? 0.7, missing })
     }
   }
   return out.sort((a, b) => b.confidence - a.confidence).slice(0, 2)

package/lib/index.js

@@ -5,7 +5,8 @@ import { registerRpc } from './rpc.js'
 import { registerCycle } from './cycle.js'
 import { syncPromptDefaults } from './prompts.js'
 import { ensureEngine, engineStatus, ENGINE_VERSION } from './engine.js'
-import { getSettings, getMemoryBackend, backupDatabases, flushState } from './state.js'
+import { getSettings, getMemoryBackend, backupDatabases, flushState, listLibraries } from './state.js'
+import { pruneOrphanEntitiesAll } from './graph.js'
 import { closeEmbeddingWorkers } from './local-embedding-manager.js'
 import { bootstrapLocalVector } from './local-vector-service.js'
 import { vectorStatus } from './vector-profile.js'
@@ -62,6 +63,35 @@ function startEngineBootstrap(ctx) {
   }
 }
 
+// 兜底扫描（老数据升级）：旧版本的永久删除只删条目本身，别的条目 links 里指向它的
+// 入边和 entities.json 里的实体节点会留下来，restoreGraphData 的部分恢复也会。这里在
+// 启动后延迟跑一次全库 pruneOrphanEntitiesAll。选「每次启动延迟扫」而不是「首次
+// getGraph + 持久化已清理标记」：扫描只是对小 JSON 文件的读 + 偶尔一次原子写，健康
+// 数据零写入（no-op），幂等且能在任何一次删除路径将来回归时自愈；持久化标记省不下
+// 多少成本，反而多一个「标记存在但数据又坏了」的状态要维护。延迟 4 秒（在 1.5s 引擎
+// 引导与 2.5s 向量引导之后）让激活路径零阻塞；新安装没有库、没有实体，扫描直接 no-op。
+// 单库失败只收集不抛出，整段失败只 warn —— 加载绝不能被一次清理绊倒。
+function startOrphanEntitySweep(ctx) {
+  try {
+    const timer = setTimeout(() => {
+      try {
+        const result = pruneOrphanEntitiesAll()
+        for (const failure of result.failed) {
+          try { ctx.logger?.warn?.(`dsh-memu-memory: orphan entity sweep skipped ${failure.libraryId}: ${failure.error}`) } catch {}
+        }
+        if (result.removed > 0) {
+          try { ctx.logger?.info?.(`dsh-memu-memory: orphan entity sweep collected ${result.removed} node(s) across ${result.scanned} library(ies)`) } catch {}
+        }
+      } catch (error) {
+        try { ctx.logger?.warn?.(`dsh-memu-memory: orphan entity sweep failed: ${error?.message || error}`) } catch {}
+      }
+    }, 4000)
+    timer.unref?.()
+  } catch (error) {
+    try { ctx.logger?.warn?.(`dsh-memu-memory: orphan entity sweep skipped: ${error?.message || error}`) } catch {}
+  }
+}
+
 export function apply(ctx, config) {
   // A conversation that names an agent preset composes its model-facing plugin set from
   // that preset's cordis.yml alone, and that subtree never delivers `agent/pre-step` to
@@ -121,6 +151,8 @@ export function apply(ctx, config) {
     registerRpc(ctx)
     startEngineBootstrap(ctx)
     startLocalVectorBootstrap(ctx)
+    // 进程级清理：preset 挂载共享同一个数据目录，只由拥有全局表面的实例扫一次。
+    startOrphanEntitySweep(ctx)
   }
 
   // The embedding worker is a process-global singleton, so it outlives any single