dsh-conversation-recycle-bin 0.2.1 → 0.2.2

package/lib/core/memory/json-index-adapter.js

@@ -9,21 +9,41 @@
  *
  * Deletion is a MOVE, never an unlink: node content files and full index
  * entries are copied into the recycle-bin bundle, the live index is rewritten
- * atomically, and the store lock is held for the whole read-modify-write. A
- * failure at any point restores the files and the original index before the
- * error leaves this module, so the caller's transaction sees an all-or-nothing
- * memory stage.
+ * atomically, and a plugin-private lock is held for the whole
+ * read-modify-write.
  *
- * Limitations are explicit: the adapter understands the documented index
- * shape and reference arrays (`edges`, `depends_on`, `derived_from`) by
- * best-effort inspection. If a store introduces another reference field, its
- * dangling references are preserved (not silently rewritten) and the adapter's
- * `probe()` remains available because the schema itself still matches.
+ * ## Why the lock is plugin-private
+ *
+ * The store's own `<index>.lock` belongs to md_cg and is a permanent zero-byte
+ * file guarded by an OS byte-range lock (`msvcrt.locking` on Windows, `flock`
+ * on POSIX). Its existence is NOT a lock, and removing it silently breaks the
+ * store's own mutual exclusion. This adapter therefore never opens, writes or
+ * deletes that file; it uses `<index>.conversation-recycle-bin.lock`, which
+ * only coordinates this plugin's Host processes. The store lock is left to
+ * md_cg.
+ *
+ * ## Index generations
+ *
+ * md_cg persists an index as a snapshot (`_index.json`) plus append-only
+ * shards under `_index_log/`, replaying the shards over the snapshot on every
+ * load. When `indexLogDir` is configured (the 灵枢 preset), this adapter
+ * appends its own records to a fresh shard: tombstones (`e: null`) for removed
+ * nodes and the updated records of peers whose references were stripped. That
+ * keeps a later snapshot+log replay from resurrecting a deleted node. The
+ * adapter still writes the snapshot itself so its own reads (counts, preview)
+ * are immediately correct.
+ *
+ * Limitations are explicit: a md_cg process that already holds the node in its
+ * unflushed `_dirty` set can still re-expose it until that process reloads; and
+ * a node updated by another writer after our tombstone wins. Callers must treat
+ * this as a best-effort offline adapter, not as md_cg's own `forget` protocol.
  */
 import * as defaultFs from 'node:fs/promises'
+import { createHash, randomBytes } from 'node:crypto'
 import { dirname, isAbsolute, join, resolve, sep } from 'node:path'
 import { resolveJsonPointer } from '../config.js'
 import { DeleteError } from '../errors.js'
+import { createFileLock } from '../file-lock.js'
 
 const REFERENCE_FIELDS = ['edges', 'depends_on', 'derived_from']
 const REFERENCE_KEYS = ['id', 'target', 'to', 'nodeId', 'node', 'targetId', 'target_id', 'ref']
@@ -56,6 +76,8 @@ function validateRelativeContentPath(rootDir, relative) {
   return target
 }
 
+const sleepDefault = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
+
 /**
  * @param {object} options
  * @param {string} options.id stable adapter id
@@ -65,8 +87,11 @@ function validateRelativeContentPath(rootDir, relative) {
  * @param {string} [options.sessionField]
  * @param {string} [options.nodesPointer]
  * @param {string|null} [options.contentField]
- * @param {boolean} [options.useStoreLock] use `<indexFile>.lock` (the store's own lock) instead of a plugin-private lock
+ * @param {string|null} [options.indexLogDir] md_cg `_index_log` directory; null = no shard records
  * @param {(index: any) => string | null} [options.validateIndex] return an error string to mark the store unsupported
+ * @param {number} [options.lockTimeoutMs]
+ * @param {number} [options.staleLockMs]
+ * @param {number} [options.lockHeartbeatMs]
  * @param {() => number} [options.now]
  * @param {object} [options.fsApi]
  */
@@ -79,22 +104,40 @@ export function createJsonIndexMemoryAdapter(options) {
     sessionField = 'session',
     nodesPointer = '/nodes',
     contentField = 'path',
-    useStoreLock = false,
+    indexLogDir = null,
     validateIndex = null,
-    now = Date.now,
-    lockTimeoutMs = 6_000,
+    lockTimeoutMs = 15_000,
     staleLockMs = 120_000,
-    // A zero-byte lock carries no owner pid. If it is older than this, the
-    // writer almost certainly died; waiting the full two minutes would make
-    // every delete fail while the dialog says there is nothing to export.
-    ownerlessLockStaleMs = 5_000,
+    lockHeartbeatMs = 5_000,
+    now = Date.now,
+    sleep = sleepDefault,
     fsApi = defaultFs,
   } = options
 
-  const lockFile = useStoreLock ? `${indexFile}.lock` : `${indexFile}.delete-conversation.lock`
+  // Deliberately NOT `<indexFile>.lock`: that file belongs to md_cg.
+  const lockPath = `${indexFile}.conversation-recycle-bin.lock`
   const bundleFileName = 'bundle.json'
+  const lock = createFileLock({
+    lockPath,
+    timeoutMs: lockTimeoutMs,
+    staleMs: staleLockMs,
+    heartbeatMs: lockHeartbeatMs,
+    now,
+    sleep,
+    fsApi,
+  })
 
-  async function readIndex() {
+  function withLock(operation) {
+    return lock.acquire().then(async (release) => {
+      try {
+        return await operation()
+      } finally {
+        await release()
+      }
+    })
+  }
+
+  async function readSnapshot() {
     const raw = await fsApi.readFile(indexFile, 'utf8')
     let index
     try {
@@ -115,7 +158,12 @@ export function createJsonIndexMemoryAdapter(options) {
     if (nodes === null || typeof nodes !== 'object') {
       throw new DeleteError('E_MEMORY_FAILED', `Memory index has no node map at ${nodesPointer}`, { indexFile, nodesPointer })
     }
-    return { index, nodes }
+    return { index, nodes, sig: createHash('sha256').update(raw).digest('hex') }
+  }
+
+  async function readIndex() {
+    const snapshot = await readSnapshot()
+    return { index: snapshot.index, nodes: snapshot.nodes }
   }
 
   async function writeIndexAtomic(index) {
@@ -124,65 +172,62 @@ export function createJsonIndexMemoryAdapter(options) {
     await fsApi.rename(temporary, indexFile)
   }
 
-  /** Acquire the store lock; a stale lock is broken after `staleLockMs`. */
-  async function withLock(operation) {
-    const started = now()
-    let handle = null
-    // A zero-byte lock is the crashed-writer signal, and the grace that proves
-    // it must fit INSIDE the wait budget. A content-bearing lock keeps the
-    // caller's (shorter) timeout, because it is a live writer, not a corpse.
-    let waitBudgetMs = lockTimeoutMs
-    while (handle === null) {
-      try {
-        handle = await fsApi.open(lockFile, 'wx')
-      } catch (error) {
-        if (error?.code !== 'EEXIST') throw error
-        let stale = false
-        try {
-          const info = await fsApi.stat(lockFile)
-          stale = now() - info.mtimeMs > staleLockMs
-          if (info.size === 0) {
-            waitBudgetMs = Math.max(waitBudgetMs, ownerlessLockStaleMs + 500)
-            if (!stale && now() - info.mtimeMs > ownerlessLockStaleMs) stale = true
-          }
-        } catch {
-          stale = false
-        }
-        if (stale) {
-          await fsApi.rm(lockFile, { force: true }).catch(() => {})
-          continue
-        }
-        if (now() - started > waitBudgetMs) {
-          throw new DeleteError('E_MEMORY_FAILED', 'The memory store is busy; try again later.', { lockFile })
-        }
-        await new Promise((resolvePromise) => setTimeout(resolvePromise, 25))
-        continue
-      }
-      // The file now exists. A failed owner write must not leave a lock this
-      // process will never release (the exact zero-byte lock that blocked every
-      // later delete until it aged out).
-      try {
-        await handle.writeFile(`${process.pid}\n${now()}\n`, 'utf8')
-      } catch (error) {
-        try {
-          await handle.close()
-        } catch {
-          // already broken; the lock file is removed below either way
-        }
-        await fsApi.rm(lockFile, { force: true }).catch(() => {})
-        throw error
-      }
-    }
+  async function newestLogMtimeMs() {
+    if (indexLogDir === null) return 0
+    let entries
     try {
-      return await operation()
-    } finally {
+      entries = await fsApi.readdir(indexLogDir, { withFileTypes: true })
+    } catch {
+      return 0
+    }
+    let newest = 0
+    for (const entry of entries) {
+      if (!entry.isFile() || !entry.name.endsWith('.log')) continue
       try {
-        await handle?.close()
+        const info = await fsApi.stat(join(indexLogDir, entry.name))
+        if (info.mtimeMs > newest) newest = info.mtimeMs
       } catch {
-        // closing a lock handle must never mask the operation outcome
+        // A shard cleared by a concurrent compact is simply absent.
       }
-      await fsApi.rm(lockFile, { force: true }).catch(() => {})
     }
+    return newest
+  }
+
+  /**
+   * Append records to a fresh md_cg shard.
+   *
+   * The shard format is the one `md_cg.fsutil.ShardedLog` writes:
+   * `{ id, e, _t, _s }` per line. `e: null` is a tombstone; a dict is an
+   * upsert. A fresh shard avoids sharing a write point with md_cg, and `_t`
+   * starts after the newest existing shard so the records replay last.
+   *
+   * @param {Array<{ id: string, e: object | null }>} records
+   * @returns {Promise<string|null>} shard path, or null when disabled/empty
+   */
+  async function appendIndexLog(records) {
+    if (indexLogDir === null || records.length === 0) return null
+    await fsApi.mkdir(indexLogDir, { recursive: true })
+    const newest = await newestLogMtimeMs()
+    const base = Math.max(now() / 1000, newest / 1000 + 0.001)
+    const shardPath = join(indexLogDir, `${process.pid}-${randomBytes(4).toString('hex')}.log`)
+    const lines = records.map((record, index) => JSON.stringify({
+      id: record.id,
+      e: record.e,
+      _t: base + index / 1_000_000,
+      _s: index + 1,
+    }))
+    try {
+      await fsApi.writeFile(shardPath, `${lines.join('\n')}\n`, 'utf8')
+      return shardPath
+    } catch (error) {
+      await fsApi.rm(shardPath, { force: true }).catch(() => {})
+      throw error
+    }
+  }
+
+  async function removeQuietly(target) {
+    if (target === null || target === undefined) return
+    await fsApi.rm(target, { recursive: true, force: true }).catch(() => {})
   }
 
   return {
@@ -190,6 +235,7 @@ export function createJsonIndexMemoryAdapter(options) {
     label,
     indexFile,
     rootDir,
+    lockPath,
 
     async probe() {
       try {
@@ -213,7 +259,7 @@ export function createJsonIndexMemoryAdapter(options) {
       const { nodes } = await readIndex()
       const wanted = new Set(sessionIds)
       const counts = new Map()
-      for (const id of wanted) counts.set(id, 0)
+      for (const wantedId of wanted) counts.set(wantedId, 0)
       let protectedCount = 0
       for (const node of Object.values(nodes)) {
         if (node === null || typeof node !== 'object') continue
@@ -245,20 +291,27 @@ export function createJsonIndexMemoryAdapter(options) {
         links: [],
       })
 
-      // Fast path: a conversation with nothing to move must not depend on the
-      // store lock. A stale lock left by a crashed writer (or a long write)
-      // used to make "delete this conversation" fail with "memory store is
-      // busy" even though the dialog showed zero linked memories.
-      const before = await readIndex()
+      // Fast path: a conversation with nothing to move must not take the lock.
+      const before = await readSnapshot()
       const hasLinked = Object.values(before.nodes).some(
         (node) => node !== null && typeof node === 'object' && node[sessionField] === sessionId && !isProtected(node),
       )
       if (!hasLinked) return emptyBundle()
 
       return withLock(async () => {
-        const { index, nodes } = await readIndex()
-        // Snapshot before ANY mutation: compensation must be able to put the
-        // original index back byte-for-byte in meaning (key order preserved).
+        const current = await readSnapshot()
+        // Another writer changed the snapshot while we waited for the lock.
+        // Recomputing against `current` would silently miss a node added after
+        // `before`; refusing is the only safe answer.
+        if (current.sig !== before.sig) {
+          throw new DeleteError('E_MEMORY_FAILED', 'The memory index changed while waiting; retry the deletion.', {
+            indexFile,
+            retryable: true,
+          })
+        }
+        const { index, nodes } = current
+        // Snapshot before ANY mutation: compensation must put the original
+        // index back byte-for-byte in meaning (key order preserved).
         const pristine = structuredClone(index)
         const bundleDir = join(targetDir, 'memory', id)
         const filesDir = join(bundleDir, 'files')
@@ -274,6 +327,7 @@ export function createJsonIndexMemoryAdapter(options) {
 
         const movedFiles = []
         let indexWritten = false
+        let logShard = null
         try {
           await fsApi.mkdir(filesDir, { recursive: true })
           for (const nodeId of ids) {
@@ -321,15 +375,27 @@ export function createJsonIndexMemoryAdapter(options) {
           bundle.bundlePath = bundleFile
           // Persist the restore bundle BEFORE removing the entries from the
           // live index: a crash in between leaves a recoverable bundle plus an
-          // index that still lists its nodes, which is inspectable. The reverse
-          // order could drop entries whose only copy was the rewritten index.
+          // index that still lists its nodes, which is inspectable.
           await fsApi.writeFile(bundleFile, `${JSON.stringify(bundle, null, 2)}\n`, 'utf8')
           await writeIndexAtomic(index)
           indexWritten = true
+
+          // Snapshot taken. Now make the deletion survive a md_cg
+          // snapshot+log replay: tombstones for the removed nodes and the
+          // updated records of the peers whose references we stripped.
+          const records = [...ids].map((nodeId) => ({ id: nodeId, e: null }))
+          const seenPeers = new Set()
+          for (const link of bundle.links) {
+            if (seenPeers.has(link.nodeId) || nodes[link.nodeId] === undefined) continue
+            seenPeers.add(link.nodeId)
+            records.push({ id: link.nodeId, e: nodes[link.nodeId] })
+          }
+          logShard = await appendIndexLog(records)
           return bundle
         } catch (error) {
-          // Compensate inside the memory stage: the caller must see either a
+          // Compensation inside the memory stage: the caller must see either a
           // fully moved session or an untouched store.
+          await removeQuietly(logShard)
           if (indexWritten) {
             await writeIndexAtomic(pristine).catch(() => {})
           }
@@ -337,6 +403,10 @@ export function createJsonIndexMemoryAdapter(options) {
             await fsApi.mkdir(dirname(moved.from), { recursive: true }).catch(() => {})
             await fsApi.rename(moved.to, moved.from).catch(() => {})
           }
+          // A bundle written before the failure is not referenced by any
+          // manifest; leaving it behind made the failed entry impossible to
+          // discard (the discard guard sees files the manifest does not know).
+          await removeQuietly(bundleDir)
           throw error instanceof DeleteError
             ? error
             : new DeleteError('E_MEMORY_FAILED', 'Failed to move memory entries to the recycle bin.', {
@@ -352,7 +422,9 @@ export function createJsonIndexMemoryAdapter(options) {
     async restoreBundle(bundle) {
       if (bundle === null || typeof bundle !== 'object' || !Array.isArray(bundle.entries) || bundle.entries.length === 0) return
       await withLock(async () => {
-        const { index, nodes } = await readIndex()
+        const current = await readSnapshot()
+        const { index, nodes } = current
+        const pristine = structuredClone(index)
         const alreadyRestored = new Set()
         for (const entry of bundle.entries) {
           const destination = entry.storedFile !== null && entry.originalPath !== null && rootDir !== null
@@ -383,6 +455,8 @@ export function createJsonIndexMemoryAdapter(options) {
           }
         }
         const movedBack = []
+        let indexWritten = false
+        let logShard = null
         try {
           for (const entry of bundle.entries) {
             if (alreadyRestored.has(entry.nodeId)) continue
@@ -412,7 +486,28 @@ export function createJsonIndexMemoryAdapter(options) {
             }
           }
           await writeIndexAtomic(index)
+          indexWritten = true
+
+          // Mirror the restored records into md_cg's shard log so a later
+          // snapshot+log replay keeps them (a stale tombstone would otherwise
+          // win). Peers whose links came back are re-recorded too.
+          const records = []
+          for (const entry of bundle.entries) {
+            if (alreadyRestored.has(entry.nodeId)) continue
+            records.push({ id: entry.nodeId, e: entry.node })
+          }
+          const seenPeers = new Set()
+          for (const link of bundle.links ?? []) {
+            if (seenPeers.has(link.nodeId) || nodes[link.nodeId] === undefined) continue
+            seenPeers.add(link.nodeId)
+            records.push({ id: link.nodeId, e: nodes[link.nodeId] })
+          }
+          logShard = await appendIndexLog(records)
         } catch (error) {
+          await removeQuietly(logShard)
+          if (indexWritten) {
+            await writeIndexAtomic(pristine).catch(() => {})
+          }
           for (const moved of movedBack.reverse()) {
             await fsApi.rename(moved.from, moved.to).catch(() => {})
           }

package/lib/core/memory/lingshu.js

@@ -15,15 +15,21 @@ import { createJsonIndexMemoryAdapter } from './json-index-adapter.js'
 export function createLingshuAdapter({ rootDir, now, fsApi }) {
   return createJsonIndexMemoryAdapter({
     id: 'lingshu',
-    label: '灵枢认知图 (md_cg)',
+    // Neutral label: the Client localizes by adapter id, so a hard-coded
+    // Chinese/English mix here showed up untranslated in the other language.
+    label: 'lingshu (md_cg)',
     indexFile: join(rootDir, '_index.json'),
     rootDir,
     sessionField: 'session',
     nodesPointer: '/nodes',
     contentField: 'path',
-    // md_cg serializes its own writers with this exact lock file; taking it is
-    // the cooperative way to avoid a lost update against the Python process.
-    useStoreLock: true,
+    // md_cg persists the index as `_index.json` + append-only shards under
+    // `_index_log/` and replays the shards on every load. Our tombstones and
+    // restored-node records go into a fresh shard so that replay cannot
+    // resurrect a deleted node. We deliberately do NOT touch the store's
+    // `_index.json.lock`: that is md_cg's permanent OS-lock file, not a
+    // presence lock.
+    indexLogDir: join(rootDir, '_index_log'),
     validateIndex: (index) => {
       if (index.schema === 2) return null
       return `unsupported md_cg index schema ${String(index.schema)} (expected 2)`

package/lib/core/memory/none.js

@@ -9,9 +9,11 @@
 export function createNoneAdapter() {
   return {
     id: 'none',
-    label: 'No memory store',
+    // Neutral label + a stable reason key: the Client localizes both by id, so
+    // an English sentence here used to appear untranslated in a Chinese UI.
+    label: 'none',
     async probe() {
-      return { available: false, reason: 'No conversation-linked memory store was detected.', storePath: null }
+      return { available: false, reason: 'no-memory-store', storePath: null }
     },
     async countSessions(sessionIds) {
       const counts = new Map()

package/lib/core/memory/registry.js

@@ -163,6 +163,7 @@ export function createMemoryService({ adapters, config, now = Date.now }) {
     /** Restore every adapter bundle recorded in a trash manifest memory summary. */
     async restoreEntry(summary) {
       if (summary === null || typeof summary !== 'object' || !Array.isArray(summary.adapters)) return
+      const consumed = []
       for (const entry of [...summary.adapters].reverse()) {
         if (typeof entry.bundlePath !== 'string' || entry.bundlePath === '') continue
         const adapter = byId.get(entry.id)
@@ -174,12 +175,14 @@ export function createMemoryService({ adapters, config, now = Date.now }) {
         }
         const bundle = JSON.parse(await readFile(entry.bundlePath, 'utf8'))
         await adapter.restoreBundle(bundle)
-        // The payload is back in the live store, so the copy in the recycle bin
-        // is redundant. Removing it matters: `performDiscard` refuses an
-        // incomplete entry that still holds memory files, and a leftover
-        // `bundle.json` made every failed delete (memory included) permanently
-        // un-cleanable even though its data had been put back.
-        await rm(dirname(entry.bundlePath), { recursive: true, force: true }).catch(() => {})
+        consumed.push(entry.bundlePath)
+      }
+      // Remove the consumed bundles only after EVERY adapter succeeded. An
+      // eager per-adapter removal made a later adapter's failure
+      // unretryable: the first bundle was already gone while the manifest still
+      // referenced it.
+      for (const bundlePath of consumed) {
+        await rm(dirname(bundlePath), { recursive: true, force: true }).catch(() => {})
       }
     },
   }

package/lib/core/policy.js

@@ -19,10 +19,10 @@ import { DeleteError } from './errors.js'
 export function normalizePathForCompare(input) {
   if (typeof input !== 'string' || input === '') return ''
   let value = input.replace(/\\/g, '/').replace(/\/+$/g, '')
-  // A drive-lettered path is Windows: comparisons there are case-insensitive.
-  // POSIX paths stay case-sensitive so a root check cannot become broader than
-  // the filesystem's own identity rules.
-  if (/^[A-Za-z]:\//.test(value)) value = value.toLowerCase()
+  // Windows paths compare case-insensitively: drive-lettered paths and UNC
+  // roots (`\\server\share`, `\\?\C:\...`). POSIX paths stay case-sensitive so
+  // a root check cannot become broader than the filesystem's own identity rules.
+  if (/^[A-Za-z]:\//.test(value) || value.startsWith('//')) value = value.toLowerCase()
   return value
 }
 

package/lib/core/redact.js

@@ -0,0 +1,66 @@
+/**
+ * Redact host filesystem paths from error details before they cross the HTTP
+ * boundary.
+ *
+ * The Client contract is "ids, counts and status — no filesystem handles".
+ * Deleting and memory failures legitimately carry diagnostic paths in their
+ * `details`/`cause`; those are useful in the Host log, not in the browser.
+ */
+import { basename } from 'node:path'
+
+const PATH_KEYS = new Set([
+  'indexFile',
+  'lockFile',
+  'bundlePath',
+  'filePath',
+  'path',
+  'from',
+  'to',
+  'cwd',
+  'trashDir',
+  'auditFile',
+  'dataDir',
+  'rootDir',
+])
+
+const WINDOWS_ABSOLUTE = /(?:[A-Za-z]:[\\/]|\\\\)[^\s"'<>|]+/g
+const POSIX_ABSOLUTE = /(^|[\s("'=])\/(?:[^\s"'<>/]+\/)+[^\s"'<>]+/g
+
+function shortPath(value) {
+  const normalized = String(value).replace(/\\/g, '/').replace(/\/+$/g, '')
+  if (normalized === '') return ''
+  return basename(normalized)
+}
+
+function redactText(value) {
+  if (typeof value !== 'string') return value
+  return value
+    .replace(WINDOWS_ABSOLUTE, (match) => shortPath(match))
+    .replace(POSIX_ABSOLUTE, (match, prefix) => `${prefix}${shortPath(match.slice(prefix.length))}`)
+}
+
+/**
+ * Recursively redact path-like values. Unknown keys are preserved so the
+ * Client can still branch on structured fields (counts, ids, codes).
+ * @param {unknown} value
+ * @returns {unknown}
+ */
+export function redactDetails(value) {
+  if (value === null || value === undefined) return value
+  if (typeof value === 'string') return redactText(value)
+  if (Array.isArray(value)) return value.map((item) => (typeof item === 'string' ? redactText(item) : redactDetails(item)))
+  if (typeof value !== 'object') return value
+  const result = {}
+  for (const [key, item] of Object.entries(value)) {
+    if (PATH_KEYS.has(key) && typeof item === 'string') {
+      result[key] = shortPath(item)
+      continue
+    }
+    if (key === 'cause' || key === 'message') {
+      result[key] = typeof item === 'string' ? redactText(item) : redactDetails(item)
+      continue
+    }
+    result[key] = typeof item === 'string' ? item : redactDetails(item)
+  }
+  return result
+}