@remnic/core 9.7.14 → 9.8.0

package/src/maintenance/rebuild-memory-lifecycle-ledger-cli.ts

@@ -0,0 +1,77 @@
+import path from "node:path";
+
+import { probeEncryptedRegularFileHeader, SECURE_STORE_ENVELOPE_OVERHEAD_BYTES } from "../secure-store/secure-fs.js";
+import { STATE_FILE_MAX_DECRYPT_BYTES } from "../storage/secure-line-reader.js";
+import type { StorageManager } from "../index.js";
+import {
+  rebuildMemoryLifecycleLedger,
+  type RebuildMemoryLifecycleLedgerResult,
+} from "./rebuild-memory-lifecycle-ledger.js";
+
+export interface RebuildMemoryLifecycleLedgerCliCommandOptions {
+  memoryDir: string;
+  write?: boolean;
+  now?: Date;
+  /**
+   * Live storage for secure-store deployments (#2033). When the store is
+   * encrypted at rest, the rebuild must read encrypted memories and re-encrypt
+   * the rewritten ledger through this unlocked StorageManager; a keyless manager
+   * would read with no key and rewrite the ledger as plaintext. The command
+   * refuses when the on-disk ledger is encrypted and no unlocked key is present.
+   */
+  storage?: StorageManager;
+  /**
+   * Read/decrypt cap the rewritten ledger must land strictly below. Defaults to
+   * {@link STATE_FILE_MAX_DECRYPT_BYTES} (the reader's whole-file refusal cap).
+   * Exposed as an override so recovery tooling and tests can bound the rewrite
+   * deterministically, mirroring the scheduler's `lifecycleLedgerMaxBytes` seam.
+   */
+  maxLedgerBytesCap?: number;
+}
+
+export async function runRebuildMemoryLifecycleLedgerCliCommand(
+  options: RebuildMemoryLifecycleLedgerCliCommandOptions,
+): Promise<RebuildMemoryLifecycleLedgerResult> {
+  const storage = options.storage;
+  const ledgerPath = path.join(options.memoryDir, "state", "memory-lifecycle-ledger.jsonl");
+  const encrypted = await probeEncryptedRegularFileHeader(ledgerPath);
+  // Refuse a keyless plaintext rewrite of an encrypted-at-rest ledger (#2033):
+  // without an unlocked secure StorageManager the rebuild would read encrypted
+  // memories with no key and rewrite the ledger as plaintext, defeating
+  // encryption-at-rest. Fail safely and point the operator at the unlock step.
+  if (encrypted && !(storage?.isSecureStoreUnlocked() ?? false)) {
+    throw new Error(
+      "rebuild-memory-lifecycle-ledger: secure store is locked; refusing to rebuild the "
+      + "lifecycle ledger, which would read encrypted memories with no key and rewrite the "
+      + "ledger as plaintext. Run `remnic secure-store unlock` first, then re-run with --write.",
+    );
+  }
+  // This command is the advertised recovery for an over-cap ledger, so it must
+  // give the SAME guarantees as background compaction (#2033): (1) preserve
+  // append-only history frontmatter cannot reconstruct (explicit_capture_accepted,
+  // imported, promoted) instead of silently dropping it, and (2) bound the
+  // rewritten ledger under the read/decrypt cap so the repaired ledger is
+  // readable. Bound the PLAINTEXT payload, reserving the secure-store envelope
+  // (+1 byte) whenever the REPLACEMENT will be encrypted — NOT only when the
+  // current header is encrypted. A plaintext ledger rewritten through an unlocked
+  // encrypt-on-write store becomes encrypted, so it needs the reserve even though
+  // its current header is plaintext; base the decision on
+  // `willEncryptStateWrites()` exactly as auto-compaction does (#2033 write-mode
+  // finding), so the on-disk file lands STRICTLY under the cap.
+  const cap = options.maxLedgerBytesCap ?? STATE_FILE_MAX_DECRYPT_BYTES;
+  const replacementEncrypted = encrypted || (storage?.willEncryptStateWrites() ?? false);
+  const maxLedgerBytes = replacementEncrypted
+    ? cap - SECURE_STORE_ENVELOPE_OVERHEAD_BYTES - 1
+    : cap - 1;
+  return rebuildMemoryLifecycleLedger({
+    memoryDir: options.memoryDir,
+    dryRun: options.write !== true,
+    now: options.now,
+    // Carry the live (unlocked) secure StorageManager so encrypted memories
+    // decrypt on read and the rewritten ledger stays encrypted at rest; a
+    // plaintext store passes a plaintext manager here, which is correct.
+    storage,
+    preserveExistingEvents: true,
+    maxLedgerBytes,
+  });
+}

package/src/maintenance/rebuild-memory-lifecycle-ledger.ts

@@ -2,18 +2,80 @@ import path from "node:path";
 import { stat } from "node:fs/promises";
 import { StorageManager } from "../storage.js";
 import { log } from "../logger.js";
+import { displayErrorDetail } from "../runtime/better-sqlite.js";
 import type { MemoryLifecycleEvent } from "../types.js";
 import { toBackupStamp } from "./backup-stamp.js";
-import { writeFileAtomically } from "./atomic-file.js";
+import { copyExistingFileToBackup, writeFileAtomically } from "./atomic-file.js";
 import {
   buildLifecycleEventsForMemory,
+  compareMemoryLifecycleEvents,
   sortMemoryLifecycleEvents,
+  memoryLifecycleLedgerLockPath,
+  MEMORY_LIFECYCLE_LEDGER_LOCK_STALE_MS,
 } from "../memory-lifecycle-ledger-utils.js";
+import { withHeldFileLock, type HeldFileLockOptions, type HeldFileLockController } from "../utils/serialize-mutations.js";
+import { probeEncryptedRegularFileHeader } from "../secure-store/secure-fs.js";
+
+/**
+ * Event types `buildLifecycleEventsForMemory` reconstructs from frontmatter.
+ * Everything else in the ledger is append-only history with no frontmatter
+ * equivalent and must be carried over by a preserving rebuild (issue #1910).
+ */
+const FRONTMATTER_DERIVED_EVENT_TYPES: Record<string, true> = {
+  created: true,
+  updated: true,
+  superseded: true,
+  archived: true,
+};
 
 export interface RebuildMemoryLifecycleLedgerOptions {
   memoryDir: string;
   dryRun?: boolean;
   now?: Date;
+  /**
+   * Active storage context (issue #1910). When provided, memories are read
+   * through it (so encrypted-at-rest memories in a secure-store deployment are
+   * decrypted with the live key) and the rebuilt ledger is rewritten through
+   * its secure writer (so it stays encrypted at rest). Its `dir` MUST equal
+   * `memoryDir`. When omitted, a fresh plaintext `StorageManager` is used and
+   * the ledger is written as plaintext — the CLI default.
+   */
+  storage?: StorageManager;
+  /**
+   * Preserve append-only lifecycle events that have no frontmatter equivalent
+   * (issue #1910). `buildLifecycleEventsForMemory` only reconstructs
+   * `created`/`updated`/`superseded`/`archived` from frontmatter, so a bare
+   * rebuild silently drops history like `explicit_capture_accepted`,
+   * `explicit_capture_queued`, `imported`, `promoted`, `merged`, `restored`,
+   * and `rejected`. When true, the existing ledger's non-reconstructable rows
+   * are read (via the same secure/permissive `readAllMemoryLifecycleEvents`
+   * path), merged with the rebuilt rows, deduplicated by `eventId`, and sorted
+   * canonically. Background auto-compaction sets this so it never loses events;
+   * the manual/CLI repair rebuild leaves it off (reconstruct purely from
+   * frontmatter).
+   */
+  preserveExistingEvents?: boolean;
+  /**
+   * Override the ledger-lock acquisition timing (issue #2033). Only
+   * `maxWaitMs`/`pollMs` are honored; `staleMs` stays fixed at
+   * `MEMORY_LIFECYCLE_LEDGER_LOCK_STALE_MS`. Used by tests to force the
+   * lock-acquisition-failure branch (`acquired=false`) deterministically
+   * without a multi-second wait.
+   */
+  lockOptions?: Pick<HeldFileLockOptions, "maxWaitMs" | "pollMs">;
+  /**
+   * Upper bound (bytes) on the rewritten ledger when preserving events (issue
+   * #2033). A preserving compaction carries over ALL append-only history, which
+   * — if the history alone is large — could leave the rewritten ledger over the
+   * whole-file read/decrypt cap (STATE_FILE_MAX_DECRYPT_BYTES), the exact
+   * unreadable-ledger failure compaction exists to prevent. When set and the
+   * preserved payload would exceed it, the OLDEST rows are dropped from the
+   * active ledger until it fits; because the full pre-compaction ledger is
+   * copied verbatim to the timestamped backup first, those rows are archived,
+   * not lost. Ignored when `preserveExistingEvents` is off (a frontmatter-only
+   * rebuild is already bounded by the corpus). `0`/undefined disables bounding.
+   */
+  maxLedgerBytes?: number;
 }
 
 export interface SkippedLifecycleBlankIdMemory {
@@ -34,12 +96,84 @@ export interface RebuildMemoryLifecycleLedgerResult {
   backupPath?: string;
   skippedBlankIdMemories: SkippedLifecycleBlankIdMemory[];
   skippedDuplicateEvents: SkippedDuplicateLifecycleEvent[];
+  /**
+   * Count of existing-ledger rows carried over beyond the frontmatter
+   * reconstruction (only set when `preserveExistingEvents` is on; issue #1910):
+   * append-only history with no frontmatter equivalent, plus any
+   * frontmatter-derived row that raced in after the corpus scan and so was not
+   * reproduced by the reconstruction (#2033).
+   */
+  preservedAppendOnlyRows?: number;
+  /**
+   * Count of oldest rows dropped from the ACTIVE ledger to keep it under
+   * `maxLedgerBytes` (issue #2033). They remain in the verbatim timestamped
+   * backup, so this is a bound-and-archive, not a loss. Set only when bounding
+   * actually trimmed rows.
+   */
+  archivedOverflowRows?: number;
+  /**
+   * Whether the active ledger was actually backed up and rewritten (issue #2033
+   * thread PRRT_kwDORJXyws6SExst). A preserving write-mode rebuild that would
+   * reproduce the current ledger byte-for-byte (same decrypted content AND same
+   * at-rest format) skips the backup+rewrite, so callers can distinguish a real
+   * compaction from a no-op and avoid re-archiving an unchanged ledger every
+   * interval. `true` for every dry run's non-write preview is never set — only
+   * write-mode results carry it; defaults to `false` when no rewrite happened.
+   */
+  rewritten?: boolean;
 }
 
 function isNodeError(err: unknown): err is NodeJS.ErrnoException {
   return typeof err === "object" && err !== null && "code" in err;
 }
 
+/** Serialized on-disk size (bytes, incl. trailing newline) of one ledger row. */
+function lifecycleRowBytes(event: MemoryLifecycleEvent): number {
+  return Buffer.byteLength(`${JSON.stringify(event)}\n`, "utf8");
+}
+
+/**
+ * Keep the NEWEST canonically-ordered events whose serialized payload fits
+ * strictly under `cap` bytes (issue #2033). Recency is by `timestamp` (newest
+ * first), tie-broken by the canonical comparator, so a large append-only
+ * history is trimmed to its most recent rows rather than to a single memoryId
+ * group (the ledger's on-disk order is by memoryId then time).
+ *
+ * A row too large to fit the remaining budget is SKIPPED, and scanning
+ * continues to older rows that still fit — one oversized (or future-dated,
+ * hence recency-first) row cannot truncate the entire older history. The kept
+ * subset is returned in the input's canonical order; the caller relies on the
+ * verbatim backup to retain the dropped rows.
+ */
+export function boundLifecycleEventsToByteCap(
+  events: MemoryLifecycleEvent[],
+  cap: number,
+): { kept: MemoryLifecycleEvent[]; dropped: number } {
+  if (!(cap > 0)) return { kept: events, dropped: 0 };
+  const sizes = new Map<MemoryLifecycleEvent, number>();
+  let total = 0;
+  for (const event of events) {
+    const bytes = lifecycleRowBytes(event);
+    sizes.set(event, bytes);
+    total += bytes;
+  }
+  if (total <= cap) return { kept: events, dropped: 0 };
+  const byRecency = [...events].sort((a, b) => {
+    if (a.timestamp !== b.timestamp) return b.timestamp.localeCompare(a.timestamp);
+    return -compareMemoryLifecycleEvents(a, b);
+  });
+  const keep = new Set<MemoryLifecycleEvent>();
+  let running = 0;
+  for (const event of byRecency) {
+    const bytes = sizes.get(event)!;
+    if (running + bytes > cap) continue; // skip this row, keep scanning older rows that still fit
+    running += bytes;
+    keep.add(event);
+  }
+  const kept = events.filter((event) => keep.has(event));
+  return { kept, dropped: events.length - kept.length };
+}
+
 export async function backupExistingLedger(
   memoryDir: string,
   outputPath: string,
@@ -71,7 +205,14 @@ export async function rebuildMemoryLifecycleLedger(
   const dryRun = options.dryRun !== false;
   const now = options.now ?? new Date();
   const outputPath = path.join(options.memoryDir, "state", "memory-lifecycle-ledger.jsonl");
-  const storage = new StorageManager(options.memoryDir);
+  const storage = options.storage ?? new StorageManager(options.memoryDir);
+  if (options.storage && path.resolve(storage.dir) !== path.resolve(options.memoryDir)) {
+    throw new Error(
+      `rebuildMemoryLifecycleLedger: storage.dir (${storage.dir}) must match `
+      + `memoryDir (${options.memoryDir})`,
+    );
+  }
+  const secureRewrite = options.storage !== undefined;
   const tiers = [
     await storage.readAllMemories(),
     await storage.readAllColdMemories(),
@@ -119,24 +260,247 @@ export async function rebuildMemoryLifecycleLedger(
     events.push(event);
   }
 
+  let finalEvents = events;
+  let preservedAppendOnlyRows: number | undefined;
+  let archivedOverflowRows: number | undefined;
   let backupPath: string | undefined;
+  let rewritten = false;
+  // The preserve-read + whole-file rewrite are serialized against concurrent
+  // lifecycle appends ONLY in write mode (issue #1910, codex):
+  // appendMemoryLifecycleEvents holds this same cross-process lock, so an event
+  // appended after the preserve-read but before the atomic rename waits and
+  // lands on the compacted ledger instead of being clobbered by the rename. A
+  // DRY RUN takes no lock — it reads the ledger read-only to compute the
+  // preserve/merge/byte-cap PREVIEW and never rewrites, so it must not block
+  // appends (#2033 Cursor). The corpus scan above stays outside the lock in
+  // both modes — frontmatter-derived rows come from the memory files.
+  const runLedgerCriticalSection = async (
+    acquired: boolean,
+    lock: HeldFileLockController | null,
+  ): Promise<void> => {
+    // A WRITE that read or rewrote the ledger without the lock could race a
+    // concurrent append/rewrite and lose data, so REFUSE the unlocked fallback
+    // withHeldFileLock takes on acquisition timeout (issue #2033 CodeRabbit
+    // Critical / codex). A dry run passes lock=null and acquired=true: it never
+    // reaches the rewrite, so the unlocked preview read is safe.
+    if (!dryRun && !acquired) {
+      throw new Error(
+        "lifecycle ledger rebuild aborted: could not acquire the ledger lock within the "
+        + "acquisition budget; refusing to read or rewrite unlocked so a concurrent append "
+        + "or compaction cannot be clobbered (issue #1910).",
+      );
+    }
+    if (options.preserveExistingEvents) {
+      // Merge the frontmatter reconstruction (`events`) with the CURRENT ledger
+      // so compaction never deletes a lifecycle row that raced the scan (issue
+      // #1910, #2033). Read the existing ledger through the uncapped BUFFER
+      // path — it decrypts with the live key when `storage` carries the secure
+      // context and bypasses the whole-file string decrypt cap, so an oversize
+      // encrypted ledger (the exact file compaction must shrink) is still
+      // readable here rather than aborting (#2033 Cursor High).
+      // preserveExistingEvents means "never lose history": if the current
+      // ledger genuinely cannot be read (corruption, or a locked required
+      // store), ABORT rather than rewrite from frontmatter only and silently
+      // drop that history. An absent ledger reads as [] (ENOENT swallowed
+      // downstream), so a throw here is always a real read failure.
+      let existing: MemoryLifecycleEvent[];
+      try {
+        existing = await storage.readAllMemoryLifecycleEventsForCompaction();
+      } catch (err) {
+        throw new Error(
+          `lifecycle ledger rebuild aborted: cannot read existing events to preserve `
+          + `(${displayErrorDetail(err) || "unknown error"})`,
+        );
+      }
+      // The corpus scan (`events`) runs OUTSIDE the lock and is therefore a
+      // stale snapshot; `existing` is read here under the lock and is
+      // authoritative for what the ledger holds at rewrite time. A
+      // frontmatter-derived row (created/updated/superseded/archived) is safe to
+      // collapse into the reconstruction ONLY when the scan actually reproduced
+      // the same logical event — same (memoryId, eventType, timestamp), which is
+      // exactly what the live append stamps (fm.created/updated/...) and what
+      // makeRebuiltMemoryLifecycleEvent emits. A frontmatter-derived row whose
+      // content key the reconstruction did NOT produce raced in after the scan
+      // (a memory created/updated between the scan and this locked read); the
+      // reconstruction has no equivalent, so dropping it by eventType alone
+      // would delete it. Preserve it instead, deduped by content key so a
+      // retried append of the same logical event still collapses to one. Rows
+      // that are not frontmatter-derived are append-only history with no
+      // reconstruction — always carried over, deduped by eventId in the merge.
+      const contentKey = (event: MemoryLifecycleEvent): string =>
+        `${event.memoryId}\u0000${event.eventType}\u0000${event.timestamp}`;
+      const reconstructedKeys = new Set(events.map(contentKey));
+      const racedFrontmatterByKey = new Map<string, MemoryLifecycleEvent>();
+      const appendOnly: MemoryLifecycleEvent[] = [];
+      for (const event of existing) {
+        if (!FRONTMATTER_DERIVED_EVENT_TYPES[event.eventType]) {
+          appendOnly.push(event);
+          continue;
+        }
+        const key = contentKey(event);
+        if (reconstructedKeys.has(key)) continue; // collapsed into reconstruction
+        if (!racedFrontmatterByKey.has(key)) racedFrontmatterByKey.set(key, event);
+      }
+      const mergedById = new Map<string, MemoryLifecycleEvent>();
+      for (const event of sortMemoryLifecycleEvents([
+        ...events,
+        ...racedFrontmatterByKey.values(),
+        ...appendOnly,
+      ])) {
+        if (!mergedById.has(event.eventId)) mergedById.set(event.eventId, event);
+      }
+      finalEvents = [...mergedById.values()];
+      preservedAppendOnlyRows = finalEvents.length - events.length;
+      // Bound the rewritten ledger so a preserving compaction can never leave it
+      // over the whole-file read/decrypt cap (#2033). This must run whenever a
+      // cap is set — NOT only when `existing.length > 0`: an oversized on-disk
+      // ledger whose every row fails validation reads as `existing === []`, yet
+      // `finalEvents` (the frontmatter reconstruction) can itself exceed the cap.
+      // Overflow (oldest) rows are dropped from the active ledger but survive
+      // both in the timestamped backup (a verbatim copy when the ledger is
+      // plaintext, re-encrypted under the backup path's own AAD when it is
+      // encrypted so it still decrypts there, #2033) and, for frontmatter-derived
+      // rows, in the memory files they were reconstructed from — so a later full
+      // rebuild regenerates them.
+      if (options.maxLedgerBytes && options.maxLedgerBytes > 0) {
+        const bounded = boundLifecycleEventsToByteCap(finalEvents, options.maxLedgerBytes);
+        if (bounded.dropped > 0) {
+          finalEvents = bounded.kept;
+          archivedOverflowRows = bounded.dropped;
+          log.warn(
+            `lifecycle ledger rebuild bounded to ${options.maxLedgerBytes} bytes: `
+            + `dropped ${bounded.dropped} oldest row(s) from the active ledger `
+            + `(preserved in the backup) to stay under the read/decrypt cap.`,
+          );
+        }
+      }
+    }
+
+    // Re-assert the lock immediately before the destructive active-ledger
+    // rewrite. Everything above (the preserve read, merge, sort, byte-cap
+    // bound, and JSON serialize) is CPU-bound and blocks the event loop, so the
+    // timer heartbeat inside withHeldFileLock cannot refresh the lock's mtime;
+    // a peer that judged the lock stale within the 30s window could have broken
+    // it and appended to the current ledger. `lock.refresh()` (a) reports the
+    // lock LOST so we ABORT rather than rename our compacted file over that
+    // peer's append, and (b) when still held, re-stamps the mtime so the bounded
+    // write below cannot itself be judged stale mid-rename (issue #1910/#2033).
+    const abortIfLockLost = async (): Promise<void> => {
+      if (lock && await lock.refresh()) return;
+      throw new Error(
+        "lifecycle ledger rebuild aborted: lost the ledger lock during the compaction "
+        + "rewrite (a peer stale-broke it while the event loop was blocked by CPU-bound "
+        + "merge/serialize); refusing to clobber a concurrent append (issue #1910/#2033).",
+      );
+    };
+    if (!dryRun) {
+      const payload = finalEvents.map((event) => JSON.stringify(event)).join("\n");
+      const content = payload.length > 0 ? `${payload}\n` : "";
+      // No-op skip (#2033 thread PRRT_kwDORJXyws6SExst): a preserving rebuild
+      // bounds the ACTIVE ledger to the read/decrypt cap, so a VALID ledger that
+      // sits above the configured compaction trigger but below the cap would
+      // otherwise be rewritten to identical content and re-archived on every
+      // maintenance interval — unbounded growth of archive/memory-lifecycle-ledger
+      // plus pointless disk/CPU churn. Skip the backup+rewrite when it would
+      // change neither the logical (decrypted) content NOR the at-rest format.
+      // The raw read here doubles as the source for the encrypted verbatim backup
+      // below, so the current ledger is read at most once. An absent ledger reads
+      // as ENOENT (a create is always a real change) and any other read/probe
+      // error propagates rather than masking a needed rewrite.
+      let currentEncrypted = false;
+      let priorRawBuffer: Buffer | null = null;
+      try {
+        currentEncrypted = await probeEncryptedRegularFileHeader(outputPath);
+        priorRawBuffer = await storage.readMemoryLifecycleLedgerRawBufferForCompaction();
+      } catch (err) {
+        if (!isNodeError(err) || err.code !== "ENOENT") throw err;
+      }
+      // Preserve encryption at rest (#2033). An already-encrypted ledger (or its
+      // encrypted backup) MUST be rewritten encrypted even when the
+      // `secureStoreEncryptOnWrite` policy is paused — otherwise the compaction
+      // silently DOWNGRADES encrypted state to plaintext. Force encryption
+      // whenever the current ledger is encrypted and the store is unlocked (a
+      // key is available); reaching this point with an encrypted ledger already
+      // implies an unlocked store, since the raw read above decrypts and would
+      // have thrown under a locked key. A plaintext ledger keeps the ordinary
+      // policy (`willEncryptStateWrites`), so existing plaintext behavior holds.
+      const forceEncrypt =
+        secureRewrite && currentEncrypted && storage.isSecureStoreUnlocked();
+      const replacementEncrypted =
+        secureRewrite && (storage.willEncryptStateWrites() || forceEncrypt);
+      // Compare as buffers, never decoding the raw ledger to one giant string:
+      // a multi-hundred-MB decrypted ledger can exceed V8's max string length,
+      // so `priorRawBuffer.equals(...)` (byte compare) must be used, NOT
+      // `.toString()` (#2033 giant-string invariant). `content` is the freshly
+      // serialized replacement, already a string; Buffer.from() reuses it.
+      if (
+        priorRawBuffer !== null &&
+        currentEncrypted === replacementEncrypted &&
+        priorRawBuffer.equals(Buffer.from(content, "utf8"))
+      ) {
+        // Nothing to change: leave the active ledger and archive untouched.
+        rewritten = false;
+      } else {
+        const desiredBackup = await backupExistingLedger(options.memoryDir, outputPath, now);
+        if (secureRewrite) {
+          // Preserve encrypted-at-rest (issue #1910). The BACKUP needs care under
+          // secure-store (#2033): when the existing ledger is encrypted, its bytes
+          // are path-bound via AAD, so a byte-for-byte copy to the archive path
+          // cannot be decrypted there — it would silently orphan the overflow rows
+          // a bounded rewrite claims to preserve. Reuse the RAW decrypted ledger
+          // content read above (no event parsing, so malformed/truncated/future
+          // rows survive exactly as the plaintext verbatim copy keeps them) and
+          // re-encrypt it under the backup path's own AAD, yielding a directly
+          // decryptable backup that retains every original byte. A PLAINTEXT
+          // existing ledger is already readable, so a verbatim copy is correct.
+          // Then rewrite the active ledger through the secure writer.
+          if (desiredBackup && currentEncrypted) {
+            const rawForBackup =
+              priorRawBuffer ?? (await storage.readMemoryLifecycleLedgerRawBufferForCompaction());
+            await storage.writeMemoryLifecycleLedgerContent(rawForBackup, desiredBackup, forceEncrypt);
+            backupPath = desiredBackup;
+          } else {
+            backupPath = desiredBackup
+              ? await copyExistingFileToBackup(outputPath, desiredBackup)
+              : undefined;
+          }
+          await abortIfLockLost();
+          await storage.writeMemoryLifecycleLedgerContent(content, undefined, forceEncrypt);
+        } else {
+          await abortIfLockLost();
+          backupPath = await writeFileAtomically(outputPath, content, desiredBackup);
+        }
+        rewritten = true;
+      }
+    }
+  };
+  // Write mode holds the ledger lock across the preserve-read and atomic
+  // rewrite so a concurrent append lands on the compacted ledger instead of
+  // being clobbered. A dry run computes the SAME preserve/merge/byte-cap
+  // preview but WITHOUT the lock (read-only, no rewrite), so `rebuiltRows`,
+  // `preservedAppendOnlyRows`, and `archivedOverflowRows` report what a --write
+  // would apply while never blocking concurrent appends (#2033 Cursor). A dry
+  // run with nothing to preserve touches nothing at all.
   if (!dryRun) {
-    backupPath = await backupExistingLedger(options.memoryDir, outputPath, now);
-    const payload = events.map((event) => JSON.stringify(event)).join("\n");
-    backupPath = await writeFileAtomically(
-      outputPath,
-      payload.length > 0 ? `${payload}\n` : "",
-      backupPath,
+    await withHeldFileLock(
+      memoryLifecycleLedgerLockPath(outputPath),
+      { staleMs: MEMORY_LIFECYCLE_LEDGER_LOCK_STALE_MS, ...options.lockOptions },
+      runLedgerCriticalSection,
     );
+  } else if (options.preserveExistingEvents) {
+    await runLedgerCriticalSection(true, null);
   }
 
   return {
     dryRun,
     scannedMemories: allMemories.length,
-    rebuiltRows: events.length,
+    rebuiltRows: finalEvents.length,
     outputPath,
     backupPath,
     skippedBlankIdMemories,
     skippedDuplicateEvents,
+    preservedAppendOnlyRows,
+    archivedOverflowRows,
+    rewritten,
   };
 }

package/src/memory-cache.ts

@@ -284,6 +284,19 @@ export function setCachedQmdSearch(cacheKey: string, results: unknown[]): void {
   }
 }
 
+/**
+ * Mutation scopes (issue #1904). A cache layer declares (via `evictOn`) which
+ * of these mutation kinds evict it, so the invalidation chokepoint can clear
+ * only the layers a given write actually affects instead of nuking every
+ * layer on every write (the permanently-cold-cache regression #1904 fixes).
+ */
+export type MemoryMutationScope =
+  | "memory-create" // a new memory doc added; no existing doc's recall-visibility changes
+  | "memory-mutate" // existing doc changed/moved/removed/status-changed (update, supersede,
+  //   archive, invalidate, frontmatter, cold-tier change, bulk delete)
+  | "entity-write" // an entity file changed
+  | "secure-key-change"; // encryption key rotated; everything is now unreadable → evict all
+
 /**
  * Cache-layer registry (issue #1535).
  *
@@ -305,6 +318,11 @@ export interface MemoryCacheLayer {
    *  the simple-and-correct choice — these caches are re-warmable
    *  (issue #1535, implementation guide option (b)). */
   readonly scope: "dir" | "global";
+  /** Which mutation scopes evict this layer (issue #1904). A `memory-create`
+   *  only refreshes the full-memory-list views (hot + derived episode/rule); a
+   *  `memory-mutate`/`entity-write`/`secure-key-change` evicts every layer whose
+   *  content it can affect. Adding a layer REQUIRES declaring this. */
+  readonly evictOn: Partial<Record<MemoryMutationScope, true>>;
   /** Evict entries for baseDir ("dir" scope) or everything ("global" scope). */
   readonly invalidateForDir: (baseDir: string) => void;
   /** Evict every entry across all dirs. */
@@ -318,6 +336,7 @@ export const ALL_CACHE_LAYERS: readonly MemoryCacheLayer[] = [
   {
     name: "hot-memories",
     scope: "dir",
+    evictOn: { "memory-create": true, "memory-mutate": true, "secure-key-change": true },
     invalidateForDir: (baseDir) => void hotCacheByDir.delete(baseDir),
     clearAll: () => hotCacheByDir.clear(),
     hasEntriesFor: (baseDir) => hotCacheByDir.has(baseDir),
@@ -325,6 +344,7 @@ export const ALL_CACHE_LAYERS: readonly MemoryCacheLayer[] = [
   {
     name: "archive-memories",
     scope: "dir",
+    evictOn: { "memory-mutate": true, "secure-key-change": true },
     invalidateForDir: (baseDir) => void archiveCacheByDir.delete(baseDir),
     clearAll: () => archiveCacheByDir.clear(),
     hasEntriesFor: (baseDir) => archiveCacheByDir.has(baseDir),
@@ -332,6 +352,7 @@ export const ALL_CACHE_LAYERS: readonly MemoryCacheLayer[] = [
   {
     name: "entities",
     scope: "dir",
+    evictOn: { "entity-write": true, "secure-key-change": true },
     invalidateForDir: (baseDir) => invalidateCachedEntities(baseDir),
     clearAll: () => entityCacheByDir.clear(),
     hasEntriesFor: (baseDir) => {
@@ -345,6 +366,7 @@ export const ALL_CACHE_LAYERS: readonly MemoryCacheLayer[] = [
   {
     name: "derived-episode-map",
     scope: "dir",
+    evictOn: { "memory-create": true, "memory-mutate": true, "secure-key-change": true },
     invalidateForDir: (baseDir) => void episodeMapByDir.delete(baseDir),
     clearAll: () => episodeMapByDir.clear(),
     hasEntriesFor: (baseDir) => episodeMapByDir.has(baseDir),
@@ -352,6 +374,7 @@ export const ALL_CACHE_LAYERS: readonly MemoryCacheLayer[] = [
   {
     name: "derived-rule-memories",
     scope: "dir",
+    evictOn: { "memory-create": true, "memory-mutate": true, "secure-key-change": true },
     invalidateForDir: (baseDir) => void ruleMemoriesByDir.delete(baseDir),
     clearAll: () => ruleMemoriesByDir.clear(),
     hasEntriesFor: (baseDir) => ruleMemoriesByDir.has(baseDir),
@@ -359,6 +382,7 @@ export const ALL_CACHE_LAYERS: readonly MemoryCacheLayer[] = [
   {
     name: "qmd-search",
     scope: "global",
+    evictOn: { "memory-mutate": true, "entity-write": true, "secure-key-change": true },
     invalidateForDir: () => qmdSearchCache.clear(),
     clearAll: () => qmdSearchCache.clear(),
     hasEntriesFor: () => qmdSearchCache.size > 0,
@@ -366,6 +390,7 @@ export const ALL_CACHE_LAYERS: readonly MemoryCacheLayer[] = [
   {
     name: "qmd-recall",
     scope: "global",
+    evictOn: { "memory-mutate": true, "entity-write": true, "secure-key-change": true },
     invalidateForDir: () => clearQmdRecallCache(),
     clearAll: () => clearQmdRecallCache(),
     hasEntriesFor: () => qmdRecallCacheSize() > 0,
@@ -389,6 +414,37 @@ export function invalidateAllForDir(baseDir: string): void {
   }
 }
 
+/**
+ * Scope-aware invalidation chokepoint (issue #1904). Evicts only the layers
+ * whose `evictOn` includes `scope`, so the highest-frequency write
+ * (`memory-create`, fired continuously by extraction) stops nuking the QMD
+ * result caches and the version-keyed entity cache it cannot affect. This is
+ * the single scoped entry point — like `invalidateAllForDir`, it iterates the
+ * `ALL_CACHE_LAYERS` registry rather than clearing any layer ad hoc, so the
+ * single-chokepoint invariant (#1535) holds: a new layer only participates by
+ * declaring `evictOn`.
+ */
+export function invalidateForScope(baseDir: string, scope: MemoryMutationScope): void {
+  for (const layer of ALL_CACHE_LAYERS) {
+    if (layer.evictOn[scope]) layer.invalidateForDir(baseDir);
+  }
+}
+
+/**
+ * Companion to {@link invalidateForScope} for the single-file write fast path
+ * (issue #1902 + #1904): it patches the hot-memories entry in place and
+ * re-keys it, so this evicts every OTHER layer the scope would clear but leaves
+ * hot-memories alone. A `memory-create` thus evicts only the derived
+ * episode/rule views (hot patched, QMD/entities/archive kept warm), which is
+ * the whole point of #1904 — a create must not evict the QMD result caches.
+ */
+export function invalidateForScopeExceptHot(baseDir: string, scope: MemoryMutationScope): void {
+  for (const layer of ALL_CACHE_LAYERS) {
+    if (layer.name === "hot-memories") continue;
+    if (layer.evictOn[scope]) layer.invalidateForDir(baseDir);
+  }
+}
+
 /**
  * Invalidate every cache layer for `baseDir` EXCEPT the hot-memories layer
  * (issue #1902). Single-file write paths use this instead of
@@ -429,6 +485,20 @@ export function clearMemoryCache(baseDir?: string): void {
   }
 }
 
+/**
+ * Clear ONLY the two global QMD result caches (qmd-search + qmd-recall), leaving
+ * every dir-scoped layer (hot/archive/entities/derived) untouched. Called after
+ * a successful QMD maintenance update+embed (#1904, Codex): a newly-persisted
+ * fact becomes searchable at that moment, so any cached pre-index recall/search
+ * bundle is now stale and must not be served. This lets the create path keep the
+ * QMD caches warm during index lag (perf win) while guaranteeing invalidation
+ * the moment the fact is actually indexed — no extended stale window.
+ */
+export function clearQmdResultCaches(): void {
+  qmdSearchCache.clear();
+  clearQmdRecallCache();
+}
+
 export function getMemoryCacheStats(baseDir: string): {
   hotSize: number;
   archiveSize: number;