@remnic/core 9.69.33 → 9.69.35

package/src/storage/deletion-revision-store.ts

@@ -2,7 +2,7 @@ import { createHash, randomUUID } from "node:crypto";
 import { lstat, mkdir, open, readFile, rename, unlink, type FileHandle } from "node:fs/promises";
 import path from "node:path";
 import { resolveSafeStoragePath } from "../storage-paths.js";
-import type { MemoryFile } from "../types.js";
+import type { MemoryFile, MemoryFrontmatter } from "../types.js";
 import { isErrnoCode } from "../utils/errno.js";
 import { withHeldFileLock, type HeldFileLockController } from "../utils/serialize-mutations.js";
 import { log } from "../logger.js";
@@ -63,6 +63,80 @@ export function invalidationCommitFingerprint(memory: Pick<MemoryFile, "content"
     .digest("hex");
 }
 
+/**
+ * #2813 (P1): revision identity stamped onto an Error thrown by a
+ * compare-and-swap write AFTER the durable mutation landed (a post-commit
+ * step failed). Body equality cannot attribute a standing body — a
+ * concurrent writer's identical deterministic merge is byte-for-byte the
+ * same — so throw-path ownership must come from this receipt, never from
+ * comparing content afterward.
+ */
+export function markCasCommittedRevision(err: unknown, revision: string): void {
+  if (err instanceof Error) {
+    (err as Error & { casCommittedRevision?: string }).casCommittedRevision = revision;
+  }
+}
+
+export function casCommittedRevisionOf(err: unknown): string | undefined {
+  return err instanceof Error
+    ? (err as Error & { casCommittedRevision?: string }).casCommittedRevision
+    : undefined;
+}
+
+/**
+ * #2813 (P1, round 3): the CAS receipt mint — the token the next durable
+ * content write records in the per-target sidecar (CasRevisionStore). NOT a
+ * bare wall-clock read: the stamp is strictly greater than the target's
+ * previous token (max(clock, prev + 1ms)), so two commits to the same target
+ * can never share a receipt — not within one millisecond, not across a
+ * backward clock step. It is a per-target monotonic sequence wearing
+ * ISO-8601 clothes; public `frontmatter.updated` never carries it (#2807).
+ */
+export function nextCasRevisionIso(previous: string | undefined, now = new Date()): string {
+  const previousMs = previous !== undefined ? new Date(previous).getTime() : Number.NaN;
+  if (Number.isFinite(previousMs) && now.getTime() <= previousMs) {
+    return new Date(previousMs + 1).toISOString();
+  }
+  return now.toISOString();
+}
+
+/**
+ * #2813 P1 (#2807 CI repair): whether a durable frontmatter rewrite changed
+ * anything SEMANTIC. Comparison is STRUCTURAL, through the same
+ * canonicalizing fingerprint the invalidation proof uses — a `!==` field
+ * loop would flag every array-valued field (tags, lineage, sources) because
+ * two parses of identical bytes are never reference-equal. Access
+ * telemetry (`accessCount`/`lastAccessed` — exactly the fields the
+ * fingerprint strips) is not semantic: an access bump must not advance the
+ * target's CAS revision token, or it would invalidate every pending
+ * conditional write. Every other change — status flips, caller-supplied
+ * `updated`, provenance — is a semantic mutation and mints a new token at
+ * the write chokepoint, so a receipt issued before it can never
+ * "recognise" the record afterwards (#2807 round 5).
+ */
+export function isSemanticFrontmatterChange(
+  before: MemoryFrontmatter,
+  after: MemoryFrontmatter,
+): boolean {
+  return invalidationCommitFingerprint({ content: "", frontmatter: before })
+    !== invalidationCommitFingerprint({ content: "", frontmatter: after });
+}
+
+/**
+ * Runs a compare-and-swap write's post-commit steps. The durable mutation
+ * has already landed by the time `run` executes, so any throw it raises is
+ * stamped with the commit receipt — callers must attribute the standing
+ * body by that identity, never by body equality (#2813 P1).
+ */
+export async function withCasCommitReceipt<T>(revision: string, run: () => Promise<T>): Promise<T> {
+  try {
+    return await run();
+  } catch (err) {
+    markCasCommittedRevision(err, revision);
+    throw err;
+  }
+}
+
 function isValidInvalidationCommitTimestamp(value: unknown): value is number {
   return (
     typeof value === "number" &&
@@ -80,7 +154,7 @@ function isValidMemoryId(value: unknown): value is string {
   return typeof value === "string" && value.length > 0 && !value.includes("\0");
 }
 
-function isValidDeletionRevisionPath(value: unknown): value is string {
+export function isValidManagedStoragePath(value: unknown): value is string {
   return (
     typeof value === "string" &&
     value.length > 0 &&
@@ -167,7 +241,7 @@ export class DeletionRevisionStore {
       const entry = rawEntry as Record<string, unknown>;
       if (
         Object.keys(entry).sort().join(",") !== "mtimeMs,path" ||
-        !isValidDeletionRevisionPath(entry.path) ||
+        !isValidManagedStoragePath(entry.path) ||
         typeof entry.mtimeMs !== "number" ||
         !Number.isFinite(entry.mtimeMs) ||
         entry.mtimeMs < 0 ||
@@ -428,7 +502,7 @@ export class DeletionRevisionStore {
     ) {
       throw new Error("Deletion revision timestamp is invalid.");
     }
-    if (!isValidDeletionRevisionPath(relativePath)) {
+    if (!isValidManagedStoragePath(relativePath)) {
       throw new Error("Deletion revision path is invalid.");
     }
     await this.withDeletionRevisionLock(async (lock) => {
@@ -452,7 +526,7 @@ export class DeletionRevisionStore {
 
   async writeManagedStorageFile(filePath: string, write: () => Promise<void>): Promise<void> {
     const { relativePath } = await this.resolveManagedStoragePath(filePath, "storage.writeManagedStorageFile");
-    if (!isValidDeletionRevisionPath(relativePath)) {
+    if (!isValidManagedStoragePath(relativePath)) {
       throw new Error("Deletion revision path is invalid.");
     }
     await this.withDeletionRevisionLock(async (lock) => {
@@ -492,7 +566,7 @@ export class DeletionRevisionStore {
         if (isErrnoCode(error, "ENOENT")) return false;
         throw error;
       }
-      if (!isValidDeletionRevisionPath(relativePath)) {
+      if (!isValidManagedStoragePath(relativePath)) {
         throw new Error("Deletion revision path is invalid.");
       }
       const revision = deletionMtimeMs === null ? undefined : (deletionMtimeMs ?? Date.now());

package/src/storage/tombstone-blocked-capture-index.ts

@@ -533,6 +533,21 @@ export class TombstoneBlockedCaptureIndex {
     await this.failOpen("invalidateMemory", () => this.rebuildIfLoaded(ownedMarker));
   }
 
+  /**
+   * Drop an already-committed write marker without re-reading the corpus. The
+   * offline-sync mutation path commits its marker around the file write, then
+   * must clear it even when the write cannot have changed the blocked key set
+   * (neither side tombstone-blocked) — otherwise the marker lingers until some
+   * later rebuild clears it. Mirrors the tail of `add`/`sync`.
+   */
+  async clearCommittedWriteMarker(rebuildMarker: string): Promise<void> {
+    await this.clearRebuildRequired([rebuildMarker]);
+    // getIndex() may have rebuilt while this marker was still pending and
+    // left authoritative=false. After the marker is gone, restore authority
+    // if no other uncommitted marker remains.
+    await this.setAuthoritative(true);
+  }
+
   async syncUpdatedMemory(
     before: MemoryFile,
     frontmatter: MemoryFrontmatter,

package/src/storage/tombstone-blocked-capture-mutation.ts

@@ -4,6 +4,7 @@ import { mkdir } from "node:fs/promises";
 import path from "node:path";
 import { withHeldFileLock } from "../utils/serialize-mutations.js";
 import type { MemoryFile, MemoryFrontmatter } from "../types.js";
+import type { CasRevisionTransaction } from "./cas-revision-store.js";
 const TOMBSTONE_CAPTURE_WRITE_LOCK_STALE_MS = 60_000;
 const CAPTURE_WRITE_LOCK_MAX_ATTEMPTS = 3;
 const CAPTURE_WRITE_LOCK_RETRY_BASE_MS = 25;
@@ -196,6 +197,7 @@ export type TombstoneBlockedMutationHost = {
   writeStorageSecureFile: (pathname: string, fileContent: string) => Promise<void>;
   withCaptureWriteLock: (task: () => Promise<void>, identity: string | readonly string[]) => Promise<void>;
   logWarning: (message: string) => void;
+  beginDurableMemoryRevision?: (pathname: string, expectedContent?: string | Buffer | null) => Promise<CasRevisionTransaction | undefined>;
 };
 
 export type TombstoneBlockedMutation = {
@@ -206,15 +208,17 @@ export type TombstoneBlockedMutation = {
   updateIndex: (rebuildMarker?: string, current?: MemoryFile | null) => Promise<void>;
   beforeIndexUpdate?: () => Promise<void>;
   coordinate?: boolean;
+  shouldMintRevision?: (current: MemoryFile | null) => boolean;
 };
 
 export async function runTombstoneBlockedMutation(
   host: TombstoneBlockedMutationHost,
   mutation: TombstoneBlockedMutation
-): Promise<void> {
+): Promise<string | undefined> {
   let lockIdentity: string | readonly string[] = mutation.identity;
   for (;;) {
     let retryIdentity: string | undefined;
+    let mintedRevision: string | undefined;
     const mutate = async (): Promise<void> => {
       const current = await host.readCurrent();
       const currentBlocked = host.isBlocked(current);
@@ -228,19 +232,73 @@ export async function runTombstoneBlockedMutation(
         }
       }
       const rebuildMarker = mutation.blocked || currentBlocked ? await host.prepareWrite() : undefined;
+      const discardRebuildMarker = async (): Promise<void> => {
+        if (!rebuildMarker) return;
+        try {
+          await host.discardWrite(rebuildMarker);
+        } catch (cleanupError) {
+          host.markUntrusted();
+          host.logWarning(`storage.tombstoneBlocked failed to clear write marker: ${cleanupError}`);
+        }
+      };
+      let transaction: CasRevisionTransaction | undefined;
       try {
+        // #2813 (P1 C): a two-phase receipt transaction under this same
+        // path lock. The PENDING reservation is minted BEFORE the durable
+        // file publish, so a failed reservation leaves the memory file
+        // untouched — never a mutated record without a receipt. The
+        // COMMITTED receipt publishes only AFTER the write lands, so a
+        // failed write can never leave the reservation posing as
+        // ownership.
+        transaction = mutation.shouldMintRevision?.(current)
+          ? await host.beginDurableMemoryRevision?.(mutation.pathname, mutation.fileContent)
+          : undefined;
         await host.writeStorageSecureFile(mutation.pathname, mutation.fileContent);
       } catch (err) {
-        if (rebuildMarker) {
+        // The memory write failed (or never ran): abort the reservation so
+        // no pending marker survives a write that did not land. A failed
+        // abort keeps the marker — ambiguity, recovered later.
+        if (transaction) {
           try {
-            await host.discardWrite(rebuildMarker);
-          } catch (cleanupError) {
-            host.markUntrusted();
-            host.logWarning(`storage.tombstoneBlocked failed to clear write marker: ${cleanupError}`);
+            await transaction.abort();
+          } catch (abortError) {
+            host.logWarning(
+              `storage.tombstoneBlocked failed to abort the reserved receipt (pending marker preserved): ${abortError}`,
+            );
           }
         }
+        await discardRebuildMarker();
         throw err;
       }
+      if (transaction) {
+        // #2807: record the post-write fingerprint in the PENDING marker
+        // BEFORE publishing, so a crash between the durable write and the
+        // COMMITTED receipt is decisively recoverable. The evidence only
+        // guards that crash window — a recording failure must not fail the
+        // landed write, so commit still runs and publishes truthfully.
+        try {
+          await transaction.writeLanded();
+        } catch (evidenceError) {
+          host.logWarning(
+            `storage.tombstoneBlocked failed to record receipt crash evidence (pending marker preserved): ${evidenceError}`,
+          );
+        }
+        try {
+          await transaction.commit();
+        } catch (err) {
+          // #2813 (P1 C): the memory file LANDED but the COMMITTED receipt
+          // never published. The PENDING marker stays: readers must see
+          // unavailable/recovery-needed — never ownership, never absence —
+          // and promotion/rollback refuse until recovery reconciles. The
+          // reservation must NOT be aborted: the write is on disk.
+          host.logWarning(
+            `storage.tombstoneBlocked receipt publication failed after the memory write (pending marker preserved for recovery): ${err}`,
+          );
+          await discardRebuildMarker();
+          throw err;
+        }
+      }
+      mintedRevision = transaction?.pendingRevision;
       if (rebuildMarker) {
         try {
           await host.commitWrite(rebuildMarker);
@@ -252,13 +310,15 @@ export async function runTombstoneBlockedMutation(
       if (rebuildMarker) await mutation.beforeIndexUpdate?.();
       await mutation.updateIndex(rebuildMarker, currentBlocked && current !== null ? current : undefined);
     };
+
     await host.withCaptureWriteLock(mutate, lockIdentity);
-    if (retryIdentity === undefined) return;
+    if (retryIdentity === undefined) return mintedRevision;
     const identities = Array.isArray(lockIdentity) ? [...lockIdentity, retryIdentity] : [lockIdentity, retryIdentity];
     lockIdentity = [...new Set(identities)];
   }
 }
 
+
 export function tombstoneBlocked(frontmatter: MemoryFrontmatter): boolean {
   return frontmatter.status === "pending_review" && Boolean(frontmatter.blockedBy);
 }

package/src/storage/tombstone-blocked-capture-sync.ts

@@ -5,13 +5,12 @@ import { isDeepStrictEqual } from "node:util";
 import { log } from "../logger.js";
 import type { MemoryFile, MemoryFrontmatter } from "../types.js";
 import { markProjectedMemoryPathInvalid } from "../memory-projection-store.js";
-import { RECALL_FALLBACK_DIRS } from "../utils/category-dir.js";
-import { isErrnoCode } from "../utils/errno.js";
 import { pathMayCarryEntityRefs, requestEntityCanonicalIdReconcile } from "./entity-canonical-id-references.js";
 import { withRawEntityPageMutation } from "./entity-canonical-id-lock.js";
 import { readMaybeEncryptedFileFromChunks, writeMaybeEncryptedFileFromChunks } from "../secure-store/secure-fs.js";
 import * as archiveMutation from "../archive-mutation-version.js";
-import { invalidationCommitFingerprint } from "./deletion-revision-store.js";
+import { invalidationCommitFingerprint, isSemanticFrontmatterChange } from "./deletion-revision-store.js";
+import type { CasRevisionTransaction } from "./cas-revision-store.js";
 import {
   buildExplicitCaptureDedupKey,
   buildCapturePathLockIdentity,
@@ -212,6 +211,16 @@ export abstract class TombstoneBlockedCaptureIndexHost {
   protected abstract deleteManagedStorageFile(filePath: string, deletionMtimeMs?: number | null): Promise<boolean>;
   protected abstract writeManagedStorageFile(filePath: string, write: () => Promise<void>): Promise<void>;
 
+  /** #2813 (P1 C, #2807): reserve the target's next durable CAS revision
+   * token as a two-phase receipt transaction — a PENDING write-ahead
+   * marker under the mutation's path lock, published COMMITTED only after
+   * the durable file write lands. Receipt identity lives in the per-target
+   * sidecar, never in public `frontmatter.updated`. The default host keeps
+   * no receipt identity; StorageManager records it. */
+  protected async beginDurableMemoryRevision(_pathname: string, _expectedContent?: string | Buffer | null): Promise<CasRevisionTransaction | undefined> {
+    return undefined;
+  }
+
   private async writeStorageSecureFileChunks(filePath: string, chunks: AsyncIterable<Buffer>): Promise<void> {
     const options = this.tombstoneBlockedCaptureIndexOptions();
     await writeMaybeEncryptedFileFromChunks(
@@ -244,7 +253,7 @@ export abstract class TombstoneBlockedCaptureIndexHost {
 
   async withMemorySnapshotsIfUnchanged<T>(
     expected: readonly MemoryFile[],
-    task: (memories: MemoryFile[]) => Promise<T>,
+    task: (memories: MemoryFile[]) => Promise<T>
   ): Promise<T | null> {
     if (expected.length === 0) return await task([]);
     const pathnames = [...new Set(expected.map((memory) => memory.path))];
@@ -282,8 +291,9 @@ export abstract class TombstoneBlockedCaptureIndexHost {
     identity: string | readonly string[],
     updateIndex: (rebuildMarker?: string, current?: MemoryFile | null) => Promise<void>,
     beforeIndexUpdate?: () => Promise<void>,
-    coordinate = false
-  ): Promise<void> {
+    coordinate = false,
+    shouldMintRevision?: (current: MemoryFile | null) => boolean
+  ): Promise<string | undefined> {
     const host: TombstoneBlockedMutationHost = {
       readCurrent: () => this.readMemoryByPath(pathname),
       isBlocked: (memory) => this.isTombstoneBlockedMemory(memory),
@@ -296,8 +306,10 @@ export abstract class TombstoneBlockedCaptureIndexHost {
       writeStorageSecureFile: (target, content) => this.writeStorageSecureFile(target, content),
       withCaptureWriteLock: (task, lockIdentity) => this.withTombstoneBlockedCaptureWriteLock(task, lockIdentity),
       logWarning: (message) => log.warn(message),
+      beginDurableMemoryRevision: (targetPath, expectedContent) =>
+        this.beginDurableMemoryRevision(targetPath, expectedContent),
     };
-    await runTombstoneBlockedMutation(host, {
+    return await runTombstoneBlockedMutation(host, {
       blocked,
       pathname,
       fileContent,
@@ -305,6 +317,7 @@ export abstract class TombstoneBlockedCaptureIndexHost {
       updateIndex,
       beforeIndexUpdate,
       coordinate,
+      shouldMintRevision,
     });
   }
 
@@ -400,8 +413,8 @@ export abstract class TombstoneBlockedCaptureIndexHost {
     frontmatter: MemoryFrontmatter,
     content: string,
     beforeIndexUpdate?: () => Promise<void>
-  ): Promise<void> {
-    await this.writeTombstoneBlockedMutation(
+  ): Promise<string | undefined> {
+    return await this.writeTombstoneBlockedMutation(
       tombstoneBlocked(before.frontmatter) || tombstoneBlocked(frontmatter),
       before.path,
       fileContent,
@@ -418,7 +431,8 @@ export abstract class TombstoneBlockedCaptureIndexHost {
           rebuildMarker
         ),
       beforeIndexUpdate,
-      true
+      true,
+      () => true
     );
   }
 
@@ -427,8 +441,8 @@ export abstract class TombstoneBlockedCaptureIndexHost {
     fileContent: string,
     frontmatter: MemoryFrontmatter,
     beforeIndexUpdate?: () => Promise<void>
-  ): Promise<void> {
-    await this.writeTombstoneBlockedMutation(
+  ): Promise<string | undefined> {
+    return await this.writeTombstoneBlockedMutation(
       tombstoneBlocked(before.frontmatter) || tombstoneBlocked(frontmatter),
       before.path,
       fileContent,
@@ -440,10 +454,12 @@ export abstract class TombstoneBlockedCaptureIndexHost {
       (rebuildMarker, current) =>
         this.getTombstoneBlockedCaptureIndex().syncUpdatedFrontmatter(current ?? before, frontmatter, rebuildMarker),
       beforeIndexUpdate,
-      true
+      true,
+      (current) => isSemanticFrontmatterChange((current ?? before).frontmatter, frontmatter)
     );
   }
 
+
   private isTombstoneBlockedMemory(memory: MemoryFile | null): memory is MemoryFile {
     return memory !== null && memory.frontmatter.status === "pending_review" && Boolean(memory.frontmatter.blockedBy);
   }
@@ -490,7 +506,8 @@ export abstract class TombstoneBlockedCaptureIndexHost {
   private async invalidateAfterOfflineSyncMutation(
     filePath: string,
     ownedMarker?: string,
-    archiveChanged = true
+    archiveChanged = true,
+    blockedKeySetMayHaveChanged = true
   ): Promise<void> {
     this.invalidateAllMemoriesCache();
     this.invalidateKnowledgeIndexCache();
@@ -498,7 +515,22 @@ export abstract class TombstoneBlockedCaptureIndexHost {
     if (filePath.includes(`${path.sep}cold${path.sep}`)) {
       this.invalidateColdMemoriesCache();
     }
-    await this.rebuildTombstoneBlockedCaptureAfterInvalidationForPath(filePath, ownedMarker);
+    if (blockedKeySetMayHaveChanged) {
+      await this.rebuildTombstoneBlockedCaptureAfterInvalidationForPath(filePath, ownedMarker);
+    } else if (ownedMarker !== undefined) {
+      // The index only holds tombstone-blocked explicit-capture keys, so a
+      // write where neither side is blocked cannot have changed it. Rebuilding
+      // here re-reads the whole corpus per replicated file — measured 15-31s
+      // per write against a ~190k-file corpus, which turns a boot-scale
+      // `converge apply` into a multi-week run. Clear the committed marker and
+      // keep the index as-is.
+      try {
+        await this.getTombstoneBlockedCaptureIndex().clearCommittedWriteMarker(ownedMarker);
+      } catch (err) {
+        this.getTombstoneBlockedCaptureIndex().markUntrusted();
+        log.warn(`storage.offlineSyncFile committed write succeeded; marker cleanup failed: ${err}`);
+      }
+    }
     if (filePath.includes(`${path.sep}artifacts${path.sep}`)) {
       this.bumpArtifactWriteVersion();
     }
@@ -675,7 +707,12 @@ export abstract class TombstoneBlockedCaptureIndexHost {
               log.warn(`storage.offlineSyncFile committed write marker failed: ${err}`);
             }
           }
-          await this.invalidateAfterOfflineSyncMutation(target, marker, changed !== false);
+          await this.invalidateAfterOfflineSyncMutation(
+            target,
+            marker,
+            changed !== false,
+            this.isTombstoneBlockedMemory(current) || this.isTombstoneBlockedMemory(after)
+          );
         } catch (err) {
           if (marker && !durable) {
             try {
@@ -704,10 +741,8 @@ export abstract class TombstoneBlockedCaptureIndexHost {
     const options = this.tombstoneBlockedCaptureIndexOptions();
     const after = options.parseMemory?.(target, content) ?? null;
     await withRawEntityPageMutation(path.dirname(options.stateDir), target, async () => {
-      await this.runTombstoneBlockedOfflineSyncMutation(
-        target,
-        after,
-        () => this.writeManagedStorageFile(target, () => this.writeStorageSecureFile(target, content))
+      await this.runTombstoneBlockedOfflineSyncMutation(target, after, () =>
+        this.writeManagedStorageFile(target, () => this.writeStorageSecureFile(target, content))
       );
     });
   }

package/src/storage-contract/failure-semantics.test.ts

@@ -147,7 +147,7 @@ test("failure-semantics: selected memory snapshot tasks hold every member lock",
 
     releaseTask.resolve();
     assert.deepEqual(await snapshotTask, [first.id, second.id]);
-    assert.equal(await writePromise, true);
+    assert.ok(await writePromise);
   } finally {
     releaseTask.resolve();
     await cleanup();