@remnic/core 9.29.1 → 9.30.1

package/src/testing/subjects/tombstone-blocked-capture.test.ts

@@ -0,0 +1,181 @@
+/**
+ * Tombstone-blocked-capture lifecycle subject for the scenario-matrix harness
+ * (issue #1993). Every row drives the real durable targeted index so coverage
+ * cannot be satisfied by mapping this stateful path to an unrelated subject.
+ */
+
+import assert from "node:assert/strict";
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+
+import type { MemoryFile } from "../../types.js";
+import { StorageManager } from "../../storage.js";
+import { ContentHashIndex } from "../../storage/content-hash-index.js";
+import { buildExplicitCaptureDedupKey } from "../../storage/tombstone-blocked-capture-index.js";
+import { type LifecycleSubject, type MatrixRow, runLifecycleMatrix } from "../lifecycle-matrix.js";
+import { cleanupDir, mkTempMemoryDir } from "../orchestrator-lite.js";
+
+interface TombstoneLifecycleState {
+  readonly memoryDir: string;
+  readonly stateDir: string;
+  storage: StorageManager;
+}
+
+async function readMemory(storage: StorageManager, id: string): Promise<MemoryFile> {
+  const memory = (await storage.readAllMemories()).find((entry) => entry.frontmatter.id === id);
+  assert.ok(memory, `expected persisted lifecycle memory ${id}`);
+  return memory;
+}
+
+async function addBlocked(
+  state: TombstoneLifecycleState,
+  content: string,
+  sourceConnector: string | undefined
+): Promise<MemoryFile> {
+  const result = await state.storage.writeMemory("fact", content, {
+    source: "explicit-inline-review",
+    confidence: 0.2,
+    tags: ["review"],
+    status: "pending_review",
+    contentHashSource: content,
+    ...(sourceConnector === undefined ? {} : { sourceConnector }),
+  });
+  const pending = await readMemory(state.storage, result.id);
+  await state.storage.writeMemoryFrontmatter(pending, {
+    status: "pending_review",
+    blockedBy: "tombstone-1",
+    tombstoneBlockTier: "exact",
+  });
+  return await readMemory(state.storage, result.id);
+}
+
+async function assertMembership(
+  state: TombstoneLifecycleState,
+  content: string,
+  sourceConnector: string | undefined,
+  expected: boolean
+): Promise<void> {
+  const result = await state.storage.checkTombstoneBlockedExplicitCapture(content, "fact", sourceConnector);
+  assert.equal(result.has, expected);
+  assert.equal(result.authoritative, true, "a published index must answer authoritatively");
+}
+
+const subject: LifecycleSubject<TombstoneLifecycleState> = {
+  async setup(row: MatrixRow): Promise<TombstoneLifecycleState> {
+    const memoryDir = await mkTempMemoryDir(`tombstone-${row.id}`);
+    try {
+      const stateDir = path.join(memoryDir, "state");
+      return {
+        memoryDir,
+        stateDir,
+        storage: new StorageManager(memoryDir),
+      };
+    } catch (err) {
+      await cleanupDir(memoryDir);
+      throw err;
+    }
+  },
+  async exercise(state: TombstoneLifecycleState, row: MatrixRow): Promise<void> {
+    const content = `The tombstone lifecycle row is ${row.id}.`;
+    switch (row.id) {
+      case "explicit-provider-identity":
+        await addBlocked(state, content, "openclaw");
+        return;
+      case "sparse-metadata-with-binding":
+        await addBlocked(state, content, undefined);
+        return;
+      case "sparse-metadata-without-binding":
+        await addBlocked(state, content, "remembered-provider");
+        return;
+      case "provider-rebinding": {
+        const before = await addBlocked(state, content, "provider-a");
+        await state.storage.writeMemoryFrontmatter(before, {
+          sourceConnector: "provider-b",
+        });
+        return;
+      }
+      case "restart-reload-recovery": {
+        await addBlocked(state, content, "openclaw");
+        state.storage = new StorageManager(state.memoryDir);
+        return;
+      }
+      case "compaction-flush":
+        await addBlocked(state, `${content} First.`, "openclaw");
+        await addBlocked(state, `${content} Second.`, "openclaw");
+        return;
+      case "before-reset": {
+        const before = await addBlocked(state, content, "openclaw");
+        await state.storage.writeMemoryFrontmatter(before, {
+          status: "active",
+          blockedBy: undefined,
+          tombstoneBlockTier: undefined,
+        });
+        return;
+      }
+      case "session-end":
+        await addBlocked(state, content, "openclaw");
+        await state.storage.checkTombstoneBlockedExplicitCapture(content, "fact", "openclaw");
+        return;
+      case "dedupe-replay":
+        await addBlocked(state, content, "openclaw");
+        await addBlocked(state, content, "openclaw");
+        return;
+      default: {
+        const exhaustive: never = row.id;
+        throw new Error(`unhandled row ${String(exhaustive)}`);
+      }
+    }
+  },
+
+  async invariants(state: TombstoneLifecycleState, row: MatrixRow): Promise<void> {
+    const content = `The tombstone lifecycle row is ${row.id}.`;
+    switch (row.id) {
+      case "explicit-provider-identity":
+      case "sparse-metadata-with-binding":
+      case "restart-reload-recovery":
+      case "session-end": {
+        const sourceConnector = row.id === "sparse-metadata-with-binding" ? undefined : "openclaw";
+        await assertMembership(state, content, sourceConnector, true);
+        return;
+      }
+      case "sparse-metadata-without-binding":
+        await assertMembership(state, content, undefined, false);
+        await assertMembership(state, content, "remembered-provider", true);
+        return;
+      case "provider-rebinding":
+        await assertMembership(state, content, "provider-a", false);
+        await assertMembership(state, content, "provider-b", true);
+        return;
+      case "compaction-flush":
+        await assertMembership(state, `${content} First.`, "openclaw", true);
+        await assertMembership(state, `${content} Second.`, "openclaw", true);
+        return;
+      case "before-reset":
+        await assertMembership(state, content, "openclaw", false);
+        return;
+      case "dedupe-replay": {
+        await assertMembership(state, content, "openclaw", true);
+        const raw = await readFile(path.join(state.stateDir, "tombstone-blocked-capture", "fact-hashes.txt"), "utf8");
+        const keyHash = ContentHashIndex.computeHash(buildExplicitCaptureDedupKey(content, "fact", "openclaw"));
+        assert.equal(
+          raw
+            .trim()
+            .split("\n")
+            .filter((line) => line === keyHash).length,
+          1
+        );
+        return;
+      }
+      default: {
+        const exhaustive: never = row.id;
+        throw new Error(`unhandled row ${String(exhaustive)}`);
+      }
+    }
+  },
+
+  async teardown(state: TombstoneLifecycleState): Promise<void> {
+    await cleanupDir(state.memoryDir);
+  },
+};
+
+runLifecycleMatrix("tombstone-blocked-capture", subject);

package/src/utils/serialize-mutations.test.ts

@@ -851,17 +851,20 @@ test("withHeldFileLock runs task(false) best-effort when the lock directory cann
     const lockPath = path.join(blocker, "nested.lock");
 
     let acquired = true; // expect false
+    let failure: "timeout" | "error" | undefined;
     let ran = false;
     await withHeldFileLock(
       lockPath,
       { staleMs: 5_000, maxWaitMs: 50 },
-      async (a) => {
+      async (a, lock) => {
         acquired = a;
+        failure = lock.failure;
         ran = true;
       },
     );
     assert.equal(ran, true, "task ran despite lock-dir setup failure");
     assert.equal(acquired, false, "task ran best-effort (acquired=false) without the lock");
+    assert.equal(failure, "error", "lock setup failures must be distinguishable from contention timeouts");
   } finally {
     await rm(dir, { recursive: true, force: true });
   }

package/src/utils/serialize-mutations.ts

@@ -230,6 +230,12 @@ export interface HeldFileLockController {
    * `false`) this always resolves `false` — there is no lock to hold.
    */
   refresh(): Promise<boolean>;
+  /**
+   * Why the lock was not acquired. `timeout` means a busy peer consumed the
+   * bounded wait; `error` means acquisition failed due to filesystem setup or
+   * I/O and must not be retried as contention.
+   */
+  readonly failure?: "timeout" | "error";
 }
 
 /** Default bounded acquisition wait, mirroring the catalog. */
@@ -249,6 +255,7 @@ interface HeldLock {
   readonly path: string;
   readonly ownerId: string;
 }
+type LockAcquisitionFailure = { readonly failure: "timeout" | "error" };
 
 /**
  * Run `task` under an exclusive on-disk lock at `lockPath`.
@@ -264,13 +271,11 @@ interface HeldLock {
  * id / timestamp, so its content differs and is left untouched.
  *
  * `task` receives `acquired: boolean` — `true` when we hold the lock, `false`
- * when acquisition timed out (best-effort). The signature takes
- * `(acquired) => Promise<T>` rather than the issue's sketched `() => Promise<T>`
- * so this can be the SINGLE lock home (issue: "do NOT leave two lock
- * implementations; pick one home"): the catalog's touch path needs to DROP on
- * timeout, which requires knowing whether the lock was acquired. A caller that
- * ignores the flag is still assignable (`() => Promise<T>` ⊆
- * `(acquired: boolean) => Promise<T>` in TypeScript).
+ * when acquisition is unavailable. The controller's `failure` field
+ * distinguishes a bounded busy-lock timeout from a permanent filesystem
+ * acquisition error.
+ * This explicit callback keeps the lock utility as the single lock home while
+ * allowing callers to choose their timeout and failure policy.
  *
  * Release is ownership-checked: we only `unlink` a lock whose content still
  * identifies THIS acquirer (same owner id), so a replacement created after we
@@ -344,13 +349,11 @@ export async function withHeldFileLock<T>(
   const ownerId = randomUUID();
   const lockDir = path.dirname(lockPath);
 
-  const held = await acquireLock(lockPath, lockDir, ownerId, opts, maxWaitMs, pollMs);
-  if (!held) {
-    // Best-effort: run the task WITHOUT the lock. The caller decides what to
-    // do (the catalog touch path will drop its append); we never crash the
-    // primary op on contention.
-    return task(false, { refresh: async () => false });
+  const acquisition = await acquireLock(lockPath, lockDir, ownerId, opts, maxWaitMs, pollMs);
+  if ("failure" in acquisition) {
+    return task(false, { refresh: async () => false, failure: acquisition.failure });
   }
+  const held = acquisition;
 
   // Heartbeat: while WE hold the lock, refresh its mtime so age-based stale
   // detection sees an active holder and does not break us out from under
@@ -437,10 +440,9 @@ function formatInvalidNumber(value: unknown): string {
 }
 
 /**
- * Atomically create the lock file, looping until acquired/stale-broken/timeout.
- * Returns the held-lock handle on success, or `undefined` on bounded-timeout.
- * Unexpected FS errors proceed best-effort (return undefined) rather than
- * crashing the guarded op, matching the catalog.
+ * Atomically create the lock file, looping until acquired or the bounded
+ * deadline. Reports `timeout` for a busy lock and `error` for permanent
+ * filesystem setup/I/O failures so callers can choose the safe policy.
  */
 async function acquireLock(
   lockPath: string,
@@ -449,15 +451,12 @@ async function acquireLock(
   opts: HeldFileLockOptions,
   maxWaitMs: number,
   pollMs: number,
-): Promise<HeldLock | undefined> {
+): Promise<HeldLock | LockAcquisitionFailure> {
   try {
     await mkdir(lockDir, { recursive: true });
   } catch {
-    // Lock-directory setup failure (e.g. an intermediate path is a file, or
-    // permissions deny mkdir) must NOT crash the guarded op — the advisory
-    // lock contract is best-effort. Return undefined so task(false) runs
-    // instead of rejecting (codex P2 review).
-    return undefined;
+    // Lock-directory setup failure is a permanent acquisition error.
+    return { failure: "error" };
   }
   const deadline = Date.now() + maxWaitMs;
   for (;;) {
@@ -486,18 +485,18 @@ async function acquireLock(
       }
       if (!wroteMeta) {
         await unlink(lockPath).catch(() => undefined);
-        return undefined;
+        return { failure: "error" };
       }
       return { path: lockPath, ownerId };
     } catch (err) {
       if ((err as NodeJS.ErrnoException | undefined)?.code !== "EEXIST") {
-        // Unexpected FS error — proceed best-effort without the lock.
-        return undefined;
+        // Unexpected filesystem errors are permanent acquisition failures.
+        return { failure: "error" };
       }
       // Lock exists: break it if stale, then poll. breakStaleLock is
       // replacement-safe (NG7Bg) and never throws.
       await breakStaleLock(lockPath, opts.staleMs, opts.onBeforeBreakStaleUnlinkForTest);
-      if (Date.now() >= deadline) return undefined;
+      if (Date.now() >= deadline) return { failure: "timeout" };
       // Cap the sleep to the remaining budget so a large pollMs cannot block
       // acquisition far past maxWaitMs (e.g. maxWaitMs=1000, pollMs=60000
       // would otherwise block ~60s instead of 1s — codex P2).