@remnic/core 9.3.700 → 9.3.701

package/src/namespaces/storage.ts

@@ -6,6 +6,7 @@ import type { PluginConfig } from "../types.js";
 import { ALL_CATEGORY_DIRS } from "../utils/category-dir.js";
 import { namespaceIdentityToken, normalizeNamespaceIdentity } from "./identity.js";
 import type { NamespaceCatalog } from "./catalog.js";
+import { MutationSerializer } from "../utils/serialize-mutations.js";
 
 async function exists(p: string): Promise<boolean> {
   try {
@@ -209,21 +210,22 @@ export class NamespaceStorageRouter {
   // rebuilds. We fire the hook only when the (namespace, storageDir) pair is new
   // or its dir changed, so a steady-state cache hit is a no-op for the catalog.
   private readonly notifiedResolved = new Map<string, string>();
-  // In-flight resolve-hook dedup (NFJV-, codex P2). The catalog's `onResolve`
-  // hook is ASYNC (it returns `registerResolved(...)`), so `notifiedResolved` is
-  // only set after the hook's promise SETTLES. Without tracking the in-flight
-  // window, a burst of `storageFor()` cache hits for the SAME namespace before
-  // the first registration finishes would each pass the `notifiedResolved` guard
-  // and fire their OWN `onResolve` — queueing N duplicate catalog touches + lock
-  // acquisitions despite the once-per-namespace intent. We therefore record the
-  // (namespace → storageDir) being registered BEFORE awaiting the hook so a
-  // concurrent call for the same pair skips firing. On SUCCESS the pair is
-  // promoted to `notifiedResolved` (future calls skip permanently); on `false`
-  // (dropped touch — e.g. rebuild-lock timeout) OR rejection the in-flight marker
-  // is CLEARED so a later `storageFor()` can RETRY the dropped registration. The
-  // entry is always removed when the promise settles, so the map cannot grow
-  // unbounded (one transient entry per concurrently-resolving namespace).
-  private readonly inFlightResolved = new Map<string, string>();
+  // Instance-scoped serializer for the resolve hook (issue #1524 adoption).
+  // Replaces the bespoke `inFlightResolved` marker-then-clear pattern: the
+  // serializer's per-key chain strictly orders concurrent notifications for the
+  // SAME namespace, and the task re-checks `notifiedResolved` once it runs, so a
+  // burst of cache hits before the first hook settles collapses to a single
+  // hook invocation (the once-per-namespace intent). Recovery is preserved —
+  // one rejected hook never poisons subsequent notifications for that key.
+  private readonly resolveSerializer = new MutationSerializer();
+  // (namespace, storageDir) pairs whose resolve hook is currently pending. Set SYNCHRONOUSLY in
+  // notifyResolved (before queueing) so a burst of concurrent storageFor()
+  // cache hits collapses onto the one in-flight hook instead of each enqueuing
+  // its own task. Cleared when the queued hook settles. Without this, a dropped
+  // hook (returns false on a rebuild-lock timeout) leaves notifiedResolved
+  // unset, so every queued sibling task would re-run the hook — N serial lock
+  // waits (cursor Medium 06f58a7c, codex P2).
+  private readonly inFlightResolveHooks = new Set<string>();
   // Tracks every in-flight resolve-hook promise so callers can deterministically
   // await the fire-and-forget registrations that `storageFor()` kicks off (see
   // `whenResolveHooksSettled`). Entries are removed as each hook settles, so the
@@ -308,77 +310,82 @@ export class NamespaceStorageRouter {
   /**
    * Fire the resolve hook defensively. A hook failure (e.g. a catalog write
    * error) MUST NOT crash storage resolution — see CLAUDE.md gotcha #13.
+   *
+   * Issue #1524 adoption: hook invocations for the SAME namespace are now
+   * strictly ordered through the shared `MutationSerializer` rather than the
+   * bespoke `inFlightResolved` marker-then-clear pattern. The serializer
+   * guarantees one in-flight hook per namespace; a concurrent burst of
+   * `storageFor()` cache hits collapses to a single hook invocation because
+   * each queued task re-checks `notifiedResolved` before invoking the hook. A
+   * dropped (`false`) or rejected hook leaves `notifiedResolved` unset so the
+   * next `storageFor()` retries — the serializer recovers from the rejection,
+   * so a failed hook never poisons later notifications.
    */
   private notifyResolved(namespace: string, storageDir: string): void {
     const hook = this.hooks.onResolve;
     if (!hook) return;
-    // Skip when we've already SUCCESSFULLY notified this exact (namespace,
-    // storageDir) — a steady-state cache hit must not re-append to the catalog
-    // log (NCNL2). A changed dir (rare: migration/realignment) still re-fires
-    // once. We mark the pair as notified ONLY AFTER the hook succeeds, and CLEAR
-    // it on failure, so a dropped registration (e.g. rebuild-lock timeout) is
-    // RETRIED on the next cache hit instead of being suppressed forever (round 6,
-    // cursor Medium — ND3EJ).
+    // Permanent dedup: skip once we've SUCCESSFULLY notified this exact
+    // (namespace, storageDir). A changed dir (rare: migration/realignment)
+    // still re-fires once. The mark is set ONLY AFTER the hook succeeds, so a
+    // dropped registration (e.g. rebuild-lock timeout) is RETRIED on the next
+    // cache hit instead of being suppressed forever (round 6, cursor Medium —
+    // ND3EJ).
     if (this.notifiedResolved.get(namespace) === storageDir) return;
-    // In-flight dedup (NFJV-, codex P2): if a registration for this exact
-    // (namespace, storageDir) is already AWAITING its async hook, do not fire a
-    // second one. Without this, concurrent cache-hit bursts before the first
-    // append settles each pass the `notifiedResolved` guard above and queue
-    // duplicate catalog touches/lock acquisitions. A pair with a DIFFERENT
-    // in-flight dir (rare mid-migration realignment) still fires once.
-    if (this.inFlightResolved.get(namespace) === storageDir) return;
-    try {
-      // Handle BOTH synchronous throws and asynchronous rejections (round 6,
-      // codex P2 — NDo8C). The hook may be `async`; its rejected promise would
-      // bypass this try/catch and, where unhandled rejections are fatal, crash
-      // storage resolution. Mark the dedup pair as notified ONLY when the hook
-      // resolves to a PERSISTED result (round 6, codex P2 — NEFoX): a result of
-      // `false` means the registration was dropped/no-op (e.g. rebuild-lock
-      // timeout), so we must NOT suppress its retry. `void`/`undefined` is treated
-      // as success for legacy hooks. On rejection we leave it un-notified to retry.
-      //
-      // Record the in-flight marker BEFORE awaiting so concurrent calls for the
-      // same pair skip (NFJV-). It is always cleared once the promise settles, so
-      // the map holds at most one transient entry per concurrently-resolving
-      // namespace and cannot grow unbounded.
-      this.inFlightResolved.set(namespace, storageDir);
-      const hookResult = Promise.resolve(hook(namespace, storageDir));
-      // Track the in-flight promise so `whenResolveHooksSettled()` can await it.
-      this.pendingResolveHooks.add(hookResult);
-      hookResult.then(
-        (persisted) => {
-          // Clear the in-flight marker ONLY if it is still ours (a newer resolve
-          // for a different dir may have replaced it).
-          if (this.inFlightResolved.get(namespace) === storageDir) {
-            this.inFlightResolved.delete(namespace);
-          }
-          if (persisted !== false) {
-            this.notifiedResolved.set(namespace, storageDir);
-          }
-          // On `false` (dropped touch) we intentionally do NOT mark notified, so
-          // a later `storageFor()` retries the registration. Clearing the
-          // in-flight marker above is what re-enables that retry.
-          this.pendingResolveHooks.delete(hookResult);
-        },
-        () => {
-          // Registration failed — clear in-flight AND do NOT mark as notified, so
-          // it is retried on the next cache hit.
-          if (this.inFlightResolved.get(namespace) === storageDir) {
-            this.inFlightResolved.delete(namespace);
-          }
-          if (this.notifiedResolved.get(namespace) === storageDir) {
-            this.notifiedResolved.delete(namespace);
-          }
-          this.pendingResolveHooks.delete(hookResult);
-        },
-      );
-    } catch {
-      // Synchronous throw: clear any in-flight marker we just set and leave the
-      // pair un-notified so a later resolve retries.
-      if (this.inFlightResolved.get(namespace) === storageDir) {
-        this.inFlightResolved.delete(namespace);
+    // In-flight dedup (cursor Medium 06f58a7c, codex P2): if a hook for THIS
+    // namespace is already pending, collapse this call onto it instead of
+    // enqueueing another task. The serializer strictly orders queued tasks, but
+    // ordering alone is not enough — when the first hook returns `false`
+    // (dropped touch, e.g. rebuild-lock timeout), `notifiedResolved` stays
+    // unset, so each queued sibling task would pass its re-check and re-run the
+    // hook. With the real catalog hook each retry can spend the full lock wait,
+    // so N cache hits during a rebuild leave N serial background lock attempts.
+    // Collapsing here means at most ONE hook runs per in-flight registration;
+    // its result (set `notifiedResolved` on success, leave unset on drop)
+    // decides whether a LATER `storageFor()` retries — collapsing loses
+    // nothing because the drop is already retried on the next cache hit.
+    // Keyed by the composite (namespace, storageDir) so a CHANGED dir
+    // (migration/realignment) for the same namespace still gets its own hook —
+    // it is NOT collapsed onto the old dir's pending registration (cursor
+    // Medium, codex P2).
+    const inFlightKey = namespace + "\u0000" + storageDir;
+    if (this.inFlightResolveHooks.has(inFlightKey)) return;
+    this.inFlightResolveHooks.add(inFlightKey);
+    // Queue through the serializer. Concurrent calls for the same namespace
+    // strictly order here; the 2nd call's task runs only after the 1st's hook
+    // settles, by which point `notifiedResolved` is either set (no-op) or still
+    // unset (retry). The returned promise is tracked for
+    // `whenResolveHooksSettled()`. Rejections from the task body are caught so
+    // they never reach the caller (best-effort hook contract); the serializer's
+    // recovered tail still lets subsequent notifications run.
+    const task = async (): Promise<void> => {
+      // Re-check after queueing: a prior task in this same chain may have just
+      // marked the pair as notified.
+      if (this.notifiedResolved.get(namespace) === storageDir) return;
+      try {
+        // Hook may be sync or async; Promise.resolve normalizes both. A result
+        // of `false` means the registration was dropped/no-op (e.g. rebuild-lock
+        // timeout) — leave notifiedResolved UNSET so the next storageFor retries
+        // (round 6, codex P2 — NEFoX). `void`/`undefined` is success for legacy
+        // hooks. A rejection leaves notifiedResolved unset for the same reason.
+        const persisted = await Promise.resolve(hook(namespace, storageDir));
+        if (persisted !== false) {
+          this.notifiedResolved.set(namespace, storageDir);
+        }
+      } catch {
+        // Best-effort: a hook failure MUST NOT crash storage resolution. Leave
+        // notifiedResolved unset so a later storageFor retries the registration.
       }
-    }
+    };
+    const queued = this.resolveSerializer.serialize(namespace, task);
+    this.pendingResolveHooks.add(queued);
+    // Clear the in-flight marker when the hook settles so a subsequent
+    // storageFor() can retry after a drop, or short-circuit via notifiedResolved
+    // after a success.
+    const cleanup = (): void => {
+      this.inFlightResolveHooks.delete(inFlightKey);
+      this.pendingResolveHooks.delete(queued);
+    };
+    void queued.then(cleanup, cleanup);
   }
 
   /**

package/src/summary-snapshot.test.ts

@@ -2,7 +2,7 @@ import test from "node:test";
 import assert from "node:assert/strict";
 import os from "node:os";
 import path from "node:path";
-import { mkdtemp, mkdir, open, readFile, unlink, writeFile } from "node:fs/promises";
+import { mkdtemp, mkdir, open, readFile, rm, unlink, writeFile } from "node:fs/promises";
 import { HourlySummarizer } from "./summarizer.js";
 import {
   readSummarySnapshot,
@@ -756,3 +756,65 @@ test("hourly transcript lookup reads encoded transcript channel directories", as
     ["encoded transcript"],
   );
 });
+
+// ── Issue #1524 adoption prove-fail: summary-snapshot upserts must run through
+// the shared serializeMutations chain (rejection-recovering). The defect class
+// is a naive bare-.then(fn) chain that silently drops subsequent tasks after a
+// rejection: if upsert A rejects (e.g. transient I/O error) and the chain does
+// not recover, upsert B queued behind it never runs — its hour goes missing
+// from the snapshot. We seed the failure by rejecting the first upsert (a
+// hostile lock directory that throws on mkdir), then upsert B must STILL land
+// its hour. Pre-fix (a poison chain) B would be skipped.
+test("upsertSummarySnapshot recovers after a prior upsert rejects (issue #1524 poison-chain prove-fail)", async () => {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "engram-summary-poison-chain-"));
+  const sessionKey = "session-poison";
+  try {
+    // The lock directory for sessionKey is <root>/state/summaries/<encoded>.lock.
+    // Pre-create a FILE at the summaries root so the upsert's mkdir(dirname)
+    // inside writeSummarySnapshot throws ENOTDIR — a deterministic rejection
+    // that does not depend on filesystem timing. Both upserts target the SAME
+    // sessionKey so they serialize through the same chain.
+    const summariesRoot = path.join(memoryDir, "state", "summaries");
+    await mkdir(path.dirname(summariesRoot), { recursive: true });
+    await writeFile(summariesRoot, "not-a-dir", "utf-8");
+
+    const summaryA = {
+      hour: "2026-03-26T08:00:00.000Z",
+      sessionKey,
+      bullets: ["A"],
+      turnCount: 1,
+      generatedAt: "2026-03-26T08:15:00.000Z",
+    };
+    const summaryB = {
+      hour: "2026-03-26T09:00:00.000Z",
+      sessionKey,
+      bullets: ["B"],
+      turnCount: 1,
+      generatedAt: "2026-03-26T09:15:00.000Z",
+    };
+
+    // Upsert A rejects (hostile summaries root). We MUST catch — the contract
+    // is that the chain recovers, not that the failing op silently succeeds.
+    await upsertSummarySnapshot(memoryDir, summaryA).then(
+      () => assert.fail("upsert A should have rejected on the hostile summaries root"),
+      () => undefined,
+    );
+
+    // Restore a usable summaries root so upsert B can actually write.
+    await unlink(summariesRoot);
+    await mkdir(summariesRoot, { recursive: true });
+
+    // Upsert B MUST still run despite A's rejection. With a poison chain B
+    // would be skipped (no snapshot, no B-hour bullet).
+    await upsertSummarySnapshot(memoryDir, summaryB);
+
+    const snapshot = await readSummarySnapshot(memoryDir, sessionKey);
+    assert.ok(snapshot, "upsert B landed after upsert A rejected (chain recovered)");
+    assert.ok(
+      snapshot.some((s) => s.bullets.includes("B")),
+      "upsert B's hour survived — the chain was not poisoned by A's rejection",
+    );
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});

package/src/summary-snapshot.ts

@@ -1,18 +1,14 @@
-import {
-  mkdir,
-  open,
-  readFile,
-  stat,
-  unlink,
-  utimes,
-  writeFile,
-} from "node:fs/promises";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
 import path from "node:path";
 import { z } from "zod";
 import {
   encodeStoragePathSegment,
   resolvePathInsideStorageRoot,
 } from "./storage-paths.js";
+import {
+  serializeMutations,
+  withHeldFileLock,
+} from "./utils/serialize-mutations.js";
 import type { HourlySummary } from "./types.js";
 
 const summarySnapshotSchemaVersion = 1;
@@ -34,7 +30,9 @@ const SummarySnapshotSchema = z.object({
 
 type SummarySnapshot = z.infer<typeof SummarySnapshotSchema>;
 
-const summarySnapshotUpserts = new Map<string, Promise<void>>();
+// Lock timings for the cross-process summary-snapshot file lock. These are
+// passed straight through to the shared `withHeldFileLock` utility (issue
+// #1524 adoption); the previous bespoke lock used the same numbers.
 const summarySnapshotLockTimeoutMs = 5_000;
 const summarySnapshotLockStaleMs = 30_000;
 const summarySnapshotLockHeartbeatMs = Math.max(
@@ -137,31 +135,65 @@ export async function writeSummarySnapshot(
   await writeFile(filePath, JSON.stringify(payload, null, 2), "utf-8");
 }
 
+// ── Concurrency primitives (issue #1524 adoption) ─────────────────────────
+//
+// Summary-snapshot upserts were previously guarded by TWO bespoke serializers:
+//   1. an in-process `summarySnapshotUpserts` map keyed by sessionKey that
+//      chained each upsert off the prior one's release (mirroring
+//      `serializeMutations`), and
+//   2. an on-disk `withExclusiveSummarySnapshotFileLock` that re-implemented
+//      `open(path, "wx")` acquire, mtime heartbeat, stale-break, and
+//      ownership-checked release.
+// Both now delegate to the shared utility so there is ONE home for each
+// primitive (`serializeMutations` for in-process, `withHeldFileLock` for
+// cross-process). The behavior contract is preserved:
+//   - in-process upserts for the SAME sessionKey are strictly serialized
+//     (read-merge-write cannot interleave);
+//   - a cross-process holder blocks up to `summarySnapshotLockTimeoutMs`, then
+//     the upsert FAILS (a snapshot is advisory; we never race a read-merge-write
+//     unlocked). To preserve this strictness on top of a best-effort utility,
+//     the work callback throws when `acquired === false`.
+
 async function withSummarySnapshotLock<T>(
   memoryDir: string,
   sessionKey: string,
   work: () => Promise<T>,
 ): Promise<T> {
-  const previous = summarySnapshotUpserts.get(sessionKey) ?? Promise.resolve();
-  let release!: () => void;
-  const current = new Promise<void>((resolve) => {
-    release = resolve;
-  });
-  const chained = previous.then(() => current);
-  summarySnapshotUpserts.set(sessionKey, chained);
-
-  await previous;
-  try {
-    return await withExclusiveSummarySnapshotFileLock(
+  // In-process serialization (one upsert per sessionKey at a time). The
+  // serializer recovers from rejection, so a failed upsert never poisons the
+  // next one — matching the previous chain's `.then(noop, noop)` recovery.
+  return serializeMutations(`summary-snapshot:${sessionKey}`, () =>
+    // Cross-process mutex via the shared utility. The lock path, stale
+    // threshold, bounded wait, and heartbeat cadence all flow through; the
+    // utility's replacement-safe stale break (NG7Bg) and ownership-checked
+    // release are stronger than the bare `unlink` this module used before.
+    withHeldFileLock(
       summarySnapshotLockPath(memoryDir, sessionKey),
-      work,
-    );
-  } finally {
-    release();
-    if (summarySnapshotUpserts.get(sessionKey) === chained) {
-      summarySnapshotUpserts.delete(sessionKey);
-    }
-  }
+      {
+        staleMs: summarySnapshotLockStaleMs,
+        maxWaitMs: summarySnapshotLockTimeoutMs,
+        heartbeatMs: summarySnapshotLockHeartbeatMs,
+      },
+      async (acquired) => {
+        // Strict-fail when the lock could not be acquired: the upsert is a
+        // read-merge-write, so a best-effort unlocked run would clobber a
+        // concurrent writer. The utility's `acquired === false` covers BOTH a
+        // genuine contention timeout AND a filesystem acquire failure (lock-dir
+        // mkdir/open/permission errors — the advisory lock is best-effort, so
+        // the util degrades rather than throwing). The bespoke lock this
+        // replaced propagated fs errors verbatim and reserved the timeout
+        // message for contention; we no longer claim "timed out" for an fs
+        // failure (cursor Low 25143f4f) — the message names both causes so
+        // upstream fail-open (runHourly) is unchanged but debugging is honest.
+        if (!acquired) {
+          throw new Error(
+            "could not acquire summary snapshot lock (contention timeout or filesystem error)",
+          );
+        }
+        return work();
+      },
+    ),
+  );
 }
 
 export async function upsertSummarySnapshot(
@@ -185,54 +217,3 @@ export async function upsertSummarySnapshot(
     await writeSummarySnapshot(memoryDir, summary.sessionKey, next);
   });
 }
-
-async function withExclusiveSummarySnapshotFileLock<T>(
-  lockPath: string,
-  callback: () => Promise<T>,
-): Promise<T> {
-  await mkdir(path.dirname(lockPath), { recursive: true });
-  const startedAt = Date.now();
-
-  while (true) {
-    try {
-      const handle = await open(lockPath, "wx");
-      let heartbeat: NodeJS.Timeout | null = null;
-      if (summarySnapshotLockHeartbeatMs > 0) {
-        heartbeat = setInterval(() => {
-          void utimes(lockPath, new Date(), new Date()).catch(() => undefined);
-        }, summarySnapshotLockHeartbeatMs);
-        heartbeat.unref?.();
-      }
-      try {
-        return await callback();
-      } finally {
-        if (heartbeat) clearInterval(heartbeat);
-        await handle.close().catch(() => undefined);
-        await unlink(lockPath).catch(() => undefined);
-      }
-    } catch (error) {
-      if (!isAlreadyExistsError(error)) throw error;
-      try {
-        const lockStat = await stat(lockPath);
-        if (Date.now() - lockStat.mtimeMs > summarySnapshotLockStaleMs) {
-          await unlink(lockPath).catch(() => undefined);
-          continue;
-        }
-      } catch {
-        continue;
-      }
-      if (Date.now() - startedAt > summarySnapshotLockTimeoutMs) {
-        throw new Error("timed out acquiring summary snapshot lock");
-      }
-      await sleep(10);
-    }
-  }
-}
-
-function isAlreadyExistsError(error: unknown): boolean {
-  return typeof error === "object" && error !== null && "code" in error && error.code === "EEXIST";
-}
-
-function sleep(ms: number): Promise<void> {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}

package/src/utils/serialize-mutations.ts

@@ -670,10 +670,14 @@ async function lockHeldBySelf(held: HeldLock): Promise<boolean> {
 }
 
 function sleep(ms: number): Promise<void> {
-  const { promise, resolve } = Promise.withResolvers<void>();
-  // NOT unref'd: this polls inside an awaited acquire loop, so the caller's
-  // await chain keeps the loop alive; unref would let Node exit mid-poll when
-  // nothing else is pending (the heartbeat interval IS unref'd separately).
-  setTimeout(resolve, ms);
-  return promise;
+  // Manual deferred instead of Promise.withResolvers (ES2024) — plugin-openclaw's
+  // standalone tsconfig targets ES2022 lib and this module is reachable from its
+  // type graph, so withResolvers would TS2550 there (same fix as
+  // extraction-faithfulness.ts:467).
+  return new Promise<void>((resolve) => {
+    // NOT unref'd: this polls inside an awaited acquire loop, so the caller's
+    // await chain keeps the loop alive; unref would let Node exit mid-poll when
+    // nothing else is pending (the heartbeat interval IS unref'd separately).
+    setTimeout(resolve, ms);
+  });
 }