@remnic/core 9.7.15 → 9.8.0

package/src/utils/path-containment.test.ts

@@ -0,0 +1,75 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { lstat, mkdir, mkdtemp, readdir, rm, stat, symlink, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+
+import { ensureContainedSpillDir, listContainedSpillFiles } from "./path-containment.js";
+
+test("ensureContainedSpillDir creates and accepts a real (absent then present) directory", async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-spill-real-dir-"));
+  try {
+    const spillDir = path.join(root, "state", "ledger.jsonl.pending.d");
+    // Absent: created recursively.
+    await ensureContainedSpillDir(spillDir);
+    assert.equal((await stat(spillDir)).isDirectory(), true, "spill dir created as a real directory");
+    // A file may be written into it.
+    await writeFile(path.join(spillDir, "a.jsonl"), "row\n", "utf8");
+    // Idempotent: an already-present real directory is accepted unchanged.
+    await ensureContainedSpillDir(spillDir);
+    assert.deepEqual(await readdir(spillDir), ["a.jsonl"], "existing contents preserved");
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
+test("ensureContainedSpillDir refuses a symlinked spill directory before any write", async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-spill-symlink-dir-"));
+  const outside = await mkdtemp(path.join(os.tmpdir(), "remnic-spill-symlink-outside-"));
+  try {
+    const spillDir = path.join(root, "ledger.jsonl.pending.d");
+    await symlink(outside, spillDir);
+    await assert.rejects(
+      () => ensureContainedSpillDir(spillDir),
+      /symlinked or non-directory/,
+      "a symlinked spill directory must be refused",
+    );
+    assert.equal((await lstat(spillDir)).isSymbolicLink(), true, "symlink left intact");
+    assert.deepEqual(await readdir(outside), [], "nothing written through the symlink");
+  } finally {
+    await rm(root, { recursive: true, force: true });
+    await rm(outside, { recursive: true, force: true });
+  }
+});
+
+test("ensureContainedSpillDir refuses a non-directory path at the spill location", async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-spill-nondir-"));
+  try {
+    // A regular file sitting where the spill directory should be.
+    const spillDir = path.join(root, "ledger.jsonl.pending.d");
+    await writeFile(spillDir, "not a dir\n", "utf8");
+    await assert.rejects(
+      () => ensureContainedSpillDir(spillDir),
+      /symlinked or non-directory|EEXIST|ENOTDIR/,
+      "a non-directory spill location must be refused",
+    );
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
+test("listContainedSpillFiles and ensureContainedSpillDir enforce the same symlink guard", async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-spill-parity-"));
+  const outside = await mkdtemp(path.join(os.tmpdir(), "remnic-spill-parity-outside-"));
+  try {
+    await writeFile(path.join(outside, "leak.jsonl"), "leak\n", "utf8");
+    const spillDir = path.join(root, "ledger.jsonl.pending.d");
+    await symlink(outside, spillDir);
+    // Read side skips a symlinked dir (returns []); write side refuses it.
+    assert.deepEqual(await listContainedSpillFiles(spillDir), [], "read side skips symlinked dir");
+    await assert.rejects(() => ensureContainedSpillDir(spillDir), /symlinked or non-directory/);
+  } finally {
+    await rm(root, { recursive: true, force: true });
+    await rm(outside, { recursive: true, force: true });
+  }
+});

package/src/utils/path-containment.ts

@@ -14,6 +14,8 @@
  */
 
 import path from "node:path";
+import { lstat, mkdir, readdir, realpath } from "node:fs/promises";
+import { isErrnoCode } from "./errno.js";
 
 /**
  * True when `child` is `parent` itself or nested underneath it. Both arguments
@@ -38,3 +40,77 @@ export function assertPathInsideRoot(
     throw new Error(`Refusing to scan memory path outside memoryDir: ${originalPath}`);
   }
 }
+
+/**
+ * List the regular-file children of `dir` whose name ends in `suffix` (default
+ * `.jsonl`) in sorted name order as absolute paths, rejecting a symlinked/
+ * non-directory `dir` and skipping any entry that is a symlink or whose realpath
+ * escapes `dir`. Returns `[]` when the directory is absent or cannot be safely
+ * enumerated. Used by the durable pending-spill drains (lifecycle ledger +
+ * recall impressions, #2033) so a poisoned link planted in a spill directory
+ * cannot redirect a secure read — or a later unlink — outside that directory.
+ * The `suffix` override lets the lifecycle drain enumerate crash-orphaned
+ * `*.jsonl.claimed` files with the SAME containment guard as live spills.
+ */
+export async function listContainedSpillFiles(dir: string, suffix = ".jsonl"): Promise<string[]> {
+  let dirReal: string;
+  try {
+    const dirStat = await lstat(dir);
+    if (dirStat.isSymbolicLink() || !dirStat.isDirectory()) return [];
+    dirReal = await realpath(dir);
+  } catch (err) {
+    if (isErrnoCode(err, "ENOENT")) return [];
+    throw err;
+  }
+  let names: string[];
+  try {
+    names = (await readdir(dir)).filter((name) => name.endsWith(suffix)).sort();
+  } catch (err) {
+    if (isErrnoCode(err, "ENOENT")) return [];
+    throw err;
+  }
+  const files: string[] = [];
+  for (const name of names) {
+    const filePath = path.join(dir, name);
+    let entryStat;
+    try {
+      entryStat = await lstat(filePath);
+    } catch (err) {
+      if (isErrnoCode(err, "ENOENT")) continue; // vanished between readdir and lstat
+      throw err;
+    }
+    // Reject a symlinked entry or one whose real target escapes the spill dir
+    // BEFORE any secure read/delete: a poisoned link must never redirect a
+    // decrypt or a later unlink outside the directory (#2033).
+    if (entryStat.isSymbolicLink() || !entryStat.isFile()) continue;
+    let entryReal: string;
+    try {
+      entryReal = await realpath(filePath);
+    } catch (err) {
+      if (isErrnoCode(err, "ENOENT")) continue;
+      throw err;
+    }
+    if (!pathIsInside(dirReal, entryReal)) continue;
+    files.push(filePath);
+  }
+  return files;
+}
+
+/**
+ * Create `dir` (recursively, if absent) and assert it is a REAL directory —
+ * never a symlink — BEFORE any spill file is written into it (#2033). This
+ * mirrors the lstat guard {@link listContainedSpillFiles} applies on the read
+ * side so the durable-spill WRITE and its later drain enforce the SAME
+ * symlink-safe containment: a symlink planted at the spill-directory path (e.g.
+ * `<ledger>.pending.d` -> /etc) cannot redirect a write/rename/unlink outside
+ * the intended location. `mkdir` recursive is a no-op when the path already
+ * resolves through a symlink, so the post-mkdir lstat is what actually refuses
+ * the poisoned link. Throws on a symlinked or non-directory target.
+ */
+export async function ensureContainedSpillDir(dir: string): Promise<void> {
+  await mkdir(dir, { recursive: true });
+  const dirStat = await lstat(dir);
+  if (dirStat.isSymbolicLink() || !dirStat.isDirectory()) {
+    throw new Error(`Refusing to write spill file into symlinked or non-directory path: ${dir}`);
+  }
+}

package/src/utils/serialize-mutations.test.ts

@@ -948,6 +948,128 @@ test("heartbeat does NOT refresh a REPLACEMENT lock's mtime (ownership check, co
   }
 });
 
+test("held-lock controller.refresh() re-stamps the mtime and reports still-held (#1910/#2033)", async () => {
+  // A CPU-bound critical section blocks the timer heartbeat, so a caller about
+  // to perform a destructive write re-stamps the lock manually via refresh().
+  // Real platform clock (mtime) — no fake-timer analog for fs.stat().mtimeMs.
+  const dir = await mkTmpDir();
+  try {
+    const lockPath = path.join(dir, "refresh-held.lock");
+    let held: boolean | undefined;
+    let bumped = false;
+    await withHeldFileLock(
+      lockPath,
+      { staleMs: 60_000, heartbeatMs: 30_000 },
+      async (acquired, lock) => {
+        assert.equal(acquired, true, "test holder acquires the lock");
+        const before = (await stat(lockPath)).mtimeMs;
+        await delay(50); // let the clock advance so a bump is observable
+        held = await lock.refresh();
+        bumped = (await stat(lockPath)).mtimeMs > before;
+      },
+    );
+    assert.equal(held, true, "refresh reports the lock still held");
+    assert.ok(bumped, "refresh re-stamped the lock mtime");
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+test("held-lock controller.refresh() reports LOST and never stamps a peer replacement lock (#1910/#2033)", async () => {
+  // If our event loop was blocked long enough for a peer to judge us stale,
+  // break the lock, and replace it, refresh() MUST report the lock lost (so the
+  // caller aborts its destructive write) and MUST NOT refresh the replacement's
+  // mtime. Real platform clock (mtime): no fake-timer analog.
+  const dir = await mkTmpDir();
+  try {
+    const lockPath = path.join(dir, "refresh-lost.lock");
+    let held: boolean | undefined;
+    let ageAfter = 0;
+    await withHeldFileLock(
+      lockPath,
+      { staleMs: 60_000, heartbeatMs: 30_000 },
+      async (acquired, lock) => {
+        assert.equal(acquired, true, "test holder acquires the lock");
+        // Peer stale-break + replacement: overwrite with a DIFFERENT owner id
+        // and an old mtime.
+        const replacement = `999999 deadbeef-0000-4000-8000-000000000000 ${new Date().toISOString()}\n`;
+        const old = new Date(Date.now() - 5_000);
+        await writeFile(lockPath, replacement, "utf8");
+        await utimes(lockPath, old, old);
+        held = await lock.refresh();
+        ageAfter = Date.now() - (await stat(lockPath)).mtimeMs;
+      },
+    );
+    assert.equal(held, false, "refresh reports the lock lost to a peer replacement");
+    assert.ok(
+      ageAfter > 4_000,
+      `replacement lock mtime NOT bumped by refresh (age=${ageAfter}ms; expected >4000ms)`,
+    );
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+test("held-lock controller.refresh() reports LOST when the mtime bump fails (#2033)", async () => {
+  // The ownership pre-check passes (still ours), but a peer deletes the lock
+  // before the mtime bump so `utimes` throws. Fresh evidence: the old refresh
+  // only warned and returned true, letting the caller rewrite on a lock it can
+  // no longer prove it holds and clobber a peer's append. It MUST report LOST.
+  const dir = await mkTmpDir();
+  try {
+    const lockPath = path.join(dir, "refresh-utimes-fail.lock");
+    let held: boolean | undefined;
+    let warned = false;
+    await withHeldFileLock(
+      lockPath,
+      {
+        staleMs: 60_000,
+        heartbeatMs: 30_000,
+        onLockWarning: () => { warned = true; },
+        // Delete our lock file AFTER the ownership check, BEFORE the bump, so
+        // utimes(held.path) throws ENOENT.
+        onBeforeRefreshUtimesForTest: async () => { await rm(lockPath, { force: true }); },
+      },
+      async (acquired, lock) => {
+        assert.equal(acquired, true, "test holder acquires the lock");
+        held = await lock.refresh();
+      },
+    );
+    assert.equal(held, false, "refresh reports LOST when the mtime bump throws");
+    assert.equal(warned, true, "the bump failure is surfaced via the warning hook");
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+test("held-lock controller.refresh() reports LOST when a peer replaces the lock between the check and the bump (#2033)", async () => {
+  // The ownership pre-check passes, then a peer replaces the lock with a
+  // different owner BEFORE our bump. `utimes` would then refresh the
+  // replacement's mtime; the post-bump ownership RE-CHECK must catch this and
+  // report LOST so the caller aborts rather than clobbering the peer.
+  const dir = await mkTmpDir();
+  try {
+    const lockPath = path.join(dir, "refresh-replaced-midbump.lock");
+    let held: boolean | undefined;
+    const replacement = `999999 deadbeef-0000-4000-8000-000000000000 ${new Date().toISOString()}\n`;
+    await withHeldFileLock(
+      lockPath,
+      {
+        staleMs: 60_000,
+        heartbeatMs: 30_000,
+        onBeforeRefreshUtimesForTest: async () => { await writeFile(lockPath, replacement, "utf8"); },
+      },
+      async (acquired, lock) => {
+        assert.equal(acquired, true, "test holder acquires the lock");
+        held = await lock.refresh();
+      },
+    );
+    assert.equal(held, false, "refresh reports LOST when a peer replaces the lock mid-bump");
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
 test("breakStaleLock uses atomic rename so a replacement lock is never unlinked by a second contender", async () => {
   // Two contenders both judge the same stale lock. One atomically renames
   // it away and acquires a replacement; the other's break must NOT unlink

package/src/utils/serialize-mutations.ts

@@ -193,6 +193,14 @@ export interface HeldFileLockOptions {
    * production. Used to prove the pre-check prevents the rename entirely.
    */
   readonly onAfterReleaseRenameForTest?: () => Promise<void> | void;
+  /**
+   * Test seam (#2033): fires inside {@link HeldFileLockController.refresh} AFTER
+   * the ownership pre-check passes and BEFORE the `utimes` mtime bump —
+   * simulating a peer that deletes the lock (making the bump throw) or replaces
+   * it (a different owner the post-bump re-check must catch). No-op in
+   * production; proves refresh reports LOST rather than clobbering a peer.
+   */
+  readonly onBeforeRefreshUtimesForTest?: () => Promise<void> | void;
   /**
    * Best-effort hook for non-fatal lock warnings (heartbeat refresh failure,
    * release-time ownership check failure). Never throws into the caller. If
@@ -202,6 +210,28 @@ export interface HeldFileLockOptions {
   readonly onLockWarning?: (message: string, err: unknown) => void;
 }
 
+/**
+ * Control surface handed to a held-lock task so a long, CPU-bound critical
+ * section can re-assert ownership immediately before its destructive write.
+ */
+export interface HeldFileLockController {
+  /**
+   * Re-verify THIS acquirer still owns the lock and, when it does, refresh the
+   * lock's mtime. Returns `true` only while we still hold it.
+   *
+   * The timer heartbeat cannot fire while a synchronous, CPU-bound section
+   * (a large parse/merge/sort/serialize) blocks the event loop, so a peer can
+   * judge the lock stale, break it, and start its own write within the stale
+   * window. A caller about to perform a destructive rewrite MUST call this
+   * first: `false` means the lock was stale-broken/replaced and the caller MUST
+   * abort rather than clobber the peer that now holds it; `true` also re-stamps
+   * the mtime so the bounded write that immediately follows cannot itself be
+   * judged stale mid-write. On the best-effort unlocked path (`acquired` was
+   * `false`) this always resolves `false` — there is no lock to hold.
+   */
+  refresh(): Promise<boolean>;
+}
+
 /** Default bounded acquisition wait, mirroring the catalog. */
 const DEFAULT_MAX_WAIT_MS = 5_000;
 /** Default busy-lock poll interval, mirroring the catalog. */
@@ -254,7 +284,7 @@ interface HeldLock {
 export async function withHeldFileLock<T>(
   lockPath: string,
   opts: HeldFileLockOptions,
-  task: (acquired: boolean) => Promise<T>,
+  task: (acquired: boolean, lock: HeldFileLockController) => Promise<T>,
 ): Promise<T> {
   if (typeof lockPath !== "string" || lockPath.length === 0) {
     throw new TypeError("withHeldFileLock: lockPath must be a non-empty string");
@@ -319,7 +349,7 @@ export async function withHeldFileLock<T>(
     // Best-effort: run the task WITHOUT the lock. The caller decides what to
     // do (the catalog touch path will drop its append); we never crash the
     // primary op on contention.
-    return task(false);
+    return task(false, { refresh: async () => false });
   }
 
   // Heartbeat: while WE hold the lock, refresh its mtime so age-based stale
@@ -344,8 +374,11 @@ export async function withHeldFileLock<T>(
   }, heartbeatMs);
   // Don't keep the event loop alive solely for the heartbeat.
   heartbeat.unref?.();
+  const controller: HeldFileLockController = {
+    refresh: () => refreshHeldLock(held, warn, opts.onBeforeRefreshUtimesForTest),
+  };
   try {
-    return await task(true);
+    return await task(true, controller);
   } finally {
     clearInterval(heartbeat);
     await releaseLock(held, warn, opts.onAfterReleaseRenameForTest);
@@ -669,6 +702,38 @@ async function lockHeldBySelf(held: HeldLock): Promise<boolean> {
   }
 }
 
+/**
+ * Re-assert ownership and refresh the lock mtime for {@link HeldFileLockController.refresh}.
+ * Returns `true` only while THIS acquirer still owns the lock. When ownership
+ * was lost (a peer stale-broke and replaced the lock while a CPU-bound section
+ * blocked the event loop), returns `false` WITHOUT touching the replacement's
+ * mtime — mirroring the heartbeat's ownership guard.
+ *
+ * FAIL-CLOSED on the mtime bump (#2033): a `utimes` failure means we cannot
+ * prove the lock is still ours AND fresh — the file may have been stale-broken,
+ * deleted, or replaced between the ownership check and the bump — so we report
+ * lost ownership rather than let the caller rewrite the ledger on an unrefreshed
+ * (soon-stale) or already-replaced lock. After a successful bump we RE-CHECK
+ * ownership: a peer could have broken and replaced the lock in the window, in
+ * which case `utimes` just refreshed the replacement's mtime and we must not
+ * report held.
+ */
+async function refreshHeldLock(
+  held: HeldLock,
+  warn: (message: string, err: unknown) => void,
+  onBeforeRefreshUtimesForTest?: () => Promise<void> | void,
+): Promise<boolean> {
+  if (!(await lockHeldBySelf(held))) return false;
+  if (onBeforeRefreshUtimesForTest) await onBeforeRefreshUtimesForTest();
+  try {
+    await utimes(held.path, new Date(), new Date());
+  } catch (err) {
+    warn("withHeldFileLock manual refresh failed", err);
+    return false;
+  }
+  return lockHeldBySelf(held);
+}
+
 function sleep(ms: number): Promise<void> {
   // Manual deferred instead of Promise.withResolvers (ES2024) — plugin-openclaw's
   // standalone tsconfig targets ES2022 lib and this module is reachable from its

package/src/wearables/memory-gen.test.ts

@@ -10,6 +10,7 @@ import {
   writeDailyDigestMemory,
   type WearableMemoryWriter,
 } from "./memory-gen.js";
+import { sealedWriteToLegacyArgs } from "../write-envelope.js";
 import { emptySpeakerRegistry } from "./speakers.js";
 import type { WearableConversation, WearableSourceSettings } from "./types.js";
 
@@ -68,6 +69,25 @@ interface WriteCall {
   options: Record<string, unknown>;
 }
 
+
+/**
+ * Wrap a legacy writeMemory-shaped recorder as writeSealedMemory using the
+ * PRODUCTION mapper (AGENTS.md §21 — stub behavior cannot drift from what
+ * StorageManager.writeSealedMemory does with the envelope).
+ */
+function asSealedWrite(
+  writeMemory: (
+    category: string,
+    content: string,
+    options: Record<string, unknown>,
+  ) => Promise<{ id: string; tombstoneBlocked: boolean; blockedBy?: string }>,
+): WearableMemoryWriter["writeSealedMemory"] {
+  return (envelope, extras) => {
+    const { category, content, options } = sealedWriteToLegacyArgs(envelope, extras as Record<string, unknown>);
+    return writeMemory(category, content, options);
+  };
+}
+
 function makeWriter(existingHashes: string[] = []): {
   writer: WearableMemoryWriter;
   writes: WriteCall[];
@@ -77,10 +97,10 @@ function makeWriter(existingHashes: string[] = []): {
   return {
     writes,
     writer: {
-      async writeMemory(category, content, options) {
-        writes.push({ category, content, options: options as Record<string, unknown> });
+      writeSealedMemory: asSealedWrite(async (category, content, options) => {
+        writes.push({ category, content, options });
         return { id: `id-${writes.length}`, tombstoneBlocked: false };
-      },
+      }),
       async hasFactContentHash(content) {
         return existing.has(content);
       },
@@ -173,7 +193,9 @@ test("review mode writes pending_review with full wearable provenance", async ()
   assert.equal(write.category, "decision");
   assert.equal(write.options.status, "pending_review");
   assert.equal(write.options.source, "wearable:limitless");
-  assert.equal(write.options.validAt, "2026-06-10T15:00:00Z");
+  // The envelope canonicalizes validAt at compose time (epoch round-trip) —
+  // the same form storage's own normalization would persist.
+  assert.equal(write.options.validAt, "2026-06-10T15:00:00.000Z");
   const attrs = write.options.structuredAttributes as Record<string, string>;
   assert.equal(attrs.wearableSource, "limitless");
   assert.equal(attrs.wearableDate, "2026-06-10");
@@ -764,10 +786,10 @@ test("#1645: a tombstone-blocked wearable write counts as blocked, not created",
   // blocked content as successfully created active memories.
   const writes: Array<{ category: string; content: string; options: Record<string, unknown> }> = [];
   const writer: WearableMemoryWriter = {
-    async writeMemory(category, content, options) {
-      writes.push({ category, content, options: options as Record<string, unknown> });
+    writeSealedMemory: asSealedWrite(async (category, content, options) => {
+      writes.push({ category, content, options });
       return { id: `id-${writes.length}`, tombstoneBlocked: true, blockedBy: "tomb-1" };
-    },
+    }),
     async hasFactContentHash() {
       return false;
     },
@@ -793,9 +815,9 @@ test("#1645: a tombstone-blocked wearable write counts as blocked, not created",
 
 test("#1645: a tombstone-blocked native import counts as blocked, not imported", async () => {
   const writer: WearableMemoryWriter = {
-    async writeMemory() {
+    writeSealedMemory: asSealedWrite(async () => {
       return { id: "blocked-native-1", tombstoneBlocked: true, blockedBy: "tomb-1" };
-    },
+    }),
     async hasFactContentHash() {
       return false;
     },