@vellumai/assistant 0.6.3 → 0.6.5

package/src/backup/__tests__/snapshot-lock.test.ts

@@ -0,0 +1,352 @@
+/**
+ * Tests for the cross-process snapshot lock helper. Each test gets a fresh
+ * temp directory so runs never collide with the real `~/.vellum/backups`
+ * directory and so parallel test workers never see each other's lock files.
+ *
+ * The interesting corners covered here are:
+ *   - Acquire → release round-trip leaves the filesystem clean
+ *   - A second acquire against a held lock throws with the expected prefix
+ *   - A dead-PID lock file (simulated by writing a garbage PID that is not
+ *     alive on this host) is taken over transparently
+ *   - The release function is idempotent — calling it twice is a no-op
+ *   - The lock file is created with mode `0o600` so an unprivileged
+ *     peer on the same machine cannot read the holder PID
+ *   - **TOCTOU mutual exclusion**: two concurrent acquires against a stale
+ *     lock end up with exactly one winner (no double-acquire, no lost lock)
+ *   - **Rename-aside**: takeover does not unlink-then-reacquire, so an
+ *     interleaved second acquirer cannot destroy the fresh lock
+ */
+
+import {
+  existsSync,
+  mkdtempSync,
+  readdirSync,
+  rmSync,
+  statSync,
+  writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+
+import { acquireSnapshotLock } from "../snapshot-lock.js";
+
+// ---------------------------------------------------------------------------
+// Fixtures
+// ---------------------------------------------------------------------------
+
+let ROOT: string;
+let LOCK: string;
+
+beforeEach(() => {
+  ROOT = mkdtempSync(join(tmpdir(), "vellum-snapshot-lock-"));
+  LOCK = join(ROOT, ".snapshot.lock");
+});
+
+afterEach(() => {
+  try {
+    rmSync(ROOT, { recursive: true, force: true });
+  } catch {
+    // best-effort
+  }
+});
+
+// ---------------------------------------------------------------------------
+// Happy path
+// ---------------------------------------------------------------------------
+
+describe("acquireSnapshotLock — happy path", () => {
+  test("acquire creates the lock file; release removes it", async () => {
+    expect(existsSync(LOCK)).toBe(false);
+
+    const release = await acquireSnapshotLock(LOCK);
+    expect(existsSync(LOCK)).toBe(true);
+
+    await release();
+    expect(existsSync(LOCK)).toBe(false);
+  });
+
+  test("lock file is created with mode 0o600", async () => {
+    const release = await acquireSnapshotLock(LOCK);
+    try {
+      const stats = statSync(LOCK);
+      // mask to permission bits only
+      expect(stats.mode & 0o777).toBe(0o600);
+    } finally {
+      await release();
+    }
+  });
+
+  test("acquire creates the parent directory if missing", async () => {
+    // Point the lock at a nested path whose parent does not exist yet so
+    // we exercise the mkdir-on-demand code path.
+    const nested = join(ROOT, "missing-parent", ".snapshot.lock");
+    expect(existsSync(join(ROOT, "missing-parent"))).toBe(false);
+
+    const release = await acquireSnapshotLock(nested);
+    try {
+      expect(existsSync(nested)).toBe(true);
+    } finally {
+      await release();
+    }
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Conflict detection
+// ---------------------------------------------------------------------------
+
+describe("acquireSnapshotLock — conflicts", () => {
+  test("two acquires against a live holder: second throws 'snapshot in progress'", async () => {
+    const release = await acquireSnapshotLock(LOCK);
+    try {
+      await expect(acquireSnapshotLock(LOCK)).rejects.toThrow(
+        /snapshot in progress/,
+      );
+    } finally {
+      await release();
+    }
+  });
+
+  test("conflict error includes the holder PID", async () => {
+    const release = await acquireSnapshotLock(LOCK);
+    try {
+      await expect(acquireSnapshotLock(LOCK)).rejects.toThrow(
+        new RegExp(`snapshot in progress \\(locked by pid ${process.pid}\\)`),
+      );
+    } finally {
+      await release();
+    }
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Stale-lock takeover
+// ---------------------------------------------------------------------------
+
+describe("acquireSnapshotLock — stale locks", () => {
+  test("dead PID: acquire takes over and creates a fresh lock", async () => {
+    // PID 2^31 - 1 is virtually guaranteed to be dead on any sane host —
+    // the platform PID_MAX is typically much smaller. Writing it as the
+    // lock holder simulates a crashed prior writer whose process has since
+    // exited without releasing.
+    const deadPid = 2_147_483_647;
+    writeFileSync(LOCK, `${deadPid} ${Date.now()}\n`, { mode: 0o600 });
+    expect(existsSync(LOCK)).toBe(true);
+
+    const release = await acquireSnapshotLock(LOCK);
+    try {
+      expect(existsSync(LOCK)).toBe(true);
+    } finally {
+      await release();
+    }
+    expect(existsSync(LOCK)).toBe(false);
+  });
+
+  test("unparseable lock file: acquire refuses takeover and surfaces conflict", async () => {
+    // A lock file with no parseable PID is indistinguishable from an
+    // in-progress partial write by a live holder (both read back as
+    // `null`). We conservatively refuse takeover in both cases — we
+    // only ever take over a lock whose holder PID is readable AND
+    // confirmed not running. Garbage files require manual cleanup; the
+    // alternative (silently unlinking) is what introduced the TOCTOU
+    // race flagged on PR #24896.
+    writeFileSync(LOCK, "not a pid at all\n", { mode: 0o600 });
+
+    await expect(acquireSnapshotLock(LOCK)).rejects.toThrow(
+      /^snapshot in progress/,
+    );
+    expect(existsSync(LOCK)).toBe(true);
+  });
+
+  test("empty lock file: acquire refuses takeover and preserves the file", async () => {
+    // An empty lock file is ambiguous: it could be debris from a dead
+    // writer, or it could belong to a *live* holder that just won
+    // `O_EXCL` but has not yet flushed its PID. Taking it over would
+    // allow a second process to acquire and run concurrently — the
+    // exact TOCTOU race this module exists to prevent. The correct
+    // behavior is to surface a conflict and leave the file alone.
+    writeFileSync(LOCK, "", { mode: 0o600 });
+
+    await expect(acquireSnapshotLock(LOCK)).rejects.toThrow(
+      /^snapshot in progress/,
+    );
+    expect(existsSync(LOCK)).toBe(true);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Release semantics
+// ---------------------------------------------------------------------------
+
+describe("acquireSnapshotLock — release", () => {
+  test("release is idempotent: calling twice is safe", async () => {
+    const release = await acquireSnapshotLock(LOCK);
+    await release();
+    // Second call must not throw, even though the file is already gone.
+    await release();
+    expect(existsSync(LOCK)).toBe(false);
+  });
+
+  test("release tolerates an externally-unlinked lock file", async () => {
+    const release = await acquireSnapshotLock(LOCK);
+    // Simulate another process (or a rogue admin) removing our lock file
+    // out from under us. Release must still return without throwing.
+    rmSync(LOCK, { force: true });
+    await release();
+  });
+
+  test("after release, the lock can be acquired again", async () => {
+    const release1 = await acquireSnapshotLock(LOCK);
+    await release1();
+
+    const release2 = await acquireSnapshotLock(LOCK);
+    try {
+      expect(existsSync(LOCK)).toBe(true);
+    } finally {
+      await release2();
+    }
+  });
+});
+
+// ---------------------------------------------------------------------------
+// TOCTOU / race safety — regression tests for the rename-aside takeover fix
+// ---------------------------------------------------------------------------
+
+describe("acquireSnapshotLock — TOCTOU mutual exclusion", () => {
+  test("sequential stale-takeover calls: first wins, second sees the fresh lock", async () => {
+    // Two sequential takeover attempts against the same stale lock. The
+    // first caller wins via rename-aside and holds the fresh lock. The
+    // second caller observes the fresh lock (owned by process.pid) and
+    // must throw "snapshot in progress (locked by pid <process.pid>)"
+    // — not quietly unlink-and-reacquire.
+    const deadPid = 2_147_483_647;
+    writeFileSync(LOCK, `${deadPid} ${Date.now()}\n`, { mode: 0o600 });
+
+    const release1 = await acquireSnapshotLock(LOCK);
+    try {
+      // The fresh lock exists and belongs to this process.
+      expect(existsSync(LOCK)).toBe(true);
+      // A second acquire must see the fresh lock — not the dead one — and
+      // refuse takeover because the holder is alive (process.pid === our pid).
+      await expect(acquireSnapshotLock(LOCK)).rejects.toThrow(
+        new RegExp(`snapshot in progress \\(locked by pid ${process.pid}\\)`),
+      );
+    } finally {
+      await release1();
+    }
+    expect(existsSync(LOCK)).toBe(false);
+  });
+
+  test("Promise.all of two stale-takeover attempts: exactly one wins", async () => {
+    // Proof-of-fix for the original TOCTOU race:
+    //
+    // Before the fix, two processes observing the same stale lock would
+    // both call unlinkSync and both succeed at O_EXCL — ending up with
+    // independent beliefs that they hold the lock. The rename-aside pattern
+    // makes the takeover atomic: only one rename wins, and the loser
+    // retries the acquire loop and sees the winner's fresh lock.
+    //
+    // We can't literally race two processes in a unit test, but within a
+    // single process we can stress the same event-loop interleaving via
+    // Promise.all. The assertion is: one succeeds, one throws with the
+    // expected prefix, and the winner's PID is the live process.
+    const deadPid = 2_147_483_647;
+    writeFileSync(LOCK, `${deadPid} ${Date.now()}\n`, { mode: 0o600 });
+
+    const results = await Promise.allSettled([
+      acquireSnapshotLock(LOCK),
+      acquireSnapshotLock(LOCK),
+    ]);
+
+    const fulfilled = results.filter((r) => r.status === "fulfilled");
+    const rejected = results.filter((r) => r.status === "rejected");
+    // Exactly one winner, exactly one loser. Mutual exclusion holds.
+    expect(fulfilled).toHaveLength(1);
+    expect(rejected).toHaveLength(1);
+
+    // The loser's error must start with the documented "snapshot in progress"
+    // prefix so the HTTP 409 mapping in backup-routes.ts picks it up.
+    const err = (rejected[0] as PromiseRejectedResult).reason as Error;
+    expect(err.message).toMatch(/^snapshot in progress/);
+
+    // The winning acquire returned a release function — make sure we clean up.
+    const release = (fulfilled[0] as PromiseFulfilledResult<() => Promise<void>>)
+      .value;
+    await release();
+    expect(existsSync(LOCK)).toBe(false);
+  });
+
+  test("live PID in a stale lock refuses takeover", async () => {
+    // Regression test for "take over too aggressively": if the lock file's
+    // holder PID is ALIVE (e.g. the current process for this test), the
+    // takeover path must not fire. This also catches any reversed alive /
+    // dead-check logic in the stale-takeover branch.
+    writeFileSync(LOCK, `${process.pid} ${Date.now()}\n`, { mode: 0o600 });
+    try {
+      await expect(acquireSnapshotLock(LOCK)).rejects.toThrow(
+        new RegExp(`snapshot in progress \\(locked by pid ${process.pid}\\)`),
+      );
+      // The pre-existing lock must still be there.
+      expect(existsSync(LOCK)).toBe(true);
+    } finally {
+      // Clean up the hand-rolled lock file so afterEach's rmSync doesn't
+      // collide with a leftover fd.
+      try {
+        rmSync(LOCK, { force: true });
+      } catch {
+        // best-effort
+      }
+    }
+  });
+
+  test("zero-length lock file from partial write is NOT taken over", async () => {
+    // Regression test for the TOCTOU race flagged on PR #24896.
+    //
+    // Window: process A has just succeeded at `openSync(O_EXCL)` but has
+    // not yet flushed `<pid> <timestamp>` to the file. Process B opens
+    // the file and sees zero bytes. If B takes over, it unlinks / renames
+    // A's lock and acquires its own — now both A and B believe they own
+    // the lock and both snapshots run concurrently (WAL corruption, path
+    // clobber, racing retention pruner).
+    //
+    // The fix bounds how many times we re-read an empty file and refuses
+    // takeover if it is still empty after the retry budget is exhausted.
+    // We simulate the live-holder case here by leaving the file empty
+    // throughout the acquire attempt — acquire must throw "snapshot in
+    // progress" and must NOT touch the existing file.
+    writeFileSync(LOCK, "", { mode: 0o600 });
+    expect(statSync(LOCK).size).toBe(0);
+    const inodeBefore = statSync(LOCK).ino;
+
+    await expect(acquireSnapshotLock(LOCK)).rejects.toThrow(
+      /^snapshot in progress/,
+    );
+
+    // The original empty file must still exist and must not have been
+    // replaced — if the inode changed, something unlinked/recreated it.
+    expect(existsSync(LOCK)).toBe(true);
+    expect(statSync(LOCK).ino).toBe(inodeBefore);
+    expect(statSync(LOCK).size).toBe(0);
+  });
+
+  test("rename-aside sideband file is cleaned up after takeover", async () => {
+    // After a successful stale takeover, no `.snapshot.lock.stale.*` debris
+    // should remain in the parent directory — the takeover unlinks the
+    // sideband file after the fresh lock is in place. This asserts the
+    // cleanup path is wired up so we don't accumulate orphaned files on
+    // every crash.
+    const deadPid = 2_147_483_647;
+    writeFileSync(LOCK, `${deadPid} ${Date.now()}\n`, { mode: 0o600 });
+
+    const release = await acquireSnapshotLock(LOCK);
+    try {
+      const entries = readdirSync(ROOT);
+      // Only the fresh lock file should remain; no `.stale.*` sidebands.
+      const sidebandLeftover = entries.filter((e) => e.includes(".stale."));
+      expect(sidebandLeftover).toEqual([]);
+    } finally {
+      await release();
+    }
+  });
+});

package/src/backup/__tests__/stream-crypt.test.ts

@@ -0,0 +1,228 @@
+import { randomBytes } from "node:crypto";
+import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { open } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+
+import {
+  decryptFile,
+  ENCRYPTED_HEADER_SIZE,
+  encryptFile,
+  GCM_TAG_SIZE,
+  verifyEncryptedFile,
+} from "../stream-crypt.js";
+
+// ---------------------------------------------------------------------------
+// Fixtures
+// ---------------------------------------------------------------------------
+
+let TEST_DIR: string;
+
+beforeEach(() => {
+  TEST_DIR = join(
+    tmpdir(),
+    `vellum-stream-crypt-test-${randomBytes(6).toString("hex")}`,
+  );
+  mkdirSync(TEST_DIR, { recursive: true });
+});
+
+afterEach(() => {
+  try {
+    rmSync(TEST_DIR, { recursive: true, force: true });
+  } catch {
+    // best-effort
+  }
+});
+
+function makeKey(seed: number): Buffer {
+  const buf = Buffer.alloc(32);
+  for (let i = 0; i < 32; i++) {
+    buf[i] = (seed + i) & 0xff;
+  }
+  return buf;
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("stream-crypt", () => {
+  test("round-trips a 1 KB random file", async () => {
+    const key = randomBytes(32);
+    const plaintext = randomBytes(1024);
+    const plainPath = join(TEST_DIR, "plain.bin");
+    const encPath = join(TEST_DIR, "enc.bin");
+    const roundTripPath = join(TEST_DIR, "roundtrip.bin");
+
+    writeFileSync(plainPath, plaintext);
+    await encryptFile(plainPath, encPath, key);
+    await decryptFile(encPath, roundTripPath, key);
+
+    const result = readFileSync(roundTripPath);
+    expect(result.equals(plaintext)).toBe(true);
+
+    // Encrypted file has the IV + tag overhead
+    const encBytes = readFileSync(encPath);
+    expect(encBytes.length).toBe(
+      plaintext.length + ENCRYPTED_HEADER_SIZE + GCM_TAG_SIZE,
+    );
+  });
+
+  test("round-trips a 10 MB file (streaming across chunk boundaries)", async () => {
+    const key = randomBytes(32);
+    const size = 10 * 1024 * 1024;
+    const plaintext = randomBytes(size);
+    const plainPath = join(TEST_DIR, "plain.bin");
+    const encPath = join(TEST_DIR, "enc.bin");
+    const roundTripPath = join(TEST_DIR, "roundtrip.bin");
+
+    writeFileSync(plainPath, plaintext);
+    await encryptFile(plainPath, encPath, key);
+    await decryptFile(encPath, roundTripPath, key);
+
+    const result = readFileSync(roundTripPath);
+    expect(result.length).toBe(size);
+    expect(result.equals(plaintext)).toBe(true);
+  });
+
+  test("auth tag verification: flipping a byte in the ciphertext causes decrypt to throw", async () => {
+    const key = randomBytes(32);
+    const plaintext = randomBytes(2048);
+    const plainPath = join(TEST_DIR, "plain.bin");
+    const encPath = join(TEST_DIR, "enc.bin");
+    const outPath = join(TEST_DIR, "out.bin");
+
+    writeFileSync(plainPath, plaintext);
+    await encryptFile(plainPath, encPath, key);
+
+    // Flip one byte somewhere in the middle of the ciphertext body
+    // (not in the IV header or the trailing auth tag).
+    const ciphertextByteOffset =
+      ENCRYPTED_HEADER_SIZE + Math.floor(plaintext.length / 2);
+    const fh = await open(encPath, "r+");
+    try {
+      const one = Buffer.alloc(1);
+      await fh.read(one, 0, 1, ciphertextByteOffset);
+      one[0] = one[0] ^ 0xff;
+      await fh.write(one, 0, 1, ciphertextByteOffset);
+    } finally {
+      await fh.close();
+    }
+
+    await expect(decryptFile(encPath, outPath, key)).rejects.toThrow();
+  });
+
+  test("decrypting with the wrong key throws", async () => {
+    const keyA = makeKey(1);
+    const keyB = makeKey(99);
+    const plaintext = randomBytes(4096);
+    const plainPath = join(TEST_DIR, "plain.bin");
+    const encPath = join(TEST_DIR, "enc.bin");
+    const outPath = join(TEST_DIR, "out.bin");
+
+    writeFileSync(plainPath, plaintext);
+    await encryptFile(plainPath, encPath, keyA);
+
+    await expect(decryptFile(encPath, outPath, keyB)).rejects.toThrow();
+  });
+
+  test("passing a 16-byte key throws the typed error", async () => {
+    const badKey = randomBytes(16);
+    const plainPath = join(TEST_DIR, "plain.bin");
+    const encPath = join(TEST_DIR, "enc.bin");
+
+    writeFileSync(plainPath, Buffer.from("hello world", "utf-8"));
+
+    await expect(encryptFile(plainPath, encPath, badKey)).rejects.toThrow(
+      "Backup encryption key must be 32 bytes",
+    );
+  });
+
+  test("IV uniqueness: encrypting the same file twice yields different outputs", async () => {
+    const key = randomBytes(32);
+    const plaintext = randomBytes(4096);
+    const plainPath = join(TEST_DIR, "plain.bin");
+    const encPathA = join(TEST_DIR, "enc-a.bin");
+    const encPathB = join(TEST_DIR, "enc-b.bin");
+
+    writeFileSync(plainPath, plaintext);
+    await encryptFile(plainPath, encPathA, key);
+    await encryptFile(plainPath, encPathB, key);
+
+    const a = readFileSync(encPathA);
+    const b = readFileSync(encPathB);
+
+    expect(a.equals(b)).toBe(false);
+    // The first 12 bytes are the IV — they must differ with overwhelming
+    // probability (collision chance is 1/2^96 for random 12-byte IVs).
+    expect(
+      a.subarray(0, ENCRYPTED_HEADER_SIZE).equals(
+        b.subarray(0, ENCRYPTED_HEADER_SIZE),
+      ),
+    ).toBe(false);
+  });
+
+  test("verifyEncryptedFile returns true for a valid bundle and false for a tampered one", async () => {
+    const key = randomBytes(32);
+    const plaintext = randomBytes(1024);
+    const plainPath = join(TEST_DIR, "plain.bin");
+    const encPath = join(TEST_DIR, "enc.bin");
+
+    writeFileSync(plainPath, plaintext);
+    await encryptFile(plainPath, encPath, key);
+
+    expect(await verifyEncryptedFile(encPath, key)).toBe(true);
+
+    // Tamper the ciphertext and re-verify
+    const fh = await open(encPath, "r+");
+    try {
+      const flipOffset = ENCRYPTED_HEADER_SIZE + 10;
+      const one = Buffer.alloc(1);
+      await fh.read(one, 0, 1, flipOffset);
+      one[0] = one[0] ^ 0x01;
+      await fh.write(one, 0, 1, flipOffset);
+    } finally {
+      await fh.close();
+    }
+
+    expect(await verifyEncryptedFile(encPath, key)).toBe(false);
+  });
+
+  test("verifyEncryptedFile returns true for a valid bundle even when tmpdir is unwritable", async () => {
+    const key = randomBytes(32);
+    const plaintext = randomBytes(2048);
+    const plainPath = join(TEST_DIR, "plain.bin");
+    const encPath = join(TEST_DIR, "enc.bin");
+
+    writeFileSync(plainPath, plaintext);
+    await encryptFile(plainPath, encPath, key);
+
+    // Point tmpdir at a path that cannot be written to. The implementation
+    // must authenticate the bundle without writing any scratch file — a full
+    // or read-only /tmp must not block restore for healthy backups.
+    const originalTmpdir = process.env.TMPDIR;
+    const originalTmp = process.env.TMP;
+    const originalTemp = process.env.TEMP;
+    process.env.TMPDIR = "/dev/null/does-not-exist";
+    process.env.TMP = "/dev/null/does-not-exist";
+    process.env.TEMP = "/dev/null/does-not-exist";
+    try {
+      expect(await verifyEncryptedFile(encPath, key)).toBe(true);
+    } finally {
+      if (originalTmpdir === undefined) delete process.env.TMPDIR;
+      else process.env.TMPDIR = originalTmpdir;
+      if (originalTmp === undefined) delete process.env.TMP;
+      else process.env.TMP = originalTmp;
+      if (originalTemp === undefined) delete process.env.TEMP;
+      else process.env.TEMP = originalTemp;
+    }
+  });
+
+  test("verifyEncryptedFile rethrows filesystem errors (e.g. ENOENT) instead of masking them as tamper", async () => {
+    const key = randomBytes(32);
+    const missingPath = join(TEST_DIR, "does-not-exist.bin");
+
+    await expect(verifyEncryptedFile(missingPath, key)).rejects.toThrow();
+  });
+});