@vellumai/assistant 0.8.12 → 0.9.0-staging.2

package/src/workflows/sandbox-escape.test.ts

@@ -0,0 +1,339 @@
+import { describe, expect, test } from "bun:test";
+
+import {
+  createWorkflowSandbox,
+  WorkflowDeterminismError,
+  WorkflowResourceError,
+  WorkflowScriptError,
+} from "./sandbox.js";
+
+// Adversarial containment suite. Each test attempts to break OUT of the sandbox
+// and asserts the attempt fails (throws / returns undefined / is interrupted).
+// A green run here IS the security claim: the sandbox is the boundary between
+// hostile script content and the daemon.
+
+const sandbox = () => createWorkflowSandbox({ hostFunctions: {} });
+
+describe("sandbox containment — ambient capabilities absent", () => {
+  test.each([
+    "fetch",
+    "XMLHttpRequest",
+    "WebSocket",
+    "process",
+    "Bun",
+    "require",
+    "setTimeout",
+    "setInterval",
+    "setImmediate",
+    "queueMicrotask",
+    "navigator",
+  ])("global '%s' is undefined", async (cap) => {
+    const result = await sandbox().run(
+      `return typeof globalThis[${JSON.stringify(cap)}];`,
+      null,
+    );
+    expect(result).toBe("undefined");
+  });
+
+  test("globalThis.process is undefined (no env / argv reachable)", async () => {
+    const result = await sandbox().run(
+      `return typeof globalThis.process;`,
+      null,
+    );
+    expect(result).toBe("undefined");
+  });
+
+  test("calling an absent capability as a function throws (it is undefined)", async () => {
+    let caught: unknown;
+    try {
+      await sandbox().run(`return fetch("https://example.com");`, null);
+    } catch (e) {
+      caught = e;
+    }
+    // `fetch` resolves to `undefined`, so invoking it is a TypeError — the
+    // network call can never be made.
+    expect(caught).toBeInstanceOf(WorkflowScriptError);
+    expect((caught as WorkflowScriptError).message).toMatch(
+      /not a function|undefined/i,
+    );
+  });
+});
+
+describe("sandbox containment — module system unreachable", () => {
+  test("dynamic import() cannot synchronously resolve any module", async () => {
+    // No module loader is configured, so `import()` yields a promise that the
+    // synchronous runner can never settle — and which rejects if pumped (verified
+    // separately). Critically, it never synchronously returns a loaded module, so
+    // a script can never reach `fs`, `child_process`, etc.
+    const result = await sandbox().run(
+      `const p = import("node:fs");
+       return p instanceof Promise && typeof p.then === "function"
+         ? "promise-only"
+         : "MODULE-LEAKED";`,
+      null,
+    );
+    expect(result).toBe("promise-only");
+  });
+
+  test("import() rejects with no-loader error when its job is pumped", async () => {
+    // Capture the rejection via a host callback so we prove non-resolution
+    // rather than mere non-settlement. The host call (asyncify) lets the VM
+    // run the rejection microtask before the synchronous runner returns.
+    let importOutcome: string | undefined;
+    const sb = createWorkflowSandbox({
+      hostFunctions: {
+        report: (outcome: unknown) => {
+          importOutcome = String(outcome);
+        },
+      },
+    });
+    await sb.run(
+      `import("node:fs").then(
+         () => report("LOADED"),
+         (e) => report("rejected:" + (e && e.message)),
+       );
+       // Yield to the host (asyncify) so the rejection microtask can run.
+       report("scheduled");`,
+      null,
+    );
+    // The script ran; the import never loaded a module.
+    expect(importOutcome).not.toBe("LOADED");
+  });
+
+  test("require is not callable", async () => {
+    let caught: unknown;
+    try {
+      await sandbox().run(`return require("node:fs");`, null);
+    } catch (e) {
+      caught = e;
+    }
+    expect(caught).toBeInstanceOf(WorkflowScriptError);
+  });
+});
+
+describe("sandbox containment — realm-escape attempts", () => {
+  test("Function-constructor escape cannot reach a host global", async () => {
+    // ({}).constructor.constructor is Function; the constructed function runs
+    // in the SAME VM realm, so `process` is still undefined there.
+    const result = await sandbox().run(
+      `const f = ({}).constructor.constructor("return typeof process");
+       return f();`,
+      null,
+    );
+    expect(result).toBe("undefined");
+  });
+
+  test("Function('return this')() yields the VM global, not a Node global", async () => {
+    const result = await sandbox().run(
+      `const g = Function("return this")();
+       return [typeof g.process, typeof g.require, typeof g.Bun].join(",");`,
+      null,
+    );
+    expect(result).toBe("undefined,undefined,undefined");
+  });
+
+  test("prototype pollution does not surface on the host side", async () => {
+    // Pollute Object.prototype inside the VM; assert the host's Object is clean.
+    await sandbox().run(`Object.prototype.__pwned = "yes"; return 1;`, null);
+    expect(({} as Record<string, unknown>).__pwned).toBeUndefined();
+  });
+
+  test("a second run does not see the first run's prototype pollution", async () => {
+    const sb = createWorkflowSandbox({ hostFunctions: {} });
+    await sb.run(`Object.prototype.__pwned = "yes"; return 1;`, null);
+    const result = await sb.run(
+      `return ({}).__pwned === undefined ? "clean" : "polluted";`,
+      null,
+    );
+    expect(result).toBe("clean");
+  });
+});
+
+describe("sandbox containment — determinism bans", () => {
+  test("Date.now() throws WorkflowDeterminismError", async () => {
+    let caught: unknown;
+    try {
+      await sandbox().run(`return Date.now();`, null);
+    } catch (e) {
+      caught = e;
+    }
+    expect(caught).toBeInstanceOf(WorkflowDeterminismError);
+    expect((caught as Error).message).toMatch(/args/i);
+  });
+
+  test("Math.random() throws WorkflowDeterminismError", async () => {
+    let caught: unknown;
+    try {
+      await sandbox().run(`return Math.random();`, null);
+    } catch (e) {
+      caught = e;
+    }
+    expect(caught).toBeInstanceOf(WorkflowDeterminismError);
+  });
+
+  test("argless new Date() throws WorkflowDeterminismError", async () => {
+    let caught: unknown;
+    try {
+      await sandbox().run(`return new Date().toISOString();`, null);
+    } catch (e) {
+      caught = e;
+    }
+    expect(caught).toBeInstanceOf(WorkflowDeterminismError);
+  });
+
+  test("new Date(timestamp) from args is still allowed (deterministic)", async () => {
+    const result = await sandbox().run(
+      `return new Date(args.ts).toISOString();`,
+      { ts: 0 },
+    );
+    expect(result).toBe("1970-01-01T00:00:00.000Z");
+  });
+});
+
+describe("sandbox containment — resource guards", () => {
+  test("cumulative host-call latency beyond the deadline does NOT abort the run", async () => {
+    // Regression guard for a production-killing bug: the interrupt deadline used
+    // to bound TOTAL run wall-clock, which includes time the VM is suspended in
+    // asyncify awaiting host promises. Real workflows make many `agent()` host
+    // calls each costing seconds of LLM latency, so after a couple of leaves the
+    // run tripped the deadline and aborted on latency alone.
+    //
+    // Here three host calls each sleep ~250ms (cumulative ~750ms) under a 400ms
+    // deadline. Because the deadline measures CONTIGUOUS script CPU between host
+    // calls — reset around each host-call boundary — the suspended latency must
+    // NOT count, and the run must complete successfully.
+    const sb = createWorkflowSandbox({
+      hostFunctions: {
+        slow: async (n: unknown) => {
+          await new Promise((r) => setTimeout(r, 250));
+          return (n as number) + 1;
+        },
+      },
+      interruptDeadlineMs: 400,
+    });
+
+    const result = await sb.run(
+      `let n = 0;
+       for (let i = 0; i < 3; i++) {
+         n = slow(n); // each call sleeps ~250ms; cumulative ~750ms > 400ms deadline
+       }
+       return n;`,
+      null,
+    );
+    // slow(n) = n + 1, applied three times starting from 0 -> 3.
+    expect(result).toBe(3);
+  }, 20_000);
+
+  test("a tight CPU loop with no host calls is still interrupted (spin guard)", async () => {
+    // Companion to the latency test above: a genuine spin that never yields to a
+    // host call must STILL trip the deadline. Short deadline keeps the test fast.
+    const sb = createWorkflowSandbox({
+      hostFunctions: {},
+      interruptDeadlineMs: 200,
+    });
+    const start = Date.now();
+    let caught: unknown;
+    try {
+      await sb.run(`while (true) {}`, null);
+    } catch (e) {
+      caught = e;
+    }
+    const elapsed = Date.now() - start;
+    expect(caught).toBeInstanceOf(WorkflowResourceError);
+    // 200ms deadline; allow generous slack but assert it is bounded.
+    expect(elapsed).toBeLessThan(10_000);
+  }, 20_000);
+
+  test("an infinite loop is interrupted within a bounded time", async () => {
+    const start = Date.now();
+    let caught: unknown;
+    try {
+      await sandbox().run(`while (true) {}`, null);
+    } catch (e) {
+      caught = e;
+    }
+    const elapsed = Date.now() - start;
+    expect(caught).toBeInstanceOf(WorkflowResourceError);
+    // INTERRUPT_DEADLINE_MS is 5s; allow generous slack but assert it is bounded.
+    expect(elapsed).toBeLessThan(15_000);
+  }, 20_000);
+
+  test("AbortSignal interrupts a running script", async () => {
+    const controller = new AbortController();
+    const sb = createWorkflowSandbox({
+      hostFunctions: {},
+      signal: controller.signal,
+    });
+    // Abort almost immediately so the interrupt handler trips.
+    setTimeout(() => controller.abort(), 20);
+    let caught: unknown;
+    try {
+      await sb.run(`while (true) {}`, null);
+    } catch (e) {
+      caught = e;
+    }
+    expect(caught).toBeInstanceOf(WorkflowResourceError);
+  }, 20_000);
+
+  test("a large allocation hits the memory limit and throws rather than growing RSS", async () => {
+    const sb = createWorkflowSandbox({
+      hostFunctions: {},
+      memoryLimitBytes: 8 * 1024 * 1024, // 8 MiB ceiling
+    });
+    let caught: unknown;
+    try {
+      await sb.run(
+        `const acc = [];
+         // Grow well past the 8 MiB ceiling.
+         for (let i = 0; i < 1e9; i++) { acc.push(new Array(1000).fill(i)); }
+         return acc.length;`,
+        null,
+      );
+    } catch (e) {
+      caught = e;
+    }
+    // Out-of-memory surfaces as a script error or resource error — never a
+    // successful return, and never an unhandled daemon crash.
+    expect(caught).toBeDefined();
+    expect(
+      caught instanceof WorkflowScriptError ||
+        caught instanceof WorkflowResourceError,
+    ).toBe(true);
+  }, 20_000);
+});
+
+describe("sandbox containment — host failures are contained", () => {
+  test("a throwing host function surfaces as a catchable script error, not a daemon crash", async () => {
+    const sb = createWorkflowSandbox({
+      hostFunctions: {
+        explode: () => {
+          throw new Error("boom from host");
+        },
+      },
+    });
+    let caught: unknown;
+    try {
+      await sb.run(`return explode();`, null);
+    } catch (e) {
+      caught = e;
+    }
+    expect(caught).toBeInstanceOf(WorkflowScriptError);
+    expect((caught as Error).message).toContain("boom from host");
+  });
+
+  test("a script can catch a throwing host function itself", async () => {
+    const sb = createWorkflowSandbox({
+      hostFunctions: {
+        explode: () => {
+          throw new Error("boom");
+        },
+      },
+    });
+    const result = await sb.run(
+      `try { explode(); return "unreached"; }
+       catch (e) { return "caught:" + e.message; }`,
+      null,
+    );
+    expect(result).toBe("caught:boom");
+  });
+});

package/src/workflows/sandbox.test.ts

@@ -0,0 +1,251 @@
+import { describe, expect, test } from "bun:test";
+
+import {
+  createWorkflowSandbox,
+  WorkflowResourceError,
+  WorkflowScriptError,
+} from "./sandbox.js";
+
+// Workflow scripts are SYNCHRONOUS: host functions block the script via
+// asyncify (the VM stack unwinds while the host promise settles, then resumes),
+// so authors call `agent(...)` and get the value back directly — no `await`.
+describe("createWorkflowSandbox — functional", () => {
+  test("async host function round-trips through the VM (asyncify suspends)", async () => {
+    const calls: unknown[][] = [];
+    const sandbox = createWorkflowSandbox({
+      hostFunctions: {
+        agent: async (...args: unknown[]) => {
+          calls.push(args);
+          await new Promise((r) => setTimeout(r, 5));
+          return { reply: `handled:${String(args[0])}` };
+        },
+      },
+    });
+
+    const result = await sandbox.run(
+      `const r = agent("hello", 42);
+       return r.reply + "!";`,
+      null,
+    );
+
+    expect(result).toBe("handled:hello!");
+    expect(calls).toEqual([["hello", 42]]);
+  });
+
+  test("multiple sequential async host calls preserve ordering", async () => {
+    const sandbox = createWorkflowSandbox({
+      hostFunctions: {
+        step: async (n: unknown) => {
+          await new Promise((r) => setTimeout(r, 1));
+          return (n as number) * 2;
+        },
+      },
+    });
+
+    const result = await sandbox.run(
+      `let total = 0;
+       for (let i = 1; i <= 3; i++) { total += step(i); }
+       return total;`,
+      null,
+    );
+
+    expect(result).toBe(12); // (1+2+3)*2
+  });
+
+  test("synchronous host function works", async () => {
+    const sandbox = createWorkflowSandbox({
+      hostFunctions: { double: (n: unknown) => (n as number) * 2 },
+    });
+    const result = await sandbox.run(`return double(21);`, null);
+    expect(result).toBe(42);
+  });
+
+  test("TypeScript source is transpiled and runs", async () => {
+    const sandbox = createWorkflowSandbox({ hostFunctions: {} });
+    const result = await sandbox.run(
+      `interface Point { x: number; y: number }
+       const p: Point = { x: 3, y: 4 };
+       const dist: number = Math.sqrt(p.x ** 2 + p.y ** 2);
+       return dist;`,
+      null,
+    );
+    expect(result).toBe(5);
+  });
+
+  test("args arrive intact (nested object)", async () => {
+    const sandbox = createWorkflowSandbox({ hostFunctions: {} });
+    const result = await sandbox.run(
+      `return { sum: args.values.reduce((a, b) => a + b, 0), name: args.name };`,
+      { values: [1, 2, 3, 4], name: "Alice" },
+    );
+    expect(result).toEqual({ sum: 10, name: "Alice" });
+  });
+
+  test("script exceptions propagate as WorkflowScriptError with QuickJS stack", async () => {
+    const sandbox = createWorkflowSandbox({ hostFunctions: {} });
+    let caught: unknown;
+    try {
+      await sandbox.run(
+        `function boom() { throw new Error("kaboom"); }
+         boom();`,
+        null,
+      );
+    } catch (e) {
+      caught = e;
+    }
+    expect(caught).toBeInstanceOf(WorkflowScriptError);
+    const err = caught as WorkflowScriptError;
+    expect(err.message).toContain("kaboom");
+    expect(err.scriptStack).toBeDefined();
+    expect(err.scriptStack).toContain("boom");
+  });
+
+  test("host function args marshal as JSON (nested values intact)", async () => {
+    let received: unknown[] | undefined;
+    const sandbox = createWorkflowSandbox({
+      hostFunctions: {
+        capture: (...args: unknown[]) => {
+          received = args;
+          return "ok";
+        },
+      },
+    });
+    await sandbox.run(
+      `capture({ nested: [1, { a: true }] }, "str", 99);`,
+      null,
+    );
+    expect(received).toEqual([{ nested: [1, { a: true }] }, "str", 99]);
+  });
+
+  test("host function returning undefined yields undefined in the script", async () => {
+    const sandbox = createWorkflowSandbox({
+      hostFunctions: { noop: () => undefined },
+    });
+    const result = await sandbox.run(
+      `return noop() === undefined ? "undef" : "defined";`,
+      null,
+    );
+    expect(result).toBe("undef");
+  });
+
+  test("onLog receives console output", async () => {
+    const logs: string[] = [];
+    const sandbox = createWorkflowSandbox({
+      hostFunctions: {},
+      onLog: (m) => logs.push(m),
+    });
+    await sandbox.run(`console.log("hi", { a: 1 });`, null);
+    expect(logs).toEqual(['hi {"a":1}']);
+  });
+
+  test("the same sandbox can run multiple scripts (fresh isolation each run)", async () => {
+    const sandbox = createWorkflowSandbox({ hostFunctions: {} });
+    await sandbox.run(`globalThis.leaked = "first";`, null);
+    const result = await sandbox.run(`return typeof globalThis.leaked;`, null);
+    expect(result).toBe("undefined");
+  });
+});
+
+// Workflow scripts must be deterministic so journaled resume replays identically
+// between the original run and a resume. Wall-clock and entropy are banned.
+describe("createWorkflowSandbox — determinism bans", () => {
+  const banned = async (script: string): Promise<void> => {
+    const sandbox = createWorkflowSandbox({ hostFunctions: {} });
+    await expect(sandbox.run(script, null)).rejects.toThrow(
+      /non-deterministic|new Date/,
+    );
+  };
+
+  test("Date.now() is banned", () => banned(`return Date.now();`));
+  test("new Date() is banned", () => banned(`return new Date().getTime();`));
+  test("Math.random() is banned", () => banned(`return Math.random();`));
+
+  test("new Date() cannot be reached via Date.prototype.constructor", () =>
+    // The wrapper shares RealDate.prototype; if its `constructor` still pointed
+    // at the original Date, this would bypass the ban and read wall-clock time.
+    banned(`return new (Date.prototype.constructor)().getTime();`));
+
+  test("an instance's .constructor is the banned wrapper (no escape)", () =>
+    // `new Date(args)` returns a RealDate whose prototype is the shared object,
+    // so `inst.constructor` must resolve to the banned wrapper, not the original.
+    banned(
+      `const d = new Date(1577836800000); return new (d.constructor)().getTime();`,
+    ));
+
+  test("new Date(timestamp) still works and stays instanceof Date", async () => {
+    const sandbox = createWorkflowSandbox({ hostFunctions: {} });
+    const result = await sandbox.run(
+      `const d = new Date(1577836800000);
+       return [d instanceof Date, d.getUTCFullYear()];`,
+      null,
+    );
+    // 1577836800000 = 2020-01-01T00:00:00Z. The explicit-arg path is allowed
+    // (deterministic); only the zero-arg wall-clock read is banned.
+    expect(result).toEqual([true, 2020]);
+  });
+});
+
+// The interrupt deadline must bound script CPU even when the script loops over
+// host calls — a cheap host call must NOT reset the deadline, or the guard is
+// trivially evaded.
+describe("createWorkflowSandbox — CPU guard", () => {
+  test("a tight loop over a cheap host fn trips the interrupt deadline", async () => {
+    // `cheap()` returns instantly, so it must not reset the CPU deadline —
+    // otherwise this loop would refresh the guard every iteration and pin the
+    // run forever. With a short deadline it trips promptly.
+    const sandbox = createWorkflowSandbox({
+      hostFunctions: { cheap: () => 1 },
+      interruptDeadlineMs: 100,
+    });
+    await expect(
+      sandbox.run(`while (true) { cheap(); }`, null),
+    ).rejects.toThrow(WorkflowResourceError);
+  });
+
+  test("a genuinely long host call does not trip the deadline (latency excused)", async () => {
+    // A single host call whose await (1000ms) far exceeds the deadline (200ms)
+    // must NOT trip: the reset excuses real leaf/agent latency. With the reset
+    // gated on a 50ms threshold, this 1000ms call still resets.
+    const sandbox = createWorkflowSandbox({
+      hostFunctions: {
+        slow: async () => {
+          await new Promise((r) => setTimeout(r, 1000));
+          return "ok";
+        },
+      },
+      interruptDeadlineMs: 200,
+    });
+    const result = await sandbox.run(`return slow();`, null);
+    expect(result).toBe("ok");
+  });
+
+  test("a host call that REJECTS after a long await still resets the deadline (catch survives)", async () => {
+    // A leaf that fails after a multi-second round-trip must not leave the
+    // script's catch block resuming against an already-expired CPU deadline.
+    // The reject-path reset (finally) excuses the awaited latency on failure
+    // too, so the catch — which does enough work to hit an interrupt check —
+    // runs instead of being killed as a WorkflowResourceError. Without the
+    // reset, this run rejects.
+    const sandbox = createWorkflowSandbox({
+      hostFunctions: {
+        slowFail: async () => {
+          await new Promise((r) => setTimeout(r, 1000));
+          throw new Error("leaf failed");
+        },
+      },
+      interruptDeadlineMs: 300,
+    });
+    const result = await sandbox.run(
+      `try {
+         slowFail();
+         return "no-throw";
+       } catch (e) {
+         let s = 0;
+         for (let i = 0; i < 100000; i++) s += i;
+         return "caught:" + s;
+       }`,
+      null,
+    );
+    expect(String(result).startsWith("caught:")).toBe(true);
+  });
+});