failproofai 1.0.7-beta.1 → 1.0.7-beta.2

package/.next/standalone/sdk/typescript/test/evaluator-protocol.test.ts

@@ -0,0 +1,225 @@
+import { describe, expect, it } from "vitest";
+
+import {
+  Assertion,
+  ConditionResult,
+  EvalResult,
+  Evaluator,
+  Metric,
+  Score,
+} from "../src/evaluator/authoring.js";
+import {
+  ExecutionMode,
+  PROTOCOL_VERSION,
+  ProtocolError,
+  ResultKind,
+  UnsupportedProtocolVersion,
+  assignmentDefinitionFromWire,
+  assignmentFromWire,
+  claimResponseFromWire,
+  planResponseFromWire,
+  registerResponseFromWire,
+  sessionTranscriptFromWire,
+} from "../src/evaluator/protocol.js";
+import { transcriptWire } from "./helpers.js";
+
+/**
+ * Every `fromWire` is a VALIDATOR, not a cast. The server is across a network
+ * boundary: a field that arrives as a number where a string was promised has to
+ * fail here, naming the field — not three frames later as an undefined property
+ * nobody can explain.
+ */
+
+describe("protocol version", () => {
+  it("refuses a version it does not implement", () => {
+    expect(() =>
+      registerResponseFromWire({ protocol_version: "3", evaluator_instance_id: "x" }),
+    ).toThrow(UnsupportedProtocolVersion);
+  });
+
+  it("accepts its own", () => {
+    expect(PROTOCOL_VERSION).toBe("2");
+  });
+});
+
+describe("wire validation", () => {
+  const assignment = {
+    assignment_id: "a1",
+    lease_generation: 1,
+    lease_expires_at: "2026-01-01T00:02:00Z",
+    session_id: "s1",
+    session_revision_id: "r1",
+    agent_id: "main",
+    environment: "dev",
+    trigger_reason: "session_end",
+    event_count: 3,
+    transcript_url: "/v1/evaluator/assignments/a1/transcript",
+  };
+
+  it("reads a well-formed assignment", () => {
+    expect(assignmentFromWire(assignment)).toMatchObject({ assignmentId: "a1", eventCount: 3 });
+  });
+
+  it("names the offending field", () => {
+    expect(() => assignmentFromWire({ ...assignment, session_id: 7 })).toThrow(
+      /session_id must be a string/,
+    );
+    expect(() => assignmentFromWire({ ...assignment, lease_generation: 0 })).toThrow(
+      /lease_generation must be greater than zero/,
+    );
+    expect(() => assignmentFromWire({ ...assignment, event_count: -1 })).toThrow(
+      /event_count must not be negative/,
+    );
+  });
+
+  it("requires execution_mode explicitly rather than defaulting to local", () => {
+    // Coercing a missing value to `local` silently runs a server-authored
+    // definition down the customer-local path, or the reverse.
+    const definition = {
+      eval_key: "k",
+      display_name: "K",
+      eval_version: "1",
+      result_kind: "score",
+      labels: [],
+    };
+    expect(() => assignmentDefinitionFromWire(definition)).toThrow(ProtocolError);
+    expect(
+      assignmentDefinitionFromWire({ ...definition, execution_mode: "local" }).executionMode,
+    ).toBe(ExecutionMode.LOCAL);
+    // `python` is the wire spelling of "the server authored this, sandbox it".
+    expect(
+      assignmentDefinitionFromWire({ ...definition, execution_mode: "python" }).executionMode,
+    ).toBe(ExecutionMode.SANDBOX);
+  });
+
+  it("refuses a transcript whose event_count disagrees with its events", () => {
+    const wire = transcriptWire([{ type: "tool_use" }]);
+    wire.event_count = 5;
+    expect(() => sessionTranscriptFromWire(wire)).toThrow(/event_count is 5/);
+  });
+
+  it("refuses a transcript schema version it does not implement", () => {
+    const wire = transcriptWire();
+    wire.schema_version = "9";
+    expect(() => sessionTranscriptFromWire(wire)).toThrow(/unsupported transcript schema version/);
+  });
+
+  it("round-trips a transcript through toWire", () => {
+    const wire = transcriptWire([{ type: "tool_use", payload: { a: 1 } }]);
+    expect(sessionTranscriptFromWire(sessionTranscriptFromWire(wire).toWire()).events).toHaveLength(1);
+  });
+
+  it("refuses a non-boolean idempotent_replay", () => {
+    expect(() =>
+      planResponseFromWire({
+        protocol_version: "2",
+        assignment_id: "a1",
+        assignment_status: "planned",
+        runs: [],
+        idempotent_replay: "yes",
+      }),
+    ).toThrow(/idempotent_replay must be a boolean/);
+  });
+
+  it("refuses a non-array assignments list", () => {
+    expect(() => claimResponseFromWire({ protocol_version: "2", assignments: {} })).toThrow(
+      /assignments must be an array/,
+    );
+  });
+});
+
+describe("transcript helpers", () => {
+  const session = sessionTranscriptFromWire(
+    transcriptWire([{ type: "tool_use" }, { type: "tool_result" }, { type: "tool_result" }]),
+  );
+
+  it("filters and counts by event type", () => {
+    expect(session.count("tool_result")).toBe(2);
+    expect(session.eventsOfType("tool_use")).toHaveLength(1);
+    expect(session.count("absent")).toBe(0);
+  });
+});
+
+describe("authoring bounds", () => {
+  it("keeps a score inside 0..1", () => {
+    expect(() => new Score(1.5)).toThrow(/between 0 and 1/);
+    expect(() => new Score(Number.NaN)).toThrow(/must be finite/);
+    expect(new Score(0.5).unit).toBe("ratio");
+  });
+
+  it("refuses control characters the server would 422 on", () => {
+    // A non-retryable 422 loses a SUCCESSFUL evaluation and dead-letters its
+    // assignment, so this has to fail at the line that wrote it.
+    expect(() => new EvalResult({ reasoning: "line\u0000break" })).toThrow(
+      /must not contain control characters/,
+    );
+    // Real multi-line reasoning is fine.
+    expect(() => new EvalResult({ reasoning: "line\nbreak\ttab" })).not.toThrow();
+  });
+
+  it("normalizes and de-duplicates labels", () => {
+    expect(new EvalResult({ score: new Score(1), labels: ["b", "a"] }).labels).toEqual(["a", "b"]);
+    expect(() => new EvalResult({ labels: ["a", "a"] })).toThrow(/must be unique/);
+  });
+
+  it("requires at least one result", () => {
+    expect(() => new EvalResult({}).resultItems("k")).toThrow(/must contain a score, metric/);
+  });
+
+  it("attaches the eval's reasoning to the PRIMARY result of a non-score eval", () => {
+    const items = new EvalResult({
+      metrics: { latency: 12, other: 3 },
+      reasoning: "because",
+    }).resultItems("latency");
+    expect(items.find((item) => item.resultKey === "latency")!.reasoning).toBe("because");
+    expect(items.find((item) => item.resultKey === "other")!.reasoning).toBeNull();
+  });
+
+  it("coerces a bare number or boolean into a Metric or an Assertion", () => {
+    const items = new EvalResult({ metrics: { a: 1 }, assertions: { b: true } }).resultItems("a");
+    expect(items.map((item) => item.resultKind)).toEqual([ResultKind.METRIC, ResultKind.ASSERTION]);
+    expect(new Metric(1).unit).toBe("");
+    expect(new Assertion(true).passed).toBe(true);
+  });
+
+  it("validates a condition's reason code as a key", () => {
+    expect(() => new ConditionResult(false, "Not A Key")).toThrow(/must match/);
+    expect(new ConditionResult(false).reasonCode).toBe("condition_false");
+  });
+});
+
+describe("Evaluator registry", () => {
+  it("validates the eval key shape", () => {
+    const app = new Evaluator({ name: "n", version: "1" });
+    expect(() => app.eval("Bad-Key", { version: "1" }, () => new EvalResult())).toThrow(/must match/);
+  });
+
+  it("refuses a duplicate key", () => {
+    const app = new Evaluator({ name: "n", version: "1" });
+    app.eval("a", { version: "1" }, () => new EvalResult());
+    expect(() => app.eval("a", { version: "2" }, () => new EvalResult())).toThrow(/duplicate/);
+  });
+
+  it("derives a display name from the key", () => {
+    const app = new Evaluator({ name: "n", version: "1" });
+    app.eval("tool_success_rate", { version: "1" }, () => new EvalResult());
+    expect(app.definition("tool_success_rate").displayName).toBe("Tool success rate");
+  });
+
+  it("produces a stable catalog revision that changes with the catalog", () => {
+    const build = (version: string): string => {
+      const app = new Evaluator({ name: "n", version: "1" });
+      app.eval("b", { version }, () => new EvalResult());
+      app.eval("a", { version: "1" }, () => new EvalResult());
+      return app.catalogRevision;
+    };
+    expect(build("1")).toBe(build("1"));
+    expect(build("1")).not.toBe(build("2"));
+    expect(build("1")).toMatch(/^sha256:[0-9a-f]{64}$/);
+    // Sorted, so registration order cannot change the hash.
+    const app = new Evaluator({ name: "n", version: "1" });
+    app.eval("z", { version: "1" }, () => new EvalResult());
+    app.eval("a", { version: "1" }, () => new EvalResult());
+    expect(app.catalog().map((item) => item.evalKey)).toEqual(["a", "z"]);
+  });
+});

package/.next/standalone/sdk/typescript/test/events.test.ts

@@ -0,0 +1,193 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+import { EventNamespace } from "../src/events.js";
+import { agent, session } from "../src/scopes.js";
+import { runtime } from "../src/runtime.js";
+import { setLogger } from "../src/logger.js";
+import { flushed, useSpool } from "./helpers.js";
+import type { Spool } from "./helpers.js";
+
+let spool: Spool;
+
+beforeEach(() => {
+  spool = useSpool();
+});
+
+afterEach(async () => {
+  await spool.cleanup();
+  setLogger(null);
+});
+
+const event = (): EventNamespace => runtime.event;
+
+describe("identity resolution", () => {
+  it("resolves sessionId and agentId from the enclosing scope", async () => {
+    await session({ sessionId: "s1" }, async () => {
+      await agent("planner", async () => {
+        event().toolUse({ toolName: "t", toolCallId: "c1" });
+      });
+    });
+    const events = await flushed(spool);
+    const toolUse = events.find((e) => e.type === "tool_use")!;
+    expect(toolUse.session_id).toBe("s1");
+    expect(toolUse.agent_id).toBe("planner");
+  });
+
+  it("falls back to 'main' for agentId but never invents a sessionId", async () => {
+    await session({ sessionId: "s1" }, () => {
+      event().toolUse({ toolName: "t", toolCallId: "c1" });
+    });
+    expect((await flushed(spool))[0]!.agent_id).toBe("main");
+  });
+
+  it("throws, naming the fix, when nothing is bound", () => {
+    expect(() => event().toolUse({ toolName: "t", toolCallId: "c1" })).toThrow(
+      /sessionId is required and nothing is bound/,
+    );
+  });
+
+  it("refuses an empty sessionId, which the server would accept and silently merge", () => {
+    expect(() => event().toolUse({ sessionId: "  ", toolName: "t", toolCallId: "c" })).toThrow(
+      /must not be empty/,
+    );
+  });
+
+  it("refuses a non-string sessionId, which the server skips at 200 OK", () => {
+    expect(() =>
+      event().toolUse({ sessionId: 42 as unknown as string, toolName: "t", toolCallId: "c" }),
+    ).toThrow(TypeError);
+  });
+});
+
+describe("reserved and promoted fields", () => {
+  it("refuses a reserved name as a custom field", () => {
+    expect(() =>
+      event().toolUse({ sessionId: "s", toolName: "t", toolCallId: "c", type: "nope" }),
+    ).toThrow(/Reserved field names/);
+  });
+
+  it("reports the reserved-field fault before the identity fault", () => {
+    // Both are true here. The reserved name reads identically from anywhere, so
+    // it is the more useful of the two to hear.
+    expect(() => event().toolUse({ toolName: "t", toolCallId: "c", timestamp: "x" })).toThrow(
+      /Reserved field names/,
+    );
+  });
+
+  it("drops a nullish promoted extra with a warning rather than NULLing the column", async () => {
+    const warn = vi.fn();
+    setLogger({ debug: vi.fn(), info: vi.fn(), warn, error: vi.fn() });
+    event().agentStart({ sessionId: "s", model: null });
+    expect(warn).toHaveBeenCalledWith(expect.stringContaining("model was passed as null"));
+    const [emitted] = await flushed(spool);
+    expect("model" in emitted!).toBe(false);
+  });
+
+  it("refuses a non-integer token count, which the server stores as NULL", () => {
+    expect(() => event().modelResponse({ sessionId: "s", inputTokens: 1.5 })).toThrow(TypeError);
+    expect(() => event().modelResponse({ sessionId: "s", outputTokens: -1 })).toThrow(RangeError);
+    expect(() => event().modelResponse({ sessionId: "s", inputTokens: 2 ** 32 })).toThrow(RangeError);
+  });
+
+  it("refuses a non-string promoted extra", () => {
+    expect(() =>
+      event().agentStart({ sessionId: "s", tool_name: 7 as unknown as string }),
+    ).toThrow(TypeError);
+  });
+});
+
+describe("duration pairing", () => {
+  it("computes duration_ms from the matching start event", async () => {
+    event().toolUse({ sessionId: "s", toolName: "t", toolCallId: "c1" });
+    await new Promise((resolve) => setTimeout(resolve, 12));
+    event().toolResult({ sessionId: "s", toolName: "t", toolCallId: "c1" });
+    const result = (await flushed(spool)).find((e) => e.type === "tool_result")!;
+    expect(typeof result.duration_ms).toBe("number");
+    expect(result.duration_ms as number).toBeGreaterThanOrEqual(10);
+  });
+
+  it("refuses a caller-supplied duration_ms on every paired event", () => {
+    const cases: Array<() => void> = [
+      () => event().toolResult({ sessionId: "s", toolName: "t", toolCallId: "c", duration_ms: 1 }),
+      () => event().agentResume({ sessionId: "s", pauseId: "p", duration_ms: 1 }),
+      () => event().hookCompleted({ sessionId: "s", hookName: "h", hookId: "h", duration_ms: 1 }),
+      () => event().humanInput({ sessionId: "s", inputId: "i", duration_ms: 1 }),
+    ];
+    for (const run of cases) expect(run).toThrow(/auto-computed/);
+  });
+
+  it("keys the pair by kind AND session, so a shared step id cannot cross-pair", async () => {
+    // A tool call and a hook sharing an id is not exotic — both are frequently
+    // the harness's own step id.
+    event().toolUse({ sessionId: "s", toolName: "t", toolCallId: "step-1" });
+    event().hookCompleted({ sessionId: "s", hookName: "h", hookId: "step-1" });
+    event().toolResult({ sessionId: "s", toolName: "t", toolCallId: "step-1" });
+
+    const events = await flushed(spool);
+    const hook = events.find((e) => e.type === "hook_completed")!;
+    const tool = events.find((e) => e.type === "tool_result")!;
+    // The hook never opened, so it has no duration; the tool's survived.
+    expect("duration_ms" in hook).toBe(false);
+    expect(typeof tool.duration_ms).toBe("number");
+  });
+
+  it("does not key the pair by agent, so a tool opened and closed under different agents still pairs", async () => {
+    event().toolUse({ sessionId: "s", agentId: "planner", toolName: "t", toolCallId: "c1" });
+    event().toolResult({ sessionId: "s", agentId: "worker", toolName: "t", toolCallId: "c1" });
+    const result = (await flushed(spool)).find((e) => e.type === "tool_result")!;
+    expect(typeof result.duration_ms).toBe("number");
+  });
+
+  it("separates identical step ids in different sessions", async () => {
+    event().toolUse({ sessionId: "a", toolName: "t", toolCallId: "step" });
+    event().toolUse({ sessionId: "b", toolName: "t", toolCallId: "step" });
+    event().toolResult({ sessionId: "a", toolName: "t", toolCallId: "step" });
+    const result = (await flushed(spool)).find((e) => e.type === "tool_result")!;
+    expect(result.session_id).toBe("a");
+    expect(typeof result.duration_ms).toBe("number");
+  });
+});
+
+describe("timestamps", () => {
+  it("formats with six fractional digits, matching the Python SDK and the ingest parser", async () => {
+    event().agentStart({ sessionId: "s" });
+    const [emitted] = await flushed(spool);
+    expect(emitted!.timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/);
+  });
+
+  it("orders a burst inside one millisecond: every timestamp strictly after the last", async () => {
+    // A fast agent emits model_response, tool_use, tool_result and the next
+    // model_request inside one millisecond. With the last three digits always
+    // 000 they all carried the same timestamp, and the dashboard — which sorts
+    // on it — showed tool_result before its tool_use.
+    vi.useFakeTimers({ toFake: ["Date"] });
+    try {
+      vi.setSystemTime(new Date("2026-09-23T15:55:44.134Z"));
+      event().toolUse({ sessionId: "s", toolName: "t", toolCallId: "c1" });
+      event().toolResult({ sessionId: "s", toolName: "t", toolCallId: "c1" });
+      event().modelRequest({ sessionId: "s" });
+      event().agentEnd({ sessionId: "s" });
+    } finally {
+      vi.useRealTimers();
+    }
+    const stamps = (await flushed(spool)).map((e) => e.timestamp as string);
+    expect(stamps).toHaveLength(4);
+    for (const stamp of stamps) expect(stamp.startsWith("2026-09-23T15:55:44.134")).toBe(true);
+    expect([...stamps].sort()).toEqual(stamps);
+    expect(new Set(stamps).size).toBe(4);
+  });
+
+  it("re-anchors to the wall clock when it steps back, rather than freezing", async () => {
+    vi.useFakeTimers({ toFake: ["Date"] });
+    try {
+      vi.setSystemTime(new Date("2026-09-23T16:00:00.000Z"));
+      event().agentStart({ sessionId: "s" });
+      vi.setSystemTime(new Date("2026-09-23T15:00:00.000Z")); // NTP stepped back an hour
+      event().agentEnd({ sessionId: "s" });
+    } finally {
+      vi.useRealTimers();
+    }
+    const [, end] = await flushed(spool);
+    expect((end!.timestamp as string).startsWith("2026-09-23T15:00:00.000")).toBe(true);
+  });
+});

package/.next/standalone/sdk/typescript/test/expression.test.ts

@@ -0,0 +1,181 @@
+import { describe, expect, it } from "vitest";
+
+import {
+  EvaluationBudgetExceeded,
+  MAX_POW_EXPONENT,
+  UnsafeEvaluatorSource,
+  compileExpression,
+} from "../src/evaluator/expression.js";
+import { sessionTranscriptFromWire } from "../src/evaluator/protocol.js";
+import { transcriptWire } from "./helpers.js";
+
+/**
+ * The language is the correctness boundary for server-authored source — the
+ * `worker_threads` sandbox around it bounds RESOURCES, not reach. So the cases
+ * that matter most here are the ones that must be refused, and every one of
+ * them is a real escape in a naive `eval`- or `vm`-based design.
+ */
+
+const session = sessionTranscriptFromWire(
+  transcriptWire([
+    { type: "tool_use", payload: { tool_name: "search" } },
+    { type: "tool_result", payload: { error: "timeout" } },
+    { type: "tool_result", payload: { output: "ok" } },
+  ]),
+);
+
+const GLOBALS = ["session", "Math", "Object", "Array", "Number", "JSON"] as const;
+
+function run(source: string): unknown {
+  const compiled = compileExpression(source, {
+    fieldName: "evaluator_source",
+    maximumBytes: 128 * 1024,
+    globalNames: GLOBALS,
+  });
+  return compiled({ session });
+}
+
+function compileOnly(source: string): void {
+  compileExpression(source, {
+    fieldName: "evaluator_source",
+    maximumBytes: 128 * 1024,
+    globalNames: GLOBALS,
+  });
+}
+
+describe("what the language can express", () => {
+  it("reads the transcript surface", () => {
+    expect(run("session.eventCount")).toBe(3);
+    expect(run("session.count('tool_result')")).toBe(2);
+    expect(run("session.eventsOfType('tool_use').length")).toBe(1);
+    expect(run("session.agentId")).toBe("main");
+  });
+
+  it("reads arbitrary payload keys, which no static allowlist could enumerate", () => {
+    expect(run("session.eventsOfType('tool_result')[0].payload.error")).toBe("timeout");
+    expect(run("session.eventsOfType('tool_result')[1].payload['output']")).toBe("ok");
+  });
+
+  it("supports `!= null`, the idiom for an optional payload field", () => {
+    // Mapping `!=` to `!==` would silently stop this matching a missing key,
+    // so an evaluation over a payload that omits the field would report the
+    // opposite of the truth.
+    expect(
+      run("session.eventsOfType('tool_result').filter(e => e.payload.error != null).length"),
+    ).toBe(1);
+    expect(run("session.eventsOfType('tool_result')[1].payload.error == null")).toBe(true);
+  });
+
+  it("supports arrow functions, arithmetic, comparison and the conditional operator", () => {
+    expect(run("[1, 2, 3].map(x => x * 2).filter(x => x > 2)")).toEqual([4, 6]);
+    expect(run("session.eventCount > 0 ? 'busy' : 'idle'")).toBe("busy");
+    expect(run("Math.round(1 / 3 * 100) / 100")).toBe(0.33);
+    expect(run("[1, 2, 3].reduce((a, b) => a + b, 0)")).toBe(6);
+  });
+
+  it("supports template strings and object literals", () => {
+    expect(run("`saw ${session.eventCount} events`")).toBe("saw 3 events");
+    expect(run("({ total: session.eventCount, ok: true })")).toEqual({ total: 3, ok: true });
+  });
+
+  it("supports the namespace globals it advertises", () => {
+    expect(run("Object.keys({ a: 1, b: 2 })")).toEqual(["a", "b"]);
+    expect(run("Array.isArray(session.events)")).toBe(true);
+    expect(run("Number.isInteger(session.eventCount)")).toBe(true);
+    expect(run("JSON.stringify({ a: 1 })")).toBe('{"a":1}');
+  });
+});
+
+describe("escapes that must be refused", () => {
+  const rejected: Array<[string, string]> = [
+    ["a function constructor", "(x => x).constructor"],
+    ["a literal computed constructor", "session['constructor']"],
+    ["prototype", "session.prototype"],
+    ["__proto__", "session.__proto__"],
+    ["call", "session.count.call"],
+    ["bind", "session.count.bind"],
+    ["toString, which runs arbitrary code", "session.toString"],
+    ["a reference to process", "process.env"],
+    ["a reference to globalThis", "globalThis"],
+    ["require", "require('fs')"],
+    ["import", "import('fs')"],
+    ["new", "new Object()"],
+    ["a function expression", "function () { return 1 }"],
+    ["assignment", "session.eventCount = 5"],
+    ["a statement", "if (true) 1"],
+    ["a block-bodied arrow", "(x) => { return x }"],
+    ["an unbound identifier", "somethingElse"],
+    ["a private name", "_secret"],
+    ["a regular expression literal", "/abc/.test('abc')"],
+    ["a comment, which the grammar does not accept", "1 // trailing"],
+    ["a compile-time exponent bomb", "10 ** 99"],
+    ["a generator", "function* () { yield 1 }"],
+    ["a BigInt literal", "1n"],
+    ["optional chaining onto a forbidden key", "session?.constructor"],
+  ];
+
+  for (const [label, source] of rejected) {
+    it(`refuses ${label}`, () => {
+      expect(() => compileOnly(source)).toThrow(UnsafeEvaluatorSource);
+    });
+  }
+
+  it("refuses a bare method reference, which is only useful for smuggling one out", () => {
+    expect(() => compileOnly("session.count")).toThrow(/only to call it/);
+    expect(() => compileOnly("[].map")).toThrow(/only to call it/);
+  });
+
+  it("names the exponent ceiling in the message", () => {
+    expect(() => compileOnly("2 ** 100")).toThrow(
+      new RegExp(`integer constant in 0\\.\\.${MAX_POW_EXPONENT}`),
+    );
+  });
+
+  it("refuses a RUNTIME-computed forbidden key, which no source check can see", () => {
+    // This is why the boundary is `readProperty` and not the parser.
+    expect(() => run("session[['con','structor'].join('')]")).toThrow(UnsafeEvaluatorSource);
+    expect(() => run("session['con' + 'structor']")).toThrow(UnsafeEvaluatorSource);
+  });
+
+  it("refuses a non-allowlisted method on a built-in prototype", () => {
+    expect(() => run("[1, 2].sort()")).toThrow(UnsafeEvaluatorSource);
+    expect(() => run("'abc'.matchAll('a')")).toThrow(UnsafeEvaluatorSource);
+  });
+
+  it("refuses a namespace member outside its set", () => {
+    expect(() => run("Object.getPrototypeOf({})")).toThrow(UnsafeEvaluatorSource);
+    expect(() => run("Math.random()")).toThrow(UnsafeEvaluatorSource);
+  });
+
+  it("does not walk a prototype for a plain object's missing key", () => {
+    expect(run("({}).somethingAbsent")).toBeUndefined();
+  });
+});
+
+describe("budgets", () => {
+  it("bounds unbounded self-application", () => {
+    // `(f => f(f))(f => f(f))` is reachable with nothing but arrow functions,
+    // and runs forever.
+    expect(() => run("(f => f(f))(f => f(f))")).toThrow(EvaluationBudgetExceeded);
+  });
+
+  it("bounds a string bomb", () => {
+    expect(() => run("'x'.repeat(64) .repeat(64) .repeat(64) .repeat(64)")).toThrow(
+      EvaluationBudgetExceeded,
+    );
+  });
+
+  it("refuses source above the byte ceiling", () => {
+    expect(() =>
+      compileExpression(`'${"x".repeat(200)}'`, {
+        fieldName: "evaluator_source",
+        maximumBytes: 100,
+        globalNames: GLOBALS,
+      }),
+    ).toThrow(/exceeds 100 bytes/);
+  });
+
+  it("refuses empty source", () => {
+    expect(() => compileOnly("   ")).toThrow(/must not be empty/);
+  });
+});

package/.next/standalone/sdk/typescript/test/global-setup.ts

@@ -0,0 +1,26 @@
+import { existsSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+/**
+ * Point the evaluator sandbox at this checkout's built worker.
+ *
+ * The tests run against `src/` through Vitest's TypeScript loader, but a
+ * `worker_threads` Worker is a real Node process entry and cannot load a `.ts`
+ * file. In an installed package `source.ts` resolves `@failproofai/sdk/sandbox-worker`
+ * through the exports map; here there is no installed package, so the env
+ * override — the same one the code documents for bundled consumers — points at
+ * `dist/`. `npm test` builds first, which is why this can assert rather than
+ * skip: a sandbox test that silently did not run is worse than no sandbox test.
+ */
+export default function setup(): void {
+  const root = dirname(dirname(fileURLToPath(import.meta.url)));
+  const worker = resolve(root, "dist/cjs/evaluator/sandbox-worker.js");
+  if (!existsSync(worker)) {
+    throw new Error(
+      `the evaluator sandbox worker is not built at ${worker}. Run \`npm run build\` first ` +
+        "(`npm test` does it for you).",
+    );
+  }
+  process.env.FAILPROOFAI_SDK_SANDBOX_WORKER = worker;
+}