@workos/quickstudy 0.0.1

package/src/runner/run-lock.ts

@@ -0,0 +1,32 @@
+import { mkdirSync, openSync, readFileSync, closeSync, writeFileSync, unlinkSync } from "node:fs";
+import { join } from "node:path";
+/** Refuse simultaneous writers; recover a lock only when its process is gone. */
+export function acquireRunLock(root: string, runId: string): () => void {
+  const dir = join(root, runId);
+  mkdirSync(dir, { recursive: true });
+  const path = join(dir, "runner.lock");
+  const create = () => {
+    const fd = openSync(path, "wx", 0o600);
+    writeFileSync(fd, String(process.pid));
+    closeSync(fd);
+  };
+  try {
+    create();
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err;
+    const pid = Number(readFileSync(path, "utf8"));
+    if (!Number.isInteger(pid) || pid < 1) throw new Error(`invalid run lock for ${runId}`);
+    let alive = true;
+    try {
+      process.kill(pid, 0);
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code === "ESRCH") alive = false;
+    }
+    if (alive) throw new Error(`run ${runId} is already active (pid ${pid})`);
+    unlinkSync(path);
+    create();
+  }
+  return () => {
+    unlinkSync(path);
+  };
+}

package/src/runner/scheduler.ts

@@ -0,0 +1,62 @@
+import { hashString } from "../hash.ts";
+import type { AttemptPlan } from "../plan.ts";
+
+/** Seeded order changes dispatch timing, never trial coordinates. */
+export function orderedPlan(plan: AttemptPlan[], seed: string): AttemptPlan[] {
+  return [...plan].sort((a, b) =>
+    hashString(`${seed}:${JSON.stringify(a)}`).localeCompare(hashString(`${seed}:${JSON.stringify(b)}`)),
+  );
+}
+export async function schedule<T>(
+  items: T[],
+  concurrency: number,
+  work: (item: T) => Promise<void>,
+  options: {
+    signal?: AbortSignal;
+    capacity?: (item: T) => { key: string; limit: number } | undefined;
+  } = {},
+): Promise<void> {
+  if (!Number.isInteger(concurrency) || concurrency < 1) throw new Error("concurrency must be a positive integer");
+  const capacities = new Map<T, { key: string; limit: number } | undefined>();
+  const limits = new Map<string, number>();
+  for (const item of items) {
+    const cap = options.capacity?.(item);
+    if (cap) {
+      if (!cap.key || !Number.isInteger(cap.limit) || cap.limit < 1)
+        throw new Error("runtime capacity must have a key and positive integer limit");
+      if (limits.has(cap.key) && limits.get(cap.key) !== cap.limit)
+        throw new Error(`conflicting runtime capacity for ${cap.key}`);
+      limits.set(cap.key, cap.limit);
+    }
+    capacities.set(item, cap);
+  }
+  const remaining = [...items];
+  const running = new Set<Promise<void>>();
+  const active = new Map<string, number>();
+  let failure: unknown;
+  while (remaining.length || running.size) {
+    while (!failure && !options.signal?.aborted && running.size < concurrency) {
+      const index = remaining.findIndex((item) => {
+        const cap = capacities.get(item);
+        return !cap || (active.get(cap.key) ?? 0) < cap.limit;
+      });
+      if (index < 0) break;
+      const item = remaining.splice(index, 1)[0]!;
+      const cap = capacities.get(item);
+      if (cap) active.set(cap.key, (active.get(cap.key) ?? 0) + 1);
+      const task = Promise.resolve()
+        .then(() => work(item))
+        .catch((err) => {
+          failure = err;
+        })
+        .finally(() => {
+          running.delete(task);
+          if (cap) active.set(cap.key, (active.get(cap.key) ?? 1) - 1);
+        });
+      running.add(task);
+    }
+    if (!running.size) break;
+    await Promise.race(running);
+  }
+  if (failure) throw failure;
+}

package/src/runner/score-worker.ts

@@ -0,0 +1,107 @@
+import { Worker } from "node:worker_threads";
+import type { EvalContext, EvalResult } from "../evals/types.ts";
+
+/** Scorers run in a terminable worker: a deadline also stops late direct file mutations. */
+export async function scoreInWorker(
+  scorerPath: string,
+  context: EvalContext,
+  signal: AbortSignal,
+): Promise<EvalResult> {
+  signal.throwIfAborted();
+  const worker = new Worker(
+    new URL(import.meta.url.includes("/$bunfs/") ? "./runner/scorer-worker.js" : "./scorer-worker.ts", import.meta.url),
+  );
+  const methods: Array<(...args: any[]) => unknown> = [];
+  function serialize(value: any, depth = 0): any {
+    if (depth > 8) throw new Error("client capability exceeds serialization depth");
+    if (typeof value === "function") {
+      const id = methods.length;
+      methods.push(value);
+      return { __quickstudyMethod: id };
+    }
+    if (Array.isArray(value)) return value.map((v) => serialize(v, depth + 1));
+    if (value && typeof value === "object")
+      return Object.fromEntries(
+        Object.entries(value).map(([key, item]) => [
+          key,
+          serialize(typeof item === "function" ? item.bind(value) : item, depth + 1),
+        ]),
+      );
+    return value;
+  }
+  let settled = false;
+  try {
+    return await new Promise<EvalResult>((resolve, reject) => {
+      const abort = () => {
+        void worker.terminate();
+        reject(signal.reason);
+      };
+      signal.addEventListener("abort", abort, { once: true });
+      worker.once("exit", () => {
+        signal.removeEventListener("abort", abort);
+        reject(new Error("scorer worker exited before returning a result"));
+      });
+      worker.once("error", reject);
+      worker.on("message", async (message) => {
+        if (message.type === "result") {
+          settled = true;
+          resolve(message.result);
+          return;
+        }
+        if (message.type === "failure") {
+          settled = true;
+          reject(Object.assign(new Error(message.error), { judgments: message.judgments }));
+          return;
+        }
+        let response: { value?: unknown; error?: string };
+        try {
+          signal.throwIfAborted();
+          const [first, second] = message.args;
+          const value =
+            message.method === "exec"
+              ? await context.exec(first, second)
+              : message.method === "query"
+                ? await context.query(first)
+                : message.method === "redact"
+                  ? ((await context.redact?.(first)) ?? first)
+                  : message.method === "getClient"
+                    ? context.getClient(first)
+                    : message.method === "clientMethod" && methods[first]
+                      ? await methods[first]!(...second)
+                      : (() => {
+                          throw new Error("unsupported scorer capability");
+                        })();
+          response = { value: serialize(value) };
+        } catch (error) {
+          response = { error: error instanceof Error ? error.message : String(error) };
+        }
+        if (settled || signal.aborted) return;
+        if (message.type === "sync") {
+          const state = new Int32Array(message.buffer, 0, 2);
+          let encoded = new TextEncoder().encode(JSON.stringify(response));
+          if (encoded.length > message.buffer.byteLength - 8)
+            encoded = new TextEncoder().encode('{"error":"client capability exceeds 1 MiB"}');
+          new Uint8Array(message.buffer, 8, encoded.length).set(encoded);
+          state[1] = encoded.length;
+          Atomics.store(state, 0, 1);
+          Atomics.notify(state, 0);
+        } else if (!signal.aborted) worker.postMessage({ type: "reply", id: message.id, ...response });
+      });
+      worker.postMessage({
+        type: "start",
+        scorerPath,
+        context: {
+          evalId: context.evalId,
+          experimentId: context.experimentId,
+          trialIndex: context.trialIndex,
+          framework: context.framework,
+          workspaceDir: context.workspaceDir,
+          agentOutput: context.agentOutput,
+        },
+      });
+    });
+  } finally {
+    settled = true;
+    await worker.terminate();
+  }
+}

package/src/runner/scorer-worker.ts

@@ -0,0 +1,61 @@
+import { parentPort } from "node:worker_threads";
+import { importEvalScorer } from "../evals/discovery.ts";
+import { buildEvalContext } from "./context.ts";
+
+if (!parentPort) throw new Error("scorer worker requires a parent");
+const port = parentPort;
+let sequence = 0;
+const pending = new Map<number, { resolve: (value: unknown) => void; reject: (error: Error) => void }>();
+function revive(value: any): any {
+  if (value && typeof value === "object") {
+    if (typeof value.__quickstudyMethod === "number")
+      return (...args: unknown[]) => rpc("clientMethod", [value.__quickstudyMethod, args]);
+    if (Array.isArray(value)) return value.map(revive);
+    return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, revive(item)]));
+  }
+  return value;
+}
+function rpc(method: string, args: unknown[]): Promise<any> {
+  const id = sequence++;
+  return new Promise((resolve, reject) => {
+    pending.set(id, { resolve, reject });
+    port.postMessage({ type: "rpc", id, method, args });
+  });
+}
+function getClient(name: string): unknown {
+  const buffer = new SharedArrayBuffer(1024 * 1024);
+  const state = new Int32Array(buffer, 0, 2);
+  port.postMessage({ type: "sync", method: "getClient", args: [name], buffer });
+  Atomics.wait(state, 0, 0);
+  const result = JSON.parse(new TextDecoder().decode(new Uint8Array(buffer, 8, state[1]!)));
+  if (result.error) throw new Error(result.error);
+  return revive(result.value);
+}
+port.on("message", async (message) => {
+  if (message.type === "reply") {
+    const waiter = pending.get(message.id);
+    pending.delete(message.id);
+    if (message.error) waiter?.reject(new Error(message.error));
+    else waiter?.resolve(revive(message.value));
+    return;
+  }
+  if (message.type !== "start") return;
+  try {
+    const scorer = await importEvalScorer(message.scorerPath);
+    const context = buildEvalContext({
+      ...message.context,
+      exec: (cmd, opts) => rpc("exec", [cmd, opts]),
+      query: (request) => rpc("query", [request]),
+      redact: (text) => rpc("redact", [text]),
+      getClient,
+    });
+    const result = await scorer(context);
+    port.postMessage({ type: "result", result });
+  } catch (error) {
+    port.postMessage({
+      type: "failure",
+      error: error instanceof Error ? error.message : String(error),
+      judgments: (error as { judgments?: unknown })?.judgments,
+    });
+  }
+});

package/src/runtime/types.ts

@@ -0,0 +1,89 @@
+/**
+ * The v3 runtime capability surface: everything the environment offers the
+ * agent and the scorer — per-attempt provisioning, container image selection,
+ * egress policy, MCP configuration, native-web policy, CLI availability, and
+ * scorer capabilities — declared as optional methods on the experiment's
+ * runtime instead of scattered per-surface special cases.
+ *
+ * Identity note: these are FUNCTIONS, and the manifest hashes the runtime as
+ * plain data (`{kind, image, command, config}` via canonical JSON, which
+ * drops functions). Anything identity-bearing a capability closes over MUST
+ * also appear in `Runtime.config` — the harness cannot see inside a closure.
+ */
+
+import type { EvalMetadata } from "../evals/types.ts";
+import type { McpServerConfig } from "../isolation/mcp.ts";
+
+/** What a runtime's `provision` receives, per attempt. */
+export interface RuntimeProvisionContext {
+  /** Aborted when the current stage expires. Implementations must stop work. */
+  signal?: AbortSignal;
+  attemptId: string;
+  evalId: string;
+  /** The eval's frontmatter metadata (`product`/`framework` drive provisioning). */
+  metadata: EvalMetadata;
+}
+
+/** What one provisioned attempt gets. `env` values are secrets — never log. */
+export interface ProvisionedEnvironment {
+  /** Env vars delivered to the attempt container via the 0600 env-file, never argv. */
+  env?: Record<string, string>;
+  /** Credential identifier for teardown/leak accounting — never the secret itself. */
+  credentialRef?: string;
+  /**
+   * Scrub what provision created. Failures are recorded explicitly on the
+   * attempt, never silently swallowed, and never crash the run.
+   */
+  teardown?(signal?: AbortSignal): Promise<void>;
+}
+
+/** What `scorerCapabilities` receives when the runner builds the scorer's context. */
+export interface ScorerCapabilityContext {
+  /** Aborted when the current stage expires. Implementations must stop work. */
+  signal?: AbortSignal;
+  attemptId: string;
+  evalId: string;
+  experimentId: string;
+  metadata: EvalMetadata;
+  /** The provisioned env (empty when the runtime does not provision). */
+  env: Record<string, string>;
+}
+
+/**
+ * Capability implementations handed to the scorer's EvalContext. A helper the
+ * runtime does not provide throws at call time — there is no binding phase.
+ */
+export interface ScorerCapabilities {
+  query?(request: unknown): Promise<unknown>;
+  getClient?(name: string): unknown;
+}
+
+/** Native web-browsing policy for the agent (recorded; enforced by adapters). */
+export type WebPolicy = "native-web-allowed" | "native-web-blocked";
+
+/**
+ * The optional capability methods a runtime may implement. All optional:
+ * a plain-data runtime (`{kind: "host"}`) is a complete runtime. The
+ * identity-bearing counterpart is `Runtime.config` (see identity note above).
+ */
+export interface RuntimeCapabilities {
+  /** Provision per-attempt resources BEFORE the sandbox exists (fail = no agent spend). */
+  provision?(ctx: RuntimeProvisionContext): Promise<ProvisionedEnvironment>;
+  /**
+   * Runtime-level teardown, used when `provision`'s result carries no
+   * `teardown` of its own (and attempted after a failed provision).
+   */
+  teardown?(ctx: RuntimeProvisionContext): Promise<void>;
+  /** Resolve the container image for one eval (overridden by a static `image`). */
+  containerImage?(metadata: EvalMetadata): string;
+  /** Hosts this attempt may reach when the egress proxy is enabled. */
+  egressHosts?(metadata: EvalMetadata): readonly string[];
+  /** MCP servers the environment offers; authed entries ride the token proxy. */
+  mcpServers?(): Record<string, McpServerConfig>;
+  /** Native web tool policy for the agent. */
+  webPolicy?(): WebPolicy;
+  /** Directory prepended to the container PATH (the CLI treatment), if any. */
+  pathTreatment?(): string | undefined;
+  /** Capabilities (`query`, `getClient`) exposed to the eval's scorer. */
+  scorerCapabilities?(ctx: ScorerCapabilityContext): ScorerCapabilities;
+}

package/src/secrets.ts

@@ -0,0 +1,151 @@
+/**
+ * Container environment assembly and secret hygiene.
+ *
+ * Two sources feed an attempt's environment:
+ *   (a) shared provider keys, read from the HOST env per selected adapter
+ *       (ANTHROPIC_API_KEY for claude, OPENAI_API_KEY for codex; each
+ *       adapter declares its own `requiredHostEnv`, nothing here is
+ *       per-vendor) — validated fail-fast before any container is created,
+ *       and each attempt only receives the keys its own adapter requires;
+ *   (b) per-attempt credentials from the experiment runtime's provision
+ *       hook.
+ *
+ * Invariants: secret VALUES are never logged and never written to artifacts —
+ * `credential_ref` (an identifier, never the secret) is what lands in the
+ * attempt row, and every transcript/stderr write passes through
+ * {@link redactSecrets} first.
+ */
+
+/** A provider key required by a selected adapter is absent from the host env. */
+export class MissingProviderKeyError extends Error {
+  readonly adapter: string;
+  readonly envVar: string;
+
+  constructor(adapter: string, envVar: string) {
+    super(
+      `agent "${adapter}" requires the ${envVar} environment variable, which is not set — ` +
+        `export it before running (it is injected into attempt containers at create, never mounted or persisted)`,
+    );
+    this.name = "MissingProviderKeyError";
+    this.adapter = adapter;
+    this.envVar = envVar;
+  }
+}
+
+/** A provider key is present but the provider's API rejected it. */
+export class InvalidProviderKeyError extends Error {
+  constructor(adapter: string, envVars: readonly string[], host: string, status: number) {
+    super(
+      `run not starting: agent "${adapter}" key (${envVars.join(", ")}) was rejected by ${host} (HTTP ${status}) — ` +
+        `nothing was created; fix or re-export the key and re-run`,
+    );
+    this.name = "InvalidProviderKeyError";
+  }
+}
+
+export interface ProviderKeyRequirement {
+  /** The adapter (agent axis value) that needs the keys. */
+  name: string;
+  /** Host env var names the adapter requires. */
+  requiredHostEnv: readonly string[];
+}
+
+/** One preflight probe: a cheap authenticated GET a provider will 401 on a bad key. */
+export interface ProviderKeyProbe {
+  /** The adapter whose key is being probed. */
+  name: string;
+  /** The env var names the probe's key came from (for messages; values never appear). */
+  envVars: readonly string[];
+  url: string;
+  headers: Record<string, string>;
+}
+
+/** Probe responses take at most this long before the run proceeds unvalidated. */
+const KEY_PROBE_TIMEOUT_MS = 10_000;
+
+/**
+ * Live provider-key preflight: probe each adapter's endpoint once, BEFORE any
+ * container or sidecar exists. Only an explicit auth rejection (401/403)
+ * stops the run — a key that is present but rejected would otherwise burn a
+ * container per attempt just to fail identically inside it. Network failures
+ * and non-auth statuses warn and proceed: an unreachable or rate-limiting
+ * probe is not evidence the key is bad.
+ */
+export async function validateProviderKeys(
+  probes: readonly ProviderKeyProbe[],
+  opts: { fetchImpl?: typeof fetch; log?: (line: string) => void; timeoutMs?: number } = {},
+): Promise<void> {
+  const fetchImpl = opts.fetchImpl ?? fetch;
+  const log = opts.log ?? (() => {});
+  const timeoutMs = opts.timeoutMs ?? KEY_PROBE_TIMEOUT_MS;
+  for (const probe of probes) {
+    const host = new URL(probe.url).host;
+    let status: number;
+    try {
+      const response = await fetchImpl(probe.url, {
+        method: "GET",
+        headers: probe.headers,
+        signal: AbortSignal.timeout(timeoutMs),
+      });
+      status = response.status;
+    } catch (err) {
+      log(
+        `warning: could not validate ${probe.envVars.join(", ")} against ${host} ` +
+          `(${err instanceof Error ? err.message : String(err)}) — starting anyway`,
+      );
+      continue;
+    }
+    if (status === 401 || status === 403) {
+      throw new InvalidProviderKeyError(probe.name, probe.envVars, host, status);
+    }
+    if (status >= 200 && status < 300) {
+      log(`preflight: ${probe.envVars.join(", ")} accepted by ${host}`);
+    } else {
+      log(`preflight: ${probe.envVars.join(", ")} check against ${host} inconclusive (HTTP ${status}) — starting anyway`);
+    }
+  }
+}
+
+/**
+ * Read every required provider key from the host env, failing fast — before
+ * any container is created — on the first missing or empty variable, naming
+ * both the variable and the agent that needs it.
+ */
+export function collectProviderKeys(
+  requirements: readonly ProviderKeyRequirement[],
+  hostEnv: Record<string, string | undefined> = process.env,
+): Record<string, string> {
+  const keys: Record<string, string> = {};
+  for (const requirement of requirements) {
+    for (const envVar of requirement.requiredHostEnv) {
+      const value = hostEnv[envVar];
+      if (value === undefined || value === "") {
+        throw new MissingProviderKeyError(requirement.name, envVar);
+      }
+      keys[envVar] = value;
+    }
+  }
+  return keys;
+}
+
+/** Values shorter than this are never redacted (they would shred ordinary text). */
+const MIN_REDACTABLE_LENGTH = 6;
+
+/**
+ * Replace every known secret value in `text` with `[REDACTED:<name>]`.
+ * Applied to transcripts and stderr before they are written to artifacts —
+ * an agent that echoes its env must not persist a live key.
+ */
+export function redactSecrets(text: string, env: Record<string, string>): string {
+  let redacted = text;
+  for (const [name, value] of Object.entries(env)) {
+    if (value.length < MIN_REDACTABLE_LENGTH) continue;
+    redacted = redacted.replaceAll(value, `[REDACTED:${name}]`);
+  }
+  return redacted;
+}
+
+/** Provider/judge secret env only; ordinary PATH and workspace values are evidence. */
+export function hostSecretEnv(env: Record<string, string | undefined> = process.env): Record<string, string> {
+  return Object.fromEntries(Object.entries(env).filter(([name, value]) => /key|token|password|secret|credential/i.test(name) && typeof value === "string")) as Record<string, string>;
+}