@workos/quickstudy 0.0.1

package/src/evals/prompt.ts

@@ -0,0 +1,190 @@
+/**
+ * PROMPT.md parser: YAML frontmatter (the eval's metadata) followed by the
+ * task body, which is passed through verbatim. The frontmatter schema is
+ * deliberately tiny and closed — unknown keys are rejected with a pointer at
+ * the offending line, because new metadata fields require a proven
+ * filtering/reporting use case before they exist. Two keys are control flow
+ * for discovery rather than metadata: `frameworks` (expand the directory
+ * into one eval per framework) and `local` (a shared per-framework
+ * starting-state root for variant evals). Both are consumed by discovery
+ * and never appear on the expanded metadata.
+ */
+
+import { isAbsolute } from "node:path";
+import YAML from "yaml";
+import { EVAL_SUITES, type EvalMetadata, type EvalSuite } from "./types.ts";
+
+/** A PROMPT.md failed to parse or validate. Carries file + 1-based line. */
+export class EvalLoadError extends Error {
+  readonly file: string;
+  readonly line: number;
+
+  constructor(file: string, line: number, detail: string) {
+    super(`${file}:${line}: ${detail}`);
+    this.name = "EvalLoadError";
+    this.file = file;
+    this.line = line;
+  }
+}
+
+export interface ParsedPrompt {
+  metadata: EvalMetadata;
+  /** Everything after the closing frontmatter fence, verbatim. */
+  promptBody: string;
+  /** `frameworks:` variant list — discovery expands one eval per entry. */
+  frameworks?: string[];
+  /** `local:` shared starting-state root, relative to the eval directory. */
+  local?: string;
+}
+
+const KNOWN_KEYS = ["id", "suite", "product", "framework", "frameworks", "local", "topics"] as const;
+
+/** Framework tokens become id suffixes (`<dirname>-<fw>`), so they must be kebab-case. */
+const FRAMEWORK_TOKEN = /^[a-z0-9]+(-[a-z0-9]+)*$/;
+
+/** Parse a PROMPT.md source string. `file` is used for error messages only. */
+export function parsePrompt(source: string, file: string): ParsedPrompt {
+  const lines = source.split(/\r?\n/);
+  if (lines[0]?.trim() !== "---") {
+    throw new EvalLoadError(
+      file,
+      1,
+      `expected YAML frontmatter opening "---", found ${JSON.stringify(lines[0] ?? "<empty file>")}`,
+    );
+  }
+  const fmClose = lines.findIndex((l, i) => i > 0 && l.trim() === "---");
+  if (fmClose === -1) {
+    throw new EvalLoadError(file, lines.length, `frontmatter never closed: expected a "---" line`);
+  }
+
+  let raw: unknown;
+  try {
+    raw = YAML.parse(lines.slice(1, fmClose).join("\n"));
+  } catch (err) {
+    const yamlLine = err instanceof YAML.YAMLParseError ? (err.linePos?.[0]?.line ?? 1) : 1;
+    throw new EvalLoadError(file, 1 + yamlLine, `invalid YAML in frontmatter: ${(err as Error).message.split("\n")[0]}`);
+  }
+  if (raw === null || typeof raw !== "object" || Array.isArray(raw)) {
+    throw new EvalLoadError(file, 2, `frontmatter must be a YAML mapping (id, suite, ...)`);
+  }
+  const fm = raw as Record<string, unknown>;
+
+  // 1-based line of the `key:` line, for precise errors.
+  const keyLine = (key: string): number => {
+    const idx = lines.findIndex((l, index) => index > 0 && index < fmClose && l.startsWith(`${key}:`));
+    return idx === -1 ? 2 : idx + 1;
+  };
+
+  for (const key of Object.keys(fm)) {
+    if (!(KNOWN_KEYS as readonly string[]).includes(key)) {
+      throw new EvalLoadError(
+        file,
+        keyLine(key),
+        `unknown frontmatter key "${key}" (allowed: ${KNOWN_KEYS.join(", ")}); ` +
+          `new metadata fields need a proven filtering or reporting use case first`,
+      );
+    }
+  }
+
+  const id = fm["id"];
+  if (typeof id !== "string" || id.trim() === "") {
+    throw new EvalLoadError(file, keyLine("id"), `"id" must be a non-empty string`);
+  }
+  const suite = fm["suite"];
+  if (typeof suite !== "string" || !(EVAL_SUITES as readonly string[]).includes(suite)) {
+    throw new EvalLoadError(
+      file,
+      keyLine("suite"),
+      `"suite" must be one of: ${EVAL_SUITES.join(", ")}${suite === undefined ? " (missing)" : ` (got ${JSON.stringify(suite)})`}`,
+    );
+  }
+  const optionalString = (key: "product" | "framework"): string | undefined => {
+    const value = fm[key];
+    if (value === undefined) return undefined;
+    if (typeof value !== "string" || value.trim() === "") {
+      throw new EvalLoadError(file, keyLine(key), `"${key}" must be a non-empty string`);
+    }
+    return value;
+  };
+  const product = optionalString("product");
+  const framework = optionalString("framework");
+  let frameworks: string[] | undefined;
+  if (fm["frameworks"] !== undefined) {
+    const value = fm["frameworks"];
+    if (!Array.isArray(value) || value.length === 0 || value.some((item) => typeof item !== "string")) {
+      throw new EvalLoadError(file, keyLine("frameworks"), `"frameworks" must be a non-empty list of framework tokens`);
+    }
+    const tokens = value as string[];
+    for (const token of tokens) {
+      if (!FRAMEWORK_TOKEN.test(token)) {
+        throw new EvalLoadError(
+          file,
+          keyLine("frameworks"),
+          `"frameworks" entry ${JSON.stringify(token)} must be a kebab-case token ` +
+            `(lowercase letters, digits, hyphens — it becomes the "<dirname>-<framework>" id suffix)`,
+        );
+      }
+    }
+    if (new Set(tokens).size !== tokens.length) {
+      throw new EvalLoadError(file, keyLine("frameworks"), `"frameworks" entries must be unique`);
+    }
+    if (framework !== undefined) {
+      throw new EvalLoadError(
+        file,
+        keyLine("framework"),
+        `"framework" cannot be set alongside "frameworks" — expansion sets framework per variant`,
+      );
+    }
+    frameworks = tokens;
+  }
+  let local: string | undefined;
+  if (fm["local"] !== undefined) {
+    const value = fm["local"];
+    if (typeof value !== "string" || value.trim() === "") {
+      throw new EvalLoadError(file, keyLine("local"), `"local" must be a non-empty relative path`);
+    }
+    if (isAbsolute(value)) {
+      throw new EvalLoadError(
+        file,
+        keyLine("local"),
+        `"local" must be a relative path (resolved from the eval directory), got an absolute path`,
+      );
+    }
+    if (frameworks === undefined) {
+      throw new EvalLoadError(
+        file,
+        keyLine("local"),
+        `"local" requires "frameworks" — it points at per-framework starting states ` +
+          `(<local>/<framework>); a single-directory eval uses its local/ directory`,
+      );
+    }
+    local = value;
+  }
+  let topics: string[] | undefined;
+  if (fm["topics"] !== undefined) {
+    const value = fm["topics"];
+    if (!Array.isArray(value) || value.some((item) => typeof item !== "string" || item.trim() === "")) {
+      throw new EvalLoadError(file, keyLine("topics"), `"topics" must be a list of non-empty strings`);
+    }
+    topics = value as string[];
+  }
+
+  const promptBody = lines.slice(fmClose + 1).join("\n");
+  if (promptBody.trim() === "") {
+    throw new EvalLoadError(file, fmClose + 1, `PROMPT.md has no task body after the frontmatter`);
+  }
+
+  const metadata: EvalMetadata = {
+    id,
+    suite: suite as EvalSuite,
+    ...(product !== undefined ? { product } : {}),
+    ...(framework !== undefined ? { framework } : {}),
+    ...(topics !== undefined ? { topics } : {}),
+  };
+  return {
+    metadata,
+    promptBody,
+    ...(frameworks !== undefined ? { frameworks } : {}),
+    ...(local !== undefined ? { local } : {}),
+  };
+}

package/src/evals/result.ts

@@ -0,0 +1,10 @@
+import type { EvalResult } from "./types.ts";
+export function validEvalResult(value: unknown): value is EvalResult {
+  if (!value || typeof value !== "object") return false;
+  const result = value as EvalResult;
+  if (typeof result.passed !== "boolean" || !Array.isArray(result.checks)) return false;
+  if (result.checks.some((c) => !c || typeof c.name !== "string" || c.name === "" || typeof c.passed !== "boolean")) return false;
+  if (new Set(result.checks.map((c) => c.name)).size !== result.checks.length) return false;
+  return result.judgments === undefined || (Array.isArray(result.judgments) && result.judgments.every((j) => j && typeof j === "object" && j.provenance === "semantic-judge" &&
+    typeof j.provider === "string" && typeof j.model === "string" && [j.tokensIn, j.tokensOut, j.costUsd].every((v) => v === null || (typeof v === "number" && Number.isFinite(v) && v >= 0))));
+}

package/src/evals/types.ts

@@ -0,0 +1,115 @@
+/**
+ * The v3 eval vocabulary: one directory = one eval, described by a PROMPT.md
+ * (frontmatter metadata + task body) and scored by an EVAL.ts whose default
+ * export is an {@link EvalScorer}. No dialect, no registry — the scorer is
+ * plain TypeScript handed an {@link EvalContext} over the finished workspace.
+ */
+
+/** Closed set of eval suites. `regression` never moves headline scores. */
+export const EVAL_SUITES = ["benchmark", "regression", "other"] as const;
+export type EvalSuite = (typeof EVAL_SUITES)[number];
+
+/** PROMPT.md frontmatter. New fields require a proven filtering/reporting use case. */
+export interface EvalMetadata {
+  /** Must equal the eval's directory name (prevents silent aliasing). */
+  id: string;
+  suite: EvalSuite;
+  product?: string;
+  framework?: string;
+  topics?: string[];
+}
+
+/** One discovered eval directory, validated and ready to plan. */
+export interface LoadedEval {
+  metadata: EvalMetadata;
+  /** Absolute path of the eval directory. */
+  dir: string;
+  /** Absolute path of PROMPT.md. */
+  promptPath: string;
+  /** The task body after the frontmatter, passed through verbatim. */
+  promptBody: string;
+  /** Absolute path of EVAL.ts (imported lazily at score time). */
+  scorerPath: string;
+  /** Absolute path of the optional `local/` starting-state directory, or null. */
+  localDir: string | null;
+}
+
+/** A pointer at evidence backing a check verdict (workspace-relative path). */
+export interface EvidenceReference {
+  path: string;
+  note?: string;
+}
+
+/** One named check's verdict inside an eval result. */
+export interface CheckResult {
+  name: string;
+  passed: boolean;
+  notes?: string;
+  evidence?: EvidenceReference[];
+}
+
+/** What a scorer returns for one attempt. */
+export interface JudgeRecord {
+  provenance: "semantic-judge";
+  provider: string;
+  model: string;
+  tokensIn: number | null;
+  tokensOut: number | null;
+  costUsd: number | null;
+}
+export interface AgentOutput {
+  finalReport: string | null;
+  transcript: string | null;
+  truncated: boolean;
+}
+export interface EvalResult {
+  judgments?: JudgeRecord[];
+  passed: boolean;
+  checks: CheckResult[];
+}
+
+/** Output of a command run through {@link EvalContext.exec}. */
+export interface EvalExecResult {
+  exitCode: number;
+  stdout: string;
+  stderr: string;
+  /** True when the command was killed by its wall-clock timeout. */
+  timedOut: boolean;
+}
+
+/**
+ * What a scorer gets. File helpers read the EXPORTED workspace (stable on
+ * disk); `exec` runs in the live sandbox when the experiment ran in a
+ * container — the runner keeps it alive until the scorer resolves — or in
+ * the exported workspace directory for host experiments. Capability helpers
+ * (`query`, `getClient`) are wired by experiment runtimes in a later phase;
+ * a runtime that does not provide one makes the helper throw with
+ * "this experiment runtime does not provide X".
+ */
+export interface EvalContext {
+  agentOutput?: AgentOutput;
+  /** Redact known runtime/provider secrets before sending evidence to a judge. */
+  redact?(text: string): Promise<string>;
+  /** Aborted when the current stage expires. Implementations must stop work. */
+  signal?: AbortSignal;
+  evalId: string;
+  experimentId: string;
+  trialIndex: number;
+  /** The eval's framework (`metadata.framework`); absent for framework-neutral evals. */
+  framework?: string;
+  /** Absolute path of the exported workspace being scored. */
+  workspaceDir: string;
+  /** Does this workspace-relative path exist? */
+  fileExists(path: string): Promise<boolean>;
+  /** Read a workspace-relative file as UTF-8. Throws when missing. */
+  readFile(path: string): Promise<string>;
+  /** Run a command in the sandbox (container) or workspace dir (host). */
+  exec(cmd: string[], opts?: { timeoutMs?: number }): Promise<EvalExecResult>;
+  /** Runtime-provided data query capability (wired by runtimes later). */
+  query(request: unknown): Promise<unknown>;
+  /** Runtime-provided client capability (wired by runtimes later). */
+  getClient(name: string): unknown;
+}
+
+/** EVAL.ts's default export. */
+export type EvalScorer = ((ctx: EvalContext) => Promise<EvalResult>) & { identity?: Record<string, unknown> };

package/src/execution-policy.ts

@@ -0,0 +1,71 @@
+import type { LoadedEval } from "./evals/types.ts";
+import type { LoadedExperiment } from "./experiments/types.ts";
+import type { RuntimeImageIdentity } from "./manifest.ts";
+import { composeAllowlistFromHosts } from "./isolation/proxy/allowlist.ts";
+import type { WebPolicy } from "./runtime/types.ts";
+
+export interface LifecycleBudgets {
+  provisionMs: number;
+  agentMs: number;
+  scoreMs: number;
+  exportMs: number;
+  cleanupMs: number;
+}
+export const DEFAULT_BUDGETS: LifecycleBudgets = {
+  provisionMs: 120_000,
+  agentMs: 600_000,
+  scoreMs: 180_000,
+  exportMs: 60_000,
+  cleanupMs: 30_000,
+};
+export interface PairEnvironment {
+  image: RuntimeImageIdentity | null;
+  egress: { proxy: boolean; allowlist: string[] };
+  webPolicy: WebPolicy;
+  pathTreatment: string | null;
+  mcp: string | null;
+  budgets: LifecycleBudgets;
+}
+export function pairKey(evalId: string, experimentId: string): string {
+  return JSON.stringify([evalId, experimentId]);
+}
+export function resolvePairEnvironments(options: {
+  evals: LoadedEval[];
+  experiments: LoadedExperiment[];
+  images?: Record<string, RuntimeImageIdentity>;
+  egressProxy?: boolean;
+  adapterHosts?: Record<string, readonly string[]>;
+  budgets?: Partial<LifecycleBudgets>;
+  mcpHashes?: Record<string, string>;
+  imageTags?: Record<string, string>;
+}): Record<string, PairEnvironment> {
+  const budgets = { ...DEFAULT_BUDGETS, ...options.budgets };
+  for (const [name, value] of Object.entries(budgets))
+    if (!Number.isFinite(value) || value <= 0) throw new Error(`invalid lifecycle budget ${name}`);
+  const pairs: Record<string, PairEnvironment> = {};
+  for (const loaded of options.evals)
+    for (const { experiment } of options.experiments) {
+      const runtime = experiment.runtime;
+      const tag =
+        options.imageTags?.[pairKey(loaded.metadata.id, experiment.id)] ??
+        runtime.image ??
+        runtime.containerImage?.(loaded.metadata);
+      pairs[pairKey(loaded.metadata.id, experiment.id)] = {
+        image: tag ? (options.images?.[tag] ?? { tag, id: "unresolved", repoDigests: [] }) : null,
+        // With the flag, every container is proxied. No runtime hosts means
+        // only the adapter's provider endpoints are permitted.
+        egress: {
+          proxy: runtime.kind === "container" && (options.egressProxy ?? false),
+          allowlist: composeAllowlistFromHosts([
+            ...(runtime.egressHosts?.(loaded.metadata) ?? []),
+            ...(options.adapterHosts?.[experiment.id] ?? []),
+          ]),
+        },
+        webPolicy: runtime.webPolicy?.() ?? "native-web-blocked",
+        pathTreatment: runtime.pathTreatment?.() ?? null,
+        mcp: options.mcpHashes?.[experiment.id] ?? null,
+        budgets: { ...budgets },
+      };
+    }
+  return pairs;
+}

package/src/experiments/discovery.ts

@@ -0,0 +1,76 @@
+/**
+ * Experiment discovery: import every `<root>/*.ts` module and collect its
+ * default-exported experiment. Loud on the first problem — a module without
+ * a valid default export, or two modules claiming one id, aborts the scan.
+ */
+
+import { existsSync, readdirSync, statSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import type { Experiment, LoadedExperiment } from "./types.ts";
+
+/** An experiment module failed to import or its exported shape is invalid. */
+export class ExperimentLoadError extends Error {
+  readonly file: string;
+
+  constructor(file: string, detail: string) {
+    super(`${file}: ${detail}`);
+    this.name = "ExperimentLoadError";
+    this.file = file;
+  }
+}
+
+function isExperimentShaped(value: unknown): value is Experiment {
+  if (value === null || typeof value !== "object") return false;
+  const candidate = value as Partial<Experiment>;
+  return (
+    typeof candidate.id === "string" &&
+    candidate.id !== "" &&
+    typeof candidate.agent?.adapter === "string" &&
+    (candidate.runtime?.kind === "host" || candidate.runtime?.kind === "container")
+  );
+}
+
+/** Discover every experiment under `root`, sorted by id. */
+export async function discoverExperiments(root: string): Promise<LoadedExperiment[]> {
+  const rootDir = resolve(root);
+  if (!existsSync(rootDir) || !statSync(rootDir).isDirectory()) {
+    throw new Error(`experiments root "${root}" is not a directory`);
+  }
+
+  const loaded: LoadedExperiment[] = [];
+  const files = readdirSync(rootDir)
+    .filter((name) => name.endsWith(".ts"))
+    .sort((a, b) => a.localeCompare(b));
+  for (const name of files) {
+    const path = join(rootDir, name);
+    let module: Record<string, unknown>;
+    try {
+      module = (await import(pathToFileURL(path).href)) as Record<string, unknown>;
+    } catch (err) {
+      throw new ExperimentLoadError(path, `failed to import: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    const experiment = module["default"];
+    if (!isExperimentShaped(experiment)) {
+      throw new ExperimentLoadError(
+        path,
+        `must default-export defineExperiment({ id, agent, runtime }); got ${experiment === undefined ? "no default export" : "an invalid shape"}`,
+      );
+    }
+    loaded.push({ experiment, path });
+  }
+
+  const byId = new Map<string, LoadedExperiment>();
+  for (const entry of loaded) {
+    const existing = byId.get(entry.experiment.id);
+    if (existing) {
+      throw new ExperimentLoadError(
+        entry.path,
+        `duplicate experiment id "${entry.experiment.id}" — also declared by ${existing.path}`,
+      );
+    }
+    byId.set(entry.experiment.id, entry);
+  }
+
+  return loaded.sort((a, b) => a.experiment.id.localeCompare(b.experiment.id));
+}

package/src/experiments/groups.ts

@@ -0,0 +1,119 @@
+/**
+ * Comparison groups with withhold-and-diff semantics.
+ *
+ * Experiments sharing a `comparisonGroup` claim to be treatments of ONE
+ * configuration — same agent, same model, same runtime — varying only the
+ * declared `treatment`. `compareGroups` checks that claim: members must
+ * agree on every identity component EXCEPT the treatment itself and the
+ * treatment's own runtime config subtree (by convention
+ * `runtime.config[treatment]`).
+ *
+ * On mismatch the group is marked `comparison: "withheld"` with a
+ * component-level diff — the run is NEVER blocked and raw per-experiment
+ * results are unaffected. (src/compare.ts is the diff-rendering precedent;
+ * its throw-unless---force blocking is deliberately NOT copied here.)
+ *
+ * Excluded from the comparison by construction:
+ *   - `source` — two experiment files always hash differently;
+ *   - `treatment` — the declared variation;
+ *   - `runtime.config[treatment]` — the treatment's resulting configuration;
+ *   - `harness` — equal within one manifest by construction.
+ * The manifest's opaque `runtime` hash is replaced by the raw runtime data
+ * (kind/image/command/config) so the diff can name the differing key.
+ */
+
+import { pairKey } from "../execution-policy.ts";
+import { diffIdentity, type IdentityMismatch } from "../identity-diff.ts";
+import type { RunManifest } from "../manifest.ts";
+import type { Experiment } from "./types.ts";
+
+export interface ComparisonGroup {
+  group: string;
+  /** Member experiment ids, sorted. */
+  experimentIds: string[];
+  /** "allowed" = automatic treatment comparison may proceed. */
+  comparison: "allowed" | "withheld";
+  /** Component-level mismatches (empty when allowed). */
+  mismatches: IdentityMismatch[];
+  /** Rendered diff lines naming the members and component (empty when allowed). */
+  details: string[];
+}
+
+/**
+ * The comparable view of one experiment: manifest identity components where
+ * they are precise (agent fields, resolved image), raw runtime data where
+ * the manifest is opaque (a single hash), minus every excluded component.
+ */
+function comparableView(experiment: Experiment, manifest: RunManifest): Record<string, unknown> {
+  const components = manifest.experiments[experiment.id]?.components;
+  const config: Record<string, unknown> = { ...experiment.runtime.config };
+  if (experiment.treatment !== undefined) delete config[experiment.treatment];
+  const environments = Object.fromEntries(Object.keys(manifest.evals).sort().map((evalId) => {
+    const value = { ...manifest.pairs?.[pairKey(evalId, experiment.id)] };
+    for (const field of experiment.runtime.treatmentPolicyFields ?? []) delete value[field];
+    return [evalId, value];
+  }));
+  return {
+    environments,
+    treatmentPolicyFields: [...(experiment.runtime.treatmentPolicyFields ?? [])].sort(),
+    agent: components?.agent ?? {
+      adapter: experiment.agent.adapter,
+      provider: experiment.agent.provider ?? "unspecified",
+      model: experiment.agent.model ?? "unspecified",
+      reasoning: experiment.agent.reasoning ?? "unspecified",
+      cliVersion: experiment.agent.cliVersion ?? "unspecified",
+    },
+    image: components?.image ?? null,
+    runtime: {
+      kind: experiment.runtime.kind,
+      observationTargets: experiment.runtime.observationTargets ?? null,
+      image: experiment.runtime.image ?? null,
+      command: experiment.runtime.command ?? null,
+      config,
+    },
+  };
+}
+
+/**
+ * Group experiments by `comparisonGroup` and check treatment comparability.
+ * Pure — no store access, never throws, never blocks a run; the report layer
+ * consumes its output. Experiments without a group are not compared at all;
+ * a singleton group is trivially allowed.
+ */
+export function compareGroups(experiments: readonly Experiment[], manifest: RunManifest): ComparisonGroup[] {
+  const byGroup = new Map<string, Experiment[]>();
+  for (const experiment of experiments) {
+    if (experiment.comparisonGroup === undefined) continue;
+    const members = byGroup.get(experiment.comparisonGroup) ?? [];
+    members.push(experiment);
+    byGroup.set(experiment.comparisonGroup, members);
+  }
+
+  const groups: ComparisonGroup[] = [];
+  for (const [group, members] of [...byGroup.entries()].sort(([a], [b]) => a.localeCompare(b))) {
+    const sorted = [...members].sort((a, b) => a.id.localeCompare(b.id));
+    const baseline = sorted[0] as Experiment;
+    const baselineView = comparableView(baseline, manifest);
+
+    const mismatches: IdentityMismatch[] = [];
+    const details: string[] = [];
+    for (const member of sorted.slice(1)) {
+      for (const mismatch of diffIdentity(baselineView, comparableView(member, manifest), "identity")) {
+        mismatches.push(mismatch);
+        details.push(
+          `${group}: ${baseline.id} vs ${member.id}: ${mismatch.path} changed ` +
+            `(${JSON.stringify(mismatch.a)} -> ${JSON.stringify(mismatch.b)})`,
+        );
+      }
+    }
+
+    groups.push({
+      group,
+      experimentIds: sorted.map((member) => member.id),
+      comparison: mismatches.length === 0 ? "allowed" : "withheld",
+      mismatches,
+      details,
+    });
+  }
+  return groups;
+}