@workos/quickstudy 0.4.0 → 0.6.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@workos/quickstudy",
-  "version": "0.4.0",
+  "version": "0.6.0",
   "description": "An open-source, company-agnostic eval harness for measuring whether coding agents can complete real integration tasks, across an agent x feature x framework x surface matrix.",
   "license": "MIT",
   "type": "module",

package/src/adapters/echo.ts

@@ -13,6 +13,8 @@ import type { AgentAdapter, AttemptContext, AttemptOutcome, TranscriptEvent } fr
 
 export class EchoAdapter implements AgentAdapter {
   readonly name: string;
+  /** Does nothing, so every eval must fail it: a scorer it passes rewards doing nothing. */
+  readonly role = "negative-control" as const;
   readonly identity = {
     provider: "local", model: "echo-v1", reasoning: "none", cliVersion: "built-in",
     surfacePolicies: { docs: "no-network", mcp: "no-network", cli: "no-network" },

package/src/adapters/oracle.ts

@@ -27,6 +27,7 @@ function oracleMetrics(): AttemptMetrics {
 /** Host variant: the prepared fixture already carries the solution overlay. */
 export class OracleAdapter implements AgentAdapter {
   readonly name = "oracle";
+  readonly role = "positive-control" as const;
   readonly identity = ORACLE_IDENTITY;
   readonly startingState = "solution" as const;
 
@@ -50,6 +51,7 @@ export class OracleContainerAdapter implements ContainerAgentAdapter {
   readonly name = "oracle";
   readonly identity = ORACLE_IDENTITY;
   readonly startingState = "solution" as const;
+  readonly role = "positive-control" as const;
   readonly requiredHostEnv: readonly string[] = [];
 
   setupFiles(): Record<string, string> {

package/src/adapters/types.ts

@@ -60,7 +60,11 @@ export interface AttemptContext {
    * it into the artifacts store afterwards.
    */
   workspaceDir: string;
-  /** The eval's task prompt, byte-identical across experiments. */
+  /**
+   * The prompt the agent receives: the eval's task body (byte-identical
+   * across experiments), preceded by the runtime's `promptPreamble` when the
+   * treatment declares one (recorded per pair in the manifest).
+   */
   prompt: string;
   trialIndex: number;
 }
@@ -82,12 +86,24 @@ export interface AttemptOutcome {
  */
 export type StartingState = "fixture" | "solution";
 
+/**
+ * What an adapter's attempts are for. "agent" is an arm under measurement; a
+ * control checks the instrument instead: "positive-control" must pass (the
+ * built-in oracle applies each eval's reference solution) and
+ * "negative-control" must fail (the built-in echo adapter does nothing). The
+ * runner records the role of the adapter that actually runs each experiment
+ * (RunConfig.roles); the report never guesses it from a name.
+ */
+export type AdapterRole = "agent" | "positive-control" | "negative-control";
+
 export interface AgentAdapter {
   /** The adapter name experiments select via `agent.adapter` (e.g. "echo"). */
   name: string;
   identity?: AdapterIdentityConfig;
   /** Starting workspace the runner prepares for this adapter (default "fixture"). */
   startingState?: StartingState;
+  /** What the adapter's attempts are for (default "agent"). */
+  role?: AdapterRole;
   runAttempt(ctx: AttemptContext): Promise<AttemptOutcome>;
 }
 
@@ -100,7 +116,11 @@ export interface AgentAdapter {
 export interface ContainerAttemptContext {
   runId: string;
   attemptId: string;
-  /** The eval's task prompt, byte-identical across experiments. */
+  /**
+   * The prompt the agent receives: the eval's task body (byte-identical
+   * across experiments), preceded by the runtime's `promptPreamble` when the
+   * treatment declares one (recorded per pair in the manifest).
+   */
   prompt: string;
   trialIndex: number;
   /**
@@ -128,6 +148,8 @@ export interface ContainerAgentAdapter {
   identity?: AdapterIdentityConfig;
   /** Starting workspace the runner prepares for this adapter (default "fixture"). */
   startingState?: StartingState;
+  /** What the adapter's attempts are for (default "agent"). */
+  role?: AdapterRole;
   /** Command used during preflight to record the executable's exact version. */
   versionCommand?: readonly string[];
   /**

package/src/build-info.generated.ts

@@ -9,7 +9,7 @@ export const EMBEDDED_BUILD_INFO: {
   // (the `x-release-please-version` marker anchors the replacement). It is
   // restored as-is after a binary build (scripts/build-binary.ts), so the
   // committed source `--version` stays in lockstep with package.json.
-  version: "0.4.0", // x-release-please-version
+  version: "0.6.0", // x-release-please-version
   revision: null,
   dirty: null,
   sourceHash: null,

package/src/evals/result.ts

@@ -3,7 +3,7 @@ export function validEvalResult(value: unknown): value is EvalResult {
   if (!value || typeof value !== "object") return false;
   const result = value as EvalResult;
   if (typeof result.passed !== "boolean" || !Array.isArray(result.checks)) return false;
-  if (result.checks.some((c) => !c || typeof c.name !== "string" || c.name === "" || typeof c.passed !== "boolean" || (c.graded !== undefined && typeof c.graded !== "boolean"))) return false;
+  if (result.checks.some((c) => !c || typeof c.name !== "string" || c.name === "" || typeof c.passed !== "boolean" || (c.graded !== undefined && typeof c.graded !== "boolean") || (c.kind !== undefined && (typeof c.kind !== "string" || c.kind === "")))) return false;
   if (new Set(result.checks.map((c) => c.name)).size !== result.checks.length) return false;
   return result.judgments === undefined || (Array.isArray(result.judgments) && result.judgments.every((j) => j && typeof j === "object" && j.provenance === "semantic-judge" &&
     typeof j.provider === "string" && typeof j.model === "string" && [j.tokensIn, j.tokensOut, j.costUsd].every((v) => v === null || (typeof v === "number" && Number.isFinite(v) && v >= 0))));

package/src/evals/types.ts

@@ -56,6 +56,15 @@ export interface CheckResult {
    * recall from it, and the scorer must leave it out of `passed`. Absent = true.
    */
   graded?: boolean;
+  /**
+   * What a recorded check is about, so recorded checks of different natures
+   * never pool: `"sdk"` (did the agent use the product SDK) feeds SDK recall;
+   * any other tag (`"wiring"` for shape-of-code signals, …) is tallied and
+   * shown but never enters SDK recall. Absent on an ungraded check means
+   * `"sdk"` (the pre-tag convention). Free-form string; graded checks may
+   * carry one too, purely as a label.
+   */
+  kind?: string;
   notes?: string;
   evidence?: EvidenceReference[];
 }

package/src/execution-policy.ts

@@ -24,8 +24,16 @@ export interface PairEnvironment {
   webPolicy: WebPolicy;
   pathTreatment: string | null;
   mcp: string | null;
+  /** Runtime text prepended to the eval prompt for this pair (null = prompt verbatim). */
+  promptPreamble: string | null;
   budgets: LifecycleBudgets;
 }
+
+/** The prompt an agent receives: the runtime's preamble (if any), a blank line, the eval's verbatim body. */
+export function composePrompt(preamble: string | null | undefined, body: string): string {
+  const text = preamble?.trim() ?? "";
+  return text === "" ? body : `${text}\n\n${body}`;
+}
 export function pairKey(evalId: string, experimentId: string): string {
   return JSON.stringify([evalId, experimentId]);
 }
@@ -64,6 +72,10 @@ export function resolvePairEnvironments(options: {
         webPolicy: runtime.webPolicy?.() ?? "native-web-blocked",
         pathTreatment: runtime.pathTreatment?.() ?? null,
         mcp: options.mcpHashes?.[experiment.id] ?? null,
+        promptPreamble: (() => {
+          const text = runtime.promptPreamble?.(loaded.metadata)?.trim() ?? "";
+          return text === "" ? null : text;
+        })(),
         budgets: { ...budgets },
       };
     }

package/src/experiments/types.ts

@@ -41,7 +41,7 @@ export interface Runtime extends RuntimeCapabilities {
    * Images and lifecycle budgets can never be exempted. All group members must
    * declare the same fields. Source/config identity still changes per attempt.
    */
-  treatmentPolicyFields?: Array<"egress" | "webPolicy" | "pathTreatment" | "mcp">;
+  treatmentPolicyFields?: Array<"egress" | "webPolicy" | "pathTreatment" | "mcp" | "promptPreamble">;
   /** Container image tag (container runtimes; alternative to `containerImage`). */
   image?: string;
   /** Scripted command run in /workspace (container runtimes, this phase). */

package/src/report/aggregate.ts

@@ -119,13 +119,16 @@ export function collectExperimentAggregate(
 }
 
 /**
- * Leaderboard order: discriminating pass rate desc (nulls last), then the
+ * Leaderboard order: agent arms before controls (the oracle would otherwise
+ * lead every board), then discriminating pass rate desc (nulls last), then the
  * all-eval pass rate desc (nulls last), then experiment id — a total order,
  * so every consumer renders the same rows in the same sequence.
  */
 export function compareLeaderboard(a: ReportExperiment, b: ReportExperiment): number {
   const rate = (value: number | null | undefined): number => (value === null || value === undefined ? -1 : value);
+  const control = (experiment: ReportExperiment): number => ((experiment.role ?? "agent") === "agent" ? 0 : 1);
   return (
+    control(a) - control(b) ||
     rate(b.aggregate?.pass_rate_discriminating) - rate(a.aggregate?.pass_rate_discriminating) ||
     rate(b.aggregate?.pass_rate_all) - rate(a.aggregate?.pass_rate_all) ||
     a.experiment_id.localeCompare(b.experiment_id)

package/src/report/report.ts

@@ -32,10 +32,16 @@ import { activeAttempts, type AttemptRecord, type ResultsStore, type RunRecord }
 import type { SurfaceUsage } from "../surface-usage.ts";
 import { collectPassAtK, displayPassAtK, type PassAtKEntry } from "./pass-at-k.ts";
 import { estimateRunCosts, type Pricing, type RunCostEstimate } from "./costs.ts";
-import { collectBands, collectScorerHealth, type EvalBand, type ScorerHealth } from "./scorer-health.ts";
+import {
+  collectBands,
+  collectScorerHealth,
+  type EvalBand,
+  type ExperimentRole,
+  type ScorerHealth,
+} from "./scorer-health.ts";
 import { collectExperimentAggregate, compareLeaderboard, type ExperimentAggregate } from "./aggregate.ts";
 
-export type { EvalBand, ScorerHealth, ScorerHealthSuspect } from "./scorer-health.ts";
+export type { EvalBand, ExperimentRole, ScorerHealth, ScorerHealthSuspect } from "./scorer-health.ts";
 export type { ExperimentAggregate } from "./aggregate.ts";
 export { compareLeaderboard } from "./aggregate.ts";
 
@@ -47,7 +53,7 @@ export class ReportError extends Error {
   }
 }
 
-export const REPORT_SCHEMA_VERSION = 9;
+export const REPORT_SCHEMA_VERSION = 11;
 
 /**
  * Substrings that must never appear in generated report prose: at n≤5 trials
@@ -76,10 +82,13 @@ export function methodologyNote(trials: number): string {
     `those trials. Both describe this run's trials only — no wider claim. Telemetry (cost, tokens, surface ` +
     `usage) is summed only over attempts that reported it and shown as unavailable — never zero — when none did. ` +
     `SDK recall is reported per experiment as recalled / attempts, where attempts are the scored attempts that ` +
-    `recorded at least one ungraded check and recalled are those whose ungraded checks all passed. Ungraded ` +
+    `recorded at least one ungraded check of kind "sdk" (an ungraded check with no kind counts as "sdk") and recalled are those whose ungraded "sdk" checks all passed. Ungraded checks of any other kind are tallied and shown but never enter SDK recall. Ungraded ` +
     `checks are recorded and tallied but never enter pass/fail. Scorer health lists checks that failed in every ` +
     `scored attempt of an eval under every arm while another check in the same attempts passed: a statement ` +
-    `about the instrument, not about any treatment.`
+    `about the instrument, not about any treatment. Control experiments — an oracle that applies each eval's ` +
+    `reference solution, a no-op that changes nothing — are reported alongside the agent experiments and ` +
+    `tagged as controls wherever they appear; bands and scorer health are decided over the agent experiments ` +
+    `only, and controls are ranked after every agent experiment.`
   );
 }
 
@@ -112,6 +121,13 @@ export interface ReportExperiment {
   experiment_id: string;
   treatment: string | null;
   agent: ReportAgentIdentity;
+  /**
+   * "agent" for an arm; "positive-control" (the oracle) and "negative-control"
+   * (the echo no-op) never decide a band or a scorer-health suspect. Optional on
+   * the wire: absent in schema_version < 10 artifacts; readers treat absence as
+   * "agent".
+   */
+  role?: ExperimentRole;
   /**
    * Optional on the wire: absent in schema_version < 7 artifacts, which the
    * export may still carry — readers must tolerate absence.
@@ -134,6 +150,12 @@ export interface ReportCheck {
    * schema_version < 7 artifacts; readers treat absence as graded.
    */
   graded: boolean;
+  /**
+   * The `kind` the scorer recorded for this check (`"sdk"`, `"wiring"`, …),
+   * or null when no attempt tagged it. Absent in schema_version < 11
+   * artifacts; readers treat absence as null.
+   */
+  kind: string | null;
 }
 
 /** One eval × experiment aggregation. Pairs with zero attempts are ABSENT, never 0%. */
@@ -161,8 +183,9 @@ export interface ReportPair {
   flaky: boolean;
   checks: ReportCheck[];
   /**
-   * The eval's band, denormalised onto each of its pairs: decided over the
-   * eval's pairs with scored > 0; null when fewer than 2 arms scored. Absent
+   * The eval's band, denormalised onto each of its pairs (control pairs too):
+   * decided over the eval's agent pairs with scored > 0; null when fewer than 2
+   * agent arms scored (before schema 10, controls counted as arms). Absent
    * in schema_version < 8 artifacts; readers treat absence as unbanded.
    */
   band: EvalBand | null;
@@ -296,7 +319,7 @@ export interface ReportJson {
 const USAGE_VALUE_CAP = 20;
 
 export function collectPairs(attempts: AttemptRecord[]): ReportPair[] {
-  const pairs = new Map<string, { pair: ReportPair; checks: Map<string, { passed: number; total: number; graded: boolean }>; outcomes: Set<boolean> }>();
+  const pairs = new Map<string, { pair: ReportPair; checks: Map<string, { passed: number; total: number; graded: boolean; kind: string | null }>; outcomes: Set<boolean> }>();
   for (const attempt of attempts) {
     const key = `${attempt.evalId}|${attempt.experimentId}`;
     let entry = pairs.get(key);
@@ -333,10 +356,11 @@ export function collectPairs(attempts: AttemptRecord[]): ReportPair[] {
       entry.outcomes.add(false);
     }
     for (const check of attempt.result?.checks ?? []) {
-      const tally = entry.checks.get(check.name) ?? { passed: 0, total: 0, graded: true };
+      const tally = entry.checks.get(check.name) ?? { passed: 0, total: 0, graded: true, kind: null };
       tally.total += 1;
       if (check.passed) tally.passed += 1;
       if (check.graded === false) tally.graded = false;
+      if (tally.kind === null && typeof check.kind === "string" && check.kind !== "") tally.kind = check.kind;
       entry.checks.set(check.name, tally);
     }
   }
@@ -348,7 +372,7 @@ export function collectPairs(attempts: AttemptRecord[]): ReportPair[] {
     pair.pass_at_k = collectPassAtK(pair.passed, pair.scored);
     pair.flaky = outcomes.size > 1;
     pair.checks = [...checks.entries()]
-      .map(([name, tally]) => ({ name, passed: tally.passed, total: tally.total, graded: tally.graded }))
+      .map(([name, tally]) => ({ name, passed: tally.passed, total: tally.total, graded: tally.graded, kind: tally.kind }))
       .sort((a, b) => a.name.localeCompare(b.name));
     result.push(pair);
   }
@@ -394,17 +418,24 @@ function treatmentOf(run: RunRecord, experimentId: string): string | null {
  * experiment; "unspecified" is its own explicit missing-value sentinel, so
  * this never invents a value the run did not pin.
  */
+/** An ungraded check that feeds SDK recall: kind "sdk", or no kind (the pre-tag convention). */
+export function isSdkRecallCheck(check: { graded?: boolean | undefined; kind?: string | undefined }): boolean {
+  return check.graded === false && (check.kind === undefined || check.kind === "sdk");
+}
+
 /**
  * SDK recall over one experiment's scored attempts: the denominator is every
- * completed attempt that recorded at least one `graded: false` check; an
- * attempt is recalled when all of those checks passed. Null rate at zero.
+ * completed attempt that recorded at least one `graded: false` check of kind
+ * "sdk" (or untagged); an attempt is recalled when all of those checks
+ * passed. Ungraded checks of any other kind (wiring, …) never enter. Null
+ * rate at zero.
  */
 export function collectRecall(attempts: AttemptRecord[], experimentId: string): ReportRecall {
   let total = 0;
   let recalled = 0;
   for (const attempt of attempts) {
     if (attempt.experimentId !== experimentId || attempt.status !== "completed") continue;
-    const ungraded = (attempt.result?.checks ?? []).filter((check) => check.graded === false);
+    const ungraded = (attempt.result?.checks ?? []).filter(isSdkRecallCheck);
     if (ungraded.length === 0) continue;
     total += 1;
     if (ungraded.every((check) => check.passed)) recalled += 1;
@@ -412,6 +443,21 @@ export function collectRecall(attempts: AttemptRecord[], experimentId: string):
   return { attempts: total, recalled, rate: total > 0 ? recalled / total : null };
 }
 
+/**
+ * An experiment's role as the run recorded it (RunConfig.roles, from the
+ * adapter that actually ran). A run from before roles were recorded keeps no
+ * record of which adapter ran, so every experiment in it reads as an agent —
+ * exactly as before roles existed; nothing is guessed from a name or identity.
+ */
+function roleOf(run: RunRecord, experimentId: string): ExperimentRole {
+  return run.config.roles?.[experimentId] ?? "agent";
+}
+
+/** The run's control experiments, from the roles it recorded. */
+function controlExperiments(run: RunRecord): Set<string> {
+  return new Set(run.config.experimentIds.filter((id) => roleOf(run, id) !== "agent"));
+}
+
 function collectExperiments(
   run: RunRecord,
   attempts: AttemptRecord[],
@@ -425,6 +471,7 @@ function collectExperiments(
       return {
         experiment_id: experimentId,
         treatment: treatmentOf(run, experimentId),
+        role: roleOf(run, experimentId),
         agent: {
           adapter: agent?.adapter ?? "unspecified",
           provider: agent?.provider ?? "unspecified",
@@ -599,9 +646,10 @@ export function buildReport(options: BuildReportOptions): ReportJson {
   if (publication && publication.eligible.length === 0) throw new ReportError(`no eligible complete samples for run ${runId}: ${publication.omitted.map((p) => `${p.eval_id} × ${p.experiment_id}: ${p.reasons.join("; ")}`).join(" | ")}`);
   const attempts = publication ? current.filter((a) => publication.eligible.some((p) => p.eval_id === a.evalId && p.experiment_id === a.experimentId)) : current;
   const pairs = collectPairs(attempts);
-  const bands = collectBands(pairs);
+  const controls = controlExperiments(run);
+  const bands = collectBands(pairs, controls);
   for (const pair of pairs) pair.band = bands.get(pair.eval_id) ?? null;
-  const scorerHealth = collectScorerHealth(attempts, pairs);
+  const scorerHealth = collectScorerHealth(attempts, pairs, controls);
   const costEstimate = options.pricing === undefined ? undefined : estimateRunCosts(run, attempts, options.resultsDir, options.pricing);
 
   const methodology = methodologyNote(run.config.trials) + (costEstimate === undefined ? "" :
@@ -674,11 +722,12 @@ function renderLeaderboard(experiments: ReportExperiment[]): string[] {
       (typeof agg.estimated_cost_usd === "number" ? ` +$${agg.estimated_cost_usd.toFixed(4)} est.` : "");
     const recall = experiment.sdk_recall;
     const recallText = recall && recall.rate !== null ? `${recall.recalled}/${recall.attempts}` : "n/a";
-    return [experiment.experiment_id, `${model}${reasoning}`, `${all} · ${disc}`, time, tokens, cost, recallText];
+    const id = (experiment.role ?? "agent") === "agent" ? experiment.experiment_id : `${experiment.experiment_id} (control)`;
+    return [id, `${model}${reasoning}`, `${all} · ${disc}`, time, tokens, cost, recallText];
   });
   const widths = header.map((title, i) => Math.max(title.length, ...rows.map((row) => row[i]!.length)));
   const format = (cells: string[]): string => `  ${cells.map((cell, i) => cell.padEnd(widths[i]!)).join("  ").trimEnd()}`;
-  return ["leaderboard (benchmark suite, sorted by discriminating pass rate):", format(header), ...rows.map(format)];
+  return ["leaderboard (benchmark suite, sorted by discriminating pass rate; controls last):", format(header), ...rows.map(format)];
 }
 
 /** Render the report as plain text. Renders FROM the JSON, never the store. */
@@ -794,7 +843,9 @@ export function renderReportText(report: ReportJson): string {
       lines.push(`  ${group.group}: compared (raw counts per arm)`);
       for (const arm of group.arms) {
         const cells = arm.evals.map((entry) => `${entry.eval_id} ${entry.display}`).join(", ");
-        lines.push(`    ${arm.experiment_id}${arm.treatment !== null ? ` [${arm.treatment}]` : ""}: ${cells || "no scored attempts"}`);
+        const role = experiments.find((experiment) => experiment.experiment_id === arm.experiment_id)?.role ?? "agent";
+        const control = role === "agent" ? "" : " (control)";
+        lines.push(`    ${arm.experiment_id}${control}${arm.treatment !== null ? ` [${arm.treatment}]` : ""}: ${cells || "no scored attempts"}`);
       }
     }
   }

package/src/report/scorer-health.ts

@@ -13,6 +13,15 @@ import type { ReportPair } from "./report.ts";
 
 export type EvalBand = "discriminating" | "solved-by-all" | "failed-by-all";
 
+/**
+ * What an experiment is for (the run's recorded RunConfig.roles). A control
+ * says whether the instrument works — the oracle must pass, the echo no-op must
+ * fail — so bands and scorer health are decided over agent experiments only:
+ * with an oracle and a no-op in the run, no eval could otherwise be solved or
+ * failed by all.
+ */
+export type ExperimentRole = import("../adapters/types.ts").AdapterRole;
+
 export interface ScorerHealthSuspect {
   eval_id: string;
   check: string;
@@ -30,10 +39,19 @@ export interface ScorerHealth {
   summary: string;
 }
 
-/** Per eval: over pairs with scored > 0. null when fewer than 2 arms scored. */
-export function collectBands(pairs: ReportPair[]): Map<string, EvalBand | null> {
+/**
+ * Per eval: over agent pairs with scored > 0 (`controls` names the control
+ * experiments, which never decide a band). null when fewer than 2 agent arms
+ * scored. Every eval of the run gets an entry, control-only ones as null.
+ */
+export function collectBands(
+  pairs: ReportPair[],
+  controls: ReadonlySet<string> = new Set(),
+): Map<string, EvalBand | null> {
   const byEval = new Map<string, ReportPair[]>();
   for (const pair of pairs) {
+    if (!byEval.has(pair.eval_id)) byEval.set(pair.eval_id, []);
+    if (controls.has(pair.experiment_id)) continue;
     const list = byEval.get(pair.eval_id) ?? [];
     list.push(pair);
     byEval.set(pair.eval_id, list);
@@ -62,18 +80,25 @@ export function scorerHealthSummary(suspects: ScorerHealthSuspect[]): string {
 }
 
 /**
- * Suspects come only from `failed-by-all` evals (≥2 scored arms). A check is a
- * suspect when it is present and failing in every scored (completed) attempt
- * of those arms, and at least one other graded check is present and passing
- * in every one of them. A name flagged `graded: false` in any attempt (the
- * pair tally's rule) is excluded from both sides.
+ * Suspects come only from `failed-by-all` evals (≥2 scored agent arms). A
+ * check is a suspect when it is present and failing in every scored
+ * (completed) attempt of those arms, and at least one other graded check is
+ * present and passing in every one of them. A name flagged `graded: false` in
+ * any attempt (the pair tally's rule) is excluded from both sides. Control
+ * experiments (`controls`) are never arms.
  */
-export function collectScorerHealth(attempts: AttemptRecord[], pairs: ReportPair[]): ScorerHealth {
-  const bands = collectBands(pairs);
+export function collectScorerHealth(
+  attempts: AttemptRecord[],
+  pairs: ReportPair[],
+  controls: ReadonlySet<string> = new Set(),
+): ScorerHealth {
+  const bands = collectBands(pairs, controls);
   const suspects: ScorerHealthSuspect[] = [];
   for (const [evalId, band] of bands) {
     if (band !== "failed-by-all") continue;
-    const arms = pairs.filter((pair) => pair.eval_id === evalId && pair.scored > 0);
+    const arms = pairs.filter(
+      (pair) => pair.eval_id === evalId && pair.scored > 0 && !controls.has(pair.experiment_id),
+    );
     const experiments = arms.map((pair) => pair.experiment_id).sort((a, b) => a.localeCompare(b));
     const ungraded = new Set(arms.flatMap((pair) => pair.checks.filter((check) => check.graded === false).map((check) => check.name)));
     const scored = attempts.filter(

package/src/runner/execute.ts

@@ -64,7 +64,7 @@ import {
   type McpServerConfig,
 } from "../isolation/mcp.ts";
 import { hashString } from "../hash.ts";
-import { DEFAULT_BUDGETS, pairKey, resolvePairEnvironments, type LifecycleBudgets, type PairEnvironment } from "../execution-policy.ts";
+import { DEFAULT_BUDGETS, pairKey, resolvePairEnvironments, type LifecycleBudgets, type PairEnvironment, composePrompt } from "../execution-policy.ts";
 import { ulid } from "../ids.ts";
 import { attemptIdentityHash, buildRunManifest, type RunManifest, type RuntimeImageIdentity } from "../manifest.ts";
 import { isMovingModelAlias } from "../model-identity.ts";
@@ -506,6 +506,18 @@ export async function executeRunV3(options: ExecuteRunV3Options): Promise<RunSum
       // ON THE RUN: the manifest holds runtime config only as an opaque hash,
       // so the report could never recompute this after the fact.
       groups: compareGroups(options.experiments.map((entry) => entry.experiment), manifest),
+      // Roles come from the resolved adapter, never its name: a scripted
+      // container command is an agent whatever `agent.adapter` says.
+      roles: Object.fromEntries(
+        [...experimentsById.values()].map((experiment) => [
+          experiment.id,
+          experiment.runtime.kind === "host"
+            ? (hostAdapters[experiment.agent.adapter]?.role ?? "agent")
+            : (experiment.runtime.command ?? []).length > 0
+              ? "agent"
+              : (containerAgents.get(experiment.id)?.role ?? "agent"),
+        ]),
+      ),
       ...(offeredSurfaces !== undefined ? { surfaces: offeredSurfaces } : {}),
     };
 
@@ -550,7 +562,7 @@ export async function executeRunV3(options: ExecuteRunV3Options): Promise<RunSum
       try {
         disposition =
           experiment.runtime.kind === "host"
-            ? await runHostAttemptV3(options, hostAdapters, loadedEval, experiment, entry, { runId, attemptId, attemptTimeoutMs, persisted })
+            ? await runHostAttemptV3(options, hostAdapters, loadedEval, experiment, entry, { runId, attemptId, attemptTimeoutMs, persisted, environment: manifest.pairs?.[pairKey(entry.evalId, entry.experimentId)] })
             : await runContainerAttemptV3(
                 options,
                 loadedEval,
@@ -649,7 +661,7 @@ async function runHostAttemptV3(
         attemptId,
         fixtureDir: sourceDir,
         workspaceDir: scratchDir,
-        prompt: loadedEval.promptBody,
+        prompt: composePrompt(args.environment?.promptPreamble, loadedEval.promptBody),
         trialIndex: entry.trialIndex,
       }));
     } catch (err) {
@@ -856,13 +868,14 @@ async function runContainerAttemptV3(
           const promptCtx: ContainerAttemptContext = {
             runId,
             attemptId,
-            prompt: loadedEval.promptBody,
+            // The eval body is byte-identical across experiments; a runtime's
+            // promptPreamble (recorded per pair in the manifest) may precede it.
+            prompt: composePrompt(environment?.promptPreamble, loadedEval.promptBody),
             trialIndex: entry.trialIndex,
             ...(rewrittenServers !== undefined ? { mcpServers: rewrittenServers } : {}),
             webPolicy: environment?.webPolicy ?? "native-web-blocked",
           };
           // Treatment config files (e.g. the MCP treatment's vendor config).
-          // The prompt itself is byte-identical across experiments.
           await inStage("environment", async () => {
             for (const [path, content] of Object.entries(agentAdapter.setupFiles(promptCtx))) {
               await container.writeFile(path, content);

package/src/runtime/types.ts

@@ -84,6 +84,19 @@ export interface RuntimeCapabilities {
   webPolicy?(): WebPolicy;
   /** Directory prepended to the container PATH (the CLI treatment), if any. */
   pathTreatment?(): string | undefined;
+  /**
+   * Text placed BEFORE the eval's prompt for every attempt under this
+   * runtime — the one sanctioned way to tell the agent what the treatment
+   * offers ("a `acme` CLI is on PATH; run `acme --help`"). An offered surface
+   * nobody mentions is rarely discovered, and an undiscovered surface
+   * measures nothing. Identity-bearing: the resolved text is recorded per
+   * pair in the run manifest (`pairs[].promptPreamble`), so group members
+   * that differ here must list `"promptPreamble"` in `treatmentPolicyFields`
+   * (and mirror the text under `config[treatment]`) or comparison is
+   * withheld. `undefined` or an empty string means the prompt is used
+   * verbatim.
+   */
+  promptPreamble?(metadata: EvalMetadata): string | undefined;
   /** Capabilities (`query`, `getClient`) exposed to the eval's scorer. */
   scorerCapabilities?(ctx: ScorerCapabilityContext): ScorerCapabilities;
 }

package/src/store/db.ts

@@ -34,6 +34,13 @@ function toLimitKind(value: string | null): LimitKind | null {
 
 /** The run-level configuration recorded in `runs.config_json`. */
 export interface RunConfig {
+  /**
+   * Each experiment's role, from the adapter that actually ran it (a scripted
+   * container command is always "agent"). Recorded here — NOT in the manifest —
+   * like `groups`, so it never perturbs run identity. Absent on older runs: every
+   * experiment then reads as "agent".
+   */
+  roles?: Record<string, import("../adapters/types.ts").AdapterRole>;
   scheduler?: { concurrency: number; seed: string; policy: "seeded-hash-v1"; order: import("../plan.ts").AttemptPlan[] };
   trials: number;
   evalIds: string[];