@workos/quickstudy 0.7.0 → 0.8.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@workos/quickstudy",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "description": "An open-source, company-agnostic eval harness for measuring whether coding agents can complete real integration tasks, across an agent x feature x framework x surface matrix.",
   "license": "MIT",
   "type": "module",

package/src/build-info.generated.ts

@@ -9,7 +9,7 @@ export const EMBEDDED_BUILD_INFO: {
   // (the `x-release-please-version` marker anchors the replacement). It is
   // restored as-is after a binary build (scripts/build-binary.ts), so the
   // committed source `--version` stays in lockstep with package.json.
-  version: "0.7.0", // x-release-please-version
+  version: "0.8.0", // x-release-please-version
   revision: null,
   dirty: null,
   sourceHash: null,

package/src/evals/discovery.ts

@@ -44,7 +44,7 @@ export function discoverEvals(root: string): LoadedEval[] {
     const promptPath = join(dir, "PROMPT.md");
     if (!existsSync(promptPath)) continue;
 
-    const { metadata, promptBody, frameworks, local } = parsePrompt(readFileSync(promptPath, "utf8"), promptPath);
+    const { metadata, promptBody, frameworks, local, suites } = parsePrompt(readFileSync(promptPath, "utf8"), promptPath);
 
     const scorerPath = join(dir, "EVAL.ts");
     if (!existsSync(scorerPath)) {
@@ -132,7 +132,9 @@ export function discoverEvals(root: string): LoadedEval[] {
       }
       collected.push({
         loaded: {
-          metadata: { ...metadata, id: `${metadata.id}-${framework}`, framework },
+          // A `suites:` entry retiers one variant (a saturated django port can be
+          // smoke while the nextjs port stays benchmark) without renaming its id.
+          metadata: { ...metadata, suite: suites?.[framework] ?? metadata.suite, id: `${metadata.id}-${framework}`, framework },
           dir,
           promptPath,
           promptBody,

package/src/evals/prompt.ts

@@ -33,11 +33,13 @@ export interface ParsedPrompt {
   promptBody: string;
   /** `frameworks:` variant list — discovery expands one eval per entry. */
   frameworks?: string[];
+  /** Per-framework suite overrides (`suites:`); variants not named keep `suite`. */
+  suites?: Record<string, EvalSuite>;
   /** `local:` shared starting-state root, relative to the eval directory. */
   local?: string;
 }
 
-const KNOWN_KEYS = ["id", "suite", "product", "framework", "frameworks", "local", "topics"] as const;
+const KNOWN_KEYS = ["id", "suite", "product", "framework", "frameworks", "suites", "local", "topics"] as const;
 
 /** Framework tokens become id suffixes (`<dirname>-<fw>`), so they must be kebab-case. */
 const FRAMEWORK_TOKEN = /^[a-z0-9]+(-[a-z0-9]+)*$/;
@@ -137,6 +139,41 @@ export function parsePrompt(source: string, file: string): ParsedPrompt {
     }
     frameworks = tokens;
   }
+  let suites: Record<string, EvalSuite> | undefined;
+  if (fm["suites"] !== undefined) {
+    const value = fm["suites"];
+    if (frameworks === undefined) {
+      throw new EvalLoadError(
+        file,
+        keyLine("suites"),
+        `"suites" requires "frameworks" — it overrides the suite per framework variant`,
+      );
+    }
+    if (value === null || typeof value !== "object" || Array.isArray(value)) {
+      throw new EvalLoadError(file, keyLine("suites"), `"suites" must be a mapping of framework token to suite`);
+    }
+    const entries = Object.entries(value as Record<string, unknown>);
+    if (entries.length === 0) {
+      throw new EvalLoadError(file, keyLine("suites"), `"suites" must name at least one framework`);
+    }
+    for (const [token, suiteValue] of entries) {
+      if (!frameworks.includes(token)) {
+        throw new EvalLoadError(
+          file,
+          keyLine("suites"),
+          `"suites" names framework ${JSON.stringify(token)}, which is not in "frameworks"`,
+        );
+      }
+      if (typeof suiteValue !== "string" || !(EVAL_SUITES as readonly string[]).includes(suiteValue)) {
+        throw new EvalLoadError(
+          file,
+          keyLine("suites"),
+          `"suites" entry for ${JSON.stringify(token)} must be one of: ${EVAL_SUITES.join(", ")} (got ${JSON.stringify(suiteValue)})`,
+        );
+      }
+    }
+    suites = value as Record<string, EvalSuite>;
+  }
   let local: string | undefined;
   if (fm["local"] !== undefined) {
     const value = fm["local"];
@@ -185,6 +222,7 @@ export function parsePrompt(source: string, file: string): ParsedPrompt {
     metadata,
     promptBody,
     ...(frameworks !== undefined ? { frameworks } : {}),
+    ...(suites !== undefined ? { suites } : {}),
     ...(local !== undefined ? { local } : {}),
   };
 }

package/src/evals/types.ts

@@ -5,10 +5,22 @@
  * plain TypeScript handed an {@link EvalContext} over the finished workspace.
  */
 
-/** Closed set of eval suites. `regression` never moves headline scores. */
-export const EVAL_SUITES = ["benchmark", "regression", "other"] as const;
+/**
+ * Closed set of eval suites. `benchmark` and `regression` are the headline
+ * suites: the leaderboard aggregates count both. `smoke` holds evals every
+ * arm solves — run to catch breakage, never moving the headline. `other` is
+ * anything else outside the headline.
+ */
+export const EVAL_SUITES = ["benchmark", "regression", "smoke", "other"] as const;
 export type EvalSuite = (typeof EVAL_SUITES)[number];
 
+/** The suites whose attempts count toward the leaderboard aggregates. */
+export const HEADLINE_SUITES: readonly EvalSuite[] = ["benchmark", "regression"];
+
+export function isHeadlineSuite(suite: string): boolean {
+  return (HEADLINE_SUITES as readonly string[]).includes(suite);
+}
+
 /** PROMPT.md frontmatter. New fields require a proven filtering/reporting use case. */
 export interface EvalMetadata {
   /** Must equal the eval's directory name (prevents silent aliasing). */

package/src/report/aggregate.ts

@@ -2,17 +2,26 @@
  * Per-experiment leaderboard aggregate — one row per experiment, derived from
  * the same attempts and (banded) pairs as the rest of the report.
  *
- * Every number is restricted to benchmark-suite attempts, so regression and
- * other suites never move the headline row. Telemetry follows the report's
- * rule: sums and means cover only attempts that reported a value, and are
- * null (never 0) when none did.
+ * Every number is restricted to headline-suite attempts (HEADLINE_SUITES:
+ * benchmark and regression), so smoke and other suites never move the
+ * headline row. Telemetry follows the report's rule: sums and means cover
+ * only attempts that reported a value, and are null (never 0) when none did.
  */
+import { isHeadlineSuite } from "../evals/types.ts";
 import type { AttemptRecord } from "../store/db.ts";
 import type { RunCostEstimate } from "./costs.ts";
 import type { ReportExperiment, ReportPair } from "./report.ts";
 
+/**
+ * The pass-rate gate: an experiment whose headline pass rate is below this
+ * is listed after every experiment that clears it, whatever its cost or
+ * time. Pass rate is a gate, not a rank — the leaderboard orders the arms
+ * that clear it by time-to-pass, then cost-to-pass.
+ */
+export const DEFAULT_PASS_GATE = 0.8;
+
 export interface ExperimentAggregate {
-  /** Benchmark-suite attempts of this experiment (every status). */
+  /** Headline-suite attempts of this experiment (every status). */
   attempts: number;
   completed: number;
   passed: number;
@@ -21,11 +30,11 @@ export interface ExperimentAggregate {
   errors: number;
   /** Attempts whose agent phase hit the timeout limit (graded timeouts are also completed). */
   timeouts: number;
-  /** passed / (passed + failed) over benchmark-suite pairs; null when nothing scored. */
+  /** passed / (passed + failed) over headline-suite pairs; null when nothing scored. */
   pass_rate_all: number | null;
   /** Same, restricted to evals whose band is "discriminating". */
   pass_rate_discriminating: number | null;
-  /** Discriminating benchmark evals where this experiment scored at least one attempt. */
+  /** Discriminating headline evals where this experiment scored at least one attempt. */
   discriminating_evals: number;
   /** Numerator and denominator of pass_rate_discriminating. */
   discriminating_passed: number;
@@ -40,9 +49,25 @@ export interface ExperimentAggregate {
   /** Provider-reported, summed over the attempts that reported it. */
   cost_usd: number | null;
   cost_attempts: number;
+  /**
+   * Cost-to-pass: reported cost summed over the attempts that reported it,
+   * divided by the passes among those same attempts — what one success cost,
+   * failures included. null when nothing reported cost or none of it passed.
+   */
+  cost_per_pass_usd: number | null;
+  /**
+   * Time-to-pass: agent time summed over the scored attempts that recorded
+   * both stamps, divided by the passes among them. null when none did, or
+   * none of them passed.
+   */
+  agent_ms_per_pass: number | null;
+  /** pass_rate_all >= the report's pass gate; null when nothing scored. */
+  clears_gate: boolean | null;
   /** Present only when the report was built with --pricing. */
   estimated_cost_usd?: number | null;
   estimated_attempts?: number;
+  /** --pricing's estimate over the estimated attempts, divided by their passes. */
+  estimated_cost_per_pass_usd?: number | null;
 }
 
 const sum = (values: number[]): number => values.reduce((total, v) => total + v, 0);
@@ -66,33 +91,44 @@ export function collectExperimentAggregate(
   pairs: ReportPair[],
   experimentId: string,
   costEstimate?: RunCostEstimate,
+  passGate: number = DEFAULT_PASS_GATE,
 ): ExperimentAggregate {
-  const mine = attempts.filter((a) => a.experimentId === experimentId && a.suite === "benchmark");
+  const mine = attempts.filter((a) => a.experimentId === experimentId && isHeadlineSuite(a.suite));
   const completed = mine.filter((a) => a.status === "completed");
   const passed = completed.filter((a) => a.result?.passed).length;
 
-  const benchmarkPairs = pairs.filter((p) => p.experiment_id === experimentId && p.suite === "benchmark");
-  const scoredAll = sum(benchmarkPairs.map((p) => p.scored));
-  const passedAll = sum(benchmarkPairs.map((p) => p.passed));
-  const discriminating = benchmarkPairs.filter((p) => p.band === "discriminating" && p.scored > 0);
+  const headlinePairs = pairs.filter((p) => p.experiment_id === experimentId && isHeadlineSuite(p.suite));
+  const scoredAll = sum(headlinePairs.map((p) => p.scored));
+  const passedAll = sum(headlinePairs.map((p) => p.passed));
+  const discriminating = headlinePairs.filter((p) => p.band === "discriminating" && p.scored > 0);
   const discScored = sum(discriminating.map((p) => p.scored));
   const discPassed = sum(discriminating.map((p) => p.passed));
 
-  const durations = completed.flatMap((a) =>
-    typeof a.agentStartedAt === "number" && typeof a.agentFinishedAt === "number" ? [a.agentFinishedAt - a.agentStartedAt] : [],
-  );
+  const stamped = completed.filter((a) => typeof a.agentStartedAt === "number" && typeof a.agentFinishedAt === "number");
+  const durations = stamped.map((a) => (a.agentFinishedAt as number) - (a.agentStartedAt as number));
+  const stampedPasses = stamped.filter((a) => a.result?.passed).length;
   const tokensIn = mine.map((a) => a.tokensIn).filter((v): v is number => v !== null && v !== undefined);
   const tokensOut = mine.map((a) => a.tokensOut).filter((v): v is number => v !== null && v !== undefined);
-  const costs = mine.map((a) => a.costUsd).filter((v): v is number => v !== null && v !== undefined);
+  const costed = mine.filter((a) => a.costUsd !== null && a.costUsd !== undefined);
+  const costs = costed.map((a) => a.costUsd as number);
+  const costedPasses = costed.filter((a) => a.status === "completed" && a.result?.passed).length;
 
-  let estimated: Pick<ExperimentAggregate, "estimated_cost_usd" | "estimated_attempts"> = {};
+  let estimated: Pick<ExperimentAggregate, "estimated_cost_usd" | "estimated_attempts" | "estimated_cost_per_pass_usd"> = {};
   if (costEstimate !== undefined) {
-    const ids = new Set(mine.map((a) => a.id));
-    const values = costEstimate.attempts
-      .filter((e) => ids.has(e.attempt_id))
-      .flatMap((e) => (e.cost_usd === null ? [] : [e.cost_usd]));
-    estimated = { estimated_cost_usd: values.length > 0 ? sum(values) : null, estimated_attempts: values.length };
+    const byId = new Map(mine.map((a) => [a.id, a]));
+    const priced = costEstimate.attempts.filter((e) => byId.has(e.attempt_id) && e.cost_usd !== null);
+    const values = priced.map((e) => e.cost_usd as number);
+    const pricedPasses = priced.filter((e) => {
+      const attempt = byId.get(e.attempt_id);
+      return attempt?.status === "completed" && attempt.result?.passed;
+    }).length;
+    estimated = {
+      estimated_cost_usd: values.length > 0 ? sum(values) : null,
+      estimated_attempts: values.length,
+      estimated_cost_per_pass_usd: values.length > 0 && pricedPasses > 0 ? sum(values) / pricedPasses : null,
+    };
   }
+  const passRateAll = scoredAll > 0 ? passedAll / scoredAll : null;
 
   return {
     attempts: mine.length,
@@ -102,7 +138,7 @@ export function collectExperimentAggregate(
     incomplete: mine.filter((a) => a.status === "incomplete").length,
     errors: mine.filter((a) => a.status === "error").length,
     timeouts: mine.filter((a) => a.status === "completed" && a.limitHit === "timeout").length,
-    pass_rate_all: scoredAll > 0 ? passedAll / scoredAll : null,
+    pass_rate_all: passRateAll,
     pass_rate_discriminating: discScored > 0 ? discPassed / discScored : null,
     discriminating_evals: new Set(discriminating.map((p) => p.eval_id)).size,
     discriminating_passed: discPassed,
@@ -114,21 +150,41 @@ export function collectExperimentAggregate(
     token_attempts: Math.max(tokensIn.length, tokensOut.length),
     cost_usd: costs.length > 0 ? sum(costs) : null,
     cost_attempts: costs.length,
+    cost_per_pass_usd: costs.length > 0 && costedPasses > 0 ? sum(costs) / costedPasses : null,
+    agent_ms_per_pass: durations.length > 0 && stampedPasses > 0 ? sum(durations) / stampedPasses : null,
+    clears_gate: passRateAll === null ? null : passRateAll >= passGate,
     ...estimated,
   };
 }
 
+/** Ascending with nulls (and absent values) last; equal when both are absent. */
+function ascending(a: number | null | undefined, b: number | null | undefined): number {
+  const x = a ?? null;
+  const y = b ?? null;
+  if (x === null && y === null) return 0;
+  if (x === null) return 1;
+  if (y === null) return -1;
+  return x - y;
+}
+
 /**
  * Leaderboard order: agent arms before controls (the oracle would otherwise
- * lead every board), then discriminating pass rate desc (nulls last), then the
- * all-eval pass rate desc (nulls last), then experiment id — a total order,
- * so every consumer renders the same rows in the same sequence.
+ * lead every board); arms that clear the pass gate before those that do not;
+ * then time-to-pass ascending (every adapter reports agent time), then
+ * cost-to-pass ascending (null where the provider reports no dollars sorts
+ * last); then discriminating and all-eval pass rate desc as tie-breaks; then
+ * experiment id — a total order, so every consumer renders the same rows in
+ * the same sequence.
  */
 export function compareLeaderboard(a: ReportExperiment, b: ReportExperiment): number {
   const rate = (value: number | null | undefined): number => (value === null || value === undefined ? -1 : value);
   const control = (experiment: ReportExperiment): number => ((experiment.role ?? "agent") === "agent" ? 0 : 1);
+  const gate = (experiment: ReportExperiment): number => (experiment.aggregate?.clears_gate === true ? 0 : 1);
   return (
     control(a) - control(b) ||
+    gate(a) - gate(b) ||
+    ascending(a.aggregate?.agent_ms_per_pass, b.aggregate?.agent_ms_per_pass) ||
+    ascending(a.aggregate?.cost_per_pass_usd, b.aggregate?.cost_per_pass_usd) ||
     rate(b.aggregate?.pass_rate_discriminating) - rate(a.aggregate?.pass_rate_discriminating) ||
     rate(b.aggregate?.pass_rate_all) - rate(a.aggregate?.pass_rate_all) ||
     a.experiment_id.localeCompare(b.experiment_id)

package/src/report/report.ts

@@ -39,7 +39,8 @@ import {
   type ExperimentRole,
   type ScorerHealth,
 } from "./scorer-health.ts";
-import { collectExperimentAggregate, compareLeaderboard, type ExperimentAggregate } from "./aggregate.ts";
+import { collectExperimentAggregate, compareLeaderboard, DEFAULT_PASS_GATE, type ExperimentAggregate } from "./aggregate.ts";
+import { HEADLINE_SUITES } from "../evals/types.ts";
 
 export type { EvalBand, ExperimentRole, ScorerHealth, ScorerHealthSuspect } from "./scorer-health.ts";
 export type { ExperimentAggregate } from "./aggregate.ts";
@@ -53,7 +54,7 @@ export class ReportError extends Error {
   }
 }
 
-export const REPORT_SCHEMA_VERSION = 11;
+export const REPORT_SCHEMA_VERSION = 13;
 
 /**
  * Substrings that must never appear in generated report prose: at n≤5 trials
@@ -69,7 +70,7 @@ export function findBannedLanguage(text: string): string[] {
 }
 
 /** Methodology is generated prose — it must itself pass the language guard. */
-export function methodologyNote(trials: number): string {
+export function methodologyNote(trials: number, passGate: number = DEFAULT_PASS_GATE): string {
   return (
     `Pass rates are per eval × experiment pair: passed / (passed + failed). Incomplete and errored ` +
     `attempts are counted separately and never enter that denominator. Each pair has ${trials} planned ` +
@@ -88,7 +89,12 @@ export function methodologyNote(trials: number): string {
     `about the instrument, not about any treatment. Control experiments — an oracle that applies each eval's ` +
     `reference solution, a no-op that changes nothing — are reported alongside the agent experiments and ` +
     `tagged as controls wherever they appear; bands and scorer health are decided over the agent experiments ` +
-    `only, and controls are ranked after every agent experiment.`
+    `only, and controls are ranked after every agent experiment. Pass rate is a gate, not a rank: an experiment ` +
+    `whose headline pass rate is below ${Math.round(passGate * 100)}% is listed after every ` +
+    `experiment that clears it, and the experiments that clear it are ordered by time-to-pass (agent time summed ` +
+    `over the scored attempts that recorded it, divided by the passes among them) and then cost-to-pass (reported ` +
+    `cost summed over the attempts that reported it, divided by the passes among them) — what one success took, ` +
+    `failures included. Both are null, never zero, where nothing was recorded.`
   );
 }
 
@@ -134,7 +140,7 @@ export interface ReportExperiment {
    */
   sdk_recall?: ReportRecall;
   /**
-   * Leaderboard row over benchmark-suite attempts. Optional on the wire:
+   * Leaderboard row over headline-suite (benchmark + regression) attempts. Optional on the wire:
    * absent in schema_version < 9 artifacts; readers must tolerate absence.
    */
   aggregate?: ExperimentAggregate;
@@ -310,6 +316,8 @@ export interface ReportJson {
   /** v6: empty denial logs are observed zero; absent logs are unavailable. */
   egress_observation?: { available: string[]; unavailable: string[] };
   methodology: string;
+  /** The pass-rate gate the leaderboard order applies (see methodology). */
+  pass_gate: number;
 }
 
 // ---------------------------------------------------------------------------
@@ -652,7 +660,7 @@ export function buildReport(options: BuildReportOptions): ReportJson {
   const scorerHealth = collectScorerHealth(attempts, pairs, controls);
   const costEstimate = options.pricing === undefined ? undefined : estimateRunCosts(run, attempts, options.resultsDir, options.pricing);
 
-  const methodology = methodologyNote(run.config.trials) + (costEstimate === undefined ? "" :
+  const methodology = methodologyNote(run.config.trials, DEFAULT_PASS_GATE) + (costEstimate === undefined ? "" :
     " Optional token-cost estimates supplement reported cost using the supplied pricing snapshot. They cover only the selected attempts, not superseded retries, tool fees, diagnosis or infrastructure, and are not billing records.");
   const banned = findBannedLanguage(methodology);
   if (banned.length > 0) {
@@ -695,6 +703,7 @@ export function buildReport(options: BuildReportOptions): ReportJson {
       unavailable: attempts.filter((a) => !existsSync(join(options.resultsDir, runId, a.id, "egress-denials.log"))).map((a) => a.id),
     },
     methodology,
+    pass_gate: DEFAULT_PASS_GATE,
   };
 }
 
@@ -707,12 +716,15 @@ function pct(rate: number | null): string {
 }
 
 function renderLeaderboard(experiments: ReportExperiment[]): string[] {
-  const header = ["experiment", "model", "pass (all/disc)", "agent median", "tokens in", "cost", "recall"];
+  const header = ["experiment", "model", "gate", "$/pass", "time/pass", "pass (all/disc)", "agent median", "tokens in", "cost", "recall"];
   const rows = [...experiments].sort(compareLeaderboard).map((experiment) => {
     const agg = experiment.aggregate;
     const model = experiment.agent.model === "unspecified" ? "unpinned" : experiment.agent.model;
     const reasoning = experiment.agent.reasoning === "unspecified" ? "" : ` · ${experiment.agent.reasoning}`;
-    if (!agg) return [experiment.experiment_id, `${model}${reasoning}`, "n/a", "n/a", "n/a", "n/a", "n/a"];
+    if (!agg) return [experiment.experiment_id, `${model}${reasoning}`, "n/a", "n/a", "n/a", "n/a", "n/a", "n/a", "n/a", "n/a"];
+    const gate = agg.clears_gate === null ? "n/a" : agg.clears_gate ? "clears" : "below";
+    const perPass = agg.cost_per_pass_usd === null ? "n/a" : `$${agg.cost_per_pass_usd.toFixed(2)}`;
+    const timePerPass = agg.agent_ms_per_pass === null ? "n/a" : `${(agg.agent_ms_per_pass / 1000).toFixed(1)}s`;
     const all = agg.pass_rate_all === null ? "n/a" : `${agg.passed}/${agg.passed + agg.failed}`;
     const disc = agg.pass_rate_discriminating === null ? "n/a" : `${agg.discriminating_passed}/${agg.discriminating_scored}`;
     const time = agg.median_agent_ms === null ? "n/a" : `${(agg.median_agent_ms / 1000).toFixed(1)}s (${agg.agent_time_attempts})`;
@@ -723,11 +735,16 @@ function renderLeaderboard(experiments: ReportExperiment[]): string[] {
     const recall = experiment.sdk_recall;
     const recallText = recall && recall.rate !== null ? `${recall.recalled}/${recall.attempts}` : "n/a";
     const id = (experiment.role ?? "agent") === "agent" ? experiment.experiment_id : `${experiment.experiment_id} (control)`;
-    return [id, `${model}${reasoning}`, `${all} · ${disc}`, time, tokens, cost, recallText];
+    return [id, `${model}${reasoning}`, gate, perPass, timePerPass, `${all} · ${disc}`, time, tokens, cost, recallText];
   });
   const widths = header.map((title, i) => Math.max(title.length, ...rows.map((row) => row[i]!.length)));
   const format = (cells: string[]): string => `  ${cells.map((cell, i) => cell.padEnd(widths[i]!)).join("  ").trimEnd()}`;
-  return ["leaderboard (benchmark suite, sorted by discriminating pass rate; controls last):", format(header), ...rows.map(format)];
+  return [
+    `leaderboard (headline suites ${HEADLINE_SUITES.join(" + ")}; gate: pass rate ≥ ${Math.round(DEFAULT_PASS_GATE * 100)}%; ` +
+      `ordered by time-to-pass, then cost-to-pass; controls last):`,
+    format(header),
+    ...rows.map(format),
+  ];
 }
 
 /** Render the report as plain text. Renders FROM the JSON, never the store. */