@workos/quickstudy 0.6.1 → 0.7.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@workos/quickstudy",
-  "version": "0.6.1",
+  "version": "0.7.0",
   "description": "An open-source, company-agnostic eval harness for measuring whether coding agents can complete real integration tasks, across an agent x feature x framework x surface matrix.",
   "license": "MIT",
   "type": "module",

package/src/build-info.generated.ts

@@ -9,7 +9,7 @@ export const EMBEDDED_BUILD_INFO: {
   // (the `x-release-please-version` marker anchors the replacement). It is
   // restored as-is after a binary build (scripts/build-binary.ts), so the
   // committed source `--version` stays in lockstep with package.json.
-  version: "0.6.1", // x-release-please-version
+  version: "0.7.0", // x-release-please-version
   revision: null,
   dirty: null,
   sourceHash: null,

package/src/diagnose/render.ts

@@ -17,7 +17,7 @@ export function renderDiagnosisText(diagnosis: DiagnosisJson, resultsDir: string
   lines.push(`diagnosis: run ${diagnosis.run_id}`);
   lines.push(`  model: ${diagnosis.model.requested}${diagnosis.model.resolved !== null ? ` (resolved: ${diagnosis.model.resolved})` : ""}`);
   lines.push(
-    `  pairs: ${diagnosis.pairs_diagnosed} diagnosed, ${diagnosis.pairs_skipped} skipped (all passing)` +
+    `  pairs: ${diagnosis.pairs_diagnosed} diagnosed, ${diagnosis.pairs_skipped} skipped (passing or control)` +
       (diagnosis.errors.length > 0 ? `, ${diagnosis.errors.length} errored` : ""),
   );
 

package/src/diagnose/run.ts

@@ -30,6 +30,7 @@ import {
   type PairBundle,
 } from "./prompt.ts";
 import { selectDiagnosablePairs, type DiagnosablePair } from "./select.ts";
+import { controlExperiments } from "../report/report.ts";
 import {
   DIAGNOSIS_DISCLAIMER,
   DIAGNOSIS_PROMPT_VERSION,
@@ -218,10 +219,13 @@ export async function runDiagnose(options: RunDiagnoseOptions): Promise<Diagnosi
 
   const run = mustResolveRun(store, options.runId);
   const attempts = store.listAttempts(run.id);
-  const selection = selectDiagnosablePairs(attempts);
+  // Controls check the instrument, not an agent: the oracle passes and the
+  // no-op fails by construction, so their pairs carry no diagnosable cause.
+  const controls = controlExperiments(run);
+  const selection = selectDiagnosablePairs(attempts, { controls });
   log(
     `diagnose: run ${run.id} — ${selection.pairs.length} diagnosable pair(s), ` +
-      `${selection.skipped} passing pair(s) skipped`,
+      `${selection.skipped} passing pair(s) skipped, ${selection.controls} control pair(s) excluded`,
   );
 
   const findings: DiagnosisFinding[] = [];
@@ -281,7 +285,7 @@ export async function runDiagnose(options: RunDiagnoseOptions): Promise<Diagnosi
     prompt_version: DIAGNOSIS_PROMPT_VERSION,
     disclaimer: DIAGNOSIS_DISCLAIMER,
     pairs_diagnosed: selection.pairs.length - errors.length,
-    pairs_skipped: selection.skipped,
+    pairs_skipped: selection.skipped + selection.controls,
     findings,
     errors,
   };

package/src/diagnose/select.ts

@@ -47,6 +47,19 @@ export interface PairSelection {
   pairs: DiagnosablePair[];
   /** Passing-only pairs — skipped, never sent to the model. */
   skipped: number;
+  /**
+   * Control pairs (the positive-control oracle, the negative-control no-op) —
+   * excluded before classification, never sent to the model. A control's
+   * outcome is fixed by construction: asked why the no-op "failed", a model
+   * confidently reports a broken harness. The report's scorer health watches
+   * the oracle instead.
+   */
+  controls: number;
+}
+
+export interface SelectOptions {
+  /** Experiment ids the run recorded as controls (RunConfig.roles other than "agent"). */
+  controls?: ReadonlySet<string>;
 }
 
 /** Bundle ordering: failed first, then errors, then incomplete. */
@@ -58,10 +71,11 @@ const CLASS_ORDER: Record<Exclude<AttemptClass, "passed">, number> = {
 
 /**
  * Group attempts into pairs, classify, and select each diagnosable pair's
- * bundle. Deterministic: pairs sort by eval id then experiment id (the
- * report's order); within a class, attempts sort by ULID (id) order.
+ * bundle. Control experiments are excluded first (see PairSelection.controls).
+ * Deterministic: pairs sort by eval id then experiment id (the report's
+ * order); within a class, attempts sort by ULID (id) order.
  */
-export function selectDiagnosablePairs(attempts: AttemptRecord[]): PairSelection {
+export function selectDiagnosablePairs(attempts: AttemptRecord[], options: SelectOptions = {}): PairSelection {
   const byPair = new Map<string, AttemptRecord[]>();
   for (const attempt of attempts) {
     const key = `${attempt.evalId}|${attempt.experimentId}`;
@@ -72,8 +86,13 @@ export function selectDiagnosablePairs(attempts: AttemptRecord[]): PairSelection
 
   const pairs: DiagnosablePair[] = [];
   let skipped = 0;
+  let controls = 0;
 
   for (const group of byPair.values()) {
+    if (options.controls?.has(group[0]!.experimentId)) {
+      controls += 1;
+      continue;
+    }
     const report = collectPairs(group)[0]!;
     if (report.failed + report.errors + report.incomplete === 0) {
       skipped += 1;
@@ -106,5 +125,5 @@ export function selectDiagnosablePairs(attempts: AttemptRecord[]): PairSelection
   }
 
   pairs.sort((a, b) => a.evalId.localeCompare(b.evalId) || a.experimentId.localeCompare(b.experimentId));
-  return { pairs, skipped };
+  return { pairs, skipped, controls };
 }

package/src/diagnose/types.ts

@@ -85,6 +85,7 @@ export interface DiagnosisJson {
   /** Fixed self-labeling: hypotheses with evidence links, not verdicts; single-sweep scope. */
   disclaimer: string;
   pairs_diagnosed: number;
+  /** Pairs never sent to the model: passing-only pairs and control (oracle / no-op) pairs. */
   pairs_skipped: number;
   findings: DiagnosisFinding[];
   errors: PairDiagnosisError[];

package/src/report/report.ts

@@ -454,7 +454,7 @@ function roleOf(run: RunRecord, experimentId: string): ExperimentRole {
 }
 
 /** The run's control experiments, from the roles it recorded. */
-function controlExperiments(run: RunRecord): Set<string> {
+export function controlExperiments(run: RunRecord): Set<string> {
   return new Set(run.config.experimentIds.filter((id) => roleOf(run, id) !== "agent"));
 }