@workos/quickstudy 0.5.0 → 0.6.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@workos/quickstudy",
-  "version": "0.5.0",
+  "version": "0.6.1",
   "description": "An open-source, company-agnostic eval harness for measuring whether coding agents can complete real integration tasks, across an agent x feature x framework x surface matrix.",
   "license": "MIT",
   "type": "module",

package/src/adapters/types.ts

@@ -60,7 +60,11 @@ export interface AttemptContext {
    * it into the artifacts store afterwards.
    */
   workspaceDir: string;
-  /** The eval's task prompt, byte-identical across experiments. */
+  /**
+   * The prompt the agent receives: the eval's task body (byte-identical
+   * across experiments), preceded by the runtime's `promptPreamble` when the
+   * treatment declares one (recorded per pair in the manifest).
+   */
   prompt: string;
   trialIndex: number;
 }
@@ -112,7 +116,11 @@ export interface AgentAdapter {
 export interface ContainerAttemptContext {
   runId: string;
   attemptId: string;
-  /** The eval's task prompt, byte-identical across experiments. */
+  /**
+   * The prompt the agent receives: the eval's task body (byte-identical
+   * across experiments), preceded by the runtime's `promptPreamble` when the
+   * treatment declares one (recorded per pair in the manifest).
+   */
   prompt: string;
   trialIndex: number;
   /**

package/src/build-info.generated.ts

@@ -9,7 +9,7 @@ export const EMBEDDED_BUILD_INFO: {
   // (the `x-release-please-version` marker anchors the replacement). It is
   // restored as-is after a binary build (scripts/build-binary.ts), so the
   // committed source `--version` stays in lockstep with package.json.
-  version: "0.5.0", // x-release-please-version
+  version: "0.6.1", // x-release-please-version
   revision: null,
   dirty: null,
   sourceHash: null,

package/src/evals/result.ts

@@ -3,7 +3,7 @@ export function validEvalResult(value: unknown): value is EvalResult {
   if (!value || typeof value !== "object") return false;
   const result = value as EvalResult;
   if (typeof result.passed !== "boolean" || !Array.isArray(result.checks)) return false;
-  if (result.checks.some((c) => !c || typeof c.name !== "string" || c.name === "" || typeof c.passed !== "boolean" || (c.graded !== undefined && typeof c.graded !== "boolean"))) return false;
+  if (result.checks.some((c) => !c || typeof c.name !== "string" || c.name === "" || typeof c.passed !== "boolean" || (c.graded !== undefined && typeof c.graded !== "boolean") || (c.kind !== undefined && (typeof c.kind !== "string" || c.kind === "")))) return false;
   if (new Set(result.checks.map((c) => c.name)).size !== result.checks.length) return false;
   return result.judgments === undefined || (Array.isArray(result.judgments) && result.judgments.every((j) => j && typeof j === "object" && j.provenance === "semantic-judge" &&
     typeof j.provider === "string" && typeof j.model === "string" && [j.tokensIn, j.tokensOut, j.costUsd].every((v) => v === null || (typeof v === "number" && Number.isFinite(v) && v >= 0))));

package/src/evals/types.ts

@@ -56,6 +56,15 @@ export interface CheckResult {
    * recall from it, and the scorer must leave it out of `passed`. Absent = true.
    */
   graded?: boolean;
+  /**
+   * What a recorded check is about, so recorded checks of different natures
+   * never pool: `"sdk"` (did the agent use the product SDK) feeds SDK recall;
+   * any other tag (`"wiring"` for shape-of-code signals, …) is tallied and
+   * shown but never enters SDK recall. Absent on an ungraded check means
+   * `"sdk"` (the pre-tag convention). Free-form string; graded checks may
+   * carry one too, purely as a label.
+   */
+  kind?: string;
   notes?: string;
   evidence?: EvidenceReference[];
 }

package/src/execution-policy.ts

@@ -24,8 +24,16 @@ export interface PairEnvironment {
   webPolicy: WebPolicy;
   pathTreatment: string | null;
   mcp: string | null;
+  /** Runtime text prepended to the eval prompt for this pair (null = prompt verbatim). */
+  promptPreamble: string | null;
   budgets: LifecycleBudgets;
 }
+
+/** The prompt an agent receives: the runtime's preamble (if any), a blank line, the eval's verbatim body. */
+export function composePrompt(preamble: string | null | undefined, body: string): string {
+  const text = preamble?.trim() ?? "";
+  return text === "" ? body : `${text}\n\n${body}`;
+}
 export function pairKey(evalId: string, experimentId: string): string {
   return JSON.stringify([evalId, experimentId]);
 }
@@ -64,6 +72,10 @@ export function resolvePairEnvironments(options: {
         webPolicy: runtime.webPolicy?.() ?? "native-web-blocked",
         pathTreatment: runtime.pathTreatment?.() ?? null,
         mcp: options.mcpHashes?.[experiment.id] ?? null,
+        promptPreamble: (() => {
+          const text = runtime.promptPreamble?.(loaded.metadata)?.trim() ?? "";
+          return text === "" ? null : text;
+        })(),
         budgets: { ...budgets },
       };
     }

package/src/experiments/types.ts

@@ -41,7 +41,7 @@ export interface Runtime extends RuntimeCapabilities {
    * Images and lifecycle budgets can never be exempted. All group members must
    * declare the same fields. Source/config identity still changes per attempt.
    */
-  treatmentPolicyFields?: Array<"egress" | "webPolicy" | "pathTreatment" | "mcp">;
+  treatmentPolicyFields?: Array<"egress" | "webPolicy" | "pathTreatment" | "mcp" | "promptPreamble">;
   /** Container image tag (container runtimes; alternative to `containerImage`). */
   image?: string;
   /** Scripted command run in /workspace (container runtimes, this phase). */

package/src/isolation/image-contexts.generated.ts

@@ -534,13 +534,28 @@ function dialUpstream(client: Socket, host: string, port: number, ready: (upstre
   });
 }
 
-/** Bidirectional tunnel wiring shared by CONNECT and plain forwards. */
+/**
+ * Bidirectional tunnel wiring shared by CONNECT and plain forwards.
+ *
+ * \`pipe()\` already ends the destination once the source ends — after the
+ * buffered bytes have flushed. Destroying the client on the upstream's
+ * \`close\` threw that buffer away: an upstream that closes right after a
+ * large response (a registry tarball) left the client with a truncated body
+ * and an ECONNRESET, so \`npm install\` through the proxy failed at random.
+ * So the upstream's close only ENDS the client (flush, then FIN). The other
+ * direction stays a destroy: a client that is gone will never read what the
+ * upstream still has to say, and a merely ended upstream would sit paused
+ * and half-open until the origin gave up.
+ */
 function pipeBoth(client: Socket, upstream: Socket): void {
   client.pipe(upstream);
   upstream.pipe(client);
-  client.on("close", () => upstream.destroy());
-  upstream.on("close", () => client.destroy());
   client.on("error", () => upstream.destroy());
+  upstream.on("error", () => client.destroy());
+  client.on("close", () => upstream.destroy());
+  upstream.on("close", () => {
+    if (!client.writableEnded) client.end();
+  });
 }
 
 /** CONNECT host:port — the HTTPS path. The tunnel is opaque to the proxy. */

package/src/isolation/proxy/server.ts

@@ -252,13 +252,28 @@ function dialUpstream(client: Socket, host: string, port: number, ready: (upstre
   });
 }
 
-/** Bidirectional tunnel wiring shared by CONNECT and plain forwards. */
+/**
+ * Bidirectional tunnel wiring shared by CONNECT and plain forwards.
+ *
+ * `pipe()` already ends the destination once the source ends — after the
+ * buffered bytes have flushed. Destroying the client on the upstream's
+ * `close` threw that buffer away: an upstream that closes right after a
+ * large response (a registry tarball) left the client with a truncated body
+ * and an ECONNRESET, so `npm install` through the proxy failed at random.
+ * So the upstream's close only ENDS the client (flush, then FIN). The other
+ * direction stays a destroy: a client that is gone will never read what the
+ * upstream still has to say, and a merely ended upstream would sit paused
+ * and half-open until the origin gave up.
+ */
 function pipeBoth(client: Socket, upstream: Socket): void {
   client.pipe(upstream);
   upstream.pipe(client);
-  client.on("close", () => upstream.destroy());
-  upstream.on("close", () => client.destroy());
   client.on("error", () => upstream.destroy());
+  upstream.on("error", () => client.destroy());
+  client.on("close", () => upstream.destroy());
+  upstream.on("close", () => {
+    if (!client.writableEnded) client.end();
+  });
 }
 
 /** CONNECT host:port — the HTTPS path. The tunnel is opaque to the proxy. */

package/src/report/report.ts

@@ -53,7 +53,7 @@ export class ReportError extends Error {
   }
 }
 
-export const REPORT_SCHEMA_VERSION = 10;
+export const REPORT_SCHEMA_VERSION = 11;
 
 /**
  * Substrings that must never appear in generated report prose: at n≤5 trials
@@ -82,7 +82,7 @@ export function methodologyNote(trials: number): string {
     `those trials. Both describe this run's trials only — no wider claim. Telemetry (cost, tokens, surface ` +
     `usage) is summed only over attempts that reported it and shown as unavailable — never zero — when none did. ` +
     `SDK recall is reported per experiment as recalled / attempts, where attempts are the scored attempts that ` +
-    `recorded at least one ungraded check and recalled are those whose ungraded checks all passed. Ungraded ` +
+    `recorded at least one ungraded check of kind "sdk" (an ungraded check with no kind counts as "sdk") and recalled are those whose ungraded "sdk" checks all passed. Ungraded checks of any other kind are tallied and shown but never enter SDK recall. Ungraded ` +
     `checks are recorded and tallied but never enter pass/fail. Scorer health lists checks that failed in every ` +
     `scored attempt of an eval under every arm while another check in the same attempts passed: a statement ` +
     `about the instrument, not about any treatment. Control experiments — an oracle that applies each eval's ` +
@@ -150,6 +150,12 @@ export interface ReportCheck {
    * schema_version < 7 artifacts; readers treat absence as graded.
    */
   graded: boolean;
+  /**
+   * The `kind` the scorer recorded for this check (`"sdk"`, `"wiring"`, …),
+   * or null when no attempt tagged it. Absent in schema_version < 11
+   * artifacts; readers treat absence as null.
+   */
+  kind: string | null;
 }
 
 /** One eval × experiment aggregation. Pairs with zero attempts are ABSENT, never 0%. */
@@ -313,7 +319,7 @@ export interface ReportJson {
 const USAGE_VALUE_CAP = 20;
 
 export function collectPairs(attempts: AttemptRecord[]): ReportPair[] {
-  const pairs = new Map<string, { pair: ReportPair; checks: Map<string, { passed: number; total: number; graded: boolean }>; outcomes: Set<boolean> }>();
+  const pairs = new Map<string, { pair: ReportPair; checks: Map<string, { passed: number; total: number; graded: boolean; kind: string | null }>; outcomes: Set<boolean> }>();
   for (const attempt of attempts) {
     const key = `${attempt.evalId}|${attempt.experimentId}`;
     let entry = pairs.get(key);
@@ -350,10 +356,11 @@ export function collectPairs(attempts: AttemptRecord[]): ReportPair[] {
       entry.outcomes.add(false);
     }
     for (const check of attempt.result?.checks ?? []) {
-      const tally = entry.checks.get(check.name) ?? { passed: 0, total: 0, graded: true };
+      const tally = entry.checks.get(check.name) ?? { passed: 0, total: 0, graded: true, kind: null };
       tally.total += 1;
       if (check.passed) tally.passed += 1;
       if (check.graded === false) tally.graded = false;
+      if (tally.kind === null && typeof check.kind === "string" && check.kind !== "") tally.kind = check.kind;
       entry.checks.set(check.name, tally);
     }
   }
@@ -365,7 +372,7 @@ export function collectPairs(attempts: AttemptRecord[]): ReportPair[] {
     pair.pass_at_k = collectPassAtK(pair.passed, pair.scored);
     pair.flaky = outcomes.size > 1;
     pair.checks = [...checks.entries()]
-      .map(([name, tally]) => ({ name, passed: tally.passed, total: tally.total, graded: tally.graded }))
+      .map(([name, tally]) => ({ name, passed: tally.passed, total: tally.total, graded: tally.graded, kind: tally.kind }))
       .sort((a, b) => a.name.localeCompare(b.name));
     result.push(pair);
   }
@@ -411,17 +418,24 @@ function treatmentOf(run: RunRecord, experimentId: string): string | null {
  * experiment; "unspecified" is its own explicit missing-value sentinel, so
  * this never invents a value the run did not pin.
  */
+/** An ungraded check that feeds SDK recall: kind "sdk", or no kind (the pre-tag convention). */
+export function isSdkRecallCheck(check: { graded?: boolean | undefined; kind?: string | undefined }): boolean {
+  return check.graded === false && (check.kind === undefined || check.kind === "sdk");
+}
+
 /**
  * SDK recall over one experiment's scored attempts: the denominator is every
- * completed attempt that recorded at least one `graded: false` check; an
- * attempt is recalled when all of those checks passed. Null rate at zero.
+ * completed attempt that recorded at least one `graded: false` check of kind
+ * "sdk" (or untagged); an attempt is recalled when all of those checks
+ * passed. Ungraded checks of any other kind (wiring, …) never enter. Null
+ * rate at zero.
  */
 export function collectRecall(attempts: AttemptRecord[], experimentId: string): ReportRecall {
   let total = 0;
   let recalled = 0;
   for (const attempt of attempts) {
     if (attempt.experimentId !== experimentId || attempt.status !== "completed") continue;
-    const ungraded = (attempt.result?.checks ?? []).filter((check) => check.graded === false);
+    const ungraded = (attempt.result?.checks ?? []).filter(isSdkRecallCheck);
     if (ungraded.length === 0) continue;
     total += 1;
     if (ungraded.every((check) => check.passed)) recalled += 1;

package/src/runner/execute.ts

@@ -64,7 +64,7 @@ import {
   type McpServerConfig,
 } from "../isolation/mcp.ts";
 import { hashString } from "../hash.ts";
-import { DEFAULT_BUDGETS, pairKey, resolvePairEnvironments, type LifecycleBudgets, type PairEnvironment } from "../execution-policy.ts";
+import { DEFAULT_BUDGETS, pairKey, resolvePairEnvironments, type LifecycleBudgets, type PairEnvironment, composePrompt } from "../execution-policy.ts";
 import { ulid } from "../ids.ts";
 import { attemptIdentityHash, buildRunManifest, type RunManifest, type RuntimeImageIdentity } from "../manifest.ts";
 import { isMovingModelAlias } from "../model-identity.ts";
@@ -562,7 +562,7 @@ export async function executeRunV3(options: ExecuteRunV3Options): Promise<RunSum
       try {
         disposition =
           experiment.runtime.kind === "host"
-            ? await runHostAttemptV3(options, hostAdapters, loadedEval, experiment, entry, { runId, attemptId, attemptTimeoutMs, persisted })
+            ? await runHostAttemptV3(options, hostAdapters, loadedEval, experiment, entry, { runId, attemptId, attemptTimeoutMs, persisted, environment: manifest.pairs?.[pairKey(entry.evalId, entry.experimentId)] })
             : await runContainerAttemptV3(
                 options,
                 loadedEval,
@@ -661,7 +661,7 @@ async function runHostAttemptV3(
         attemptId,
         fixtureDir: sourceDir,
         workspaceDir: scratchDir,
-        prompt: loadedEval.promptBody,
+        prompt: composePrompt(args.environment?.promptPreamble, loadedEval.promptBody),
         trialIndex: entry.trialIndex,
       }));
     } catch (err) {
@@ -868,13 +868,14 @@ async function runContainerAttemptV3(
           const promptCtx: ContainerAttemptContext = {
             runId,
             attemptId,
-            prompt: loadedEval.promptBody,
+            // The eval body is byte-identical across experiments; a runtime's
+            // promptPreamble (recorded per pair in the manifest) may precede it.
+            prompt: composePrompt(environment?.promptPreamble, loadedEval.promptBody),
             trialIndex: entry.trialIndex,
             ...(rewrittenServers !== undefined ? { mcpServers: rewrittenServers } : {}),
             webPolicy: environment?.webPolicy ?? "native-web-blocked",
           };
           // Treatment config files (e.g. the MCP treatment's vendor config).
-          // The prompt itself is byte-identical across experiments.
           await inStage("environment", async () => {
             for (const [path, content] of Object.entries(agentAdapter.setupFiles(promptCtx))) {
               await container.writeFile(path, content);

package/src/runtime/types.ts

@@ -84,6 +84,19 @@ export interface RuntimeCapabilities {
   webPolicy?(): WebPolicy;
   /** Directory prepended to the container PATH (the CLI treatment), if any. */
   pathTreatment?(): string | undefined;
+  /**
+   * Text placed BEFORE the eval's prompt for every attempt under this
+   * runtime — the one sanctioned way to tell the agent what the treatment
+   * offers ("a `acme` CLI is on PATH; run `acme --help`"). An offered surface
+   * nobody mentions is rarely discovered, and an undiscovered surface
+   * measures nothing. Identity-bearing: the resolved text is recorded per
+   * pair in the run manifest (`pairs[].promptPreamble`), so group members
+   * that differ here must list `"promptPreamble"` in `treatmentPolicyFields`
+   * (and mirror the text under `config[treatment]`) or comparison is
+   * withheld. `undefined` or an empty string means the prompt is used
+   * verbatim.
+   */
+  promptPreamble?(metadata: EvalMetadata): string | undefined;
   /** Capabilities (`query`, `getClient`) exposed to the eval's scorer. */
   scorerCapabilities?(ctx: ScorerCapabilityContext): ScorerCapabilities;
 }