@workos/quickstudy 0.11.0 → 0.12.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@workos/quickstudy",
-  "version": "0.11.0",
+  "version": "0.12.0",
   "description": "An open-source, company-agnostic eval harness for measuring whether coding agents can complete real integration tasks, across an agent x feature x framework x surface matrix.",
   "license": "MIT",
   "type": "module",

package/src/build-info.generated.ts

@@ -9,7 +9,7 @@ export const EMBEDDED_BUILD_INFO: {
   // (the `x-release-please-version` marker anchors the replacement). It is
   // restored as-is after a binary build (scripts/build-binary.ts), so the
   // committed source `--version` stays in lockstep with package.json.
-  version: "0.11.0", // x-release-please-version
+  version: "0.12.0", // x-release-please-version
   revision: null,
   dirty: null,
   sourceHash: null,

package/src/evals/types.ts

@@ -8,10 +8,12 @@
 /**
  * Closed set of eval suites. `benchmark` and `regression` are the headline
  * suites: the leaderboard aggregates count both. `smoke` holds evals every
- * arm solves — run to catch breakage, never moving the headline. `other` is
- * anything else outside the headline.
+ * arm solves — run to catch breakage, never moving the headline. `discovery`
+ * holds vendor-neutral tasks whose outcome is categorical (which product was
+ * chosen) rather than pass/fail; recorded per attempt, never headline.
+ * `other` is anything else outside the headline.
  */
-export const EVAL_SUITES = ["benchmark", "regression", "smoke", "other"] as const;
+export const EVAL_SUITES = ["benchmark", "regression", "smoke", "other", "discovery"] as const;
 export type EvalSuite = (typeof EVAL_SUITES)[number];
 
 /** The suites whose attempts count toward the leaderboard aggregates. */

package/src/execution-policy.ts

@@ -53,6 +53,13 @@ export function resolvePairEnvironments(options: {
   images?: Record<string, RuntimeImageIdentity>;
   egressProxy?: boolean;
   adapterHosts?: Record<string, readonly string[]>;
+  /**
+   * Per experiment: the hosts of the agent's endpoint overrides. They join
+   * the allowlist only for pairs the proxy fronts, where the list is live and
+   * an IP-literal gateway is refused loudly by design; an open or unproxied
+   * pair never carries them, so such a gateway cannot stop the run.
+   */
+  adapterEndpointHosts?: Record<string, readonly string[]>;
   /** Per experiment: the agent's non-secret endpoint overrides (see PairEnvironment.endpoints). */
   adapterEndpoints?: Record<string, Record<string, string>>;
   budgets?: Partial<LifecycleBudgets>;
@@ -73,16 +80,23 @@ export function resolvePairEnvironments(options: {
         options.imageTags?.[pairKey(loaded.metadata.id, experiment.id)] ??
         runtime.image ??
         runtime.containerImage?.(loaded.metadata);
+      // With the flag, every container is proxied unless its runtime opts
+      // out with egress: "open". No runtime hosts means only the adapter's
+      // provider endpoints are permitted. An open pair still records the
+      // runtime and adapter hosts (inert) so identity holds if the run flag
+      // flips, but endpoint-override hosts join only a live allowlist: the
+      // proxy refuses IP literals, and an IP-literal gateway on an arm the
+      // proxy never fronts must not stop the run.
+      const proxied = runtime.kind === "container" && (options.egressProxy ?? false) && runtime.egress !== "open";
       pairs[pairKey(loaded.metadata.id, experiment.id)] = {
         ...(files ? { workspaceFiles: workspaceFilesIdentity(files) } : {}),
         image: tag ? (options.images?.[tag] ?? { tag, id: "unresolved", repoDigests: [] }) : null,
-        // With the flag, every container is proxied. No runtime hosts means
-        // only the adapter's provider endpoints are permitted.
         egress: {
-          proxy: runtime.kind === "container" && (options.egressProxy ?? false),
+          proxy: proxied,
           allowlist: composeAllowlistFromHosts([
             ...(runtime.egressHosts?.(loaded.metadata) ?? []),
             ...(options.adapterHosts?.[experiment.id] ?? []),
+            ...(proxied ? (options.adapterEndpointHosts?.[experiment.id] ?? []) : []),
           ]),
         },
         webPolicy: runtime.webPolicy?.() ?? "native-web-blocked",

package/src/experiments/types.ts

@@ -55,6 +55,15 @@ export interface Runtime extends RuntimeCapabilities {
    * declare the same fields. Source/config identity still changes per attempt.
    */
   treatmentPolicyFields?: Array<"egress" | "webPolicy" | "pathTreatment" | "mcp" | "promptPreamble" | "workspaceFiles">;
+  /**
+   * Whether this experiment's attempts go through the run's egress proxy.
+   * "open" skips the proxy sidecar and network for these attempts even when
+   * the run was started with --egress-proxy, so an arm that must reach the
+   * open internet can share a run with proxied arms. The choice is recorded
+   * in each pair environment (identity); exempt it in a comparison group with
+   * treatmentPolicyFields: ["egress"]. Container runtimes only.
+   */
+  egress?: "proxied" | "open";
   /** Attempt-local non-secret files, staged without replacing the normal adapter. */
   workspaceFiles?: WorkspaceFiles;
   /** Container image tag (container runtimes; alternative to `containerImage`). */
@@ -127,6 +136,11 @@ export function defineExperiment(experiment: Experiment): Experiment {
   if (experiment.comparisonGroup !== undefined && experiment.comparisonGroup.trim() === "") {
     throw new Error(`defineExperiment: experiment "${experiment.id}" comparisonGroup must not be empty`);
   }
+  if (kind === "host" && experiment.runtime.egress === "open") {
+    throw new Error(
+      `defineExperiment: experiment "${experiment.id}" runtime.egress "open" is only meaningful on container runtimes`,
+    );
+  }
   validateWorkspaceFilesRuntime(experiment.runtime);
   return deepFreeze(experiment);
 }

package/src/report/aggregate.ts

@@ -3,8 +3,8 @@
  * the same attempts and (banded) pairs as the rest of the report.
  *
  * Every number is restricted to headline-suite attempts (HEADLINE_SUITES:
- * benchmark and regression), so smoke and other suites never move the
- * headline row. Telemetry follows the report's rule: sums and means cover
+ * benchmark and regression), so the smoke, discovery and other suites never
+ * move the headline row. Telemetry follows the report's rule: sums and means cover
  * only attempts that reported a value, and are null (never 0) when none did.
  */
 import { isHeadlineSuite } from "../evals/types.ts";

package/src/runner/execute.ts

@@ -471,20 +471,19 @@ export async function executeRunV3(options: ExecuteRunV3Options): Promise<RunSum
       if (scorer.identity) scorerIdentities[loaded.metadata.id] = scorer.identity;
     }
     // Each agent's endpoint overrides set on the host: the non-secret base
-    // URLs join the pair environment (identity), their hosts join the egress
-    // allowlist — but only under the proxy, where the allowlist is live and an
-    // IP-literal gateway is refused loudly by design; without it the list is
-    // inert and an IP-literal override must not stop the run.
+    // URLs join the pair environment (identity); their hosts reach the egress
+    // allowlist only for pairs the proxy fronts (pair resolution decides: the
+    // run flag on and the runtime not opted out), where the list is live and
+    // an IP-literal gateway is refused loudly by design. Elsewhere the list
+    // is inert and an IP-literal override must not stop the run.
     const overridesByExperiment = new Map([...containerAgents].map(([id, adapter]) => [id, collectOptionalHostEnv([adapter])] as const));
     const adapterEndpoints = Object.fromEntries([...overridesByExperiment].flatMap(([id, overrides]) => {
       const endpoints = Object.fromEntries(Object.entries(overrides).filter(([name]) => !isSecretEnvName(name)));
       return Object.keys(endpoints).length > 0 ? [[id, endpoints] as const] : [];
     }));
     const pairs = resolvePairEnvironments({ ...options, workspaceFilesByExperiment, images, imageTags: imageTagsByPair,
-      adapterHosts: Object.fromEntries([...containerAgents].map(([id, adapter]) => [id, [
-        ...(adapter.egressHosts ?? []),
-        ...(options.egressProxy ? endpointHosts(overridesByExperiment.get(id) ?? {}) : []),
-      ]])),
+      adapterHosts: Object.fromEntries([...containerAgents].map(([id, adapter]) => [id, [...(adapter.egressHosts ?? [])]])),
+      adapterEndpointHosts: Object.fromEntries([...overridesByExperiment].map(([id, overrides]) => [id, endpointHosts(overrides)])),
       adapterEndpoints,
       budgets: { ...options.budgets, agentMs: options.attemptTimeoutMs ?? options.budgets?.agentMs ?? attemptTimeoutMs },
       mcpHashes: Object.fromEntries([...declaredByExperiment].map(([id, servers]) => [id, hashString(canonicalJson(servers))])),