@tangle-network/agent-bench 0.8.10 → 0.8.15

package/dist/{router-turn-C2wMiDoo.js → router-turn-uTYO6KQ1.js}

@@ -9,16 +9,18 @@ import "@tangle-network/agent-interface";
 */
 async function runBenchRouterTurn(config, input) {
 	if (!config.profile.model?.default) throw new Error("runBenchRouterTurn: profile.model.default is required");
+	const factory = createExecutor({
+		backend: "router",
+		routerBaseUrl: config.routerBaseUrl,
+		routerKey: config.routerKey,
+		...config.tools ? { tools: config.tools } : {}
+	});
+	const turnInput = typeof input === "string" ? { prompt: input } : { providerOptions: { messages: input.messages.map((message) => ({ ...message })) } };
 	const turn = await collectAgentTurn(streamAgentTurn({
 		kind: "executor",
-		factory: createExecutor({
-			backend: "router",
-			routerBaseUrl: config.routerBaseUrl,
-			routerKey: config.routerKey,
-			...config.tools ? { tools: config.tools } : {}
-		}),
+		factory,
 		profile: config.profile
-	}, input, {
+	}, turnInput, {
 		...config.timeoutMs === void 0 ? {} : { timeoutMs: config.timeoutMs },
 		...config.signal ? { signal: config.signal } : {}
 	}));
@@ -28,4 +30,4 @@ async function runBenchRouterTurn(config, input) {
 //#endregion
 export { runBenchRouterTurn as t };
 
-//# sourceMappingURL=router-turn-C2wMiDoo.js.map
+//# sourceMappingURL=router-turn-uTYO6KQ1.js.map

package/dist/router-turn-uTYO6KQ1.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"router-turn-uTYO6KQ1.js","names":[],"sources":["../src/router-turn.ts"],"sourcesContent":["import {\n  type AgentProfile,\n  agentProfileSchema,\n  type ReasoningEffort,\n} from '@tangle-network/agent-interface'\nimport {\n  collectAgentTurn,\n  createExecutor,\n  streamAgentTurn,\n  type AgentTurnInput,\n  type CollectedAgentTurn,\n  type ToolSpec,\n} from '@tangle-network/agent-runtime/kernel'\n\n/** Bench-local target shorthand; Runtime still executes only the exact profile below. */\nexport interface BenchRouterTarget {\n  routerBaseUrl: string\n  routerKey: string\n  profile: AgentProfile\n}\n\nexport interface BenchRouterTurnConfig extends BenchRouterTarget {\n  tools?: ReadonlyArray<ToolSpec>\n  timeoutMs?: number\n  signal?: AbortSignal\n}\n\nexport interface BenchProfileSettings {\n  systemPrompt?: string\n  temperature?: number\n  maxTokens?: number\n  retry?: {\n    maxAttempts?: number\n    initialBackoffMs?: number\n    maxBackoffMs?: number\n    jitter?: number\n    retryStatuses?: ReadonlyArray<number>\n    requestTimeoutMs?: number\n  }\n  maxTurns?: number\n  seed?: number\n  reasoningEffort?: ReasoningEffort\n  extraBody?: Readonly<Record<string, unknown>>\n  toolChoice?: 'auto' | 'required' | 'none'\n}\n\n/** Author an exact direct-Router profile for a benchmark. This is profile construction only;\n * execution still accepts no model or generation fields outside the returned AgentProfile. */\nexport function benchRouterProfile(\n  name: string,\n  model: string,\n  settings: BenchProfileSettings = {},\n): AgentProfile {\n  return withBenchProfile(\n    {\n      name,\n      harness: 'cli-base',\n      model: { provider: 'tangle-router', default: model },\n    },\n    settings,\n  )\n}\n\n/** Derive another exact profile while preserving all untouched canonical axes. */\nexport function withBenchProfile(\n  base: AgentProfile,\n  settings: BenchProfileSettings & { name?: string },\n): AgentProfile {\n  const metadata = {\n    ...(base.model?.metadata ?? {}),\n    ...(settings.temperature !== undefined ? { temperature: settings.temperature } : {}),\n    ...(settings.maxTokens !== undefined ? { maxTokens: settings.maxTokens } : {}),\n    ...(settings.retry !== undefined ? { retry: settings.retry } : {}),\n    ...(settings.maxTurns !== undefined ? { maxTurns: settings.maxTurns } : {}),\n    ...(settings.seed !== undefined ? { seed: settings.seed } : {}),\n    ...(settings.extraBody !== undefined ? { extraBody: settings.extraBody } : {}),\n    ...(settings.toolChoice !== undefined ? { toolChoice: settings.toolChoice } : {}),\n  }\n  return agentProfileSchema.parse({\n    ...base,\n    ...(settings.name ? { name: settings.name } : {}),\n    model: {\n      ...base.model,\n      ...(settings.reasoningEffort !== undefined\n        ? { reasoningEffort: settings.reasoningEffort }\n        : {}),\n      ...(Object.keys(metadata).length > 0 ? { metadata } : {}),\n    },\n    ...(settings.systemPrompt !== undefined\n      ? { prompt: { ...base.prompt, systemPrompt: settings.systemPrompt } }\n      : {}),\n  })\n}\n\nexport function benchProfileModel(profile: AgentProfile): string {\n  const model = profile.model?.default\n  if (typeof model !== 'string' || model.length === 0 || model === 'runtime-selected') {\n    throw new Error('benchmark AgentProfile.model.default must be concrete')\n  }\n  return model\n}\n\n/**\n * The benchmark-side entry to Runtime's canonical one-turn path.\n * It is only an ergonomic composition: Runtime still parses the exact profile,\n * materializes the executor, records identity/usage/result events, and refuses\n * profile axes the direct Router backend cannot carry.\n */\nexport async function runBenchRouterTurn(\n  config: BenchRouterTurnConfig,\n  input: string | { readonly messages: ReadonlyArray<Readonly<Record<string, unknown>>> },\n): Promise<CollectedAgentTurn> {\n  if (!config.profile.model?.default) {\n    throw new Error('runBenchRouterTurn: profile.model.default is required')\n  }\n  const factory = createExecutor({\n    backend: 'router',\n    routerBaseUrl: config.routerBaseUrl,\n    routerKey: config.routerKey,\n    ...(config.tools ? { tools: config.tools } : {}),\n  })\n  const turnInput: AgentTurnInput =\n    typeof input === 'string'\n      ? { prompt: input }\n      : {\n          providerOptions: {\n            messages: input.messages.map((message) => ({ ...message })),\n          },\n        }\n  const turn = await collectAgentTurn(\n    streamAgentTurn(\n      { kind: 'executor', factory, profile: config.profile },\n      turnInput,\n      {\n        ...(config.timeoutMs === undefined ? {} : { timeoutMs: config.timeoutMs }),\n        ...(config.signal ? { signal: config.signal } : {}),\n      },\n    ),\n  )\n  if (turn.status !== 'completed') {\n    throw new Error(turn.error?.message ?? `Router turn ended with status ${turn.status}`)\n  }\n  return turn\n}\n"],"mappings":";;;;;;;;;AA4GA,eAAsB,mBACpB,QACA,OAC6B;CAC7B,IAAI,CAAC,OAAO,QAAQ,OAAO,SACzB,MAAM,IAAI,MAAM,uDAAuD;CAEzE,MAAM,UAAU,eAAe;EAC7B,SAAS;EACT,eAAe,OAAO;EACtB,WAAW,OAAO;EAClB,GAAI,OAAO,QAAQ,EAAE,OAAO,OAAO,MAAM,IAAI,CAAC;CAChD,CAAC;CACD,MAAM,YACJ,OAAO,UAAU,WACb,EAAE,QAAQ,MAAM,IAChB,EACE,iBAAiB,EACf,UAAU,MAAM,SAAS,KAAK,aAAa,EAAE,GAAG,QAAQ,EAAE,EAC5D,EACF;CACN,MAAM,OAAO,MAAM,iBACjB,gBACE;EAAE,MAAM;EAAY;EAAS,SAAS,OAAO;CAAQ,GACrD,WACA;EACE,GAAI,OAAO,cAAc,KAAA,IAAY,CAAC,IAAI,EAAE,WAAW,OAAO,UAAU;EACxE,GAAI,OAAO,SAAS,EAAE,QAAQ,OAAO,OAAO,IAAI,CAAC;CACnD,CACF,CACF;CACA,IAAI,KAAK,WAAW,aAClB,MAAM,IAAI,MAAM,KAAK,OAAO,WAAW,iCAAiC,KAAK,QAAQ;CAEvF,OAAO;AACT"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.8.10",
+  "version": "0.8.15",
   "type": "module",
   "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.",
   "repository": {
@@ -25,11 +25,11 @@
     }
   },
   "dependencies": {
-    "@tangle-network/agent-eval": "0.145.11",
-    "@tangle-network/agent-interface": "0.52.0",
-    "@tangle-network/agent-knowledge": "8.0.0",
-    "@tangle-network/sandbox": "0.26.1",
-    "@tangle-network/agent-runtime": "0.134.9"
+    "@tangle-network/agent-eval": ">=0.148.0 <0.149.0",
+    "@tangle-network/agent-interface": "^1.1.0",
+    "@tangle-network/agent-knowledge": "^8.0.9",
+    "@tangle-network/sandbox": ">=0.27.1 <0.28.0",
+    "@tangle-network/agent-runtime": "^0.141.1"
   },
   "devDependencies": {
     "@arethetypeswrong/cli": "0.18.5",
@@ -73,7 +73,8 @@
     "typecheck:public": "tsc -p tsconfig.public.json",
     "verify:package": "pnpm run verify:package:static && node scripts/verify-packed-consumer.mjs",
     "verify:package:local-runtime": "pnpm run verify:package:static && node scripts/verify-packed-consumer.mjs --local-runtime",
-    "verify:package:static": "pnpm run build && publint && attw --pack --profile esm-only .",
+    "verify:package:static": "pnpm run build && publint && attw --pack --profile esm-only . && node ../scripts/check-published-ranges.mjs bench && node ../scripts/check-api-surface.mjs bench",
+    "api:surface": "node ../scripts/check-api-surface.mjs --write bench",
     "wait:published-dependencies": "node scripts/wait-for-published-dependencies.mjs",
     "verify:pier": "tsx scripts/verify-pier-pair.mts"
   }

package/src/agent-graphs-improve.mts

@@ -226,7 +226,7 @@ export async function callAuthor(
   const turn = await collectAgentTurn(
     streamAgentTurn(
       { kind: 'executor', factory, profile, agentRunName: profile.name ?? 'agent-graphs-author' },
-      prompt,
+      { prompt },
       timeoutMs === undefined ? {} : { timeoutMs },
     ),
   )

package/src/atom-mcp-e2e.mts

@@ -103,7 +103,7 @@ async function bridgeChat(opts: {
   const turn = await collectAgentTurn(
     streamAgentTurn(
       { kind: 'executor', factory, profile },
-      opts.messages.map((message) => message.content).join('\n\n'),
+      { prompt: opts.messages.map((message) => message.content).join('\n\n') },
     ),
   )
   if (turn.status !== 'completed') {

package/src/benchmarks/appworld.ts

@@ -440,7 +440,7 @@ export function appworldToolLoopClient(cfg: {
             const loop = await collectAgentTurn(
               streamAgentTurn(
                 { kind: 'executor', factory, profile },
-                `Task: ${instruction}`,
+                { prompt: `Task: ${instruction}` },
                 { signal },
               ),
             )

package/src/commit0-gate.mts

@@ -330,7 +330,7 @@ async function runShotLocal(task: BenchTask, attempt: number, cfg: ShotCfg, stee
     const turn = await collectAgentTurn(
       streamAgentTurn(
         { kind: 'executor', factory, profile: workerProfile(cfg, `commit0-local-${attempt}`) },
-        prompt,
+        { prompt },
         cfg.timeoutMs > 0 ? { timeoutMs: cfg.timeoutMs } : {},
       ),
     )

package/src/humaneval-repair-gate.mts

@@ -89,7 +89,7 @@ async function repairAttempt(cfg: BenchRouterTarget, task: HumanEvalTask, k: num
     },
   })
   const r = await collectAgentTurn(
-    streamAgentTurn({ kind: 'executor', factory, profile }, basePrompt(task)),
+    streamAgentTurn({ kind: 'executor', factory, profile }, { prompt: basePrompt(task) }),
   )
   if (r.status !== 'completed') {
     throw new Error(r.error?.message ?? `repair turn ended with status ${r.status}`)

package/src/mcp-mount-probe.mts

@@ -70,7 +70,7 @@ async function bridgeChat(messages: Array<{ role: string; content: string }>, mc
   const turn = await collectAgentTurn(
     streamAgentTurn(
       { kind: 'executor', factory, profile },
-      messages.map((message) => message.content).join('\n\n'),
+      { prompt: messages.map((message) => message.content).join('\n\n') },
     ),
   )
   if (turn.status !== 'completed') {

package/src/quant-arena/quant-loop.mts

@@ -249,7 +249,7 @@ async function profileShot(opts: {
   const turn = await collectAgentTurn(
     streamAgentTurn(
       { kind: 'executor', factory, profile: opts.profile },
-      opts.prompt,
+      { prompt: opts.prompt },
       { timeoutMs: opts.timeoutMs },
     ),
   )

package/src/router-turn.ts

@@ -7,6 +7,7 @@ import {
   collectAgentTurn,
   createExecutor,
   streamAgentTurn,
+  type AgentTurnInput,
   type CollectedAgentTurn,
   type ToolSpec,
 } from '@tangle-network/agent-runtime/kernel'
@@ -118,10 +119,18 @@ export async function runBenchRouterTurn(
     routerKey: config.routerKey,
     ...(config.tools ? { tools: config.tools } : {}),
   })
+  const turnInput: AgentTurnInput =
+    typeof input === 'string'
+      ? { prompt: input }
+      : {
+          providerOptions: {
+            messages: input.messages.map((message) => ({ ...message })),
+          },
+        }
   const turn = await collectAgentTurn(
     streamAgentTurn(
       { kind: 'executor', factory, profile: config.profile },
-      input,
+      turnInput,
       {
         ...(config.timeoutMs === undefined ? {} : { timeoutMs: config.timeoutMs }),
         ...(config.signal ? { signal: config.signal } : {}),

package/src/run-benchmarks.ts

@@ -185,13 +185,17 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB
     ...(timeoutMs ? { timeoutMs } : {}),
   })
   const harness = cell.harness ?? (cell.profile?.metadata?.backendType as string | undefined) ?? 'opencode'
-  const profile: AgentProfile =
-    cell.profile ?? {
-      name: cell.label,
-      harness: harness as AgentProfile['harness'],
-      model: { provider: 'tangle-router', default: cell.model },
-      metadata: { backendType: harness },
-    }
+  // The cell's harness and model are the identity the box must run, so they override whatever a
+  // supplied profile declared rather than being dropped when one is supplied. `buildBackendOptions`
+  // now sends this provider/model pair on the create request and refuses a conflicting override,
+  // so the two must be derived from the same profile.
+  const profileProvider = cell.profile?.model?.provider ?? 'tangle-router'
+  const profile: AgentProfile = {
+    ...(cell.profile ?? { name: cell.label }),
+    harness: harness as AgentProfile['harness'],
+    model: { ...cell.profile?.model, provider: profileProvider, default: cell.model },
+    metadata: { ...cell.profile?.metadata, backendType: harness },
+  }
   // Unique per shot: the same (adapter, task) runs concurrently across cells and reps, so the box
   // name and runId must not collide.
   const uniq = Math.random().toString(36).slice(2, 8)
@@ -202,7 +206,7 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB
     sandboxOverrides: {
       name: `bench-${adapter.name}-${task.id}-${uniq}`.replace(/[^a-zA-Z0-9_.-]/g, '_').slice(0, 60),
       environment: 'universal',
-      backend: { type: harness as never, model: { provider: 'openai', model: cell.model, baseUrl: routerBaseUrl } },
+      backend: { type: harness as never, model: { provider: profileProvider, model: cell.model, baseUrl: routerBaseUrl } },
     },
   }
   const deliverable: Deliverable<string> = {

package/src/swe-arena/arms.ts

@@ -525,7 +525,7 @@ export async function runSoloArm(spec: SoloArmSpec, ctx: ArmRunContext): Promise
   const turn = await collectAgentTurn(
     streamAgentTurn(
       { kind: 'executor', factory, profile: spec.profile },
-      prompt,
+      { prompt },
       { timeoutMs, ...(ctx.signal ? { signal: ctx.signal } : {}) },
     ),
   )

package/dist/router-turn-C2wMiDoo.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"router-turn-C2wMiDoo.js","names":[],"sources":["../src/router-turn.ts"],"sourcesContent":["import {\n  type AgentProfile,\n  agentProfileSchema,\n  type ReasoningEffort,\n} from '@tangle-network/agent-interface'\nimport {\n  collectAgentTurn,\n  createExecutor,\n  streamAgentTurn,\n  type CollectedAgentTurn,\n  type ToolSpec,\n} from '@tangle-network/agent-runtime/kernel'\n\n/** Bench-local target shorthand; Runtime still executes only the exact profile below. */\nexport interface BenchRouterTarget {\n  routerBaseUrl: string\n  routerKey: string\n  profile: AgentProfile\n}\n\nexport interface BenchRouterTurnConfig extends BenchRouterTarget {\n  tools?: ReadonlyArray<ToolSpec>\n  timeoutMs?: number\n  signal?: AbortSignal\n}\n\nexport interface BenchProfileSettings {\n  systemPrompt?: string\n  temperature?: number\n  maxTokens?: number\n  retry?: {\n    maxAttempts?: number\n    initialBackoffMs?: number\n    maxBackoffMs?: number\n    jitter?: number\n    retryStatuses?: ReadonlyArray<number>\n    requestTimeoutMs?: number\n  }\n  maxTurns?: number\n  seed?: number\n  reasoningEffort?: ReasoningEffort\n  extraBody?: Readonly<Record<string, unknown>>\n  toolChoice?: 'auto' | 'required' | 'none'\n}\n\n/** Author an exact direct-Router profile for a benchmark. This is profile construction only;\n * execution still accepts no model or generation fields outside the returned AgentProfile. */\nexport function benchRouterProfile(\n  name: string,\n  model: string,\n  settings: BenchProfileSettings = {},\n): AgentProfile {\n  return withBenchProfile(\n    {\n      name,\n      harness: 'cli-base',\n      model: { provider: 'tangle-router', default: model },\n    },\n    settings,\n  )\n}\n\n/** Derive another exact profile while preserving all untouched canonical axes. */\nexport function withBenchProfile(\n  base: AgentProfile,\n  settings: BenchProfileSettings & { name?: string },\n): AgentProfile {\n  const metadata = {\n    ...(base.model?.metadata ?? {}),\n    ...(settings.temperature !== undefined ? { temperature: settings.temperature } : {}),\n    ...(settings.maxTokens !== undefined ? { maxTokens: settings.maxTokens } : {}),\n    ...(settings.retry !== undefined ? { retry: settings.retry } : {}),\n    ...(settings.maxTurns !== undefined ? { maxTurns: settings.maxTurns } : {}),\n    ...(settings.seed !== undefined ? { seed: settings.seed } : {}),\n    ...(settings.extraBody !== undefined ? { extraBody: settings.extraBody } : {}),\n    ...(settings.toolChoice !== undefined ? { toolChoice: settings.toolChoice } : {}),\n  }\n  return agentProfileSchema.parse({\n    ...base,\n    ...(settings.name ? { name: settings.name } : {}),\n    model: {\n      ...base.model,\n      ...(settings.reasoningEffort !== undefined\n        ? { reasoningEffort: settings.reasoningEffort }\n        : {}),\n      ...(Object.keys(metadata).length > 0 ? { metadata } : {}),\n    },\n    ...(settings.systemPrompt !== undefined\n      ? { prompt: { ...base.prompt, systemPrompt: settings.systemPrompt } }\n      : {}),\n  })\n}\n\nexport function benchProfileModel(profile: AgentProfile): string {\n  const model = profile.model?.default\n  if (typeof model !== 'string' || model.length === 0 || model === 'runtime-selected') {\n    throw new Error('benchmark AgentProfile.model.default must be concrete')\n  }\n  return model\n}\n\n/**\n * The benchmark-side entry to Runtime's canonical one-turn path.\n * It is only an ergonomic composition: Runtime still parses the exact profile,\n * materializes the executor, records identity/usage/result events, and refuses\n * profile axes the direct Router backend cannot carry.\n */\nexport async function runBenchRouterTurn(\n  config: BenchRouterTurnConfig,\n  input: string | { readonly messages: ReadonlyArray<Readonly<Record<string, unknown>>> },\n): Promise<CollectedAgentTurn> {\n  if (!config.profile.model?.default) {\n    throw new Error('runBenchRouterTurn: profile.model.default is required')\n  }\n  const factory = createExecutor({\n    backend: 'router',\n    routerBaseUrl: config.routerBaseUrl,\n    routerKey: config.routerKey,\n    ...(config.tools ? { tools: config.tools } : {}),\n  })\n  const turn = await collectAgentTurn(\n    streamAgentTurn(\n      { kind: 'executor', factory, profile: config.profile },\n      input,\n      {\n        ...(config.timeoutMs === undefined ? {} : { timeoutMs: config.timeoutMs }),\n        ...(config.signal ? { signal: config.signal } : {}),\n      },\n    ),\n  )\n  if (turn.status !== 'completed') {\n    throw new Error(turn.error?.message ?? `Router turn ended with status ${turn.status}`)\n  }\n  return turn\n}\n"],"mappings":";;;;;;;;;AA2GA,eAAsB,mBACpB,QACA,OAC6B;CAC7B,IAAI,CAAC,OAAO,QAAQ,OAAO,SACzB,MAAM,IAAI,MAAM,uDAAuD;CAQzE,MAAM,OAAO,MAAM,iBACjB,gBACE;EAAE,MAAM;EAAY,SARR,eAAe;GAC7B,SAAS;GACT,eAAe,OAAO;GACtB,WAAW,OAAO;GAClB,GAAI,OAAO,QAAQ,EAAE,OAAO,OAAO,MAAM,IAAI,CAAC;EAChD,CAG8B;EAAG,SAAS,OAAO;CAAQ,GACrD,OACA;EACE,GAAI,OAAO,cAAc,KAAA,IAAY,CAAC,IAAI,EAAE,WAAW,OAAO,UAAU;EACxE,GAAI,OAAO,SAAS,EAAE,QAAQ,OAAO,OAAO,IAAI,CAAC;CACnD,CACF,CACF;CACA,IAAI,KAAK,WAAW,aAClB,MAAM,IAAI,MAAM,KAAK,OAAO,WAAW,iCAAiC,KAAK,QAAQ;CAEvF,OAAO;AACT"}

package/src/aec-gate.mts

@@ -1,238 +0,0 @@
-/**
- * Router-based gate runner for aec-bench — the fix for the null-score sandbox path.
- *
- * aec-bench is closed-form reasoning + a deterministic local verify.py judge, so a
- * sandbox is unnecessary: solve each task with one direct router chat call, then
- * judge the raw response locally (verify.py extracts the last fenced ```json block
- * itself). The prior sandbox path emitted the JSON in-stream but never fed it to
- * the judge, so every verdict.score came out null — the bug this runner fixes by
- * passing the model's full response straight to adapter.judge().
- *
- * Two paired arms over the SAME task set (loadTasks once):
- *   random@K  — K identical-base-prompt shots/task (the compute control)
- *   diverse@K — K shots, the i-th prefixed with composeStrategies(base, K)[i]
- *
- * Each attempt carries a REAL numeric verdict.score (from verify.py) + the output,
- * written as a corpus RunRecord (condition random@K / diverse@K) the existing
- * corpus-replay --selector + corpus-report consume unchanged. Fail loud on a router
- * error — never a fabricated score.
- */
-
-import { resolveAdapter } from './adapters'
-import type { BenchmarkAdapter, BenchTask } from './benchmarks/types'
-import { type AttemptRecord, appendRunRecord, buildRunRecordFromAttempts } from './corpus'
-import { composeStrategies } from './directives'
-import {
-  benchProfileModel,
-  benchRouterProfile,
-  type BenchRouterTarget,
-  runBenchRouterTurn,
-  withBenchProfile,
-} from './router-turn'
-import { pool } from './stats.mts'
-
-function must(name: string): string {
-  const v = process.env[name]
-  if (!v) throw new Error(`env ${name} is required`)
-  return v
-}
-
-interface ArmSpec {
-  /** Corpus condition label the selector/report filter on (e.g. random@4). */
-  condition: string
-  /** Per-attempt prompt builder: the i-th of K shots for a task. */
-  promptFor(task: BenchTask, i: number, k: number): string
-}
-
-interface AttemptOutcome {
-  prompt: string
-  output: string
-  score: number
-  resolved: boolean
-  costUsd?: number
-  tokensIn?: number
-  tokensOut?: number
-  wallMs: number
-  /** the router/judge call failed after retries — EXCLUDED from stats, never scored 0. */
-  infraError?: boolean
-}
-
-async function runAttempt(
-  cfg: BenchRouterTarget,
-  adapter: BenchmarkAdapter,
-  task: BenchTask,
-  prompt: string,
-): Promise<AttemptOutcome> {
-  const startedAt = Date.now()
-  // Retry transient router/judge failures (rate limits, stream drops, 5xx) with
-  // backoff; a genuine empty completion still scores a real 0 (verify.py fail-closes).
-  // Only after retries are exhausted do we record an EXCLUDED infraError — never a
-  // fabricated score, and never a throw that aborts the whole multi-model run.
-  let lastErr: unknown
-  for (let attempt = 0; attempt < 3; attempt += 1) {
-    try {
-      const res = await runBenchRouterTurn(
-        {
-          routerBaseUrl: cfg.routerBaseUrl,
-          routerKey: cfg.routerKey,
-          profile: withBenchProfile(cfg.profile, { name: 'aec-worker' }),
-        },
-        prompt,
-      )
-      const content = res.finalText
-      const verdict = await adapter.judge(task, content)
-      return {
-        prompt,
-        output: content,
-        score: verdict.score,
-        resolved: verdict.resolved,
-        wallMs: Date.now() - startedAt,
-        ...(res.usage.costUsd !== undefined ? { costUsd: res.usage.costUsd } : {}),
-        ...(res.usage.tokensKnown === false
-          ? {}
-          : { tokensIn: res.usage.input, tokensOut: res.usage.output }),
-      }
-    } catch (err) {
-      lastErr = err
-      if (attempt < 2) await new Promise((r) => setTimeout(r, 1000 * 2 ** attempt))
-    }
-  }
-  console.warn(`[aec-gate] ${task.id}: infra error after 3 tries — excluded: ${(lastErr instanceof Error ? lastErr.message : String(lastErr)).slice(0, 160)}`)
-  return { prompt, output: '', score: 0, resolved: false, wallMs: Date.now() - startedAt, infraError: true }
-}
-
-function toAttemptRecord(o: AttemptOutcome, round: number): AttemptRecord {
-  return {
-    round,
-    prompt: o.prompt,
-    output: o.output,
-    // infra-errored attempts carry NO score/valid → corpus-replay skips them.
-    ...(o.infraError ? {} : { valid: o.resolved, score: o.score }),
-    wallMs: o.wallMs,
-    eventCount: 1,
-    eventTypes: o.infraError ? { 'router.error': 1 } : { 'router.chat': 1 },
-    traceTail: o.output.slice(-600),
-    ...(o.costUsd !== undefined ? { costUsd: o.costUsd } : {}),
-    ...(o.tokensIn !== undefined ? { tokensIn: o.tokensIn } : {}),
-    ...(o.tokensOut !== undefined ? { tokensOut: o.tokensOut } : {}),
-  }
-}
-
-interface ArmResult {
-  /** mean graded score across SCORED (non-infra) attempts */
-  meanScore: number
-  /** fraction of scored attempts at full credit (score >= 1) */
-  fullCreditRate: number
-  attemptCount: number
-  /** attempts excluded as infra errors (router/judge failed after retries) */
-  infraCount: number
-}
-
-async function runArm(
-  arm: ArmSpec,
-  cfg: BenchRouterTarget,
-  adapter: BenchmarkAdapter,
-  tasks: BenchTask[],
-  k: number,
-  concurrency: number,
-  corpusPath: string,
-): Promise<ArmResult> {
-  // Flatten (task, shot) into one unit of work so the pool bounds TOTAL in-flight
-  // router calls across all tasks, not per-task.
-  const units = tasks.flatMap((task) => Array.from({ length: k }, (_, i) => ({ task, i })))
-  const outcomes = await pool(units, concurrency, (u) => runAttempt(cfg, adapter, u.task, arm.promptFor(u.task, u.i, k)))
-
-  const scored = outcomes.filter((o) => !o.infraError)
-  let scoreSum = 0
-  let fullCredit = 0
-  for (const o of scored) {
-    scoreSum += o.score
-    if (o.score >= 1) fullCredit += 1
-  }
-
-  // Group K outcomes back per task → one RunRecord/task (the controller-run shape).
-  for (let t = 0; t < tasks.length; t += 1) {
-    const task = tasks[t] as BenchTask
-    const taskOutcomes = outcomes.slice(t * k, t * k + k)
-    const attempts = taskOutcomes.map((o, i) => toAttemptRecord(o, i))
-    const record = buildRunRecordFromAttempts(attempts, {
-      benchmark: adapter.name,
-      instanceId: task.id,
-      condition: arm.condition,
-      model: benchProfileModel(cfg.profile),
-      // k-attempt outcome = any usable attempt resolved (the oracle@k ceiling for
-      // this run; the deployable selector is scored separately by corpus-replay).
-      resolved: taskOutcomes.some((o) => o.resolved),
-      // a task whose every attempt infra-errored is itself infra-errored.
-      infraError: taskOutcomes.length > 0 && taskOutcomes.every((o) => o.infraError),
-    })
-    await appendRunRecord(corpusPath, record)
-  }
-
-  return {
-    meanScore: scored.length > 0 ? scoreSum / scored.length : 0,
-    fullCreditRate: scored.length > 0 ? fullCredit / scored.length : 0,
-    infraCount: outcomes.length - scored.length,
-    attemptCount: outcomes.length,
-  }
-}
-
-async function main(): Promise<void> {
-  const n = Number(process.env.N ?? 5)
-  const k = Number(process.env.K ?? 4)
-  const model = process.env.WORKER_MODEL ?? 'deepseek-v4-flash'
-  const routerBaseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
-  const routerKey = must('TANGLE_API_KEY')
-  const concurrency = Number(process.env.CONCURRENCY ?? 6)
-  const randomCorpus = process.env.RANDOM_CORPUS ?? '/tmp/aec-r.jsonl'
-  const diverseCorpus = process.env.DIVERSE_CORPUS ?? '/tmp/aec-d.jsonl'
-
-  if (!Number.isFinite(n) || n < 1) throw new Error(`N must be a positive integer, got ${process.env.N}`)
-  if (!Number.isFinite(k) || k < 1) throw new Error(`K must be a positive integer, got ${process.env.K}`)
-
-  const cfg: BenchRouterTarget = {
-    routerBaseUrl,
-    routerKey,
-    profile: benchRouterProfile('aec-worker', model, {
-      retry: { maxAttempts: Number(process.env.MAX_ATTEMPTS ?? 3) },
-    }),
-  }
-  const bench = process.env.BENCH ?? 'aec-bench'
-  const adapter = resolveAdapter(bench)
-
-  console.log(`=== ${bench} router gate · N=${n} K=${k} model=${model} conc=${concurrency} ===`)
-  await adapter.preflight()
-  const tasks = await adapter.loadTasks({ limit: n })
-  console.log(`loaded ${tasks.length} task(s): ${tasks.map((t) => t.id).join(', ')}`)
-
-  // random arm: the task prompt verbatim, K times (the compute control).
-  const randomArm: ArmSpec = {
-    condition: `random@${k}`,
-    promptFor: (task) => task.prompt,
-  }
-  // diverse arm: the i-th shot prefixed with the i-th distinct strategy lens.
-  const diverseArm: ArmSpec = {
-    condition: `diverse@${k}`,
-    promptFor: (task, i, kk) => composeStrategies(task.prompt, kk)[i] as string,
-  }
-
-  console.log(`\n▶ random@${k} (control — identical base prompt) → ${randomCorpus}`)
-  const r = await runArm(randomArm, cfg, adapter, tasks, k, concurrency, randomCorpus)
-  console.log(`  random@${k}:  mean score ${(r.meanScore * 100).toFixed(1)}%  full-credit ${(r.fullCreditRate * 100).toFixed(1)}%  (n=${r.attemptCount} attempts${r.infraCount ? `, ${r.infraCount} infra-excluded` : ''})`)
-
-  console.log(`\n▶ diverse@${k} (K distinct strategy lenses) → ${diverseCorpus}`)
-  const d = await runArm(diverseArm, cfg, adapter, tasks, k, concurrency, diverseCorpus)
-  console.log(`  diverse@${k}: mean score ${(d.meanScore * 100).toFixed(1)}%  full-credit ${(d.fullCreditRate * 100).toFixed(1)}%  (n=${d.attemptCount} attempts${d.infraCount ? `, ${d.infraCount} infra-excluded` : ''})`)
-
-  console.log(
-    `\n=== next: read the gate ===\n` +
-      `  npx tsx src/corpus-replay.mts ${randomCorpus} --selector\n` +
-      `  npx tsx src/corpus-replay.mts ${diverseCorpus} --selector --condition=diverse\n` +
-      `  npx tsx src/corpus-report.mts ${randomCorpus} ${diverseCorpus}`,
-  )
-}
-
-main().catch((err) => {
-  console.error(`aec-gate: ${err instanceof Error ? err.message : String(err)}`)
-  process.exit(1)
-})