@tangle-network/agent-bench 0.7.1 → 0.8.0

package/src/swe-arena/capacity.ts

@@ -6,13 +6,11 @@
  * router 503-stormed. Rule encoded here: gate every arm on the endpoint that
  * arm actually calls; supervisor arms MUST include the router-path probe.
  *
- * Secrets discipline: probes spawn `dotenvx run … -- bash -c 'curl …'` from
- * the secrets dir; the API key is referenced by NAME inside the child shell
- * (single-quoted script, so it is never expanded — let alone logged — in this
- * process).
+ * The containing experiment is launched through dotenvx, so probes read the already-scoped key
+ * from this process and enter Runtime through one exact AgentProfile.
  */
 
-import { run } from './proc'
+import { runBenchRouterTurn } from '../router-turn'
 import type { SecretsEnv } from './arms'
 
 export type CapacityProbe = (signal?: AbortSignal) => Promise<boolean>
@@ -104,10 +102,10 @@ export async function waitForCapacity(gate: EndpointCapacityGate, signal?: Abort
 
 export interface HttpProbeSpec {
   url: string
+  provider: string
   /** NAME of the env var holding the bearer key (resolved in the dotenvx child). */
   apiKeyEnv: string
   model: string
-  secrets: SecretsEnv
   /** curl --max-time, seconds. Default 40 (probe-capacity.sh). */
   maxTimeS?: number
   /**
@@ -117,55 +115,54 @@ export interface HttpProbeSpec {
   maxTokens?: number
 }
 
-export const ZAI_CODING_ENDPOINT = 'https://api.z.ai/api/coding/paas/v4/chat/completions'
-export const ROUTER_ENDPOINT = 'https://router.tangle.tools/v1/chat/completions'
-
-/** Build the probe request body (probe-body.json semantics). */
-export function probeBody(model: string, maxTokens: number): string {
-  return JSON.stringify({
-    model,
-    messages: [{ role: 'user', content: 'Reply with the single word OK.' }],
-    max_tokens: maxTokens,
-    temperature: 0,
-  })
-}
+export const ZAI_CODING_ENDPOINT = 'https://api.z.ai/api/coding/paas/v4'
+export const ROUTER_ENDPOINT = 'https://router.tangle.tools/v1'
 
 /**
- * Generic chat-completions probe: true iff the endpoint returns HTTP 200
- * within the time budget. The key stays inside the child shell.
+ * Generic chat-completions probe through Runtime: true only when the selected model emits `OK`
+ * within the time budget.
  */
 export function httpCapacityProbe(spec: HttpProbeSpec): CapacityProbe {
   if (!/^[A-Z_][A-Z0-9_]*$/.test(spec.apiKeyEnv)) {
     throw new Error(`invalid apiKeyEnv name: ${spec.apiKeyEnv}`)
   }
-  const body = probeBody(spec.model, spec.maxTokens ?? 8000)
   const maxTime = spec.maxTimeS ?? 40
   return async (signal?: AbortSignal) => {
     signal?.throwIfAborted()
-    // Body via stdin (--data @-) so the payload never sits on a command line.
-    // The HTTP code is marker-anchored because dotenvx writes its injection
-    // banner to the same stdout stream.
-    const script =
-      `curl -sS -o /dev/null -w "HTTP_CODE=%{http_code}" --max-time ${maxTime} ` +
-      `-X POST "$PROBE_URL" ` +
-      `-H "Authorization: Bearer $${spec.apiKeyEnv}" -H "Content-Type: application/json" ` +
-      `--data @-`
-    const argv = ['run', ...spec.secrets.envFiles.flatMap((f) => ['-f', f]), '--', 'bash', '-c', script]
-    const res = await run('dotenvx', argv, {
-      cwd: spec.secrets.secretsDir,
-      timeoutMs: (maxTime + 20) * 1000,
-      stdin: body,
-      env: { ...process.env, PROBE_URL: spec.url },
-      signal,
-    })
-    signal?.throwIfAborted()
-    return /HTTP_CODE=200\s*$/.test(res.stdout)
+    const routerKey = process.env[spec.apiKeyEnv]
+    if (!routerKey) throw new Error(`${spec.apiKeyEnv} is required; launch through dotenvx`)
+    const turn = await runBenchRouterTurn(
+      {
+        routerBaseUrl: spec.url.replace(/\/chat\/completions\/?$/u, ''),
+        routerKey,
+        profile: {
+          name: `capacity-${spec.model}`,
+          harness: 'cli-base',
+          model: {
+            provider: spec.provider,
+            default: spec.model,
+            metadata: { temperature: 0, maxTokens: spec.maxTokens ?? 8000 },
+          },
+          prompt: { systemPrompt: 'Reply with the single word OK.' },
+        },
+        timeoutMs: maxTime * 1000,
+        signal,
+      },
+      'Capacity probe.',
+    )
+    return turn.finalText.trim() === 'OK'
   }
 }
 
 /** probe-capacity.sh's z.ai coding-plan probe (the WORKER path). */
 export function zaiCodingProbe(secrets: SecretsEnv, model = 'glm-5.2'): CapacityProbe {
-  return httpCapacityProbe({ url: ZAI_CODING_ENDPOINT, apiKeyEnv: 'ZAI_API_KEY', model, secrets })
+  void secrets
+  return httpCapacityProbe({
+    url: ZAI_CODING_ENDPOINT,
+    apiKeyEnv: 'ZAI_API_KEY',
+    provider: 'zai',
+    model,
+  })
 }
 
 /**
@@ -173,7 +170,13 @@ export function zaiCodingProbe(secrets: SecretsEnv, model = 'glm-5.2'): Capacity
  * Supervisor arms must gate on this; probing only z.ai is the proven blind spot.
  */
 export function routerProbe(secrets: SecretsEnv, model = 'glm-5.2'): CapacityProbe {
-  return httpCapacityProbe({ url: ROUTER_ENDPOINT, apiKeyEnv: 'TANGLE_API_KEY', model, secrets })
+  void secrets
+  return httpCapacityProbe({
+    url: ROUTER_ENDPOINT,
+    apiKeyEnv: 'TANGLE_API_KEY',
+    provider: 'tangle-router',
+    model,
+  })
 }
 
 /** The gates an arm must pass, by kind: solo → worker path; supervisor → BOTH paths. */

package/src/swe-arena/diagnosis-ensemble.ts

@@ -12,9 +12,8 @@
  *    gpt-5.5 / opus-4.8 slot in by editing the analyst spec list, never by a
  *    hardcoded requirement on an unrouted model.
  *  - analysts are blind: same bundle, no cross-talk, independent calls.
- *  - secrets discipline matches capacity.ts: the API key is referenced by NAME
- *    inside a dotenvx child shell; the response body lands in a file (dotenvx
- *    writes its banner to stdout, so stdout is only trusted for the marker).
+ *  - the containing run is launched through dotenvx; every analyst enters Runtime through one
+ *    exact AgentProfile and its event record is retained with the response.
  */
 
 import { mkdir, readdir, readFile, writeFile } from 'node:fs/promises'
@@ -22,7 +21,7 @@ import { join } from 'node:path'
 import { type AnalystFinding, makeFinding } from '@tangle-network/agent-eval'
 import { findSupervisorRunDir, type SecretsEnv } from './arms'
 import { ROUTER_ENDPOINT, sleepWithSignal } from './capacity'
-import { run } from './proc'
+import { runBenchRouterTurn } from '../router-turn'
 
 // ---------------------------------------------------------------------------
 // Analyst specs.
@@ -32,6 +31,8 @@ export interface AnalystSpec {
   /** Stable label, e.g. 'glm-5.2#1'. Used in fusion attribution + filenames. */
   id: string
   model: string
+  /** Provider identity stamped into the exact profile. Default: tangle-router. */
+  provider?: string
   /** Chat-completions endpoint. Default: the Tangle router. */
   url?: string
   /** NAME of the env var holding the bearer key (resolved in the dotenvx child). */
@@ -247,8 +248,7 @@ export function parseAnalystFindings(text: string): AnalystRawFinding[] {
   })
 }
 
-/** POST one blind analyst call through the router via dotenvx (key stays in
- *  the child; response body lands in `outFile`, never on shared stdout).
+/** Run one blind analyst through Runtime's exact-profile Router boundary.
  *  ONE retry on transport failure (5xx/524/timeout) — an edge flake was
  *  observed live in the calibration smoke; a parse failure is NOT retried
  *  (same prompt, same model ⇒ same bad shape). */
@@ -265,60 +265,63 @@ export async function runAnalyst(
     return { analystId: spec.id, model: spec.model, ok: false, findings: [], error: `invalid apiKeyEnv name: ${apiKeyEnv}` }
   }
   const url = spec.url ?? ROUTER_ENDPOINT
-  const body = JSON.stringify({
-    model: spec.model,
-    messages: [{ role: 'user', content: analystPrompt(bundle) }],
-    temperature: spec.temperature ?? 0,
-    max_tokens: spec.maxTokens ?? 16_000,
-  })
+  const routerKey = process.env[apiKeyEnv]
+  if (!routerKey) {
+    return {
+      analystId: spec.id,
+      model: spec.model,
+      ok: false,
+      findings: [],
+      error: `${apiKeyEnv} is required; launch through dotenvx`,
+    }
+  }
+  void secrets
   opts.signal?.throwIfAborted()
   await mkdir(scratchDir, { recursive: true })
   const outFile = join(scratchDir, `analyst-${spec.id.replace(/[^a-zA-Z0-9._-]/g, '_')}.response.json`)
   const timeoutMs = opts.timeoutMs ?? 600_000
-  // Body via stdin (--data @-) so the payload never sits on a command line;
-  // the HTTP code is marker-anchored on stdout (dotenvx banners share stdout).
-  const script =
-    `curl -sS -o "$DIAG_OUT" -w "HTTP_CODE=%{http_code}" --max-time ${Math.ceil(timeoutMs / 1000) - 20} ` +
-    `-X POST "$DIAG_URL" -H "Authorization: Bearer $${apiKeyEnv}" -H "Content-Type: application/json" --data @-`
-  const argv = ['run', ...secrets.envFiles.flatMap((f) => ['-f', f]), '--', 'bash', '-c', script]
   const attempts = 1 + (opts.retries ?? 1)
   let transportError = ''
-  let transported = false
-  for (let attempt = 0; attempt < attempts && !transported; attempt++) {
+  let content = ''
+  let tokens: AnalystReport['tokens']
+  for (let attempt = 0; attempt < attempts && content.length === 0; attempt++) {
     opts.signal?.throwIfAborted()
     if (attempt > 0) await sleepWithSignal(opts.retryDelayMs ?? 5_000, opts.signal)
-    const res = await run('dotenvx', argv, {
-      cwd: secrets.secretsDir,
-      timeoutMs,
-      stdin: body,
-      env: { ...process.env, DIAG_URL: url, DIAG_OUT: outFile },
-      signal: opts.signal,
-    })
-    opts.signal?.throwIfAborted()
-    const codeMatch = res.stdout.match(/HTTP_CODE=(\d{3})\s*$/)
-    if (codeMatch && codeMatch[1] === '200') {
-      transported = true
-    } else {
-      transportError = `router call failed (http=${codeMatch?.[1] ?? 'none'}, rc=${res.code}${res.timedOut ? ', timeout' : ''}, attempt=${attempt + 1}/${attempts})`
+    try {
+      const turn = await runBenchRouterTurn(
+        {
+          routerBaseUrl: url.replace(/\/chat\/completions\/?$/u, ''),
+          routerKey,
+          profile: {
+            name: `diagnosis-${spec.id}`,
+            harness: 'cli-base',
+            model: {
+              provider: spec.provider ?? 'tangle-router',
+              default: spec.model,
+              metadata: {
+                temperature: spec.temperature ?? 0,
+                maxTokens: spec.maxTokens ?? 16_000,
+              },
+            },
+            prompt: { systemPrompt: 'Diagnose the supplied run evidence as an independent analyst.' },
+          },
+          timeoutMs,
+          signal: opts.signal,
+        },
+        analystPrompt(bundle),
+      )
+      content = turn.finalText
+      if (turn.usage.tokensKnown !== false) {
+        tokens = { input: turn.usage.input, output: turn.usage.output }
+      }
+      await writeFile(outFile, JSON.stringify(turn, null, 2))
+    } catch (cause) {
+      transportError = `router call failed (${(cause as Error).message}, attempt=${attempt + 1}/${attempts})`
     }
   }
-  if (!transported) {
+  if (content.length === 0 && transportError) {
     return { analystId: spec.id, model: spec.model, ok: false, findings: [], error: transportError }
   }
-  let content = ''
-  let tokens: AnalystReport['tokens']
-  try {
-    const parsed = JSON.parse(await readFile(outFile, 'utf8')) as {
-      choices?: Array<{ message?: { content?: string } }>
-      usage?: { prompt_tokens?: number; completion_tokens?: number }
-    }
-    content = parsed.choices?.[0]?.message?.content ?? ''
-    if (parsed.usage) {
-      tokens = { input: parsed.usage.prompt_tokens ?? 0, output: parsed.usage.completion_tokens ?? 0 }
-    }
-  } catch (cause) {
-    return { analystId: spec.id, model: spec.model, ok: false, findings: [], error: `unparseable response body: ${(cause as Error).message}` }
-  }
   if (content.trim().length === 0) {
     return { analystId: spec.id, model: spec.model, ok: false, findings: [], error: 'empty content (max_tokens starvation?)', tokens }
   }

package/src/swe-arena/execution.test.mts

@@ -31,7 +31,12 @@ import {
   type SupervisorArmResult,
 } from './arms.ts'
 import { containerName, materializeWorkspace } from './materialize.ts'
-import { gatesForArmKind, probeBody, probeWindow, waitForCapacity, httpCapacityProbe } from './capacity.ts'
+import {
+  gatesForArmKind,
+  httpCapacityProbe,
+  probeWindow,
+  waitForCapacity,
+} from './capacity.ts'
 import { run, runOk } from './proc.ts'
 import {
   createSerializedJudge,
@@ -1065,15 +1070,6 @@ describe('capacity gate', () => {
     expect(await waitForCapacity(gate)).toBe(false)
   })
 
-  it('probe body defaults to max_tokens 8000 (glm-5.2 starves below it)', () => {
-    expect(JSON.parse(probeBody('glm-5.2', 8000))).toEqual({
-      model: 'glm-5.2',
-      messages: [{ role: 'user', content: 'Reply with the single word OK.' }],
-      max_tokens: 8000,
-      temperature: 0,
-    })
-  })
-
   it('supervisor arms gate on BOTH worker and router paths; solo on worker only', () => {
     const secrets = { secretsDir: '/dev/null', envFiles: ['agent-state.env'] }
     expect(gatesForArmKind('solo', secrets).map((g) => g.name)).toEqual(['z.ai-coding'])
@@ -1081,9 +1077,13 @@ describe('capacity gate', () => {
   })
 
   it('rejects a non-env-shaped key name (no value smuggling)', () => {
-    const secrets = { secretsDir: '/dev/null', envFiles: [] }
     expect(() =>
-      httpCapacityProbe({ url: 'https://x', apiKeyEnv: 'k; cat /etc/passwd', model: 'm', secrets }),
+      httpCapacityProbe({
+        url: 'https://x',
+        apiKeyEnv: 'k; cat /etc/passwd',
+        provider: 'test',
+        model: 'm',
+      }),
     ).toThrow(/apiKeyEnv/)
   })
 })

package/src/swe-arena/gepa-seat.mts

@@ -41,6 +41,7 @@ import { mkdir, readFile, readdir, rename, rm, writeFile } from 'node:fs/promise
 import { dirname, join } from 'node:path'
 import {
   type DispatchContext,
+  type ComparisonCost,
   createRunCostLedger,
   fsCampaignStorage,
   type GepaOptimizationMethodConfig,
@@ -233,7 +234,7 @@ export function innerSmokeJudge(): JudgeConfig<SmokeVerdict, GepaSeatScenario> {
 // ---------------------------------------------------------------------------
 
 export const GEPA_PYTHON_INSTALL_HINT =
-  'install `agent-eval-rpc==0.131.0`, then install ' +
+  'install `agent-eval-rpc==0.144.6`, then install ' +
   '`gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919db0a622e2e9f9204779b81fe00cc1b2d808f`'
 
 export type GepaMethodFactory = (
@@ -326,6 +327,7 @@ export interface GepaSeatInnerRun {
   tokenUsage: OptimizationTokenUsage
   artifactDir: string
   totalCostUsd: number
+  costProvenance: ComparisonCost['costProvenance']
   accountingComplete: boolean
   incompleteReasons: string[]
   durationMs: number
@@ -532,10 +534,7 @@ function completeProvenance(
   }
 }
 
-function assertCompleteCost(
-  cost: { totalCostUsd: number; accountingComplete: boolean; incompleteReasons: string[] },
-  seatName: string,
-): void {
+function assertCompleteCost(cost: ComparisonCost, seatName: string): void {
   if (!Number.isFinite(cost.totalCostUsd) || cost.totalCostUsd < 0) {
     throw new Error(`gepa seat '${seatName}': optimizer returned invalid total cost`)
   }
@@ -550,6 +549,13 @@ function assertCompleteCost(
   if (cost.accountingComplete !== (cost.incompleteReasons.length === 0)) {
     throw new Error(`gepa seat '${seatName}': optimizer returned inconsistent cost accounting`)
   }
+  if (cost.costProvenance.kind === 'uncaptured') {
+    if (cost.accountingComplete) {
+      throw new Error(`gepa seat '${seatName}': complete cost cannot have uncaptured provenance`)
+    }
+  } else if (cost.costProvenance.usd !== cost.totalCostUsd) {
+    throw new Error(`gepa seat '${seatName}': cost provenance does not match total cost`)
+  }
   if (!cost.accountingComplete) {
     throw new Error(
       `gepa seat '${seatName}': cost accounting is incomplete: ${cost.incompleteReasons.join('; ') || 'no reason provided'}`,
@@ -823,6 +829,7 @@ export function gepaSeatAuthor(config: OuterLoopConfig, deps: GepaSeatDeps): Aut
             : null,
         ...completeProvenance(result.provenance, spec.name),
         totalCostUsd: result.cost.totalCostUsd,
+        costProvenance: result.cost.costProvenance,
         accountingComplete: result.cost.accountingComplete,
         incompleteReasons: [...result.cost.incompleteReasons],
         durationMs: Date.now() - started,

package/src/swe-arena/gepa-seat.test.mts

@@ -34,6 +34,7 @@ import {
 import { defaultRound4Config, type OuterLoopConfig } from './outer-loop.mts'
 import { fanOutLoopsGenerator, type ProposerSpec, type SmokeRunner, type SmokeVerdict } from './proposer-fanout.mts'
 import { captureProposerProvenance } from './proposer-provenance.mts'
+import { officialOptimizerModel } from '../official-optimizer-config.mts'
 import { runOk } from './proc.ts'
 
 const SURFACE = 'extensions/pi/prompts/worker-coding-system.md'
@@ -370,6 +371,7 @@ describe('recordGepaSeatInnerRun', () => {
       tokenUsage: result.tokenUsage,
       artifactDir: result.artifactDir,
       totalCostUsd: 0.25,
+      costProvenance: { kind: 'observed', usd: 0.25 },
       accountingComplete: true,
       incompleteReasons: [],
       durationMs: 5,
@@ -426,24 +428,26 @@ describe('recordGepaSeatInnerRun', () => {
 // ---------------------------------------------------------------------------
 
 const fakeCtx = {} as unknown as DispatchContext
-const testOptimizer = {
+const testOptimizerEnv: NodeJS.ProcessEnv = {
+  TEST_OPTIMIZER_INPUT_USD_PER_MILLION: '1',
+  TEST_OPTIMIZER_CACHED_INPUT_USD_PER_MILLION: '0.1',
+  TEST_OPTIMIZER_CACHE_WRITE_USD_PER_MILLION: '1.25',
+  TEST_OPTIMIZER_OUTPUT_USD_PER_MILLION: '5',
+  TEST_OPTIMIZER_MAX_REQUESTS: '10',
+  TEST_OPTIMIZER_MAX_REQUEST_BYTES: '100000',
+  TEST_OPTIMIZER_MAX_RESPONSE_BYTES: '100000',
+}
+
+const testOptimizer = officialOptimizerModel({
+  env: testOptimizerEnv,
+  envPrefix: 'TEST_OPTIMIZER',
   model: 'optimizer-model',
   baseUrl: 'http://127.0.0.1:1/v1',
   apiKey: 'optimizer-key',
-  budget: {
-    maxCostUsd: 1,
-    maxRequests: 10,
-    maxRequestBytes: 100_000,
-    maxResponseBytes: 100_000,
-    maxOutputTokensPerRequest: 2_000,
-    pricing: {
-      inputUsdPerMillion: 1,
-      cachedInputUsdPerMillion: 0.1,
-      cacheWriteUsdPerMillion: 1.25,
-      outputUsdPerMillion: 5,
-    },
-  },
-}
+  maxCostUsd: 1,
+  maxOutputTokensPerRequest: 2_000,
+  callRef: 'test:optimizer-model',
+})
 
 const fullProvenance = (
   runId = 'gepa-run',
@@ -504,7 +508,12 @@ const fakeGepaFactory =
         }
         return {
           winnerSurface: best.surface,
-          cost: { totalCostUsd: 0.125, accountingComplete: true, incompleteReasons: [] },
+          cost: {
+            totalCostUsd: 0.125,
+            costProvenance: { kind: 'observed', usd: 0.125 },
+            accountingComplete: true,
+            incompleteReasons: [],
+          },
           durationMs: 1,
           provenance: fullProvenance(),
         }
@@ -737,7 +746,12 @@ describe('fanOutLoopsGenerator with the gepa seat', () => {
         ])
         return {
           winnerSurface: candidateA,
-          cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] },
+          cost: {
+            totalCostUsd: 0,
+            costProvenance: { kind: 'observed', usd: 0 },
+            accountingComplete: true,
+            incompleteReasons: [],
+          },
           durationMs: 1,
           provenance: fullProvenance('parallel-gepa', { evaluationCount: 2 }),
         }
@@ -823,6 +837,7 @@ describe('fanOutLoopsGenerator with the gepa seat', () => {
           winnerSurface: WINNER,
           cost: {
             totalCostUsd: 0.25,
+            costProvenance: { kind: 'uncaptured', usd: null },
             accountingComplete: false,
             incompleteReasons: ['optimizer model receipt missing'],
           },
@@ -873,7 +888,12 @@ describe('fanOutLoopsGenerator with the gepa seat', () => {
         if (rejected) throw rejected.reason
         return {
           winnerSurface: SEED,
-          cost: { totalCostUsd: 0, accountingComplete: false, incompleteReasons: [] },
+          cost: {
+            totalCostUsd: 0,
+            costProvenance: { kind: 'uncaptured', usd: null },
+            accountingComplete: false,
+            incompleteReasons: [],
+          },
           durationMs: 1,
           provenance: fullProvenance('runaway'),
         }
@@ -1009,24 +1029,16 @@ describe('integration: real adapter roundtrip', () => {
           },
           smokeInstanceId: 'astropy__astropy-13033',
           scoreSplit: null,
-          gepaOptimizer: {
+          gepaOptimizer: officialOptimizerModel({
+            env: testOptimizerEnv,
+            envPrefix: 'TEST_OPTIMIZER',
             model: 'test-optimizer',
             baseUrl: `http://127.0.0.1:${address.port}/v1`,
             apiKey: 'local-test-key',
-            budget: {
-              maxCostUsd: 1,
-              maxRequests: 10,
-              maxRequestBytes: 100_000,
-              maxResponseBytes: 100_000,
-              maxOutputTokensPerRequest: 2_000,
-              pricing: {
-                inputUsdPerMillion: 1,
-                cachedInputUsdPerMillion: 0.1,
-                cacheWriteUsdPerMillion: 1.25,
-                outputUsdPerMillion: 5,
-              },
-            },
-          },
+            maxCostUsd: 1,
+            maxOutputTokensPerRequest: 2_000,
+            callRef: 'test:integration-optimizer',
+          }),
         },
       )
       const result = await gen.generate({