@tangle-network/agent-bench 0.3.8 → 0.4.1

package/src/benchmarks/_harness.test.mts

@@ -4,7 +4,13 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
 import test from 'node:test'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
-import { resolveBenchPython, runStagedJudge } from './_harness'
+import {
+  benchRoot,
+  resolveBenchPython,
+  runStagedJudge,
+  venvBin,
+  venvBinAt,
+} from './_harness'
 
 const digest = (bytes: Uint8Array): `sha256:${string}` =>
   `sha256:${createHash('sha256').update(bytes).digest('hex')}`
@@ -31,6 +37,15 @@ test('resolveBenchPython rejects relative and empty interpreter paths', () => {
   )
 })
 
+test('venv executable paths support package-owned and external environments', () => {
+  assert.equal(venvBinAt('/srv/terminal-bench', 'tb'), join('/srv/terminal-bench', 'bin', 'tb'))
+  assert.equal(
+    venvBinAt('.venv-commit0', 'python'),
+    join(benchRoot, '.venv-commit0', 'bin', 'python'),
+  )
+  assert.equal(venvBin('python'), venvBinAt('.venv', 'python'))
+})
+
 test('runStagedJudge terminates a hung evaluator at its explicit timeout', async () => {
   const started = Date.now()
   await assert.rejects(

package/src/benchmarks/_harness.ts

@@ -82,13 +82,20 @@ export const venvPython = resolveBenchPython()
 /** Interpreter for a NAMED isolated venv (e.g. `.venv-commit0`). Benches whose pip
  *  deps conflict with the shared `.venv` (commit0 downgrades pydantic/sqlalchemy)
  *  get their own venv and pass its python explicitly — keeping the shared one clean. */
-export const venvPythonAt = (venvDir: string): string => join(benchRoot, venvDir, 'bin', 'python')
+export const venvPythonAt = (venvDir: string): string => venvBinAt(venvDir, 'python')
+
+/** Resolve an executable in an isolated venv. Relative venv paths are package-owned;
+ * absolute paths allow installed consumers to keep large environments elsewhere. */
+export function venvBinAt(venvDir: string, name: string): string {
+  return join(resolve(benchRoot, venvDir), 'bin', name)
+}
+
 /** Report/transcript reads are large; 256 MiB matches the SWE harness budget. */
 export const bigBuffer = 1024 * 1024 * 256
 
 /** Path to a named executable inside the bench venv (e.g. `venvBin('tb')`). */
 export function venvBin(name: string): string {
-  return join(benchRoot, '.venv', 'bin', name)
+  return venvBinAt('.venv', name)
 }
 
 /**

package/src/benchmarks/terminal-bench.ts

@@ -25,6 +25,7 @@ import {
   runVenvPython,
   safeRunId,
   stageFile,
+  venvBinAt,
   venvPythonAt,
 } from './_harness'
 import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'
@@ -34,7 +35,7 @@ import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './typ
 // override the env without reloading this module.
 const terminalBenchVenvDir = (): string => process.env.TERMINAL_BENCH_VENV ?? '.venv-terminal-bench'
 const terminalBenchPython = (): string => venvPythonAt(terminalBenchVenvDir())
-const terminalBenchBin = (): string => join(benchRoot, terminalBenchVenvDir(), 'bin', 'tb')
+const terminalBenchBin = (): string => venvBinAt(terminalBenchVenvDir(), 'tb')
 
 // Pinned dataset: the 0.1.1 core set is patched for terminal-bench >=0.2.4 (the
 // installed CLI) and is the published launch task set. name==version is what `tb

package/src/gate.ts

@@ -427,7 +427,7 @@ export async function runGate(opts: RunGateOptions): Promise<GateReport> {
       root: a.label,
       nodes: [],
       total: acc.get(a.label)!.spend,
-      statusCounts: { done: 0, failed: 0, cancelled: 0, pending: 0 },
+      statusCounts: { done: 0, failed: 0, cancelled: 0, pending: 0, waiting: 0 },
     },
   }))
   const equalK = equalKOnCost(equalKArms)

package/src/hev-eval.mts

@@ -27,6 +27,7 @@ async function complete(base: string, key: string, model: string, prompt: string
 async function main(): Promise<void> {
   const key = process.env.TANGLE_API_KEY
   if (!key) throw new Error('TANGLE_API_KEY required')
+  const apiKey: string = key
   const base = process.env.ROUTER_BASE ?? 'https://api.together.xyz/v1'
   const model = process.env.WORKER_MODEL ?? 'meta-llama/Meta-Llama-3-8B-Instruct-Lite'
   const instruction = process.env.INSTRUCTION_FILE
@@ -51,8 +52,10 @@ async function main(): Promise<void> {
   let i = 0
   async function worker(): Promise<void> {
     while (i < tasks.length) {
-      const t = tasks[i++]
-      const reply = await complete(base, key, model, `${instruction}\n\n\`\`\`python\n${t.prompt}\`\`\``, maxTokens)
+      const t = tasks[i]
+      i += 1
+      if (!t) continue
+      const reply = await complete(base, apiKey, model, `${instruction}\n\n\`\`\`python\n${t.prompt}\`\`\``, maxTokens)
       const { pass: p } = await runChecker(t, extractCode(reply))
       if (p === 1) pass += 1
       else fails.push(t.taskId)

package/src/hev-improve.mts

@@ -1,8 +1,6 @@
 /**
- * SELF-IMPROVEMENT on HumanEval — the prompt-sensitive, VISIBLE-ORACLE counterpart
- * to the SWE-bench run. Same machinery (improve(surface:'prompt') + gepaProposer +
- * held-out gate), but the worker is a single chat completion and the judge is the
- * deterministic Docker checker (run the function against its own hidden unit tests).
+ * Official GEPA prompt optimization on HumanEval. The worker is a single chat
+ * completion and the judge is the deterministic Docker checker.
  *
  * WHY this exists: on SWE-bench the same GEPA loop was NULL because the grading test
  * is withheld — the worker cannot verify, so prompt wording cannot move resolve.
@@ -13,11 +11,22 @@
  * Worker + reflect models call the zai coding endpoint directly (no tangle router,
  * no WAF, no 503):  TANGLE_API_KEY=$ZAI_API_KEY  ROUTER_BASE=https://api.z.ai/api/coding/paas/v4
  */
-import { improve } from '@tangle-network/agent-runtime'
-import type { AgentProfile } from '@tangle-network/agent-interface'
+import {
+  improve,
+  officialGepa,
+  type ReadonlyAgentProfile,
+} from '@tangle-network/agent-runtime'
+import {
+  canonicalCandidateDigest,
+  type AgentProfile,
+} from '@tangle-network/agent-interface'
 import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract'
-import { gepaProposer } from '@tangle-network/agent-eval/campaign'
 import { extractCode, loadHumanEval, runChecker, type HumanEvalTask } from './benchmarks/humaneval'
+import {
+  assertCompleteCost,
+  officialOptimizerModel,
+  requiredTokenPricing,
+} from './official-optimizer-config.mjs'
 
 // The SEED instruction GEPA evolves. Byte-identical to humaneval.ts basePrompt's
 // solveInstruction so the baseline arm reproduces the plain-prompt denominator.
@@ -26,7 +35,6 @@ const SEED_INSTRUCTION =
 
 interface Completion {
   text: string
-  usd: number
   tokIn: number
   tokOut: number
 }
@@ -45,10 +53,7 @@ async function complete(base: string, key: string, model: string, prompt: string
   const text = d.choices?.[0]?.message?.content ?? ''
   const tokIn = d.usage?.prompt_tokens ?? 0
   const tokOut = d.usage?.completion_tokens ?? 0
-  // zai glm pricing is ~ $0.6/M in, $2.2/M out (coding plan); a rough cost tag so the
-  // stub-guard sees a real backend. Exact cost is not the metric (pass-rate is).
-  const usd = (tokIn * 0.6 + tokOut * 2.2) / 1_000_000
-  return { text, usd, tokIn, tokOut }
+  return { text, tokIn, tokOut }
 }
 
 async function main(): Promise<void> {
@@ -63,51 +68,81 @@ async function main(): Promise<void> {
   const reflectBase = process.env.REFLECT_BASE ?? base
   const reflectKey = process.env.REFLECT_KEY ?? key
   const trainN = Number(process.env.TRAIN_N ?? 12)
-  const holdoutN = Number(process.env.HOLDOUT_N ?? 12)
+  const selectionN = Number(process.env.SELECTION_N ?? 12)
+  const testN = Number(process.env.TEST_N ?? 12)
   const offset = Number(process.env.OFFSET ?? 80)
-  // generations=1 never exercises the GEPA Pareto/combine path (the frontier
-  // needs >=1 completed generation before combine can fire) — default to a
-  // multi-generation budget so the default run measures the full loop.
-  const generations = Number(process.env.GENERATIONS ?? 6)
-  const population = Number(process.env.POPULATION ?? 4)
+  const maxEvaluations = Number(process.env.MAX_EVALUATIONS ?? 24)
+  const maxProposerCostUsd = Number(process.env.MAX_PROPOSER_COST_USD ?? 5)
   const workerMaxTokens = Number(process.env.MAX_TOKENS ?? 6000)
   const reflectMaxTokens = Number(process.env.REFLECT_MAX_TOKENS ?? 8000)
   const maxConcurrency = Number(process.env.MAX_CONCURRENCY ?? 4)
+  const runDir = process.env.RUN_DIR ?? '.runs/humaneval-official-gepa'
+  if (process.env.DRYRUN) {
+    console.log(
+      `DRYRUN: imports OK (improve=${typeof improve}, officialGepa=${typeof officialGepa})`,
+    )
+    return
+  }
+  const workerPricing = requiredTokenPricing(process.env, 'WORKER')
+  const optimizer = officialOptimizerModel({
+    env: process.env,
+    model: reflectModel,
+    baseUrl: reflectBase,
+    apiKey: reflectKey,
+    maxCostUsd: maxProposerCostUsd,
+    maxOutputTokensPerRequest: reflectMaxTokens,
+  })
 
-  // TRAIN and HOLDOUT are DISJOINT slices of the harder middle band (offset).
+  // All three partitions are disjoint slices of the harder middle band.
   const train = await loadHumanEval(trainN, offset)
-  const holdout = await loadHumanEval(holdoutN, offset + trainN)
-  const byId = new Map<string, HumanEvalTask>([...train, ...holdout].map((t) => [t.taskId, t]))
-  const allIds = [...byId.keys()]
+  const selection = await loadHumanEval(selectionN, offset + trainN)
+  const testCases = await loadHumanEval(testN, offset + trainN + selectionN)
+  const byId = new Map<string, HumanEvalTask>(
+    [...train, ...selection, ...testCases].map((t) => [t.taskId, t]),
+  )
 
-  console.log('═══ HumanEval self-improvement — VISIBLE oracle (deterministic Docker checker) ═══')
-  console.log(`worker=${workerModel}  reflect=${reflectModel}  base=${base}`)
+  console.log('=== HumanEval prompt optimization with official GEPA ===')
+  console.log(`worker=${workerModel} reflect=${reflectModel} base=${base}`)
   console.log(`train=[${train.map((t) => t.taskId).join(', ')}]`)
-  console.log(`holdout=[${holdout.map((t) => t.taskId).join(', ')}]`)
-  console.log(`generations=${generations} population=${population} offset=${offset} maxTokens=${workerMaxTokens}`)
-  console.log(`≈ ${trainN * (1 + generations * population) + 2 * holdoutN} cells (each = 1 completion + 1 Docker check)\n`)
+  console.log(`selection=[${selection.map((t) => t.taskId).join(', ')}]`)
+  console.log(`test=[${testCases.map((t) => t.taskId).join(', ')}]`)
+  console.log(`maxEvaluations=${maxEvaluations} maxProposerCostUsd=${maxProposerCostUsd} offset=${offset} maxTokens=${workerMaxTokens}`)
+  console.log(`runDir=${runDir}\n`)
 
   const stats = { n: 0 }
-  const agent = async (surface: unknown, scenario: Scenario, ctx: DispatchContext): Promise<string | null> => {
-    const instr = String(surface)
+  const agent = async (candidate: ReadonlyAgentProfile, scenario: Scenario, ctx: DispatchContext): Promise<string | null> => {
+    const instr = candidate.prompt?.systemPrompt
+    if (instr === undefined) throw new Error('agent: candidate profile has no system prompt')
     const t = byId.get(scenario.id)
     if (!t) throw new Error(`agent: unknown scenario ${scenario.id}`)
     const prompt = `${instr}\n\n\`\`\`python\n${t.prompt}\`\`\``
     const t0 = Date.now()
-    const r = await complete(base, key, workerModel, prompt, workerMaxTokens)
-    const zeroUsage = r.tokIn === 0 && r.tokOut === 0
+    const paid = await ctx.cost.runPaidCall({
+      channel: 'agent',
+      actor: 'humaneval-worker',
+      model: workerModel,
+      execute: () => complete(base, key, workerModel, prompt, workerMaxTokens),
+      receipt: (result) => {
+        const usageUnknown = result.tokIn === 0 && result.tokOut === 0
+        return {
+          model: workerModel,
+          inputTokens: result.tokIn,
+          outputTokens: result.tokOut,
+          customTokenPricing: workerPricing,
+          ...(usageUnknown ? { usageUnknown: true } : {}),
+        }
+      },
+    })
+    if (!paid.succeeded) throw paid.error
+    const r = paid.value
     const hasText = r.text.trim().length > 0
-    ctx.cost.observe(zeroUsage && hasText ? Math.max(r.usd, 0.0001) : r.usd, workerModel)
-    ctx.cost.observeTokens(
-      zeroUsage && hasText ? { input: Math.max(r.tokIn, 1), output: Math.max(r.tokOut, 1) } : { input: r.tokIn, output: r.tokOut },
-    )
     stats.n += 1
     const codeLen = extractCode(r.text).length
     console.log(`  [agent] ${scenario.id} instr=${instr.length}c code=${codeLen}b tok=in:${r.tokIn}/out:${r.tokOut} ${Math.round((Date.now() - t0) / 1000)}s`)
     return hasText ? r.text : null
   }
 
-  const judge: JudgeConfig<string, Scenario> = {
+  const judge: JudgeConfig<string | null, Scenario> = {
     name: 'humaneval-docker',
     dimensions: [{ key: 'pass', description: 'the completed function passes its hidden unit tests (deterministic Docker checker)' }],
     async score({ artifact, scenario }) {
@@ -137,49 +172,59 @@ async function main(): Promise<void> {
   }
 
   const profile: AgentProfile = { name: 'hev-solver', prompt: { systemPrompt: SEED_INSTRUCTION } }
-  const proposer = gepaProposer({
-    llm: { baseUrl: reflectBase, apiKey: reflectKey },
-    model: reflectModel,
-    target:
-      'the instruction/system prompt strategy for a SMALL model completing Python functions to pass hidden unit tests. ' +
-      'Propose SUBSTANTIALLY different strategies, not wording tweaks: e.g. require the model to first reason step-by-step ' +
-      'about the algorithm and edge cases (empty inputs, off-by-one, boundary values, types) in a brief plan or comments ' +
-      'BEFORE writing the code; provide a short worked example; or add an explicit self-check against the docstring. ' +
-      'Bold rewrites that change model BEHAVIOR beat cosmetic edits.',
-    maxTokens: reflectMaxTokens,
-    temperature: 0.7,
-  })
-
-  const scenarios: Scenario[] = allIds.map((id) => ({ id, kind: 'humaneval' }))
-  const holdoutScenarios: Scenario[] = holdout.map((t) => ({ id: t.taskId, kind: 'humaneval' }))
+  const scenario = (task: HumanEvalTask): Scenario => ({ id: task.taskId, kind: 'humaneval' })
 
-  const out = await improve(profile, [], {
+  const out = await improve(profile, {
     surface: 'prompt',
-    gate: 'holdout',
-    generator: proposer,
-    scenarios,
-    judge,
+    executionRef: canonicalCandidateDigest({
+      callback: 'bench/hev-improve',
+      model: workerModel,
+      endpoint: new URL(base).origin,
+      maxTokens: workerMaxTokens,
+      checker: 'local-python',
+    }),
+    method: officialGepa<Scenario, string | null>({
+      objective:
+        'Improve the complete instruction for a small model that writes Python functions which pass hidden unit tests.',
+      background:
+        'Prefer behavioral strategies over wording changes. Address algorithm choice, edge cases, boundary values, type behavior, and self-checking. Return only the complete instruction.',
+      recipe: {
+        kind: 'engine',
+        run: {
+          engine: 'gepa',
+          maxEvaluations,
+          maxProposerCostUsd,
+        },
+      },
+      optimizer,
+      resume: 'if-compatible',
+      trustResumeState: true,
+      describeScenario: (item) => ({ prompt: byId.get(item.id)?.prompt ?? item.id }),
+    }),
+    trainScenarios: train.map(scenario),
+    selectionScenarios: selection.map(scenario),
+    testScenarios: testCases.map(scenario),
+    judges: [judge],
     agent,
     expectUsage: 'warn',
-    // rawTraceContext stays OFF deliberately: it swaps the distilled findings for
-    // filesystem paths + grep/cat instructions (rawTraceDistiller), which only a
-    // coding harness can execute. This run's proposer is prompt-tier (gepaProposer
-    // — a single LLM call that cannot run grep), so the trace evidence arrives via
-    // the judge's traceback notes + the breakdown's `emitted` excerpt instead.
-    budget: { generations, populationSize: population, holdoutScenarios, maxConcurrency, reps: 1 },
-    llm: { baseUrl: reflectBase, apiKey: reflectKey, model: reflectModel },
+    maxConcurrency,
+    reps: 1,
+    runDir,
+    optimizationRunOptions: {
+      expectUsage: 'warn',
+      maxConcurrency,
+      reps: 1,
+    },
   })
 
-  console.log('\n═══ RESULT ═══')
-  console.log(`gateDecision=${out.gateDecision}  shipped=${out.shipped}  lift=${out.lift}`)
-  console.log(`baseline holdout pass-rate = ${out.raw.baseline.compositeMean}`)
-  console.log(`winner   holdout pass-rate = ${out.raw.winner.compositeMean}`)
-  console.log(`baseline per-scenario: ${JSON.stringify(out.raw.baseline.perScenario)}`)
-  console.log(`winner   per-scenario: ${JSON.stringify(out.raw.winner.perScenario)}`)
-  if (out.raw.winner.label) console.log(`winner label   : ${out.raw.winner.label}`)
-  if ((out.raw.winner as { surface?: unknown }).surface) {
-    console.log(`winner instruction:\n${String((out.raw.winner as { surface?: unknown }).surface).slice(0, 1200)}`)
-  }
+  assertCompleteCost('humaneval official GEPA run', out.cost)
+  console.log('\n=== RESULT ===')
+  console.log(`decision=${out.decision} lift=${out.lift} interval=[${out.liftInterval.low}, ${out.liftInterval.high}]`)
+  console.log(`baseline test pass-rate=${out.raw.best.baselineComposite}`)
+  console.log(`winner test pass-rate=${out.raw.best.winnerComposite}`)
+  console.log(`test scenarios=${JSON.stringify(out.raw.best.scenarioScores)}`)
+  console.log(`cost=${JSON.stringify(out.cost)}`)
+  console.log(`winner instruction:\n${String(out.candidate.value).slice(0, 2000)}`)
 }
 
 main().catch((e) => {

package/src/official-optimizer-config.mts

@@ -0,0 +1,89 @@
+function requiredNonNegativeNumber(
+  env: NodeJS.ProcessEnv,
+  name: string,
+): number {
+  const raw = env[name]
+  if (raw === undefined || raw.trim() === '') {
+    throw new Error(`env ${name} is required`)
+  }
+  const value = Number(raw)
+  if (!Number.isFinite(value) || value < 0) {
+    throw new Error(`env ${name} must be a finite non-negative number`)
+  }
+  return value
+}
+
+function positiveInteger(
+  env: NodeJS.ProcessEnv,
+  name: string,
+  fallback: number,
+): number {
+  const value = Number(env[name] ?? fallback)
+  if (!Number.isSafeInteger(value) || value <= 0) {
+    throw new Error(`env ${name} must be a positive integer`)
+  }
+  return value
+}
+
+export function requiredTokenPricing(
+  env: NodeJS.ProcessEnv,
+  prefix: string,
+) {
+  return {
+    inputUsdPerMillion: requiredNonNegativeNumber(
+      env,
+      `${prefix}_INPUT_USD_PER_MILLION`,
+    ),
+    cachedInputUsdPerMillion: requiredNonNegativeNumber(
+      env,
+      `${prefix}_CACHED_INPUT_USD_PER_MILLION`,
+    ),
+    cacheWriteUsdPerMillion: requiredNonNegativeNumber(
+      env,
+      `${prefix}_CACHE_WRITE_USD_PER_MILLION`,
+    ),
+    outputUsdPerMillion: requiredNonNegativeNumber(
+      env,
+      `${prefix}_OUTPUT_USD_PER_MILLION`,
+    ),
+  }
+}
+
+export function officialOptimizerModel(options: {
+  env: NodeJS.ProcessEnv
+  model: string
+  baseUrl: string
+  apiKey: string
+  maxCostUsd: number
+  maxOutputTokensPerRequest: number
+  envPrefix?: string
+}) {
+  const { env } = options
+  const envPrefix = options.envPrefix ?? 'REFLECT'
+  return {
+    model: options.model,
+    baseUrl: options.baseUrl,
+    apiKey: options.apiKey,
+    budget: {
+      maxCostUsd: options.maxCostUsd,
+      maxRequests: positiveInteger(env, `${envPrefix}_MAX_REQUESTS`, 100),
+      maxRequestBytes: positiveInteger(env, `${envPrefix}_MAX_REQUEST_BYTES`, 2_000_000),
+      maxResponseBytes: positiveInteger(env, `${envPrefix}_MAX_RESPONSE_BYTES`, 2_000_000),
+      maxOutputTokensPerRequest: options.maxOutputTokensPerRequest,
+      requestTimeoutMs: positiveInteger(env, `${envPrefix}_REQUEST_TIMEOUT_MS`, 300_000),
+      pricing: requiredTokenPricing(env, envPrefix),
+    },
+  }
+}
+
+export function assertCompleteCost(
+  label: string,
+  cost: { accountingComplete: boolean; incompleteReasons: readonly string[] },
+): void {
+  if (cost.accountingComplete) return
+  const reasons =
+    cost.incompleteReasons.length > 0
+      ? cost.incompleteReasons.join('; ')
+      : 'no incomplete reason was recorded'
+  throw new Error(`${label}: cost accounting is incomplete: ${reasons}`)
+}

package/src/official-optimizer-config.test.mts

@@ -0,0 +1,88 @@
+import { describe, expect, it } from 'vitest'
+import {
+  assertCompleteCost,
+  officialOptimizerModel,
+  requiredTokenPricing,
+} from './official-optimizer-config.mts'
+
+const pricingEnv = {
+  OPT_INPUT_USD_PER_MILLION: '1',
+  OPT_CACHED_INPUT_USD_PER_MILLION: '0.1',
+  OPT_CACHE_WRITE_USD_PER_MILLION: '1.25',
+  OPT_OUTPUT_USD_PER_MILLION: '5',
+}
+
+describe('official optimizer configuration', () => {
+  it('builds a bounded model configuration from an arbitrary environment prefix', () => {
+    const model = officialOptimizerModel({
+      env: {
+        ...pricingEnv,
+        OPT_MAX_REQUESTS: '7',
+        OPT_MAX_REQUEST_BYTES: '1000',
+        OPT_MAX_RESPONSE_BYTES: '2000',
+        OPT_REQUEST_TIMEOUT_MS: '3000',
+      },
+      envPrefix: 'OPT',
+      model: 'test-model',
+      baseUrl: 'http://127.0.0.1:8080/v1',
+      apiKey: 'test-key',
+      maxCostUsd: 2,
+      maxOutputTokensPerRequest: 4000,
+    })
+
+    expect(model).toEqual({
+      model: 'test-model',
+      baseUrl: 'http://127.0.0.1:8080/v1',
+      apiKey: 'test-key',
+      budget: {
+        maxCostUsd: 2,
+        maxRequests: 7,
+        maxRequestBytes: 1000,
+        maxResponseBytes: 2000,
+        maxOutputTokensPerRequest: 4000,
+        requestTimeoutMs: 3000,
+        pricing: {
+          inputUsdPerMillion: 1,
+          cachedInputUsdPerMillion: 0.1,
+          cacheWriteUsdPerMillion: 1.25,
+          outputUsdPerMillion: 5,
+        },
+      },
+    })
+  })
+
+  it('requires every token price instead of inventing cost data', () => {
+    expect(() =>
+      requiredTokenPricing(
+        {
+          ...pricingEnv,
+          OPT_OUTPUT_USD_PER_MILLION: undefined,
+        },
+        'OPT',
+      ),
+    ).toThrow('env OPT_OUTPUT_USD_PER_MILLION is required')
+  })
+
+  it('rejects invalid request limits before constructing the optimizer', () => {
+    expect(() =>
+      officialOptimizerModel({
+        env: { ...pricingEnv, OPT_MAX_REQUESTS: '0' },
+        envPrefix: 'OPT',
+        model: 'test-model',
+        baseUrl: 'http://127.0.0.1:8080/v1',
+        apiKey: 'test-key',
+        maxCostUsd: 2,
+        maxOutputTokensPerRequest: 4000,
+      }),
+    ).toThrow('env OPT_MAX_REQUESTS must be a positive integer')
+  })
+
+  it('rejects incomplete cost records', () => {
+    expect(() =>
+      assertCompleteCost('official optimizer', {
+        accountingComplete: false,
+        incompleteReasons: ['provider omitted usage'],
+      }),
+    ).toThrow('official optimizer: cost accounting is incomplete: provider omitted usage')
+  })
+})

package/src/profiles.ts

@@ -5,8 +5,8 @@
  * "operator type" or "analyst type" — there are profiles + the operator toolbox they use to manage
  * each other (in-process via the Scope, in a sandbox via the same verbs exposed as MCP tools).
  *
- * The directory is OPTIMIZED over time by the agent-eval RSI loop (`runImprovementLoop`): `gepaProposer`
- * evolves the prompts, a `skillOptProposer` evolves `skills`, Pareto-tracked + holdout-gated.
+ * Complete agent-eval methods optimize these profiles over explicit train,
+ * selection, and final-test partitions.
  * The trace-analyst's findings are the optimizer's input — the loop improves the profile that the
  * findings say is weak.
  */

package/src/rollout-ledger/backfill-swe-arena.test.mts

@@ -285,31 +285,35 @@ describe('backfillSweArena', () => {
     expect(stats.workerSessionsJoined).toBe(2)
   })
 
-  it('marks a session with no readable parts as an incomplete gap line', async () => {
-    await buildFixtureTree()
-    await buildFixtureDb()
-    const { DatabaseSync } = await import('node:sqlite')
-    const db = new DatabaseSync(dbPath)
-    // Session row survives; its message parts do not.
-    db.exec("DELETE FROM part; DELETE FROM message")
-    db.close()
+  it(
+    'marks a session with no readable parts as an incomplete gap line',
+    async () => {
+      await buildFixtureTree()
+      await buildFixtureDb()
+      const { DatabaseSync } = await import('node:sqlite')
+      const db = new DatabaseSync(dbPath)
+      // Session row survives; its message parts do not.
+      db.exec("DELETE FROM part; DELETE FROM message")
+      db.close()
 
-    const { lines, stats } = await backfillSweArena(outDir, {
-      opencodeDb: dbPath,
-      claudeProjectsDir: projectsDir,
-    })
-    const joinedCwd = lines.filter((l) => l.role === 'worker' && l.outcome.metrics.has_session === true)
-    expect(joinedCwd.length).toBeGreaterThan(0)
-    for (const worker of joinedCwd) {
-      expect(worker.messages).toHaveLength(0)
-      expect(worker.outcome.is_completed).toBe(false)
-      expect(worker.provenance.gap).toMatch(/no readable message parts/)
-    }
-    // The store-integrity failure is counted apart from the cwd-join failure.
-    expect(stats.workerSessionsEmpty).toBe(2)
-    expect(stats.workerCwdsMissed).toBe(1)
-    expect(stats.workerSessionsJoined).toBe(0)
-  })
+      const { lines, stats } = await backfillSweArena(outDir, {
+        opencodeDb: dbPath,
+        claudeProjectsDir: projectsDir,
+      })
+      const joinedCwd = lines.filter((l) => l.role === 'worker' && l.outcome.metrics.has_session === true)
+      expect(joinedCwd.length).toBeGreaterThan(0)
+      for (const worker of joinedCwd) {
+        expect(worker.messages).toHaveLength(0)
+        expect(worker.outcome.is_completed).toBe(false)
+        expect(worker.provenance.gap).toMatch(/no readable message parts/)
+      }
+      // The store-integrity failure is counted apart from the cwd-join failure.
+      expect(stats.workerSessionsEmpty).toBe(2)
+      expect(stats.workerCwdsMissed).toBe(1)
+      expect(stats.workerSessionsJoined).toBe(0)
+    },
+    15_000,
+  )
 
   it('fails loud on a cell cache that carries no scenarioId/rep identity', async () => {
     await buildFixtureTree()