@tangle-network/agent-bench 0.9.0 → 0.9.1

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 0.9.1
+
+Omni agent engines use the configured optimizer model through the metered Anthropic endpoint.
+Optimizer requests retain their exact output limits within the shared request ceiling.
+The Runtime dependency follows 0.199.0.
+
 ## 0.8.34
 
 The dependency range requires Knowledge 14.0.3 for the compatible Eval cohort.

package/HARNESS.md

@@ -109,6 +109,8 @@ Exact score equality is usually the wrong parity criterion for stochastic system
 - **AutoResearch / Prime Agent** — agentic search over an editable research surface. Treat the agent, tools, and external evaluator as separate identities.
 - **Meta-Harness** — search over harness or orchestration behavior; preserve the same outcome evaluator.
 - **Omni** — phase-one portfolio search followed by a fresh phase-two optimizer seeded from the best phase-one artifact. The matched phase-one budgets and the phase boundary are part of the protocol.
+  Bench enables Eval’s metered Anthropic endpoint for the CLI engines and binds their model to the selected optimizer model.
+  Each admitted optimizer request executes with its requested output limit; Eval enforces the configured ceiling.
 - **Trace analysts** — evidence producers. Measure finding quality against labeled traces before using findings to steer search.
 - **Prime Agent RLM and DSPy RLM** — alternative analyst/context engines, not optimization methods by themselves. Compare them on the same trace questions, evidence requirements, context budgets, and downstream decisions.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.9.0",
+  "version": "0.9.1",
   "type": "module",
   "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.",
   "repository": {
@@ -29,7 +29,7 @@
     "@tangle-network/agent-interface": "^2.3.0",
     "@tangle-network/agent-knowledge": "^14.0.3",
     "@tangle-network/sandbox": ">=0.36.4 <0.38.0",
-    "@tangle-network/agent-runtime": "^0.198.0"
+    "@tangle-network/agent-runtime": "^0.199.0"
   },
   "devDependencies": {
     "@arethetypeswrong/cli": "0.18.5",

package/src/official-optimizer-config.mts

@@ -7,8 +7,7 @@ import {
   canonicalAgentProfileDigest,
   canonicalCandidateDigest,
 } from '@tangle-network/agent-interface'
-import { profileOptimizerModelCall } from '../../src/runtime/profile-chat-client'
-import type { RouterSeam } from '../../src/runtime/supervise/runtime'
+import { profileOptimizerModelCall, type RouterSeam } from '@tangle-network/agent-runtime/kernel'
 
 function requiredNonNegativeNumber(
   env: NodeJS.ProcessEnv,
@@ -73,6 +72,7 @@ export function officialOptimizerModel(options: {
   temperature?: number
   reasoningEffort?: NonNullable<AgentProfile['model']>['reasoningEffort']
   callRef?: string
+  anthropicEndpoint?: boolean
   complete?: RouterSeam['complete']
 }): OpenAICompatibleOptimizerModel {
   const { env } = options
@@ -106,12 +106,20 @@ export function officialOptimizerModel(options: {
     routerKey: options.apiKey,
     ...(options.complete ? { complete: options.complete } : {}),
   }
-  const call = profileOptimizerModelCall({
-    profile,
-    context: 'official optimizer model',
-    executor,
-    pricing: budget.pricing,
-  })
+  const call: OpenAICompatibleOptimizerModel['call'] = (request) =>
+    profileOptimizerModelCall({
+      // Eval admits the request against the budget; execute its exact output limit.
+      profile: {
+        ...profile,
+        model: {
+          ...profile.model,
+          maxVisibleOutputTokens: request.request.maxTokens ?? options.maxOutputTokensPerRequest,
+        },
+      },
+      context: 'official optimizer model',
+      executor,
+      pricing: budget.pricing,
+    })(request)
   return {
     model: options.model,
     callRef:
@@ -122,6 +130,7 @@ export function officialOptimizerModel(options: {
       })}`,
     call,
     budget,
+    ...(options.anthropicEndpoint === true ? { anthropicEndpoint: true } : {}),
   }
 }
 

package/src/swe-arena/gepa-seat.mts

@@ -130,7 +130,7 @@ export type GepaSeatRecipe = Extract<
 /** Build the bounded recipe for a seat. The TOTAL inner-evaluation budget is
  *  exactly `maxMetricCalls`. The adapter's local callback enforces the sum
  *  of per-run limits, and the seat's own dispatch wrapper re-enforces it. */
-export function recipeForSeat(spec: GepaSeatSpec): GepaSeatRecipe {
+export function recipeForSeat(spec: GepaSeatSpec, optimizerModel?: string): GepaSeatRecipe {
   const calls = spec.maxMetricCalls ?? DEFAULT_MAX_METRIC_CALLS
   const cost = spec.maxProposerCostUsd ?? DEFAULT_MAX_PROPOSER_COST_USD
   if (spec.engine === 'gepa') {
@@ -145,6 +145,7 @@ export function recipeForSeat(spec: GepaSeatSpec): GepaSeatRecipe {
     engine,
     maxEvaluations: perExplore,
     maxProposerCostUsd: perRunCost,
+    ...(engine !== 'gepa' && optimizerModel ? { engineConfig: { model: optimizerModel } } : {}),
   }))
   return {
     kind: 'omni',
@@ -758,6 +759,7 @@ export function gepaSeatAuthor(config: OuterLoopConfig, deps: GepaSeatDeps): Aut
         : officialOptimizerModel({
             env: process.env,
             envPrefix: 'GEPA_OPTIMIZER',
+            anthropicEndpoint: spec.engine === 'omni',
             model: process.env.GEPA_OPTIMIZER_MODEL ?? config.arm.driverModel,
             baseUrl:
               process.env.GEPA_OPTIMIZER_BASE_URL ??
@@ -771,7 +773,7 @@ export function gepaSeatAuthor(config: OuterLoopConfig, deps: GepaSeatDeps): Aut
           }))
     const method = factory({
       name: `gepa-seat:${spec.name}`,
-      recipe,
+      recipe: recipeForSeat(spec, optimizer?.model),
       objective,
       evaluationId: gepaSeatEvaluationId({
         smokeInstanceId: deps.smokeInstanceId,

package/src/swe-arena/gepa-seat.test.mts

@@ -4,9 +4,12 @@ import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promis
 import { createServer } from 'node:http'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
-import type {
-  DispatchContext,
-  OptimizationMethodProvenance,
+import {
+  gepaOptimizationMethod,
+  createRunCostLedger,
+  fsCampaignStorage,
+  type DispatchContext,
+  type OptimizationMethodProvenance,
 } from '@tangle-network/agent-eval/campaign'
 import { afterEach, beforeEach, describe, expect, it } from 'vitest'
 import { ACTIVATION_PREDICATE_RELPATH, parseActivationPredicate } from './activation.mts'
@@ -89,6 +92,68 @@ describe('validateGepaSeat', () => {
 // ---------------------------------------------------------------------------
 
 describe('recipeForSeat', () => {
+  it.each(['gepa', 'omni'] as const)('transports the real %s optimizer configuration', async (engine) => {
+    const runDir = await mkdtemp(join(tmpdir(), 'gepa-seat-transport-'))
+    const requests: unknown[] = []
+    try {
+      const optimizer = officialOptimizerModel({
+        env: { OPT_INPUT_USD_PER_MILLION: '1', OPT_CACHED_INPUT_USD_PER_MILLION: '1', OPT_CACHE_WRITE_USD_PER_MILLION: '1', OPT_OUTPUT_USD_PER_MILLION: '1' },
+        envPrefix: 'OPT', model: 'fixture-model', baseUrl: 'http://127.0.0.1:1/v1', apiKey: 'fixture-key',
+        maxCostUsd: 1, maxOutputTokensPerRequest: 100, anthropicEndpoint: engine === 'omni',
+        complete: async (request) => {
+          requests.push(request)
+          return { model: 'fixture-model', choices: [{ message: { content: 'ok' }, finish_reason: 'stop' }], usage: { prompt_tokens: 2, completion_tokens: 1, cost: 0.000003 } }
+        },
+      })
+      const spec = seat({ engine })
+      validateGepaSeat(spec)
+      const runtime = {
+        python: { implementation: 'CPython', version: '3.12.0' },
+        bridge: { package: 'agent-eval-rpc', version: 'fixture', sourceUrl: 'https://github.com/tangle-network/agent-eval', revision: 'fixture', sourceSha256: 'a'.repeat(64) },
+        optimizer: { package: 'gepa', version: 'fixture', sourceUrl: 'https://github.com/gepa-ai/gepa', revision: 'fixture', sourceSha256: 'b'.repeat(64) },
+        engineModules: [],
+      }
+      // Only the child optimizer is substituted; the real Eval proxy calls Runtime.
+      const bridge = join(runDir, 'transport.mjs')
+      await writeFile(bridge, `
+import fs from 'node:fs'
+const input = JSON.parse(fs.readFileSync(process.argv[process.argv.indexOf('--input') + 1], 'utf8'))
+if (input.operation === 'inspect') {
+  fs.writeFileSync(process.argv[process.argv.indexOf('--output') + 1], JSON.stringify({ runtime: ${JSON.stringify(runtime)} }))
+} else {
+  const omni = input.recipe.kind === 'omni'
+  if (omni && input.recipe.explore.some(run => run.engine !== 'gepa' && run.engineConfig.model !== input.modelProxy.model)) throw new Error('wrong engine model')
+  const send = maxTokens => fetch(input.modelProxy.baseUrl + (omni ? '/messages' : '/chat/completions'), {
+    method: 'POST',
+    headers: { 'content-type': 'application/json', authorization: 'Bearer ' + input.modelProxy.apiKey, 'x-api-key': input.modelProxy.apiKey, 'anthropic-version': '2023-06-01' },
+    body: JSON.stringify({ model: input.modelProxy.model, max_tokens: maxTokens, messages: [{ role: 'user', content: 'transport proof' }] }),
+  })
+  const overBudget = await send(101)
+  if (overBudget.ok) throw new Error('output ceiling was not enforced')
+  await overBudget.text()
+  const response = await send(10)
+  if (!response.ok) throw new Error('model transport failed: ' + response.status + ' ' + await response.text())
+  await response.json()
+  throw new Error('transport-handshake-complete')
+}
+`)
+      const method = gepaOptimizationMethod({
+        recipe: recipeForSeat(spec, optimizer.model), optimizer,
+        objective: 'Check transport only', evaluationId: 'bench-transport',
+        runner: { command: process.execPath, args: [bridge] },
+      })
+      await expect(method.optimize({
+        baselineSurface: 'seed', trainScenarios: [{ id: 'train', kind: 'fixture' }], selectionScenarios: [{ id: 'selection', kind: 'fixture' }],
+        dispatchWithSurface: async () => 'unused',
+        judges: [{ name: 'fixture', dimensions: [{ key: 'score', description: 'fixture' }], score: async () => ({ dimensions: { score: 1 }, composite: 1 }) }],
+        runDir, seed: 42, runOptions: {}, costLedger: createRunCostLedger({ storage: fsCampaignStorage(), runDir: join(runDir, 'cost') }),
+      })).rejects.toThrow('transport-handshake-complete')
+      expect(requests).toHaveLength(1)
+    } finally {
+      await rm(runDir, { recursive: true, force: true })
+    }
+  })
+
   it("'gepa' is one bounded engine run carrying the full budget (default 10)", () => {
     const recipe = recipeForSeat(seat() as GepaSeatSpec)
     expect(recipe).toMatchObject({ kind: 'engine', run: { engine: 'gepa', maxEvaluations: DEFAULT_MAX_METRIC_CALLS } })