@tangle-network/agent-bench 0.7.1 → 0.8.0

package/src/humaneval-repair-gate.mts

@@ -20,9 +20,20 @@
  *     tsx src/humaneval-repair-gate.mts
  */
 import { type HumanEvalTask, basePrompt, extractCode, loadHumanEval, runChecker } from './benchmarks/humaneval'
-import { type RouterConfig, type ToolSpec, routerChatWithUsage, routerToolLoop } from '@tangle-network/agent-runtime/kernel'
+import {
+  collectAgentTurn,
+  createExecutor,
+  streamAgentTurn,
+  type ToolSpec,
+} from '@tangle-network/agent-runtime/kernel'
 import { verifierGroundedSelect } from './selector'
 import { type PairedLift, pairedLift, pool } from './stats.mts'
+import {
+  benchRouterProfile,
+  type BenchRouterTarget,
+  runBenchRouterTurn,
+  withBenchProfile,
+} from './router-turn'
 
 function must(name: string): string {
   const v = process.env[name]
@@ -51,14 +62,23 @@ const repairSystem = [
 ].join(' ')
 
 /** repair@K: one worker, up to K inference turns, steering on real test failures. */
-async function repairAttempt(cfg: RouterConfig, task: HumanEvalTask, k: number): Promise<number> {
+async function repairAttempt(cfg: BenchRouterTarget, task: HumanEvalTask, k: number): Promise<number> {
   let lastTested = ''
-  const r = await routerToolLoop(
-    cfg,
-    repairSystem,
-    basePrompt(task),
-    [runTestsTool],
-    async (name, args) => {
+  const profile = {
+    ...withBenchProfile(cfg.profile, {
+      name: 'humaneval-repair-worker',
+      systemPrompt: repairSystem,
+      maxTurns: k,
+      temperature: 0.3,
+    }),
+    tools: { run_tests: true },
+  }
+  const factory = createExecutor({
+    backend: 'router-tools',
+    routerBaseUrl: cfg.routerBaseUrl,
+    routerKey: cfg.routerKey,
+    tools: [runTestsTool],
+    executeToolCall: async (name, args) => {
       if (name !== 'run_tests') return `error: unknown tool ${name}`
       const code = extractCode(String(args.code ?? ''))
       lastTested = code
@@ -67,22 +87,37 @@ async function repairAttempt(cfg: RouterConfig, task: HumanEvalTask, k: number):
         ? 'ALL TESTS PASSED. Reply with the final function now; do not call run_tests again.'
         : `TESTS FAILED:\n${res.detail ?? 'no output'}\n\nFix the function and call run_tests again.`
     },
-    { maxTurns: k, temperature: 0.3 },
+  })
+  const r = await collectAgentTurn(
+    streamAgentTurn({ kind: 'executor', factory, profile }, basePrompt(task)),
   )
+  if (r.status !== 'completed') {
+    throw new Error(r.error?.message ?? `repair turn ended with status ${r.status}`)
+  }
   // Judge the model's final answer; fall back to the last code it tested (it may
   // report "done" without re-pasting the passing function).
-  const finalCode = extractCode(r.final) || lastTested
+  const finalCode = extractCode(r.finalText) || lastTested
   if (!finalCode) return 0
   return (await runChecker(task, finalCode)).pass
 }
 
 /** blind@K: K independent completions, verifier-grounded pick (the resample control). */
-async function blindAttempts(cfg: RouterConfig, task: HumanEvalTask, k: number): Promise<number[]> {
+async function blindAttempts(cfg: BenchRouterTarget, task: HumanEvalTask, k: number): Promise<number[]> {
   const base = basePrompt(task)
   const passes: number[] = []
   for (let i = 0; i < k; i += 1) {
-    const res = await routerChatWithUsage(cfg, [{ role: 'user', content: base }], { temperature: 0.8 })
-    passes.push((await runChecker(task, extractCode(res.content))).pass)
+    const res = await runBenchRouterTurn(
+      {
+        routerBaseUrl: cfg.routerBaseUrl,
+        routerKey: cfg.routerKey,
+        profile: withBenchProfile(cfg.profile, {
+          name: 'humaneval-blind-worker',
+          temperature: 0.8,
+        }),
+      },
+      base,
+    )
+    passes.push((await runChecker(task, extractCode(res.finalText))).pass)
   }
   return passes
 }
@@ -95,7 +130,11 @@ async function main(): Promise<void> {
   const k = Number(process.env.K ?? 3)
   const offset = Number(process.env.OFFSET ?? 82)
   const model = process.env.WORKER_MODEL ?? 'deepseek-v4-flash'
-  const cfg: RouterConfig = { routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1', routerKey: must('TANGLE_API_KEY'), model }
+  const cfg: BenchRouterTarget = {
+    routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',
+    routerKey: must('TANGLE_API_KEY'),
+    profile: benchRouterProfile('humaneval-worker', model),
+  }
   const concurrency = Number(process.env.CONCURRENCY ?? 6)
   if (k < 2) throw new Error('K must be >= 2 (repair needs at least write + one fix)')
 

package/src/mbpp-structural.mts

@@ -32,6 +32,7 @@ import { tmpdir } from 'node:os'
 import { join, resolve } from 'node:path'
 import { fileURLToPath } from 'node:url'
 import { extractCode } from './benchmarks/humaneval'
+import { runBenchRouterTurn } from './router-turn'
 import { type PairedLift, pairedLift, pool } from './stats.mts'
 
 const dockerImage = 'python:3.12-slim'
@@ -326,33 +327,40 @@ async function complete(cfg: ClientCfg, messages: Array<{ role: string; content:
   let lastErr = ''
   for (let attempt = 1; attempt <= 4; attempt += 1) {
     if (attempt > 1) await new Promise((r) => setTimeout(r, 2000 * 2 ** attempt))
-    const ctl = new AbortController()
-    const timer = setTimeout(() => ctl.abort(), 240_000)
     try {
-      const res = await fetch(`${cfg.base}/chat/completions`, {
-        method: 'POST',
-        headers: { Authorization: `Bearer ${cfg.key}`, 'Content-Type': 'application/json' },
-        body: JSON.stringify({ model: cfg.model, max_tokens: cfg.maxTokens, temperature: cfg.temperature, messages }),
-        signal: ctl.signal,
-      })
-      if (!res.ok) {
-        lastErr = `HTTP ${res.status}: ${(await res.text()).slice(0, 200)}`
-        continue
-      }
-      const d = (await res.json()) as {
-        choices?: Array<{ message?: { content?: string } }>
-        usage?: { prompt_tokens?: number; completion_tokens?: number }
-      }
-      const content = d.choices?.[0]?.message?.content ?? ''
+      const system = messages.find((message) => message.role === 'system')?.content
+      const result = await runBenchRouterTurn(
+        {
+          routerBaseUrl: cfg.base,
+          routerKey: cfg.key,
+          profile: {
+            name: 'mbpp-structural-worker',
+            harness: 'cli-base',
+            model: {
+              provider: 'tangle-router',
+              default: cfg.model,
+              metadata: { temperature: cfg.temperature, maxTokens: cfg.maxTokens },
+            },
+            ...(system ? { prompt: { systemPrompt: system } } : {}),
+          },
+          timeoutMs: Number(process.env.LLM_TIMEOUT_MS ?? 240_000),
+        },
+        { messages: messages.filter((message) => message.role !== 'system') },
+      )
+      if (result.usage.tokensKnown === false) throw new Error('provider omitted token usage')
+      const content = result.finalText
       if (content.trim() === '') {
         lastErr = 'empty content'
         continue
       }
-      return { content, attempts: attempt, tokensIn: d.usage?.prompt_tokens ?? 0, tokensOut: d.usage?.completion_tokens ?? 0 }
+      return {
+        content,
+        attempts: attempt,
+        tokensIn: result.usage.input,
+        tokensOut: result.usage.output,
+      }
     } catch (e) {
       lastErr = e instanceof Error ? e.message : String(e)
-    } finally {
-      clearTimeout(timer)
     }
   }
   throw new Error(`completion failed after retries: ${lastErr}`)

package/src/mcp-mount-probe.mts

@@ -14,6 +14,8 @@ import {
   type Agent,
   type AgentProfile,
   type AgentSpec,
+  collectAgentTurn,
+  createExecutor,
   createExecutorRegistry,
   createSupervisor,
   type Executor,
@@ -21,6 +23,7 @@ import {
   InMemoryResultBlobStore,
   InMemorySpawnJournal,
   type Scope,
+  streamAgentTurn,
   type UsageEvent,
 } from '../../src/runtime/index'
 import { serveCoordinationMcp } from '../../src/runtime/supervise/coordination-mcp'
@@ -53,18 +56,27 @@ function deliveringLeaf(name: string, out: unknown): Agent<unknown, unknown> {
 }
 
 async function bridgeChat(messages: Array<{ role: string; content: string }>, mcpUrl: string): Promise<string> {
-  const r = await fetch(`${BRIDGE.replace(/\/$/, '')}/chat/completions`, {
-    method: 'POST',
-    headers: { authorization: `Bearer ${BEARER}`, 'content-type': 'application/json' },
-    body: JSON.stringify({
-      model: MODEL,
-      messages,
-      mcp: { mcpServers: { coordination: { type: 'http', url: mcpUrl } } },
-    }),
+  if (!BEARER) throw new Error('TANGLE_API_KEY is required')
+  const profile: AgentProfile = {
+    name: 'mcp-mount-probe-supervisor',
+    model: { default: MODEL },
+    mcp: { coordination: { transport: 'http', url: mcpUrl } },
+  }
+  const factory = createExecutor({
+    backend: 'bridge',
+    bridgeUrl: BRIDGE.replace(/\/v1\/?$/u, ''),
+    bridgeBearer: BEARER,
   })
-  if (!r.ok) return `(bridge HTTP ${r.status}: ${(await r.text()).slice(0, 200)})`
-  const j = (await r.json()) as { choices?: Array<{ message?: { content?: string } }> }
-  return j.choices?.[0]?.message?.content ?? ''
+  const turn = await collectAgentTurn(
+    streamAgentTurn(
+      { kind: 'executor', factory, profile },
+      messages.map((message) => message.content).join('\n\n'),
+    ),
+  )
+  if (turn.status !== 'completed') {
+    throw new Error(turn.error?.message ?? `bridge turn ended with ${turn.status}`)
+  }
+  return turn.finalText
 }
 
 async function main(): Promise<void> {

package/src/official-optimizer-config.mts

@@ -1,3 +1,15 @@
+import type {
+  OpenAICompatibleOptimizerModel,
+  OptimizerModelBudget,
+} from '@tangle-network/agent-eval/campaign'
+import {
+  type AgentProfile,
+  canonicalAgentProfileDigest,
+  canonicalCandidateDigest,
+} from '@tangle-network/agent-interface'
+import { profileOptimizerModelCall } from '../../src/runtime/profile-chat-client'
+import type { RouterSeam } from '../../src/runtime/supervise/runtime'
+
 function requiredNonNegativeNumber(
   env: NodeJS.ProcessEnv,
   name: string,
@@ -57,22 +69,59 @@ export function officialOptimizerModel(options: {
   maxCostUsd: number
   maxOutputTokensPerRequest: number
   envPrefix?: string
-}) {
+  provider?: string
+  temperature?: number
+  reasoningEffort?: NonNullable<AgentProfile['model']>['reasoningEffort']
+  callRef?: string
+  complete?: RouterSeam['complete']
+}): OpenAICompatibleOptimizerModel {
   const { env } = options
   const envPrefix = options.envPrefix ?? 'REFLECT'
+  const budget: OptimizerModelBudget = {
+    maxCostUsd: options.maxCostUsd,
+    maxRequests: positiveInteger(env, `${envPrefix}_MAX_REQUESTS`, 100),
+    maxRequestBytes: positiveInteger(env, `${envPrefix}_MAX_REQUEST_BYTES`, 2_000_000),
+    maxResponseBytes: positiveInteger(env, `${envPrefix}_MAX_RESPONSE_BYTES`, 2_000_000),
+    maxOutputTokensPerRequest: options.maxOutputTokensPerRequest,
+    requestTimeoutMs: positiveInteger(env, `${envPrefix}_REQUEST_TIMEOUT_MS`, 300_000),
+    pricing: requiredTokenPricing(env, envPrefix),
+  }
+  const profile: AgentProfile = {
+    name: 'official-optimizer-model',
+    harness: 'cli-base',
+    model: {
+      provider: options.provider ?? new URL(options.baseUrl).hostname,
+      default: options.model,
+      ...(options.reasoningEffort ? { reasoningEffort: options.reasoningEffort } : {}),
+      metadata: {
+        maxTokens: options.maxOutputTokensPerRequest,
+        ...(options.temperature !== undefined ? { temperature: options.temperature } : {}),
+      },
+    },
+  }
+  const profileDigest = canonicalAgentProfileDigest(profile)
+  const executor: RouterSeam & { backend: 'router' } = {
+    backend: 'router',
+    routerBaseUrl: options.baseUrl,
+    routerKey: options.apiKey,
+    ...(options.complete ? { complete: options.complete } : {}),
+  }
+  const call = profileOptimizerModelCall({
+    profile,
+    context: 'official optimizer model',
+    executor,
+    pricing: budget.pricing,
+  })
   return {
     model: options.model,
-    baseUrl: options.baseUrl,
-    apiKey: options.apiKey,
-    budget: {
-      maxCostUsd: options.maxCostUsd,
-      maxRequests: positiveInteger(env, `${envPrefix}_MAX_REQUESTS`, 100),
-      maxRequestBytes: positiveInteger(env, `${envPrefix}_MAX_REQUEST_BYTES`, 2_000_000),
-      maxResponseBytes: positiveInteger(env, `${envPrefix}_MAX_RESPONSE_BYTES`, 2_000_000),
-      maxOutputTokensPerRequest: options.maxOutputTokensPerRequest,
-      requestTimeoutMs: positiveInteger(env, `${envPrefix}_REQUEST_TIMEOUT_MS`, 300_000),
-      pricing: requiredTokenPricing(env, envPrefix),
-    },
+    callRef:
+      options.callRef ??
+      `agent-runtime:${canonicalCandidateDigest({
+        profileDigest,
+        endpoint: new URL(options.baseUrl).origin,
+      })}`,
+    call,
+    budget,
   }
 }
 

package/src/official-optimizer-config.test.mts

@@ -30,10 +30,8 @@ describe('official optimizer configuration', () => {
       maxOutputTokensPerRequest: 4000,
     })
 
-    expect(model).toEqual({
+    expect(model).toMatchObject({
       model: 'test-model',
-      baseUrl: 'http://127.0.0.1:8080/v1',
-      apiKey: 'test-key',
       budget: {
         maxCostUsd: 2,
         maxRequests: 7,
@@ -49,6 +47,77 @@ describe('official optimizer configuration', () => {
         },
       },
     })
+    expect(model.call).toEqual(expect.any(Function))
+    expect(model.callRef).toMatch(/^agent-runtime:sha256:/)
+    expect(model).not.toHaveProperty('baseUrl')
+    expect(model).not.toHaveProperty('apiKey')
+  })
+
+  it('executes the published optimizer callback through the exact Runtime profile', async () => {
+    const requests: unknown[] = []
+    const model = officialOptimizerModel({
+      env: pricingEnv,
+      envPrefix: 'OPT',
+      model: 'test-model',
+      baseUrl: 'http://127.0.0.1:8080/v1',
+      apiKey: 'test-key',
+      maxCostUsd: 2,
+      maxOutputTokensPerRequest: 4000,
+      complete: async (request) => {
+        requests.push(request)
+        return {
+          model: 'test-model',
+          choices: [{ message: { content: 'ok' }, finish_reason: 'stop' }],
+          usage: {
+            prompt_tokens: 3,
+            completion_tokens: 2,
+            cost: 0.001,
+            prompt_tokens_details: { cached_tokens: 1 },
+            completion_tokens_details: { reasoning_tokens: 1 },
+          },
+        }
+      },
+    })
+
+    const result = await model.call({
+      callId: 'optimizer-call-1',
+      request: {
+        model: 'test-model',
+        messages: [{ role: 'user', content: 'hi' }],
+        maxTokens: 4000,
+      },
+      endpointFormat: 'chat-completions',
+      signal: new AbortController().signal,
+    })
+
+    expect(result.succeeded).toBe(true)
+    if (!result.succeeded) throw new Error(result.error)
+    expect(result.response).toMatchObject({
+      model: 'test-model',
+      content: 'ok',
+      usage: { promptTokens: 3, completionTokens: 2, totalTokens: 5 },
+    })
+    expect(result.receipt).toMatchObject({
+      model: 'test-model',
+      inputTokens: 2,
+      cachedTokens: 1,
+      outputTokens: 2,
+      reasoningTokens: 1,
+      actualCostUsd: 0.001,
+    })
+    expect(result.execution).toMatchObject({
+      kind: 'agent-runtime-profile-model-call',
+      executed: true,
+      succeeded: true,
+      model: 'test-model',
+      callId: 'optimizer-call-1',
+      endpointFormat: 'chat-completions',
+    })
+    expect(requests).toHaveLength(1)
+    expect(requests[0]).toMatchObject({
+      model: 'test-model',
+      max_tokens: 4000,
+    })
   })
 
   it('requires every token price instead of inventing cost data', () => {

package/src/quant-arena/profiles/quant-researcher.profile.json

@@ -1,6 +1,11 @@
 {
   "name": "quant-researcher",
-  "description": "Strategy-author lens that speaks quant language: regimes, turnover, capacity, transaction costs. Same claude harness as default-author; only the system prompt differs, so any performance delta is attributable to the lens.",
+  "description": "Pi strategy-author lens that speaks quant language: regimes, turnover, capacity, transaction costs. The model and harness match the default author; only the system prompt differs.",
+  "harness": "pi",
+  "model": {
+    "provider": "tangle-router",
+    "default": "glm-5.2"
+  },
   "prompt": {
     "systemPrompt": "You are a systematic-trading researcher at a mid-frequency equity desk. You think in regimes (trend vs chop, calm vs stress), you treat turnover as a cost center (every rebalance pays spread + impact), you size positions for capacity, and you are allergic to backtest overfitting — you prefer one robust effect over five fitted parameters. When you write a strategy you state, in comments, the economic rationale the rule is harvesting and why it should survive out-of-sample."
   }