@tangle-network/agent-bench 0.8.10 → 0.8.15

package/src/humaneval-object-ablation.mts

@@ -1,239 +0,0 @@
-/**
- * OBJECT-OF-IMPROVEMENT ablation on HumanEval — the head-on test of the claim our
- * whole self-improvement line rests on: at EQUAL budget, does adding CAPABILITY
- * (a real code-execution tool) beat adding IMPROVER-cleverness (blind prompt
- * self-refinement)? Both arms get the same model, the same K rounds per task, the
- * same held-out tasks. The ONLY difference is what the budget buys:
- *
- *   IMPROVER arm  — K rounds of "critique your own function and rewrite it",
- *                   with NO execution. The classic reflect-without-a-tool loop.
- *   CAPABILITY arm — K rounds WITH a `run_python` tool: the agent executes its
- *                   function on inputs it chooses, sees real output/errors, fixes.
- *
- * Grading is a DETERMINISTIC hidden test (never shown): the task's own `check`.
- * If the capability arm wins the held-out pass rate, the object of improvement
- * (what you can DO) dominates the improver (how cleverly you rewrite) — the
- * finding the prompt-only self-improvement runs kept nulling on.
- *
- * Fast by construction: HumanEval tasks are tiny, graded in an isolated Python
- * container with a hard timeout — seconds per task. Paired
- * McNemar over the per-task pass/fail difference gives the significance.
- *
- * Run from cwd=bench:  env WORKER_MODEL=deepseek-v4-flash N=60 K=3 \
- *   REPS=2 node_modules/.bin/tsx src/humaneval-object-ablation.mts
- */
-import {
-  loadHumanEval,
-  extractCode,
-  runPythonProgram,
-  type HumanEvalTask,
-} from './benchmarks/humaneval'
-import { runBenchRouterTurn } from './router-turn'
-
-const ROUTER = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
-const KEY = process.env.TANGLE_API_KEY
-if (!KEY) throw new Error('TANGLE_API_KEY required')
-const ROUTER_KEY = KEY
-const MODEL = process.env.WORKER_MODEL ?? 'deepseek-v4-flash'
-const N = Number(process.env.N ?? 60)
-const OFFSET = Number(process.env.OFFSET ?? 0)
-const K = Number(process.env.K ?? 3) // rounds/budget per task (equal for both arms)
-const REPS = Number(process.env.REPS ?? 2)
-const CONC = Number(process.env.CONCURRENCY ?? 6)
-const EXEC_TIMEOUT = Number(process.env.EXEC_TIMEOUT_MS ?? 8000)
-
-interface ChatMsg extends Record<string, unknown> { role: string; content: string; tool_calls?: unknown; tool_call_id?: string; name?: string }
-interface Tool { type: 'function'; function: { name: string; description: string; parameters: unknown } }
-
-async function router(messages: ChatMsg[], tools?: Tool[]): Promise<{ content: string; toolCalls: { id: string; name: string; args: Record<string, unknown> }[] }> {
-  for (let attempt = 0; ; attempt++) {
-    try {
-      const system = messages.find((message) => message.role === 'system')?.content
-      const result = await runBenchRouterTurn(
-        {
-          routerBaseUrl: ROUTER,
-          routerKey: ROUTER_KEY,
-          profile: {
-            name: 'humaneval-object-ablation-worker',
-            harness: 'cli-base',
-            model: {
-              provider: 'tangle-router',
-              default: MODEL,
-              metadata: {
-                temperature: 0.4,
-                ...(tools ? { toolChoice: 'auto' } : {}),
-              },
-            },
-            ...(system ? { prompt: { systemPrompt: system } } : {}),
-            ...(tools
-              ? { tools: Object.fromEntries(tools.map((tool) => [tool.function.name, true])) }
-              : {}),
-          },
-          ...(tools ? { tools } : {}),
-          timeoutMs: Number(process.env.LLM_TIMEOUT_MS ?? 60_000),
-        },
-        { messages: messages.filter((message) => message.role !== 'system') },
-      )
-      const toolCalls = result.toolCalls.map((call) => {
-        if (call.id === undefined) {
-          throw new Error(`router tool call '${call.name}' omitted its required id`)
-        }
-        let args: Record<string, unknown> = {}
-        try {
-          args = JSON.parse(call.arguments) as Record<string, unknown>
-        } catch {
-          // Keep the empty argument object; the tool returns a useful error.
-        }
-        return { id: call.id, name: call.name, args }
-      })
-      return { content: result.finalText, toolCalls }
-    } catch (error) {
-      const message = error instanceof Error ? error.message : String(error)
-      const status = Number(/router (\d+)/.exec(message)?.[1])
-      const transient =
-        !Number.isFinite(status) || [408, 429, 500, 502, 503, 504, 520, 522, 524].includes(status)
-      if (!transient || attempt >= 5) throw error
-      await sleep(800 * 2 ** attempt)
-    }
-  }
-}
-const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms))
-/** Run model-written Python in the shared networkless, resource-capped container. */
-async function runPython(program: string): Promise<{ stdout: string; stderr: string; ok: boolean }> {
-  const result = await runPythonProgram(program, EXEC_TIMEOUT)
-  return {
-    stdout: result.stdout.slice(0, 2000),
-    stderr: result.stderr.slice(0, 2000),
-    ok: result.exitCode === 0,
-  }
-}
-
-/** The HIDDEN judge: candidate full function + the task's own check. Never shown. */
-async function judge(task: HumanEvalTask, candidate: string): Promise<boolean> {
-  if (!candidate.trim()) return false
-  const program = `${candidate}\n\n${task.test}\n\ncheck(${task.entryPoint})\nprint("PASS")\n`
-  const r = await runPython(program)
-  return r.ok && r.stdout.includes('PASS')
-}
-
-const SYSTEM = 'You are an expert Python programmer. Output the COMPLETE function definition (signature + body, plus any imports) in a single ```python block. No tests, no prose outside the block.'
-const userPrompt = (t: HumanEvalTask) => `Complete this function:\n\n\`\`\`python\n${t.prompt}\`\`\``
-
-/** IMPROVER arm: K rounds of blind self-refinement — no execution, just "review and rewrite". */
-async function improverArm(task: HumanEvalTask): Promise<string> {
-  const messages: ChatMsg[] = [{ role: 'system', content: SYSTEM }, { role: 'user', content: userPrompt(task) }]
-  let code = ''
-  for (let round = 0; round < K; round++) {
-    const { content } = await router(messages)
-    code = extractCode(content) || content
-    if (round < K - 1) {
-      messages.push({ role: 'assistant', content })
-      messages.push({ role: 'user', content: 'Carefully review your function for correctness bugs and edge cases. If it can be improved, output the corrected COMPLETE function in a python block; if it is already correct, output it again unchanged.' })
-    }
-  }
-  return code
-}
-
-const RUN_TOOL: Tool = { type: 'function', function: { name: 'run_python', description: 'Execute a Python snippet and return its stdout/stderr. Use it to test your function on example inputs from the docstring before finalizing.', parameters: { type: 'object', properties: { code: { type: 'string', description: 'python source to run' } }, required: ['code'] } } }
-
-/** CAPABILITY arm: K rounds WITH a real code-execution tool — write, run, see real output, fix. */
-async function capabilityArm(task: HumanEvalTask): Promise<string> {
-  const messages: ChatMsg[] = [
-    { role: 'system', content: `${SYSTEM}\nYou have a run_python tool: test your function on the docstring's example inputs before giving your final answer. Fix any failures you observe.` },
-    { role: 'user', content: userPrompt(task) },
-  ]
-  let code = ''
-  // (K-1) tool-exploration rounds + 1 forced final answer = K calls total, matching
-  // the improver arm's K blind-refine rounds (equal budget).
-  for (let round = 0; round < Math.max(1, K - 1); round++) {
-    const { content, toolCalls } = await router(messages, [RUN_TOOL])
-    if (content && extractCode(content)) code = extractCode(content)
-    messages.push({ role: 'assistant', content: content || '', ...(toolCalls.length ? { tool_calls: toolCalls.map((t) => ({ id: t.id, type: 'function', function: { name: t.name, arguments: JSON.stringify(t.args) } })) } : {}) })
-    if (toolCalls.length) {
-      for (const tc of toolCalls) {
-        const snippet = String(tc.args.code ?? '')
-        const r = await runPython(snippet)
-        messages.push({ role: 'tool', tool_call_id: tc.id, name: tc.name, content: `stdout:\n${r.stdout}\nstderr:\n${r.stderr}\nexit_ok=${r.ok}` })
-      }
-    } else {
-      messages.push({ role: 'user', content: 'Test your function with run_python on the docstring examples before finalizing.' })
-    }
-  }
-  // FAIR FINAL ANSWER: the tool rounds are exploration; force one no-tool call to
-  // emit the complete function. Without this, an agent that ends mid-tool-call
-  // yields empty code and is unfairly scored 0 (an extraction artifact, not a
-  // real "the tool hurt" signal). Only override if it produces a real block.
-  {
-    messages.push({ role: 'user', content: 'Now output your FINAL complete function in a single ```python block, no tools, no prose.' })
-    const { content } = await router(messages)
-    const finalCode = extractCode(content)
-    if (finalCode.trim()) code = finalCode
-  }
-  return code
-}
-
-async function pool<T, R>(items: T[], limit: number, fn: (t: T, i: number) => Promise<R>): Promise<R[]> {
-  const out = new Array<R>(items.length)
-  let next = 0
-  await Promise.all(Array.from({ length: Math.min(limit, items.length) }, async () => {
-    while (next < items.length) { const i = next++; out[i] = await fn(items[i]!, i) }
-  }))
-  return out
-}
-
-// McNemar exact (paired): b = capability-only wins, c = improver-only wins.
-function mcnemarP(b: number, c: number): number {
-  const n = b + c; if (n === 0) return 1
-  const k = Math.min(b, c)
-  const lf = (x: number) => { let s = 0; for (let i = 2; i <= x; i++) s += Math.log(i); return s }
-  let tail = 0; for (let i = 0; i <= k; i++) tail += Math.exp(lf(n) - lf(i) - lf(n - i) - n * Math.log(2))
-  return Math.min(1, 2 * tail)
-}
-
-async function main(): Promise<void> {
-  if (['1', 'true'].includes((process.env.SMOKE ?? '').toLowerCase())) { console.error('SMOKE ok: humaneval-object-ablation loaded'); return }
-  const tasks = await loadHumanEval(N, OFFSET)
-  console.error(`=== OBJECT-OF-IMPROVEMENT ablation · HumanEval n=${tasks.length} (offset ${OFFSET}) · model=${MODEL} · K=${K} rounds · reps=${REPS} · equal budget ===`)
-  // self-check the local grader on the gold solution of task 0 (never shown to the model)
-  if (tasks[0]?.canonicalSolution) {
-    const gold = `${tasks[0].prompt}${tasks[0].canonicalSolution}`
-    console.error(`  grader self-check (gold passes): ${await judge(tasks[0], gold)}`)
-  }
-
-  // per (task, rep): run both arms; record pass/fail.
-  const units = tasks.flatMap((task) => Array.from({ length: REPS }, (_, rep) => ({ task, rep })))
-  let done = 0
-  let capEmpty = 0
-  const results = await pool(units, CONC, async ({ task }) => {
-    // Per-unit resilience: a transient model failure (e.g. a weak model emitting a
-    // malformed tool call → router 400) scores that arm 0 for this unit, never
-    // crashes the whole run. Both arms wrapped identically so neither is favored.
-    const safe = async (fn: () => Promise<string>) => { try { return await fn() } catch { return '' } }
-    const [impCode, capCode] = await Promise.all([safe(() => improverArm(task)), safe(() => capabilityArm(task))])
-    const [imp, cap] = await Promise.all([judge(task, impCode), judge(task, capCode)])
-    done++
-    // AUTOPSY: a capability-arm failure with EMPTY final code is an extraction
-    // artifact (ended mid-tool-call, never emitted a final function), NOT a real
-    // "the tool hurt" signal. Count it so the effect can be separated.
-    if (!cap && !capCode.trim()) { capEmpty++; if (imp) console.error(`  [cap-empty] ${task.taskId} (improver passed)`) }
-    if (done % 20 === 0) console.error(`  ${done}/${units.length} units`)
-    return { id: task.taskId, imp, cap }
-  })
-  console.error(`  capability-arm empty-final-code (artifact) count: ${capEmpty}/${results.length}`)
-
-  const impPass = results.filter((r) => r.imp).length
-  const capPass = results.filter((r) => r.cap).length
-  const b = results.filter((r) => r.cap && !r.imp).length // capability-only wins
-  const c = results.filter((r) => !r.cap && r.imp).length // improver-only wins
-  const p = mcnemarP(b, c)
-  const n = results.length
-  const liftPp = ((capPass - impPass) / n) * 100
-
-  console.log('')
-  console.log('=== RESULT (held-out HumanEval, equal budget) ===')
-  console.log(`  IMPROVER   (blind self-refine, no tool): ${impPass}/${n} = ${((impPass / n) * 100).toFixed(1)}%`)
-  console.log(`  CAPABILITY (self-built execution tool)  : ${capPass}/${n} = ${((capPass / n) * 100).toFixed(1)}%`)
-  console.log(`  lift = ${liftPp >= 0 ? '+' : ''}${liftPp.toFixed(1)}pp   paired McNemar: capability-only=${b} improver-only=${c} p=${p.toFixed(4)}`)
-  console.log(`  verdict: ${p < 0.05 && capPass > impPass ? 'CAPABILITY > IMPROVER (significant)' : p < 0.05 && impPass > capPass ? 'IMPROVER > CAPABILITY (significant)' : 'no significant difference'}`)
-}
-main().catch((e) => { console.error('MAIN:', e instanceof Error ? (e.stack ?? e.message) : e); process.exit(1) })

package/src/trata-gate.mts

@@ -1,243 +0,0 @@
-/**
- * Trata hedge-bench gate — direct router completions (no sandbox/opencode).
- *
- * Trata is a text-analysis benchmark: each task is a financial analysis brief
- * over embedded source docs. The worker is a single router completion (not an
- * agentic loop), which is appropriate here: tasks are scored by an LLM judge on
- * theme coverage, not on code execution or tool use.
- *
- * Usage:
- *   BENCH=trata-hedge WORKER_MODEL=deepseek-v4-flash JUDGE_MODEL=gemini-2.5-flash \
- *   TRATA_BENCH_ROOT=/tmp/trata-hedge-bench N=10 CONCURRENCY=5 \
- *   dotenvx run -f ~/company/devops/secrets/agent-state.env -- \
- *     pnpm exec tsx bench/src/trata-gate.mts
- *
- * Key env vars:
- *   BENCH              adapter key (default: trata-hedge)
- *   WORKER_MODEL       model for the analysis completion (default: deepseek-v4-flash)
- *   JUDGE_MODEL        model for the 3-stage judge (default: gemini-2.5-pro)
- *   TRATA_BENCH_ROOT   path to cloned trata-hedge-bench (default: /tmp/trata-hedge-bench)
- *   N                  how many tasks to run (default: 10; 0 = all 102)
- *   CONCURRENCY        parallel worker slots (default: 5)
- *   IDS                comma-separated task ids to run a specific subset
- *   ROUTER_BASE        router base URL (default: https://router.tangle.tools/v1)
- *   CORPUS             path to write JSONL run records (optional)
- *
- * Score is 0–4 / 4 = 0–100%; resolved = 4/4 (all themes + synthesis = sparse reward).
- * Partial credit (1-3) is reported in the score distribution.
- */
-
-import { appendFileSync } from 'node:fs'
-import { resolveAdapter } from './adapters'
-import type { BenchScore, BenchTask } from './benchmarks/types'
-import { runBenchRouterTurn } from './router-turn'
-import { runPool } from './run-pool'
-
-function must(name: string): string {
-  const v = process.env[name]
-  if (!v) throw new Error(`env ${name} is required`)
-  return v
-}
-
-// GEPA-optimised surface — +8.6pp holdout lift over the minimal baseline on deepseek-v4-flash
-// (2 independent runs, CI [0, 12.5pp]; gate holds for caution, surface ships as default).
-const ANALYST_SYSTEM = process.env.SYSTEM_PROMPT ?? [
-  'You are a senior financial analyst producing a structured investment memo.',
-  'Begin your response with exactly "ANALYSIS:" on its own line (nothing before it), then write your full analysis.',
-  'Structure your analysis with clearly labeled sections for each distinct analytical theme',
-  '(e.g., valuation, capital allocation, competitive dynamics, risk factors)',
-  'so that no major investment consideration is merged or omitted.',
-  'Benchmark the company explicitly against named sector peers, citing specific metrics such as',
-  'EV/EBITDA, P/E ratios, margin differentials, and growth premiums from the peer financial files.',
-  'Every factual claim must cite the filename (e.g., "per earnings_call/apo_q4_2025_earnings_call.txt").',
-  'Identify and verbatim-cite specific numerical targets from management guidance such as',
-  'earnings per share targets, margin percentages, growth rates, or AUM figures rather than paraphrasing approximately.',
-  'When evaluating capital allocation options, explicitly compute implied returns or internal rates of return (IRRs),',
-  'showing the arithmetic using price levels and targets from the source data.',
-  'Take a clear, decisive position — do not hedge with "it depends". Reconcile conflicting data points explicitly.',
-].join(' ')
-
-async function workerComplete(
-  task: BenchTask,
-  cfg: { routerBaseUrl: string; routerKey: string; model: string; timeoutMs: number },
-): Promise<{ answer: string; inputTokens: number; outputTokens: number; durationMs: number }> {
-  const startedAt = Date.now()
-  const result = await runBenchRouterTurn(
-    {
-      routerBaseUrl: cfg.routerBaseUrl,
-      routerKey: cfg.routerKey,
-      profile: {
-        name: 'trata-financial-analyst',
-        harness: 'cli-base',
-        model: {
-          provider: 'tangle-router',
-          default: cfg.model,
-          metadata: {
-            temperature: 0,
-            maxTokens: Number(process.env.WORKER_MAX_TOKENS ?? 4096),
-          },
-        },
-        prompt: { systemPrompt: ANALYST_SYSTEM },
-      },
-      timeoutMs: cfg.timeoutMs,
-    },
-    task.prompt,
-  )
-  if (result.usage.tokensKnown === false) throw new Error('worker provider omitted token usage')
-  return {
-    answer: result.finalText,
-    inputTokens: result.usage.input,
-    outputTokens: result.usage.output,
-    durationMs: Date.now() - startedAt,
-  }
-}
-
-interface TaskResult {
-  taskId: string
-  answer: string
-  score: BenchScore
-  rawScore: number
-  inputTokens: number
-  outputTokens: number
-  durationMs: number
-  error?: string
-}
-
-async function runTask(
-  task: BenchTask,
-  workerCfg: { routerBaseUrl: string; routerKey: string; model: string; timeoutMs: number },
-  adapter: ReturnType<typeof resolveAdapter>,
-): Promise<TaskResult> {
-  let answer = ''
-  let inputTokens = 0
-  let outputTokens = 0
-  let durationMs = 0
-  let error: string | undefined
-
-  try {
-    const result = await workerComplete(task, workerCfg)
-    answer = result.answer
-    inputTokens = result.inputTokens
-    outputTokens = result.outputTokens
-    durationMs = result.durationMs
-  } catch (err) {
-    error = err instanceof Error ? err.message : String(err)
-  }
-
-  const score = answer.trim()
-    ? await adapter.judge(task, answer).catch((err) => ({
-        resolved: false as const,
-        score: 0,
-        detail: JSON.stringify({ error: err instanceof Error ? err.message : String(err) }),
-      }))
-    : { resolved: false as const, score: 0, detail: JSON.stringify({ error: error ?? 'empty answer' }) }
-
-  const detail = score.detail ? (JSON.parse(score.detail) as { rawScore?: number }) : {}
-  return {
-    taskId: task.id,
-    answer,
-    score,
-    rawScore: detail.rawScore ?? (score.resolved ? 4 : 0),
-    inputTokens,
-    outputTokens,
-    durationMs,
-    error,
-  }
-}
-
-async function main(): Promise<void> {
-  const benchName = process.env.BENCH ?? 'trata-hedge'
-  const adapter = resolveAdapter(benchName)
-  await adapter.preflight()
-
-  const n = Number(process.env.N ?? 10)
-  const concurrency = Number(process.env.CONCURRENCY ?? 5)
-  const model = process.env.WORKER_MODEL ?? 'deepseek-v4-flash'
-  const routerBase = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
-  const routerKey = must('TANGLE_API_KEY')
-  const corpusPath = process.env.CORPUS
-  const idsEnv = process.env.IDS
-
-  const loadOpts = idsEnv
-    ? { ids: idsEnv.split(',').map((s) => s.trim()).filter(Boolean) }
-    : n > 0
-      ? { limit: n }
-      : {}
-  const tasks = await adapter.loadTasks(loadOpts)
-  if (tasks.length === 0) throw new Error('no tasks loaded')
-
-  const workerCfg = { routerBaseUrl: routerBase, routerKey, model, timeoutMs: 180_000 }
-
-  console.log(
-    `trata-gate: ${tasks.length} tasks | model=${model} | judge=${process.env.JUDGE_MODEL ?? 'gemini-2.5-pro'} | concurrency=${concurrency}`,
-  )
-
-  let done = 0
-  const results: TaskResult[] = []
-  await runPool(tasks, concurrency, async (task) => {
-    const r = await runTask(task, workerCfg, adapter)
-    results.push(r)
-    done++
-    const icon = r.rawScore === 4 ? '✓' : r.rawScore >= 2 ? '~' : '·'
-    process.stdout.write(
-      `  [${done}/${tasks.length}] ${task.id.slice(0, 60)}: ${icon} (${r.rawScore}/4)\n`,
-    )
-    return r
-  })
-
-  // Score distribution
-  const byRaw = [0, 0, 0, 0, 0] // index = rawScore 0-4
-  let totalInputTok = 0
-  let totalOutputTok = 0
-  let totalMs = 0
-  for (const r of results) {
-    byRaw[Math.min(r.rawScore, 4)]++
-    totalInputTok += r.inputTokens
-    totalOutputTok += r.outputTokens
-    totalMs += r.durationMs
-  }
-  const resolved = byRaw[4] ?? 0
-  const partial = (byRaw[2] ?? 0) + (byRaw[3] ?? 0)
-  const n_ = results.length
-  const pct = (x: number) => (n_ > 0 ? `${((x / n_) * 100).toFixed(1)}%` : 'n/a')
-  const meanScore = n_ > 0 ? results.reduce((s, r) => s + r.score.score, 0) / n_ : 0
-
-  console.log(`\n=== ${benchName} — trata-gate (n=${n_}, model=${model}) ===`)
-  console.log(`  resolved (4/4):       ${pct(resolved)}  (${resolved}/${n_})`)
-  console.log(`  partial credit (2-3): ${pct(partial)}  (${partial}/${n_})`)
-  console.log(`  mean score (0..1):    ${meanScore.toFixed(3)}`)
-  console.log(`  score dist [0..4]:    ${byRaw.join(' | ')}`)
-  console.log(
-    `  tokens (in/out):     ${(totalInputTok / 1000).toFixed(0)}k / ${(totalOutputTok / 1000).toFixed(0)}k`,
-  )
-  console.log(`  wall time:            ${(totalMs / 1000).toFixed(0)}s total`)
-  console.log(`  errors:               ${results.filter((r) => r.error).length}`)
-
-  if (corpusPath) {
-    for (const r of results) {
-      appendFileSync(
-        corpusPath,
-        JSON.stringify({
-          benchmark: benchName,
-          instanceId: r.taskId,
-          condition: `blind@1`,
-          model,
-          resolved: r.score.resolved,
-          rawScore: r.rawScore,
-          score: r.score.score,
-          detail: r.score.detail,
-          inputTokens: r.inputTokens,
-          outputTokens: r.outputTokens,
-          durationMs: r.durationMs,
-          ...(r.error ? { error: r.error } : {}),
-        }) + '\n',
-      )
-    }
-    console.log(`\ncorpus written → ${corpusPath}`)
-  }
-}
-
-main().catch((err) => {
-  console.error(err instanceof Error ? (err.stack ?? err.message) : String(err))
-  process.exit(1)
-})