@tangle-network/agent-bench 0.8.10 → 0.8.15

package/src/diverse-gate.mjs

@@ -1,112 +0,0 @@
-// The beat-blind gate, as ONE command — the experiment that decides whether the
-// diversification/selection surface (the one PR #145 defers to as "tracked separately")
-// actually beats compute-matched best-of-N. Composes only LANDED pieces:
-//   1. random@k corpus  — k identical-directive shots/instance (the compute control)
-//   2. diverse@k corpus — k DIFFERENT strategy lenses/instance (DIVERSE=1; the bet)
-//   3. selector replay  — selfConsistencySelect@k over each (corpus-replay --selector)
-//   4. paired report    — bootstrap CI + Benjamini-Hochberg over both (corpus-report)
-//
-// The decomposition it yields:
-//   random@k           = more-compute, no picking          (control)
-//   selector@k (homog) = picking over IDENTICAL attempts   (#143: −8.2pp on the committed corpus)
-//   diverse-selector@k = picking over DIVERSE attempts      (THE bet: does approach-diversity
-//                        give self-consistency the signal identical attempts don't?)
-// Beat-blind iff diverse-selector@k > random@k at significant n.
-//
-//   node diverse-gate.mjs            run it (generates corpora — a real worker run)
-//   node diverse-gate.mjs --dry      print the plan only (no run; safe while another
-//                                    sandbox run is live — zero router/sandbox contention)
-//
-// Knobs (env): BENCH (default hotpotqa) · N (default 30) · K (default 4) ·
-//   RESEARCH=1 (local opencode, default) | SANDBOX=1 (prod sandbox web worker) · MODELS ·
-//   DIVERSE_BASE (compose #145's GEPA-learned directive as the lens base — follow-on).
-
-import { spawn } from 'node:child_process'
-
-const DRY = process.argv.includes('--dry')
-const BENCH = process.env.BENCH ?? 'hotpotqa'
-const N = process.env.N ?? '30'
-const K = process.env.K ?? '4'
-const RANDOM_CORPUS = process.env.RANDOM_CORPUS ?? '/tmp/dg-random.jsonl'
-const DIVERSE_CORPUS = process.env.DIVERSE_CORPUS ?? '/tmp/dg-diverse.jsonl'
-// Worker-mode env passes through to batch-oracle unchanged (RESEARCH=1 / SANDBOX=1 / MODELS / TANGLE_API_KEY).
-const passEnv = { ...process.env, BENCH, N, K }
-
-const steps = [
-  {
-    label: 'random@k corpus (control — identical directive)',
-    cmd: 'npx',
-    args: ['tsx', 'src/run.ts', 'batch-oracle', N],
-    env: { ...passEnv, CORPUS: RANDOM_CORPUS },
-  },
-  {
-    label: 'diverse@k corpus (the bet — k distinct strategy lenses)',
-    cmd: 'npx',
-    args: ['tsx', 'src/run.ts', 'batch-oracle', N],
-    env: { ...passEnv, CORPUS: DIVERSE_CORPUS, DIVERSE: '1' },
-  },
-  {
-    label: 'selector@k over the CONTROL corpus (homogeneous)',
-    cmd: 'npx',
-    args: ['tsx', 'src/corpus-replay.mts', RANDOM_CORPUS, '--selector'],
-    env: passEnv,
-  },
-  {
-    label: 'selector@k over the DIVERSE corpus (the beat-blind number)',
-    cmd: 'npx',
-    // The diverse corpus records carry condition="diverse@4"; corpus-replay's
-    // selector filter defaults to "random", so match the diverse condition here.
-    args: ['tsx', 'src/corpus-replay.mts', DIVERSE_CORPUS, '--selector', '--condition=diverse'],
-    env: passEnv,
-  },
-  {
-    label: 'paired bootstrap CI + Benjamini-Hochberg over both corpora',
-    cmd: 'npx',
-    args: ['tsx', 'src/corpus-report.mts', RANDOM_CORPUS, DIVERSE_CORPUS],
-    env: passEnv,
-  },
-]
-
-const shellPreview = (s) => {
-  const envStr = Object.entries(s.env)
-    .filter(([k]) => ['BENCH', 'N', 'K', 'CORPUS', 'DIVERSE', 'RESEARCH', 'SANDBOX', 'MODELS'].includes(k))
-    .map(([k, v]) => `${k}=${v}`)
-    .join(' ')
-  return `  ${envStr} ${s.cmd} ${s.args.join(' ')}`.replace(/\s+/g, ' ')
-}
-
-function runStep(s) {
-  return new Promise((resolve, reject) => {
-    console.log(`\n▶ ${s.label}`)
-    const child = spawn(s.cmd, s.args, { cwd: process.cwd(), env: s.env, stdio: 'inherit' })
-    child.on('error', reject)
-    // Fail loud: a non-zero step aborts the gate (no silent partial result).
-    child.on('exit', (code) => (code === 0 ? resolve() : reject(new Error(`${s.label} exited ${code}`))))
-  })
-}
-
-async function main() {
-  console.log(`=== beat-blind gate · BENCH=${BENCH} N=${N} K=${K} ${DRY ? '(DRY — plan only)' : ''} ===`)
-  if (DRY) {
-    console.log('plan (no run — zero sandbox/router contention):')
-    for (const s of steps) console.log(shellPreview(s))
-    console.log(
-      '\nbeat-blind iff diverse-selector@k > random@k at significant n.' +
-        '\nDIVERSE_BASE=<file> composes #145\'s GEPA-learned directive as the lens base (follow-on).',
-    )
-    return
-  }
-  for (const s of steps) await runStep(s)
-  console.log(
-    '\n=== read the gate ===\n' +
-      `  random@k (control)        — from ${RANDOM_CORPUS} replay\n` +
-      `  selector@k (homogeneous)  — same corpus, the pick (#143: −8.2pp on committed finsearch)\n` +
-      `  diverse-selector@k        — ${DIVERSE_CORPUS} replay: THE bet\n` +
-      '  beat-blind iff diverse-selector@k > random@k, significant per the BH report above.',
-  )
-}
-
-main().catch((err) => {
-  console.error(`diverse-gate: ${err instanceof Error ? err.message : String(err)}`)
-  process.exit(1)
-})

package/src/hev-eval.mts

@@ -1,101 +0,0 @@
-/**
- * Minimal HumanEval evaluator: given an INSTRUCTION (env) + a fixed task set, run the
- * worker model on each task and print the pass rate + the per-task result. Used to
- * measure a baseline instruction vs a proposer-supplied instruction on the SAME
- * held-out set (the proposer proposes; this grades — kept separate for honesty).
- *
- *   INSTRUCTION="..." IDS=HumanEval/55,... WORKER_MODEL=... ROUTER_BASE=... TANGLE_API_KEY=... \
- *     HUMANEVAL_GZ=/abs/HumanEval.jsonl.gz tsx src/hev-eval.mts
- */
-import { readFileSync } from 'node:fs'
-import { extractCode, loadHumanEval, runChecker, type HumanEvalTask } from './benchmarks/humaneval'
-import { runBenchRouterTurn } from './router-turn'
-
-const SEED_INSTRUCTION =
-  'Complete the following Python function. Output the COMPLETE function definition (signature, docstring optional, body) inside a single ```python code block. Include any imports the function needs. Do not write tests or example calls.'
-
-async function complete(
-  base: string,
-  key: string,
-  model: string,
-  instruction: string,
-  prompt: string,
-  maxTokens: number,
-): Promise<string> {
-  try {
-    const turn = await runBenchRouterTurn(
-      {
-        routerBaseUrl: base,
-        routerKey: key,
-        profile: {
-          name: 'humaneval-worker',
-          harness: 'cli-base',
-          model: {
-            provider: 'tangle-router',
-            default: model,
-            metadata: { temperature: 0.2, maxTokens },
-          },
-          prompt: { systemPrompt: instruction },
-        },
-      },
-      prompt,
-    )
-    return turn.finalText
-  } catch {
-    return ''
-  }
-}
-
-async function main(): Promise<void> {
-  const key = process.env.TANGLE_API_KEY
-  if (!key) throw new Error('TANGLE_API_KEY required')
-  const apiKey: string = key
-  const base = process.env.ROUTER_BASE ?? 'https://api.together.xyz/v1'
-  const model = process.env.WORKER_MODEL ?? 'meta-llama/Meta-Llama-3-8B-Instruct-Lite'
-  const instruction = process.env.INSTRUCTION_FILE
-    ? readFileSync(process.env.INSTRUCTION_FILE, 'utf8')
-    : (process.env.INSTRUCTION ?? SEED_INSTRUCTION)
-  const maxTokens = Number(process.env.MAX_TOKENS ?? 2500)
-  const conc = Number(process.env.CONC ?? 6)
-  const offset = Number(process.env.OFFSET ?? 55)
-  const n = Number(process.env.N ?? 40)
-  const idsEnv = (process.env.IDS ?? '').split(',').map((s) => s.trim()).filter(Boolean)
-
-  const all = await loadHumanEval(164, 0)
-  const byId = new Map(all.map((t) => [t.taskId, t]))
-  const tasks: HumanEvalTask[] = idsEnv.length
-    ? idsEnv.map((id) => byId.get(id)).filter((t): t is HumanEvalTask => !!t)
-    : all.slice(offset, offset + n)
-
-  console.log(`eval model=${model} n=${tasks.length} instr_len=${instruction.length}`)
-  let pass = 0
-  const fails: string[] = []
-  // simple concurrency pool
-  let i = 0
-  async function worker(): Promise<void> {
-    while (i < tasks.length) {
-      const t = tasks[i]
-      i += 1
-      if (!t) continue
-      const reply = await complete(
-        base,
-        apiKey,
-        model,
-        instruction,
-        `\`\`\`python\n${t.prompt}\`\`\``,
-        maxTokens,
-      )
-      const { pass: p } = await runChecker(t, extractCode(reply))
-      if (p === 1) pass += 1
-      else fails.push(t.taskId)
-    }
-  }
-  await Promise.all(Array.from({ length: conc }, () => worker()))
-  console.log(`PASS ${pass}/${tasks.length} = ${((100 * pass) / tasks.length).toFixed(1)}%`)
-  console.log(`FAILED: ${fails.sort().join(', ')}`)
-}
-
-main().catch((e) => {
-  console.error(e instanceof Error ? (e.stack ?? e.message) : String(e))
-  process.exit(1)
-})

package/src/hev-improve.mts

@@ -1,245 +0,0 @@
-/**
- * Official GEPA prompt optimization on HumanEval. The worker is a single chat
- * completion and the judge is the deterministic Docker checker.
- *
- * WHY this exists: on SWE-bench the same GEPA loop was NULL because the grading test
- * is withheld — the worker cannot verify, so prompt wording cannot move resolve.
- * HumanEval hands the worker a well-specified function to complete and grades by
- * running tests, so the instruction prompt DOES move pass-rate. This run measures
- * whether self-improvement lifts a CHEAP model when the task is prompt-sensitive.
- *
- * Worker + reflect models call the zai coding endpoint directly (no tangle router,
- * no WAF, no 503):  TANGLE_API_KEY=$ZAI_API_KEY  ROUTER_BASE=https://api.z.ai/api/coding/paas/v4
- */
-import {
-  improve,
-  officialGepa,
-  type ReadonlyAgentProfile,
-} from '@tangle-network/agent-runtime'
-import {
-  canonicalCandidateDigest,
-  type AgentProfile,
-  agentProfileSchema,
-} from '@tangle-network/agent-interface'
-import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract'
-import { extractCode, loadHumanEval, runChecker, type HumanEvalTask } from './benchmarks/humaneval'
-import {
-  assertCompleteCost,
-  officialOptimizerModel,
-  requiredTokenPricing,
-} from './official-optimizer-config.mjs'
-import { runBenchRouterTurn, withBenchProfile } from './router-turn'
-
-// The SEED instruction GEPA evolves. Byte-identical to humaneval.ts basePrompt's
-// solveInstruction so the baseline arm reproduces the plain-prompt denominator.
-const SEED_INSTRUCTION =
-  'Complete the following Python function. Output the COMPLETE function definition (signature, docstring optional, body) inside a single ```python code block. Include any imports the function needs. Do not write tests or example calls.'
-
-interface Completion {
-  text: string
-  tokIn: number
-  tokOut: number
-}
-
-async function complete(
-  base: string,
-  key: string,
-  profile: AgentProfile,
-  prompt: string,
-  maxTokens: number,
-): Promise<Completion> {
-  const result = await runBenchRouterTurn(
-    {
-      routerBaseUrl: base,
-      routerKey: key,
-      profile: withBenchProfile(profile, { temperature: 0.2, maxTokens }),
-    },
-    prompt,
-  )
-  return {
-    text: result.finalText,
-    tokIn: result.usage.input,
-    tokOut: result.usage.output,
-  }
-}
-
-async function main(): Promise<void> {
-  const key = process.env.TANGLE_API_KEY
-  if (!key) throw new Error('TANGLE_API_KEY required (worker + reflect completions)')
-  const base = process.env.ROUTER_BASE ?? 'https://api.z.ai/api/coding/paas/v4'
-  const workerModel = process.env.WORKER_MODEL ?? 'glm-4.5-air'
-  const reflectModel = process.env.REFLECT_MODEL ?? 'glm-4.6'
-  // The GEPA reflector may live on a DIFFERENT endpoint than the (cheap) worker —
-  // e.g. a small worker on Together + a strong optimizer on zai. Defaults to the
-  // worker endpoint when unset.
-  const reflectBase = process.env.REFLECT_BASE ?? base
-  const reflectKey = process.env.REFLECT_KEY ?? key
-  const trainN = Number(process.env.TRAIN_N ?? 12)
-  const selectionN = Number(process.env.SELECTION_N ?? 12)
-  const testN = Number(process.env.TEST_N ?? 12)
-  const offset = Number(process.env.OFFSET ?? 80)
-  const maxEvaluations = Number(process.env.MAX_EVALUATIONS ?? 24)
-  const maxProposerCostUsd = Number(process.env.MAX_PROPOSER_COST_USD ?? 5)
-  const workerMaxTokens = Number(process.env.MAX_TOKENS ?? 6000)
-  const reflectMaxTokens = Number(process.env.REFLECT_MAX_TOKENS ?? 8000)
-  const maxConcurrency = Number(process.env.MAX_CONCURRENCY ?? 4)
-  const runDir = process.env.RUN_DIR ?? '.runs/humaneval-official-gepa'
-  if (process.env.DRYRUN) {
-    console.log(
-      `DRYRUN: imports OK (improve=${typeof improve}, officialGepa=${typeof officialGepa})`,
-    )
-    return
-  }
-  const workerPricing = requiredTokenPricing(process.env, 'WORKER')
-  const optimizer = officialOptimizerModel({
-    env: process.env,
-    model: reflectModel,
-    baseUrl: reflectBase,
-    apiKey: reflectKey,
-    maxCostUsd: maxProposerCostUsd,
-    maxOutputTokensPerRequest: reflectMaxTokens,
-  })
-
-  // All three partitions are disjoint slices of the harder middle band.
-  const train = await loadHumanEval(trainN, offset)
-  const selection = await loadHumanEval(selectionN, offset + trainN)
-  const testCases = await loadHumanEval(testN, offset + trainN + selectionN)
-  const byId = new Map<string, HumanEvalTask>(
-    [...train, ...selection, ...testCases].map((t) => [t.taskId, t]),
-  )
-
-  console.log('=== HumanEval prompt optimization with official GEPA ===')
-  console.log(`worker=${workerModel} reflect=${reflectModel} base=${base}`)
-  console.log(`train=[${train.map((t) => t.taskId).join(', ')}]`)
-  console.log(`selection=[${selection.map((t) => t.taskId).join(', ')}]`)
-  console.log(`test=[${testCases.map((t) => t.taskId).join(', ')}]`)
-  console.log(`maxEvaluations=${maxEvaluations} maxProposerCostUsd=${maxProposerCostUsd} offset=${offset} maxTokens=${workerMaxTokens}`)
-  console.log(`runDir=${runDir}\n`)
-
-  const stats = { n: 0 }
-  const agent = async (candidate: ReadonlyAgentProfile, scenario: Scenario, ctx: DispatchContext): Promise<string | null> => {
-    const instr = candidate.prompt?.systemPrompt
-    if (instr === undefined) throw new Error('agent: candidate profile has no system prompt')
-    const t = byId.get(scenario.id)
-    if (!t) throw new Error(`agent: unknown scenario ${scenario.id}`)
-    const prompt = `\`\`\`python\n${t.prompt}\`\`\``
-    const executionProfile: AgentProfile = agentProfileSchema.parse({
-      ...candidate,
-      name: candidate.name ?? 'humaneval-improvement-worker',
-      model: { ...candidate.model, provider: 'tangle-router', default: workerModel },
-    })
-    const t0 = Date.now()
-    const paid = await ctx.cost.runPaidCall({
-      channel: 'agent',
-      actor: 'humaneval-worker',
-      model: workerModel,
-      execute: () => complete(base, key, executionProfile, prompt, workerMaxTokens),
-      receipt: (result) => {
-        const usageUnknown = result.tokIn === 0 && result.tokOut === 0
-        return {
-          model: workerModel,
-          inputTokens: result.tokIn,
-          outputTokens: result.tokOut,
-          customTokenPricing: workerPricing,
-          ...(usageUnknown ? { usageUnknown: true } : {}),
-        }
-      },
-    })
-    if (!paid.succeeded) throw paid.error
-    const r = paid.value
-    const hasText = r.text.trim().length > 0
-    stats.n += 1
-    const codeLen = extractCode(r.text).length
-    console.log(`  [agent] ${scenario.id} instr=${instr.length}c code=${codeLen}b tok=in:${r.tokIn}/out:${r.tokOut} ${Math.round((Date.now() - t0) / 1000)}s`)
-    return hasText ? r.text : null
-  }
-
-  const judge: JudgeConfig<string | null, Scenario> = {
-    name: 'humaneval-docker',
-    dimensions: [{ key: 'pass', description: 'the completed function passes its hidden unit tests (deterministic Docker checker)' }],
-    async score({ artifact, scenario }) {
-      const t = byId.get(scenario.id)
-      if (!t) throw new Error(`judge: unknown scenario ${scenario.id}`)
-      const code = extractCode(String(artifact ?? ''))
-      if (!code.trim()) {
-        console.log(`  [judge] ${scenario.id} pass=0 (empty)`)
-        return { dimensions: { pass: 0 }, composite: 0, notes: 'empty' }
-      }
-      const { pass, detail } = await runChecker(t, code)
-      console.log(`  [judge] ${scenario.id} pass=${pass}`)
-      if (pass === 1) return { dimensions: { pass }, composite: pass, notes: 'passed' }
-      // Trajectory-grounded failure note: the checker's traceback/assertion tail
-      // plus the model's own emitted code, so GEPA reflection sees WHAT failed and
-      // WHAT the model wrote — not just the word 'failed'. The candidate's full
-      // raw reply additionally reaches the proposer via the campaign breakdown's
-      // `emitted` field (carried automatically from the string artifact).
-      const traceback = (detail ?? 'checker produced no output (timeout or silent non-zero exit)').slice(-800)
-      const excerpt = code.slice(0, 700)
-      return {
-        dimensions: { pass },
-        composite: pass,
-        notes: `${traceback}\n--- emitted code (first 700 chars) ---\n${excerpt}`,
-      }
-    },
-  }
-
-  const profile: AgentProfile = { name: 'hev-solver', prompt: { systemPrompt: SEED_INSTRUCTION } }
-  const scenario = (task: HumanEvalTask): Scenario => ({ id: task.taskId, kind: 'humaneval' })
-
-  const out = await improve(profile, {
-    surface: 'prompt',
-    executionRef: canonicalCandidateDigest({
-      callback: 'bench/hev-improve',
-      model: workerModel,
-      endpoint: new URL(base).origin,
-      maxTokens: workerMaxTokens,
-      checker: 'local-python',
-    }),
-    method: officialGepa<Scenario, string | null>({
-      objective:
-        'Improve the complete instruction for a small model that writes Python functions which pass hidden unit tests.',
-      background:
-        'Prefer behavioral strategies over wording changes. Address algorithm choice, edge cases, boundary values, type behavior, and self-checking. Return only the complete instruction.',
-      recipe: {
-        kind: 'engine',
-        run: {
-          engine: 'gepa',
-          maxEvaluations,
-          maxProposerCostUsd,
-        },
-      },
-      optimizer,
-      resume: 'if-compatible',
-      trustResumeState: true,
-      describeScenario: (item) => ({ prompt: byId.get(item.id)?.prompt ?? item.id }),
-    }),
-    trainScenarios: train.map(scenario),
-    selectionScenarios: selection.map(scenario),
-    testScenarios: testCases.map(scenario),
-    judges: [judge],
-    agent,
-    expectUsage: 'warn',
-    maxConcurrency,
-    reps: 1,
-    runDir,
-    optimizationRunOptions: {
-      expectUsage: 'warn',
-      maxConcurrency,
-      reps: 1,
-    },
-  })
-
-  assertCompleteCost('humaneval official GEPA run', out.cost)
-  console.log('\n=== RESULT ===')
-  console.log(`decision=${out.decision} lift=${out.lift} interval=[${out.liftInterval.low}, ${out.liftInterval.high}]`)
-  console.log(`baseline test pass-rate=${out.raw.best.baselineComposite}`)
-  console.log(`winner test pass-rate=${out.raw.best.winnerComposite}`)
-  console.log(`test scenarios=${JSON.stringify(out.raw.best.scenarioScores)}`)
-  console.log(`cost=${JSON.stringify(out.cost)}`)
-  console.log(`winner instruction:\n${String(out.candidate.value).slice(0, 2000)}`)
-}
-
-main().catch((e) => {
-  console.error(e instanceof Error ? (e.stack ?? e.message) : String(e))
-  process.exit(1)
-})