@tangle-network/agent-bench 0.1.0

package/src/trata-gate.mts

@@ -0,0 +1,243 @@
+/**
+ * Trata hedge-bench gate — direct router completions (no sandbox/opencode).
+ *
+ * Trata is a text-analysis benchmark: each task is a financial analysis brief
+ * over embedded source docs. The worker is a single router completion (not an
+ * agentic loop), which is appropriate here: tasks are scored by an LLM judge on
+ * theme coverage, not on code execution or tool use.
+ *
+ * Usage:
+ *   BENCH=trata-hedge WORKER_MODEL=deepseek-v4-flash JUDGE_MODEL=gemini-2.5-flash \
+ *   TRATA_BENCH_ROOT=/tmp/trata-hedge-bench N=10 CONCURRENCY=5 \
+ *   dotenvx run -f ~/company/devops/secrets/agent-state.env -- \
+ *     pnpm exec tsx bench/src/trata-gate.mts
+ *
+ * Key env vars:
+ *   BENCH              adapter key (default: trata-hedge)
+ *   WORKER_MODEL       model for the analysis completion (default: deepseek-v4-flash)
+ *   JUDGE_MODEL        model for the 3-stage judge (default: gemini-2.5-pro)
+ *   TRATA_BENCH_ROOT   path to cloned trata-hedge-bench (default: /tmp/trata-hedge-bench)
+ *   N                  how many tasks to run (default: 10; 0 = all 102)
+ *   CONCURRENCY        parallel worker slots (default: 5)
+ *   IDS                comma-separated task ids to run a specific subset
+ *   ROUTER_BASE        router base URL (default: https://router.tangle.tools/v1)
+ *   CORPUS             path to write JSONL run records (optional)
+ *
+ * Score is 0–4 / 4 = 0–100%; resolved = 4/4 (all themes + synthesis = sparse reward).
+ * Partial credit (1-3) is reported in the score distribution.
+ */
+
+import { appendFileSync } from 'node:fs'
+import { resolveAdapter } from './adapters'
+import type { BenchScore, BenchTask } from './benchmarks/types'
+import { runPool } from './run-pool'
+
+function must(name: string): string {
+  const v = process.env[name]
+  if (!v) throw new Error(`env ${name} is required`)
+  return v
+}
+
+// GEPA-optimised surface — +8.6pp holdout lift over the minimal baseline on deepseek-v4-flash
+// (2 independent runs, CI [0, 12.5pp]; gate holds for caution, surface ships as default).
+const ANALYST_SYSTEM = process.env.SYSTEM_PROMPT ?? [
+  'You are a senior financial analyst producing a structured investment memo.',
+  'Begin your response with exactly "ANALYSIS:" on its own line (nothing before it), then write your full analysis.',
+  'Structure your analysis with clearly labeled sections for each distinct analytical theme',
+  '(e.g., valuation, capital allocation, competitive dynamics, risk factors)',
+  'so that no major investment consideration is merged or omitted.',
+  'Benchmark the company explicitly against named sector peers, citing specific metrics such as',
+  'EV/EBITDA, P/E ratios, margin differentials, and growth premiums from the peer financial files.',
+  'Every factual claim must cite the filename (e.g., "per earnings_call/apo_q4_2025_earnings_call.txt").',
+  'Identify and verbatim-cite specific numerical targets from management guidance such as',
+  'earnings per share targets, margin percentages, growth rates, or AUM figures rather than paraphrasing approximately.',
+  'When evaluating capital allocation options, explicitly compute implied returns or internal rates of return (IRRs),',
+  'showing the arithmetic using price levels and targets from the source data.',
+  'Take a clear, decisive position — do not hedge with "it depends". Reconcile conflicting data points explicitly.',
+].join(' ')
+
+async function workerComplete(
+  task: BenchTask,
+  cfg: { routerBaseUrl: string; routerKey: string; model: string; timeoutMs: number },
+): Promise<{ answer: string; inputTokens: number; outputTokens: number; durationMs: number }> {
+  const startedAt = Date.now()
+  const res = await fetch(`${cfg.routerBaseUrl}/chat/completions`, {
+    method: 'POST',
+    signal: AbortSignal.timeout(cfg.timeoutMs),
+    headers: { 'content-type': 'application/json', authorization: `Bearer ${cfg.routerKey}` },
+    body: JSON.stringify({
+      model: cfg.model,
+      temperature: 0,
+      max_tokens: 4096,
+      messages: [
+        { role: 'system', content: ANALYST_SYSTEM },
+        { role: 'user', content: task.prompt },
+      ],
+    }),
+  })
+  if (!res.ok) {
+    const body = (await res.text()).slice(0, 300)
+    throw new Error(`router ${res.status} for ${task.id}: ${body}`)
+  }
+  const j = (await res.json()) as {
+    choices?: Array<{ message?: { content?: string } }>
+    usage?: { prompt_tokens?: number; completion_tokens?: number }
+  }
+  const answer = j.choices?.[0]?.message?.content ?? ''
+  return {
+    answer,
+    inputTokens: j.usage?.prompt_tokens ?? 0,
+    outputTokens: j.usage?.completion_tokens ?? 0,
+    durationMs: Date.now() - startedAt,
+  }
+}
+
+interface TaskResult {
+  taskId: string
+  answer: string
+  score: BenchScore
+  rawScore: number
+  inputTokens: number
+  outputTokens: number
+  durationMs: number
+  error?: string
+}
+
+async function runTask(
+  task: BenchTask,
+  workerCfg: { routerBaseUrl: string; routerKey: string; model: string; timeoutMs: number },
+  adapter: ReturnType<typeof resolveAdapter>,
+): Promise<TaskResult> {
+  let answer = ''
+  let inputTokens = 0
+  let outputTokens = 0
+  let durationMs = 0
+  let error: string | undefined
+
+  try {
+    const result = await workerComplete(task, workerCfg)
+    answer = result.answer
+    inputTokens = result.inputTokens
+    outputTokens = result.outputTokens
+    durationMs = result.durationMs
+  } catch (err) {
+    error = err instanceof Error ? err.message : String(err)
+  }
+
+  const score = answer.trim()
+    ? await adapter.judge(task, answer).catch((err) => ({
+        resolved: false as const,
+        score: 0,
+        detail: JSON.stringify({ error: err instanceof Error ? err.message : String(err) }),
+      }))
+    : { resolved: false as const, score: 0, detail: JSON.stringify({ error: error ?? 'empty answer' }) }
+
+  const detail = score.detail ? (JSON.parse(score.detail) as { rawScore?: number }) : {}
+  return {
+    taskId: task.id,
+    answer,
+    score,
+    rawScore: detail.rawScore ?? (score.resolved ? 4 : 0),
+    inputTokens,
+    outputTokens,
+    durationMs,
+    error,
+  }
+}
+
+async function main(): Promise<void> {
+  const benchName = process.env.BENCH ?? 'trata-hedge'
+  const adapter = resolveAdapter(benchName)
+  await adapter.preflight()
+
+  const n = Number(process.env.N ?? 10)
+  const concurrency = Number(process.env.CONCURRENCY ?? 5)
+  const model = process.env.WORKER_MODEL ?? 'deepseek-v4-flash'
+  const routerBase = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
+  const routerKey = must('TANGLE_API_KEY')
+  const corpusPath = process.env.CORPUS
+  const idsEnv = process.env.IDS
+
+  const loadOpts = idsEnv
+    ? { ids: idsEnv.split(',').map((s) => s.trim()).filter(Boolean) }
+    : n > 0
+      ? { limit: n }
+      : {}
+  const tasks = await adapter.loadTasks(loadOpts)
+  if (tasks.length === 0) throw new Error('no tasks loaded')
+
+  const workerCfg = { routerBaseUrl: routerBase, routerKey, model, timeoutMs: 180_000 }
+
+  console.log(
+    `trata-gate: ${tasks.length} tasks | model=${model} | judge=${process.env.JUDGE_MODEL ?? 'gemini-2.5-pro'} | concurrency=${concurrency}`,
+  )
+
+  let done = 0
+  const results: TaskResult[] = []
+  await runPool(tasks, concurrency, async (task) => {
+    const r = await runTask(task, workerCfg, adapter)
+    results.push(r)
+    done++
+    const icon = r.rawScore === 4 ? '✓' : r.rawScore >= 2 ? '~' : '·'
+    process.stdout.write(
+      `  [${done}/${tasks.length}] ${task.id.slice(0, 60)}: ${icon} (${r.rawScore}/4)\n`,
+    )
+    return r
+  })
+
+  // Score distribution
+  const byRaw = [0, 0, 0, 0, 0] // index = rawScore 0-4
+  let totalInputTok = 0
+  let totalOutputTok = 0
+  let totalMs = 0
+  for (const r of results) {
+    byRaw[Math.min(r.rawScore, 4)]++
+    totalInputTok += r.inputTokens
+    totalOutputTok += r.outputTokens
+    totalMs += r.durationMs
+  }
+  const resolved = byRaw[4] ?? 0
+  const partial = (byRaw[2] ?? 0) + (byRaw[3] ?? 0)
+  const n_ = results.length
+  const pct = (x: number) => (n_ > 0 ? `${((x / n_) * 100).toFixed(1)}%` : 'n/a')
+  const meanScore = n_ > 0 ? results.reduce((s, r) => s + r.score.score, 0) / n_ : 0
+
+  console.log(`\n=== ${benchName} — trata-gate (n=${n_}, model=${model}) ===`)
+  console.log(`  resolved (4/4):       ${pct(resolved)}  (${resolved}/${n_})`)
+  console.log(`  partial credit (2-3): ${pct(partial)}  (${partial}/${n_})`)
+  console.log(`  mean score (0..1):    ${meanScore.toFixed(3)}`)
+  console.log(`  score dist [0..4]:    ${byRaw.join(' | ')}`)
+  console.log(
+    `  tokens (in/out):     ${(totalInputTok / 1000).toFixed(0)}k / ${(totalOutputTok / 1000).toFixed(0)}k`,
+  )
+  console.log(`  wall time:            ${(totalMs / 1000).toFixed(0)}s total`)
+  console.log(`  errors:               ${results.filter((r) => r.error).length}`)
+
+  if (corpusPath) {
+    for (const r of results) {
+      appendFileSync(
+        corpusPath,
+        JSON.stringify({
+          benchmark: benchName,
+          instanceId: r.taskId,
+          condition: `blind@1`,
+          model,
+          resolved: r.score.resolved,
+          rawScore: r.rawScore,
+          score: r.score.score,
+          detail: r.score.detail,
+          inputTokens: r.inputTokens,
+          outputTokens: r.outputTokens,
+          durationMs: r.durationMs,
+          ...(r.error ? { error: r.error } : {}),
+        }) + '\n',
+      )
+    }
+    console.log(`\ncorpus written → ${corpusPath}`)
+  }
+}
+
+main().catch((err) => {
+  console.error(err instanceof Error ? (err.stack ?? err.message) : String(err))
+  process.exit(1)
+})

package/src/trata-gepa.mts

@@ -0,0 +1,434 @@
+/**
+ * trata-gepa — selfImprove (GEPA) outer loop for Trata hedge-bench.
+ *
+ * The optimization surface is the system prompt given to the financial analyst
+ * worker. GEPA reflects on which rubric themes were missed across training tasks
+ * and proposes improved system prompts — learning to instruct the model to
+ * extract specific quantitative claims, cover multiple analytical themes, and
+ * cite named evidence. Gated on a frozen holdout.
+ *
+ * The surface evolves beyond a bare system instruction: GEPA naturally discovers
+ * that it can add few-shot analytical patterns, calculation templates, and
+ * structured coverage checklists — effectively skill-creating without
+ * hand-engineering. Set K_ROUNDS=2 to add a self-critique refine pass.
+ *
+ * Usage:
+ *   TRATA_BENCH_ROOT=/tmp/trata-hedge-bench \
+ *   JUDGE_MODEL=gemini-2.5-flash WORKER_MODEL=deepseek-v4-flash \
+ *   REFLECT_MODEL=gemini-2.5-pro \
+ *   TRAIN_N=70 HOLDOUT_N=32 GENS=2 POP=3 CONCURRENCY=8 \
+ *   dotenvx run -f ~/company/devops/secrets/agent-state.env -- \
+ *     pnpm exec tsx bench/src/trata-gepa.mts
+ *
+ * Key env vars:
+ *   TRATA_BENCH_ROOT   cloned trata-hedge-bench (default /tmp/trata-hedge-bench)
+ *   WORKER_MODEL       analyst model (default deepseek-v4-flash)
+ *   JUDGE_MODEL        judge model in trata adapter (default gemini-2.5-flash)
+ *   REFLECT_MODEL      GEPA reflection model (default gemini-2.5-pro)
+ *   TRAIN_N            training tasks (default 70)
+ *   HOLDOUT_N          frozen holdout tasks (default 32)
+ *   GENS               optimization generations (default 2)
+ *   POP                candidates per generation (default 3)
+ *   REPS               reps per scenario (default 1)
+ *   CONCURRENCY        parallel worker slots (default 8)
+ *   K_ROUNDS           1=single-shot, 2=analysis+self-critique (default 1)
+ *   BASELINE_DIRECTIVE override the starting system prompt
+ *   CORPUS             path to write JSONL run records (optional)
+ */
+
+import { selfImprove } from '@tangle-network/agent-eval/contract'
+import type { CampaignResult, JudgeConfig, JudgeScore, Scenario } from '@tangle-network/agent-eval/campaign'
+import { heldoutSignificance, inMemoryCampaignStorage, pairHoldout } from '@tangle-network/agent-eval/campaign'
+import { appendFileSync, writeFileSync } from 'node:fs'
+import { createTrataHedgeAdapter } from './benchmarks/trata-hedge'
+import type { BenchTask } from './benchmarks/types'
+
+interface TrataScenario extends Scenario {
+  task: BenchTask
+}
+
+interface DiagnosedFinding {
+  claim: string
+  severity: 'critical' | 'high' | 'medium' | 'low' | 'info'
+  area?: string
+  recommended_action?: string
+}
+
+function must(name: string): string {
+  const v = process.env[name]
+  if (!v) throw new Error(`env ${name} is required`)
+  return v
+}
+
+// GEPA-optimised baseline — the best surface found across 9 runs (+8.6pp on holdout, 2 independent
+// confirmations). Future GEPA runs start from here; BASELINE_DIRECTIVE overrides if you want to
+// experiment from a different starting point.
+const DEFAULT_TRATA_SYSTEM = [
+  'You are a senior financial analyst producing a structured investment memo.',
+  'Begin your response with exactly "ANALYSIS:" on its own line (nothing before it), then write your full analysis.',
+  'Structure your analysis with clearly labeled sections for each distinct analytical theme',
+  '(e.g., valuation, capital allocation, competitive dynamics, risk factors)',
+  'so that no major investment consideration is merged or omitted.',
+  'Benchmark the company explicitly against named sector peers, citing specific metrics such as',
+  'EV/EBITDA, P/E ratios, margin differentials, and growth premiums from the peer financial files.',
+  'Every factual claim must cite the filename (e.g., "per earnings_call/apo_q4_2025_earnings_call.txt").',
+  'Identify and verbatim-cite specific numerical targets from management guidance such as',
+  'earnings per share targets, margin percentages, growth rates, or AUM figures rather than paraphrasing approximately.',
+  'When evaluating capital allocation options, explicitly compute implied returns or internal rates of return (IRRs),',
+  'showing the arithmetic using price levels and targets from the source data.',
+  'Take a clear, decisive position — do not hedge with "it depends". Reconcile conflicting data points explicitly.',
+].join(' ')
+
+// Refine round instruction — used only when K_ROUNDS >= 2.
+const REFINE_INSTRUCTION = [
+  'Review your analysis above. For each distinct analytical theme in the investment brief,',
+  'verify you have a dedicated section with at least one specific quantitative claim (a named figure,',
+  'percentage, ratio, or calculation) from the cited data files. Add any missing themes.',
+  'Rewrite any section that only gestures at a theme without a specific supporting data point.',
+].join(' ')
+
+async function chatComplete(
+  baseUrl: string,
+  key: string,
+  model: string,
+  messages: Array<{ role: string; content: string }>,
+): Promise<{ content: string; usage?: { input: number; output: number } }> {
+  const res = await fetch(`${baseUrl}/chat/completions`, {
+    method: 'POST',
+    signal: AbortSignal.timeout(180_000),
+    headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` },
+    body: JSON.stringify({ model, temperature: 0, max_tokens: 4096, messages }),
+  })
+  if (!res.ok) throw new Error(`router ${res.status}: ${(await res.text()).slice(0, 300)}`)
+  const j = (await res.json()) as {
+    choices?: Array<{ message?: { content?: string } }>
+    usage?: { prompt_tokens?: number; completion_tokens?: number }
+  }
+  const content = j.choices?.[0]?.message?.content ?? ''
+  const usage =
+    j.usage?.prompt_tokens != null
+      ? { input: j.usage.prompt_tokens, output: j.usage.completion_tokens ?? 0 }
+      : undefined
+  return { content, usage }
+}
+
+function parseFindings(content: string): DiagnosedFinding[] {
+  // Balanced-bracket scan so `]` inside string values doesn't terminate early.
+  const startIdx = content.indexOf('[')
+  if (startIdx < 0) return []
+  let depth = 0, inString = false, endIdx = -1
+  for (let i = startIdx; i < content.length; i++) {
+    const ch = content[i]
+    if (inString) {
+      if (ch === '\\') { i++; continue }
+      if (ch === '"') inString = false
+    } else {
+      if (ch === '"') inString = true
+      else if (ch === '[' || ch === '{') depth++
+      else if (ch === ']' || ch === '}') {
+        depth--
+        if (depth === 0 && ch === ']') { endIdx = i; break }
+      }
+    }
+  }
+  if (endIdx < 0) return []
+  const candidate = content.slice(startIdx, endIdx + 1)
+  let arr: unknown
+  try {
+    arr = JSON.parse(candidate)
+  } catch (e1) {
+    try { arr = JSON.parse(candidate.replace(/,(\s*[}\]])/g, '$1')) }
+    catch { console.error(`[trata-gepa] parseFindings failed: ${(e1 as Error).message} | head: ${candidate.slice(0, 120)}`); return [] }
+  }
+  if (!Array.isArray(arr)) return []
+  const sev = new Set(['critical', 'high', 'medium', 'low', 'info'])
+  return arr
+    .filter(
+      (x): x is Record<string, unknown> =>
+        typeof x === 'object' && x !== null && typeof (x as { claim?: unknown }).claim === 'string',
+    )
+    .map((x) => ({
+      claim: String(x.claim),
+      severity: (sev.has(String(x.severity)) ? String(x.severity) : 'medium') as DiagnosedFinding['severity'],
+      area: x.area !== undefined ? String(x.area) : 'failure-mode',
+      recommended_action: x.recommended_action !== undefined ? String(x.recommended_action) : undefined,
+    }))
+}
+
+async function main(): Promise<void> {
+  const adapter = createTrataHedgeAdapter()
+  await adapter.preflight()
+
+  const model = process.env.WORKER_MODEL ?? 'deepseek-v4-flash'
+  const reflectModel = process.env.REFLECT_MODEL ?? 'deepseek-v4-flash'
+  const routerBaseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
+  const routerKey = must('TANGLE_API_KEY')
+  const trainN = Number(process.env.TRAIN_N ?? 70)
+  const holdoutN = Number(process.env.HOLDOUT_N ?? 32)
+  const kRounds = Number(process.env.K_ROUNDS ?? 1)
+  const corpusPath = process.env.CORPUS
+  const baselineSurface = process.env.BASELINE_DIRECTIVE ?? DEFAULT_TRATA_SYSTEM
+
+  // Load all tasks and split deterministically.
+  // Hash-shuffle by task id so both splits carry the same difficulty mix.
+  const tasks = await adapter.loadTasks({ limit: trainN + holdoutN })
+  const idHash = (s: string): number => {
+    let h = 2166136261
+    for (let i = 0; i < s.length; i += 1) {
+      h ^= s.charCodeAt(i)
+      h = Math.imul(h, 16777619)
+    }
+    return h >>> 0
+  }
+  tasks.sort((a, b) => idHash(a.id) - idHash(b.id))
+  const train = tasks.slice(0, Math.min(trainN, tasks.length))
+  const holdout = tasks.slice(train.length, train.length + Math.min(holdoutN, tasks.length - train.length))
+  const toScenario = (t: BenchTask): TrataScenario => ({ id: t.id, kind: 'trata-hedge', task: t })
+
+  console.log(
+    `[trata-gepa] worker=${model} reflect=${reflectModel} rounds=${kRounds} train=${train.length} holdout=${holdout.length}`,
+  )
+
+  // Domain seam: run the financial analyst worker under the candidate surface.
+  // For K_ROUNDS=2, a second round asks the model to review its own coverage.
+  // Reports real token usage to ctx.cost (never fabricated).
+  const runWithSurface = async (
+    surface: string,
+    scenario: TrataScenario,
+    ctx: {
+      cost: {
+        observe(usd: number, source: string): void
+        observeTokens(u: { input: number; output: number }): void
+      }
+    },
+  ): Promise<string> => {
+    // Round 1: initial analysis under the candidate system prompt.
+    const r1 = await chatComplete(routerBaseUrl, routerKey, model, [
+      { role: 'system', content: surface },
+      { role: 'user', content: scenario.task.prompt },
+    ])
+    if (r1.usage) ctx.cost.observeTokens(r1.usage)
+    let answer = r1.content
+
+    // Round 2 (optional): self-critique for rubric coverage.
+    if (kRounds >= 2 && answer.trim()) {
+      const r2 = await chatComplete(routerBaseUrl, routerKey, model, [
+        { role: 'system', content: surface },
+        { role: 'user', content: scenario.task.prompt },
+        { role: 'assistant', content: answer },
+        { role: 'user', content: REFINE_INSTRUCTION },
+      ])
+      if (r2.usage) ctx.cost.observeTokens(r2.usage)
+      if (r2.content.trim()) answer = r2.content
+    }
+
+    if (corpusPath) {
+      appendFileSync(
+        corpusPath,
+        JSON.stringify({ benchmark: 'trata-gepa', instanceId: scenario.task.id, model, answer: answer.slice(0, 500) }) +
+          '\n',
+      )
+    }
+    return answer
+  }
+
+  // Judge: wraps the Trata 3-stage judge. Returns partial-credit score (0/0.25/0.5/0.75/1.0).
+  const judge: JudgeConfig<string, TrataScenario> = {
+    name: 'trata-hedge-judge',
+    dimensions: [
+      { key: 'score', description: 'rubric theme coverage fraction (rawScore/4)' },
+      { key: 'resolved', description: 'all themes hit + synthesis (rawScore=4)' },
+    ],
+    async score({ artifact, scenario }): Promise<JudgeScore> {
+      if (!artifact.trim()) {
+        return { dimensions: { score: 0, resolved: 0 }, composite: 0, notes: 'empty artifact' }
+      }
+      const verdict = await adapter.judge(scenario.task, artifact)
+      const sc = typeof verdict.score === 'number' ? verdict.score : verdict.resolved ? 1 : 0
+      return {
+        dimensions: { score: sc, resolved: verdict.resolved ? 1 : 0 },
+        composite: sc,
+        notes: verdict.detail ?? '',
+      }
+    },
+  }
+
+  // EYES→HANDS: diagnose FAILED runs using themesMissed from judge detail.
+  // Trata's judge returns structured per-theme failure data — richer than the
+  // generic "wrong answer" signal that other benches feed into this hook.
+  const taskById = new Map(tasks.map((t) => [t.id, t]))
+  const analyzeGeneration = async (input: {
+    generation: number
+    runDir: string
+    candidates: Array<{
+      surfaceHash: string
+      campaign: CampaignResult<string, TrataScenario>
+      composite: number
+    }>
+    history: unknown[]
+  }): Promise<DiagnosedFinding[]> => {
+    interface FailureItem {
+      question: string
+      themesMissed: string[]
+      themesHit: string[]
+      answer: string
+      note: string
+    }
+    const failures = new Map<string, FailureItem>()
+    for (const cand of input.candidates) {
+      for (const cell of cand.campaign.cells) {
+        const js = cell.judgeScores?.[judge.name]
+        if ((js?.composite ?? 0) >= 1) continue
+        if (failures.has(cell.scenarioId)) continue
+        const task = taskById.get(cell.scenarioId)
+        if (!task) continue
+        let themesMissed: string[] = []
+        let themesHit: string[] = []
+        try {
+          const d = JSON.parse(js?.notes ?? '{}') as {
+            themesMissed?: string[]
+            themesHit?: string[]
+          }
+          themesMissed = d.themesMissed ?? []
+          themesHit = d.themesHit ?? []
+        } catch {
+          // no structured detail available
+        }
+        failures.set(cell.scenarioId, {
+          question: task.prompt.slice(0, 800),
+          themesMissed,
+          themesHit,
+          answer: (typeof cell.artifact === 'string' ? cell.artifact : '').slice(-1200),
+          note: (js?.notes ?? '').slice(0, 200),
+        })
+      }
+    }
+    const items = [...failures.values()].slice(0, 8)
+    if (items.length === 0) {
+      console.log(`[trata-gepa] gen ${input.generation}: 0 failures to diagnose`)
+      return []
+    }
+    const user = items
+      .map(
+        (f, i) =>
+          `### Failure ${i + 1}\nTASK (excerpt): ${f.question}\n` +
+          (f.themesMissed.length > 0 ? `MISSED THEMES: ${f.themesMissed.join(', ')}\n` : '') +
+          (f.themesHit.length > 0 ? `HIT THEMES: ${f.themesHit.join(', ')}\n` : '') +
+          `AGENT ANSWER (tail): ${f.answer}`,
+      )
+      .join('\n\n')
+    const system =
+      'You are a failure analyst for a financial analyst agent. The agent produces investment memos ' +
+      'scored by a rubric with 4-6 analytical themes, each requiring specific quantitative claims. ' +
+      'Below are FAILED runs showing which themes were missed and the agent\'s answer. ' +
+      'Identify the COMMON failure patterns — e.g., generic statements without specific figures, ' +
+      'missing peer comparisons, no explicit calculations, ignoring certain data file types. ' +
+      'For each finding, recommend a CONCRETE change to the system instruction that would fix it. ' +
+      'Return ONLY a JSON array (no prose): [{"claim","severity":"high"|"medium"|"low","area","recommended_action"}]. Max 6.'
+    let content: string | undefined
+    for (let attempt = 1; attempt <= 4; attempt += 1) {
+      try {
+        const r = await chatComplete(routerBaseUrl, routerKey, reflectModel, [
+          { role: 'system', content: system },
+          { role: 'user', content: user },
+        ])
+        content = r.content
+        break
+      } catch (err) {
+        const msg = (err as Error).message
+        if (attempt === 4) {
+          console.error(`[trata-gepa] analyzeGeneration failed gen ${input.generation}: ${msg}`)
+          return []
+        }
+        await new Promise((r) => setTimeout(r, 1000 * 2 ** (attempt - 1)))
+      }
+    }
+    if (!content) return []
+    const findings = parseFindings(content)
+    console.log(`[trata-gepa] gen ${input.generation}: ${items.length} failures → ${findings.length} findings`)
+    return findings
+  }
+
+  const result = await selfImprove<TrataScenario, string>({
+    agent: (surface, scenario, ctx) => runWithSurface(surface as string, scenario, ctx),
+    scenarios: train.map(toScenario),
+    judge,
+    baselineSurface,
+    budget: {
+      generations: Number(process.env.GENS ?? 2),
+      populationSize: Number(process.env.POP ?? 3),
+      maxConcurrency: Number(process.env.CONCURRENCY ?? 8),
+      reps: Number(process.env.REPS ?? 1),
+      promoteTopK: Number(process.env.TOPK ?? 1),
+      holdoutScenarios: holdout.map(toScenario),
+    },
+    llm: {
+      baseUrl: routerBaseUrl,
+      apiKey: routerKey,
+      model: reflectModel,
+    },
+    driverTarget:
+      'a FINANCIAL ANALYST SYSTEM INSTRUCTION: the directive given to an agent that produces an investment memo from embedded earnings call transcripts, SEC filings, financial statements, and investor presentations. ' +
+      'The memo is scored by a rubric with 4-6 analytical themes, each requiring 2-4 specific analytical moves (quantitative claims, strategic conclusions, peer comparisons, or explicit calculations). ' +
+      'A theme is "hit" only when the agent makes the SPECIFIC move — not just gestures at the theme. ' +
+      'The directive must make the agent: (1) extract and cite specific numerical targets from management guidance, ' +
+      '(2) compute implied returns/IRRs when comparing capital allocation options, ' +
+      '(3) cover every distinct analytical theme with a dedicated paragraph, ' +
+      '(4) benchmark against named peers with specific metrics. The "ANALYSIS:" sentinel must start the response.',
+    mutationPrimitives: [
+      'instruct the agent to identify and verbatim-cite specific numerical targets in management guidance (earnings per share targets, margin percentages, growth rates, AUM figures) rather than paraphrasing in approximate terms',
+      'instruct the agent to explicitly compute implied returns or IRRs when evaluating capital allocation trade-offs — show the arithmetic using the price levels and targets from the source data',
+      'instruct the agent to structure the analysis with a clearly-labeled section for each distinct analytical theme (valuation, capital allocation, competitive dynamics, risk factors, etc.) so no major investment consideration is merged or omitted',
+      'instruct the agent to compare the company against its NAMED sector peers with specific metrics (EV/EBITDA, P/E, margin differential, growth premium) cited from the peer financials files in the data',
+    ],
+    runDir: 'improve-prompt-trata-hedge',
+    storage: inMemoryCampaignStorage(),
+    autoOnPromote: 'none',
+    analyzeGeneration,
+  })
+
+  console.log('\n=== trata-gepa RESULT ===')
+  const improved = result.gateDecision === 'ship'
+  console.log(`  baseline held-out mean: ${(result.baseline.compositeMean * 100).toFixed(1)}%`)
+  console.log(`  winner   held-out mean: ${(result.winner.compositeMean * 100).toFixed(1)}%`)
+  console.log(`  ► held-out delta:       ${(result.lift * 100).toFixed(1)} pp`)
+  console.log(`  gate decision:          ${result.gateDecision}`)
+
+  try {
+    const cellsToMap = (cells: ReadonlyArray<{ scenarioId: string; judgeScores: Record<string, JudgeScore> }>) => {
+      const m = new Map<string, Record<string, JudgeScore>>()
+      for (const c of cells) m.set(c.scenarioId, c.judgeScores)
+      return m
+    }
+    const baseMap = cellsToMap(result.raw.baselineOnHoldout.cells)
+    const winMap = cellsToMap(result.raw.winnerOnHoldout.cells)
+    const ids = new Set([...baseMap.keys()].filter((id) => winMap.has(id)))
+    const paired = pairHoldout(winMap, baseMap, ids, (s) => s.composite)
+    const sig = heldoutSignificance(paired)
+    console.log(
+      `  ► 95% CI (n=${sig.n}): [${(sig.bootstrap.low * 100).toFixed(1)}, ${(sig.bootstrap.high * 100).toFixed(1)}] pp · median ${(sig.bootstrap.median * 100).toFixed(1)}pp · significant=${sig.significant}`,
+    )
+    if (!sig.significant)
+      console.log('    (CI spans 0 — scale n or generations before promoting)')
+  } catch (err) {
+    console.log(`  (significance unavailable: ${(err instanceof Error ? err.message : String(err)).slice(0, 80)})`)
+  }
+
+  const winnerSurface = result.winner.surface as string
+  if (improved) {
+    console.log(`\n  PROMOTED SYSTEM PROMPT:\n${winnerSurface}`)
+    if (result.winner.rationale) console.log(`\n  rationale: ${result.winner.rationale}`)
+  } else {
+    console.log('  kept baseline (gate did not promote)')
+    console.log(`\n  BEST CANDIDATE SURFACE (set as BASELINE_DIRECTIVE to seed next run):\n${winnerSurface}`)
+  }
+  try {
+    writeFileSync('/tmp/trata-gepa-winner-surface.txt', winnerSurface)
+    console.log('\n  (winner surface written to /tmp/trata-gepa-winner-surface.txt)')
+  } catch { /* non-fatal */ }
+}
+
+main().catch((err) => {
+  console.error(err instanceof Error ? (err.stack ?? err.message) : String(err))
+  process.exit(1)
+})