@tangle-network/agent-bench 0.11.3 → 0.13.0

package/src/swe-arena/diagnosis-ensemble.ts

@@ -1,523 +0,0 @@
-/**
- * Diagnosis ensemble — N BLIND failure analysts over the SAME supervisor-run
- * artifacts (brain.jsonl, driver.log, workers/*.ndjson + *.patch, verify.log,
- * result.json, the delivered patch, and the official-judge outcome), each a
- * different model routed through router.tangle.tools, fused by union +
- * cross-analyst agreement ranking. Minority findings SURVIVE fusion marked as
- * competing hypotheses — the round-4 protocol treats them as candidate
- * proposal seeds, not noise.
- *
- * Design constraints (from supervisor-lab .evolve/state.json round4_design):
- *  - the model list is CONFIG — glm-5.2 is the only model proven routed today;
- *    gpt-5.5 / opus-4.8 slot in by editing the analyst spec list, never by a
- *    hardcoded requirement on an unrouted model.
- *  - analysts are blind: same bundle, no cross-talk, independent calls.
- *  - the containing run is launched through dotenvx; every analyst enters Runtime through one
- *    exact AgentProfile and its event record is retained with the response.
- */
-
-import { mkdir, readdir, readFile, writeFile } from 'node:fs/promises'
-import { join } from 'node:path'
-import { type AnalystFinding, makeFinding } from '@tangle-network/agent-eval'
-import { findSupervisorRunDir, type SecretsEnv } from './arms'
-import { ROUTER_ENDPOINT, sleepWithSignal } from './capacity'
-import { runBenchRouterTurn } from '../router-turn'
-
-// ---------------------------------------------------------------------------
-// Analyst specs.
-// ---------------------------------------------------------------------------
-
-export interface AnalystSpec {
-  /** Stable label, e.g. 'glm-5.2#1'. Used in fusion attribution + filenames. */
-  id: string
-  model: string
-  /** Provider identity stamped into the exact profile. Default: tangle-router. */
-  provider?: string
-  /** Chat-completions endpoint. Default: the Tangle router. */
-  url?: string
-  /** NAME of the env var holding the bearer key (resolved in the dotenvx child). */
-  apiKeyEnv?: string
-  /** glm-5.2 returns empty content when starved below ~8000, and at sampled
-   *  temperatures its REASONING alone can consume a full 8000 ceiling (both
-   *  measured — the calibration smoke saw out=8000 with empty content).
-   *  Default 16_000 so reasoning + the JSON answer always fit. */
-  maxTokens?: number
-  temperature?: number
-}
-
-/** N same-model analysts (the smoke default). Real rounds replace entries with
- *  other routed models — diversity comes from the config, never a hardcode.
- *  Same-model analysts at temperature 0 are byte-identical duplicates (measured
- *  in the calibration smoke: #2 and #3 returned the same 2823 output tokens),
- *  which silently inflates agreement — so only the FIRST same-model analyst
- *  runs at 0; the rest sample at 0.7 to buy real diversity. */
-export function defaultAnalysts(n = 3, model = 'glm-5.2'): AnalystSpec[] {
-  return Array.from({ length: n }, (_, i) => ({
-    id: `${model}#${i + 1}`,
-    model,
-    temperature: i === 0 ? 0 : 0.7,
-  }))
-}
-
-// ---------------------------------------------------------------------------
-// Artifact bundle — the ONE shared context every blind analyst reads.
-// ---------------------------------------------------------------------------
-
-/** One supervisor arm run to diagnose. `dir` is the run dir layout the arms
- *  write: brain.jsonl / driver.log / result.json / verify.log / ws/. */
-export interface SupRunArtifacts {
-  iid: string
-  arm: string
-  dir: string
-  /** The delivered (extracted) arm patch, when it exists. */
-  patchPath?: string
-  /** Official-judge outcome for this run, when known. `resolved: null` =
-   *  inconclusive judge — shown to analysts as such, never coerced. */
-  judge?: { resolved: boolean | null; score?: number; note?: string }
-}
-
-export interface BundleOptions {
-  /** Total bundle ceiling (chars). Default 60_000 (~15-20k tokens). */
-  maxChars?: number
-  /** Max workers whose ndjson/patch are excerpted per run. Default 6. */
-  maxWorkers?: number
-}
-
-const head = (s: string, n: number): string => (s.length <= n ? s : `${s.slice(0, n)}\n…[truncated head]`)
-const tail = (s: string, n: number): string => (s.length <= n ? s : `…[truncated tail]\n${s.slice(-n)}`)
-
-async function safeRead(path: string): Promise<string> {
-  return readFile(path, 'utf8').catch(() => '')
-}
-
-function section(title: string, body: string): string {
-  const trimmed = body.trim()
-  if (trimmed.length === 0) return ''
-  return `--- ${title} ---\n${trimmed}\n`
-}
-
-/** Build the bounded shared bundle. Every section is capped so one megabyte
- *  brain log cannot crowd out the patch the analysts must actually read. */
-export async function buildArtifactBundle(
-  runs: SupRunArtifacts[],
-  opts: BundleOptions = {},
-): Promise<string> {
-  const maxChars = opts.maxChars ?? 60_000
-  const maxWorkers = opts.maxWorkers ?? 6
-  const parts: string[] = []
-  for (const r of runs) {
-    const chunks: string[] = [`### RUN ${r.iid} arm=${r.arm}\nrun dir: ${r.dir}`]
-    if (r.judge) {
-      chunks.push(
-        `official judge: resolved=${r.judge.resolved === null ? 'INCONCLUSIVE' : r.judge.resolved}` +
-          (r.judge.score !== undefined ? ` score=${r.judge.score}` : '') +
-          (r.judge.note ? ` (${r.judge.note})` : ''),
-      )
-    }
-    chunks.push(section('result.json', head(await safeRead(join(r.dir, 'result.json')), 1_500)))
-    chunks.push(section('verify.log (tail)', tail(await safeRead(join(r.dir, 'verify.log')), 2_000)))
-    const driver = await safeRead(join(r.dir, 'driver.log'))
-    chunks.push(section('driver.log (head)', head(driver, 800)))
-    chunks.push(section('driver.log (tail)', tail(driver, 4_000)))
-    chunks.push(section('brain.jsonl (tail)', tail(await safeRead(join(r.dir, 'brain.jsonl')), 4_000)))
-
-    const supRunDir = await findSupervisorRunDir(join(r.dir, 'ws'))
-    if (supRunDir) {
-      chunks.push(section('supervisor state.json', head(await safeRead(join(supRunDir, 'state.json')), 3_500)))
-      chunks.push(section('supervisor journal.jsonl (tail)', tail(await safeRead(join(supRunDir, 'journal.jsonl')), 3_000)))
-      const workerFiles = (await readdir(join(supRunDir, 'workers')).catch(() => [] as string[])).sort()
-      const ndjson = workerFiles.filter((f) => f.endsWith('.ndjson')).slice(0, maxWorkers)
-      const patches = workerFiles.filter((f) => f.endsWith('.patch')).slice(0, maxWorkers)
-      for (const f of ndjson) {
-        chunks.push(section(`workers/${f} (tail)`, tail(await safeRead(join(supRunDir, 'workers', f)), 2_500)))
-      }
-      for (const f of patches) {
-        chunks.push(section(`workers/${f} (head)`, head(await safeRead(join(supRunDir, 'workers', f)), 3_000)))
-      }
-    }
-    if (r.patchPath) {
-      chunks.push(section('DELIVERED PATCH (head)', head(await safeRead(r.patchPath), 6_000)))
-    }
-    parts.push(chunks.filter(Boolean).join('\n'))
-  }
-  const bundle = parts.join('\n\n')
-  if (bundle.length <= maxChars) return bundle
-  // Keep the head (earliest runs) and the tail (latest run's patch) — the
-  // middle is the least diagnostic. Marked loudly so analysts know.
-  const keep = Math.floor(maxChars / 2)
-  return `${bundle.slice(0, keep)}\n\n…[BUNDLE TRUNCATED: ${bundle.length - maxChars} chars removed]…\n\n${bundle.slice(-keep)}`
-}
-
-// ---------------------------------------------------------------------------
-// One blind analyst call.
-// ---------------------------------------------------------------------------
-
-export interface AnalystRawFinding {
-  failure_class: string
-  evidence_quote: string
-  proposed_direction: string
-  /** 0..1, clamped. Defaults to 0.5 when the model omits it. */
-  confidence: number
-}
-
-export interface AnalystReport {
-  analystId: string
-  model: string
-  ok: boolean
-  findings: AnalystRawFinding[]
-  error?: string
-  /** Raw model text, kept for the audit trail. */
-  rawText?: string
-  tokens?: { input: number; output: number }
-}
-
-/** The blind-analyst instruction. Exported so the calibration smoke and tests
- *  pin the exact contract the parser expects. */
-export function analystPrompt(bundle: string): string {
-  return [
-    'You are one independent, BLIND failure analyst (other analysts see the same artifacts; you cannot see them).',
-    'The artifacts below come from runs of an automated software-engineering SUPERVISOR agent on SWE-bench Verified instances:',
-    '- a supervisor "brain" plans, spawns sandboxed workers (each in a clone of the instance repo), and settles a delivered patch;',
-    '- each run has a self-authored verify script (worker-visible reproduction check);',
-    '- after the run locks, the OFFICIAL hidden maintainer test suite grades the delivered patch (worker-blind);',
-    '- "verify_pass=true but official resolved=false" means the self-check passed while the maintainers\' tests did not.',
-    '',
-    'Diagnose the DOMINANT reasons these runs failed to resolve. Ground every finding in a verbatim quote from the artifacts.',
-    'Respond with STRICT JSON only (no markdown fences, no commentary):',
-    '{"findings":[{"failure_class":"2-6 word category","evidence_quote":"verbatim from artifacts","proposed_direction":"concrete change direction","confidence":0.0}]}',
-    'Return 1 to 5 findings, most important first.',
-    '',
-    '===== ARTIFACTS =====',
-    bundle,
-  ].join('\n')
-}
-
-/** Extract + validate the strict-JSON findings contract from model text.
- *  Tolerates code fences and leading/trailing prose; throws on anything that
- *  does not contain one parseable findings object (the caller records the
- *  analyst as failed — never a silently-empty diagnosis). */
-export function parseAnalystFindings(text: string): AnalystRawFinding[] {
-  const stripped = text.replace(/```(?:json)?/gi, '').trim()
-  const start = stripped.indexOf('{')
-  if (start === -1) throw new Error('analyst output contains no JSON object')
-  // Balanced-brace scan from the first '{' — models love trailing prose.
-  let depth = 0
-  let end = -1
-  let inString = false
-  let escaped = false
-  for (let i = start; i < stripped.length; i++) {
-    const ch = stripped[i]
-    if (inString) {
-      if (escaped) escaped = false
-      else if (ch === '\\') escaped = true
-      else if (ch === '"') inString = false
-      continue
-    }
-    if (ch === '"') inString = true
-    else if (ch === '{') depth += 1
-    else if (ch === '}') {
-      depth -= 1
-      if (depth === 0) {
-        end = i
-        break
-      }
-    }
-  }
-  if (end === -1) throw new Error('analyst output JSON object never closes')
-  let parsed: unknown
-  try {
-    parsed = JSON.parse(stripped.slice(start, end + 1))
-  } catch (cause) {
-    throw new Error(`analyst output is not valid JSON: ${(cause as Error).message}`)
-  }
-  const findings = (parsed as { findings?: unknown }).findings
-  if (!Array.isArray(findings) || findings.length === 0) {
-    throw new Error('analyst output has no findings array')
-  }
-  return findings.map((f, i) => {
-    const o = f as Record<string, unknown>
-    const failureClass = typeof o.failure_class === 'string' ? o.failure_class.trim() : ''
-    if (!failureClass) throw new Error(`finding ${i} missing failure_class`)
-    const conf = typeof o.confidence === 'number' && Number.isFinite(o.confidence) ? o.confidence : 0.5
-    return {
-      failure_class: failureClass,
-      evidence_quote: typeof o.evidence_quote === 'string' ? o.evidence_quote : '',
-      proposed_direction: typeof o.proposed_direction === 'string' ? o.proposed_direction : '',
-      confidence: Math.min(1, Math.max(0, conf)),
-    }
-  })
-}
-
-/** Run one blind analyst through Runtime's exact-profile Router boundary.
- *  ONE retry on transport failure (5xx/524/timeout) — an edge flake was
- *  observed live in the calibration smoke; a parse failure is NOT retried
- *  (same prompt, same model ⇒ same bad shape). */
-export async function runAnalyst(
-  spec: AnalystSpec,
-  bundle: string,
-  secrets: SecretsEnv,
-  scratchDir: string,
-  opts: { timeoutMs?: number; retries?: number; retryDelayMs?: number; signal?: AbortSignal } = {},
-): Promise<AnalystReport> {
-  opts.signal?.throwIfAborted()
-  const apiKeyEnv = spec.apiKeyEnv ?? 'TANGLE_API_KEY'
-  if (!/^[A-Z_][A-Z0-9_]*$/.test(apiKeyEnv)) {
-    return { analystId: spec.id, model: spec.model, ok: false, findings: [], error: `invalid apiKeyEnv name: ${apiKeyEnv}` }
-  }
-  const url = spec.url ?? ROUTER_ENDPOINT
-  const routerKey = process.env[apiKeyEnv]
-  if (!routerKey) {
-    return {
-      analystId: spec.id,
-      model: spec.model,
-      ok: false,
-      findings: [],
-      error: `${apiKeyEnv} is required; launch through dotenvx`,
-    }
-  }
-  void secrets
-  opts.signal?.throwIfAborted()
-  await mkdir(scratchDir, { recursive: true })
-  const outFile = join(scratchDir, `analyst-${spec.id.replace(/[^a-zA-Z0-9._-]/g, '_')}.response.json`)
-  const timeoutMs = opts.timeoutMs ?? 600_000
-  const attempts = 1 + (opts.retries ?? 1)
-  let transportError = ''
-  let content = ''
-  let tokens: AnalystReport['tokens']
-  for (let attempt = 0; attempt < attempts && content.length === 0; attempt++) {
-    opts.signal?.throwIfAborted()
-    if (attempt > 0) await sleepWithSignal(opts.retryDelayMs ?? 5_000, opts.signal)
-    try {
-      const turn = await runBenchRouterTurn(
-        {
-          routerBaseUrl: url.replace(/\/chat\/completions\/?$/u, ''),
-          routerKey,
-          profile: {
-            name: `diagnosis-${spec.id}`,
-            harness: 'cli-base',
-            model: {
-              provider: spec.provider ?? 'tangle-router',
-              default: spec.model,
-              metadata: {
-                temperature: spec.temperature ?? 0,
-                maxTokens: spec.maxTokens ?? 16_000,
-              },
-            },
-            prompt: { systemPrompt: 'Diagnose the supplied run evidence as an independent analyst.' },
-          },
-          timeoutMs,
-          signal: opts.signal,
-        },
-        analystPrompt(bundle),
-      )
-      content = turn.finalText
-      if (turn.usage.tokensKnown !== false) {
-        tokens = { input: turn.usage.input, output: turn.usage.output }
-      }
-      await writeFile(outFile, JSON.stringify(turn, null, 2))
-    } catch (cause) {
-      transportError = `router call failed (${(cause as Error).message}, attempt=${attempt + 1}/${attempts})`
-    }
-  }
-  if (content.length === 0 && transportError) {
-    return { analystId: spec.id, model: spec.model, ok: false, findings: [], error: transportError }
-  }
-  if (content.trim().length === 0) {
-    return { analystId: spec.id, model: spec.model, ok: false, findings: [], error: 'empty content (max_tokens starvation?)', tokens }
-  }
-  try {
-    const findings = parseAnalystFindings(content)
-    return { analystId: spec.id, model: spec.model, ok: true, findings, rawText: content, tokens }
-  } catch (cause) {
-    return { analystId: spec.id, model: spec.model, ok: false, findings: [], error: (cause as Error).message, rawText: content, tokens }
-  }
-}
-
-// ---------------------------------------------------------------------------
-// Fusion — union + agreement rank; minority findings survive flagged.
-// ---------------------------------------------------------------------------
-
-export interface FusedFinding {
-  /** Representative label (the first-seen member's failure_class). */
-  failure_class: string
-  /** Distinct analysts whose findings landed in this cluster. */
-  analysts: string[]
-  agreement: number
-  /** True when only ONE analyst surfaced it while others reported ok — a
-   *  surviving minority view, kept as an explicit competing hypothesis. */
-  competingHypothesis: boolean
-  meanConfidence: number
-  evidence: Array<{ analyst: string; quote: string }>
-  directions: Array<{ analyst: string; direction: string }>
-}
-
-const CLASS_STOPWORDS = new Set([
-  'the', 'a', 'an', 'of', 'in', 'on', 'to', 'and', 'or', 'is', 'for', 'with', 'by', 'at', 'not', 'no',
-])
-
-/** Normalized token set of a failure-class label. */
-export function classTokens(label: string): string[] {
-  return [
-    ...new Set(
-      label
-        .toLowerCase()
-        .split(/[^a-z0-9]+/)
-        .filter((t) => t.length > 1 && !CLASS_STOPWORDS.has(t)),
-    ),
-  ].sort()
-}
-
-/** Jaccard similarity of two class labels' token sets. */
-export function classSimilarity(a: string, b: string): number {
-  const ta = classTokens(a)
-  const tb = new Set(classTokens(b))
-  if (ta.length === 0 || tb.size === 0) return 0
-  let inter = 0
-  for (const t of ta) if (tb.has(t)) inter += 1
-  return inter / (ta.length + tb.size - inter)
-}
-
-/**
- * Fuse per-analyst findings: greedy clustering on failure-class similarity
- * (>= threshold joins the first matching cluster), then rank by agreement
- * (desc) and mean confidence (desc). Every input finding survives — union, not
- * intersection; a single-analyst cluster is marked `competingHypothesis` when
- * at least one OTHER analyst produced a successful report.
- */
-export function fuseFindings(
-  reports: AnalystReport[],
-  opts: { similarityThreshold?: number } = {},
-): FusedFinding[] {
-  const threshold = opts.similarityThreshold ?? 0.5
-  const okReports = reports.filter((r) => r.ok)
-  interface Cluster {
-    representative: string
-    members: Array<{ analyst: string; finding: AnalystRawFinding }>
-  }
-  const clusters: Cluster[] = []
-  for (const report of okReports) {
-    for (const finding of report.findings) {
-      const match = clusters.find((c) => classSimilarity(c.representative, finding.failure_class) >= threshold)
-      if (match) {
-        match.members.push({ analyst: report.analystId, finding })
-      } else {
-        clusters.push({ representative: finding.failure_class, members: [{ analyst: report.analystId, finding }] })
-      }
-    }
-  }
-  const fused = clusters.map((c): FusedFinding => {
-    const analysts = [...new Set(c.members.map((m) => m.analyst))]
-    return {
-      failure_class: c.representative,
-      analysts,
-      agreement: analysts.length,
-      competingHypothesis: analysts.length === 1 && okReports.length > 1,
-      meanConfidence:
-        c.members.reduce((s, m) => s + m.finding.confidence, 0) / c.members.length,
-      evidence: c.members
-        .filter((m) => m.finding.evidence_quote.length > 0)
-        .map((m) => ({ analyst: m.analyst, quote: m.finding.evidence_quote })),
-      directions: c.members
-        .filter((m) => m.finding.proposed_direction.length > 0)
-        .map((m) => ({ analyst: m.analyst, direction: m.finding.proposed_direction })),
-    }
-  })
-  return fused.sort((a, b) => b.agreement - a.agreement || b.meanConfidence - a.meanConfidence)
-}
-
-// ---------------------------------------------------------------------------
-// The ensemble.
-// ---------------------------------------------------------------------------
-
-export interface DiagnosisEnsembleResult {
-  bundleChars: number
-  reports: AnalystReport[]
-  fused: FusedFinding[]
-}
-
-/** Run every analyst (sequentially — gentle on the shared router key) over the
- *  SAME bundle and fuse. A failed analyst is recorded, never fabricated. */
-export async function runDiagnosisEnsemble(input: {
-  analysts: AnalystSpec[]
-  runs: SupRunArtifacts[]
-  secrets: SecretsEnv
-  scratchDir: string
-  maxBundleChars?: number
-  timeoutMsPerAnalyst?: number
-  /** Transport retries per analyst (router 524 storms are a measured, known
-   *  infra class — the loops brain itself runs with LOOPS_BRAIN_RETRIES=30). */
-  retriesPerAnalyst?: number
-  retryDelayMs?: number
-  onStatus?: (msg: string) => void
-  signal?: AbortSignal
-}): Promise<DiagnosisEnsembleResult> {
-  input.signal?.throwIfAborted()
-  if (input.analysts.length === 0) throw new Error('diagnosis ensemble: no analysts configured')
-  if (input.runs.length === 0) throw new Error('diagnosis ensemble: no run artifacts to diagnose')
-  const bundle = await buildArtifactBundle(
-    input.runs,
-    input.maxBundleChars !== undefined ? { maxChars: input.maxBundleChars } : {},
-  )
-  input.signal?.throwIfAborted()
-  await mkdir(input.scratchDir, { recursive: true })
-  await writeFile(join(input.scratchDir, 'bundle.txt'), bundle)
-  input.signal?.throwIfAborted()
-  const reports: AnalystReport[] = []
-  for (const spec of input.analysts) {
-    input.signal?.throwIfAborted()
-    input.onStatus?.(`analyst ${spec.id} (${spec.model}) reading ${bundle.length} chars…`)
-    const report = await runAnalyst(spec, bundle, input.secrets, input.scratchDir, {
-      ...(input.timeoutMsPerAnalyst !== undefined ? { timeoutMs: input.timeoutMsPerAnalyst } : {}),
-      ...(input.retriesPerAnalyst !== undefined ? { retries: input.retriesPerAnalyst } : {}),
-      ...(input.retryDelayMs !== undefined ? { retryDelayMs: input.retryDelayMs } : {}),
-      ...(input.signal ? { signal: input.signal } : {}),
-    })
-    input.signal?.throwIfAborted()
-    input.onStatus?.(
-      `analyst ${spec.id}: ${report.ok ? `${report.findings.length} finding(s)` : `FAILED (${report.error})`}` +
-        (report.tokens ? ` [tokens in=${report.tokens.input} out=${report.tokens.output}]` : ''),
-    )
-    reports.push(report)
-  }
-  return { bundleChars: bundle.length, reports, fused: fuseFindings(reports) }
-}
-
-/** Calibration grading: does a finding surface FIX PLACEMENT (the known truth
- *  of the round-2 django run — a locally-authored helper where the gold fix
- *  extends django/utils/encoding.py)? Assistive keyword net; the smoke always
- *  prints the raw findings so a miss here is auditable, never load-bearing. */
-export function surfacesPlacementRegex(): RegExp {
-  return /placement|misplac|wrong\s+(file|location|module|place)|different\s+(file|location|module)|expected\s+(location|file|module)|local\s+(helper|copy|implementation)|duplicat|encoding\.py|django\.utils\.encoding|utils\/encoding|belongs\s+in|should\s+(live|be\s+placed|be\s+located|go)\s+in|reimplement|instead\s+of\s+(the\s+)?(existing|shared|upstream)/i
-}
-
-/** Map fused findings onto the substrate's `AnalystFinding` envelope so they
- *  drop into the same `analyzeGeneration` slot the raw-trace distiller uses. */
-export function fusedToAnalystFindings(
-  fused: FusedFinding[],
-  evidence: { dirs: string[]; totalAnalysts: number },
-): AnalystFinding[] {
-  return fused.map((f) => {
-    const directions = f.directions.map((d) => `[${d.analyst}] ${d.direction}`).join(' | ')
-    const quotes = f.evidence
-      .slice(0, 3)
-      .map((e) => `[${e.analyst}] "${e.quote.slice(0, 240)}"`)
-      .join('; ')
-    return makeFinding({
-      analyst_id: 'diagnosis-ensemble',
-      severity: f.agreement >= 2 ? 'high' : 'medium',
-      area: 'failure-diagnosis',
-      confidence: Math.min(1, Math.max(0, f.meanConfidence)),
-      claim:
-        `${f.competingHypothesis ? '[competing hypothesis] ' : ''}` +
-        `(${f.agreement}/${evidence.totalAnalysts} analysts) ${f.failure_class}` +
-        (quotes ? ` — evidence: ${quotes}` : ''),
-      ...(directions ? { recommended_action: directions } : {}),
-      evidence_refs: evidence.dirs.map((d) => ({ kind: 'artifact' as const, uri: d })),
-      metadata: {
-        agreement: f.agreement,
-        competingHypothesis: f.competingHypothesis,
-        analysts: f.analysts,
-      },
-    })
-  })
-}