@tangle-network/agent-bench 0.11.2 → 0.13.0

package/src/swe-arena/run-report.mts

@@ -1,75 +0,0 @@
-/**
- * CLI over the supervisor-run reader. NO metric logic lives here.
- *
- * The reader itself is `@tangle-network/agent-eval/supervisor-run` — a
- * supervision tree is a rollout trace with one more dimension, so it sits in
- * the trace-analysis layer next to single-rollout analysis, beside the
- * `tangle.rollout.v1` ledger whose row type its tree nodes ARE. This file is
- * only argv parsing plus the entry points the arenas already call.
- *
- * Harness-session view of the same run (model calls, latency, stuck loops,
- * tool errors):
- * `npx @tangle-network/traces analyze --harness opencode --cwd <worker-clone-cwd>`.
- */
-
-import { pathToFileURL } from 'node:url'
-import {
-  renderSupervisorRunMarkdown,
-  reportSupervisorRound,
-  type WriteSupervisorRunOptions,
-  writeSupervisorRunReport,
-} from '@tangle-network/agent-eval/supervisor-run'
-
-const isMain =
-  process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href
-
-if (isMain) {
-  const args = process.argv.slice(2)
-  const usage =
-    'usage:\n' +
-    '  tsx run-report.mts <cellDir> [--log <run.log>] [--patch <file>] [--ledger <ledger.jsonl>] [--report-dir <dir>] [--no-opencode]\n' +
-    '  tsx run-report.mts --round <outDir> [--log <run.log>] [--ledger <ledger.jsonl>] [--report-dir <dir>] [--no-opencode]\n' +
-    '\n--report-dir writes the reports outside the run directory (use it when the run dir is READ-ONLY).\n'
-  const noOpencode = args.includes('--no-opencode')
-  const flagValue = (flag: string): string | undefined => {
-    const i = args.indexOf(flag)
-    return i >= 0 ? args[i + 1] : undefined
-  }
-  const appendHeadlineTo = flagValue('--log')
-  const patchPath = flagValue('--patch')
-  const reportDir = flagValue('--report-dir')
-  const ledgerPath = flagValue('--ledger')
-  const flagValueIndices = new Set(
-    ['--log', '--patch', '--report-dir', '--ledger', '--round']
-      .map((f) => args.indexOf(f))
-      .filter((i) => i >= 0)
-      .map((i) => i + 1),
-  )
-  const positional = args.filter((a, i) => !a.startsWith('--') && !flagValueIndices.has(i))
-  const roundIdx = args.indexOf('--round')
-  const opts: WriteSupervisorRunOptions = {
-    ...(appendHeadlineTo !== undefined ? { appendHeadlineTo } : {}),
-    ...(patchPath !== undefined ? { patchPath } : {}),
-    ...(reportDir !== undefined ? { reportDir } : {}),
-    ...(ledgerPath !== undefined ? { ledgerPath } : {}),
-    ...(noOpencode ? { opencodeDb: null } : {}),
-    echo: true,
-  }
-  if (roundIdx >= 0) {
-    const outDir = args[roundIdx + 1]
-    if (outDir === undefined) {
-      console.error(usage)
-      process.exit(2)
-    }
-    await reportSupervisorRound(outDir, opts)
-  } else {
-    const cellDir = positional[0]
-    if (cellDir === undefined) {
-      console.error(usage)
-      process.exit(2)
-    }
-    const report = await writeSupervisorRunReport(cellDir, opts)
-    console.log('')
-    console.log(renderSupervisorRunMarkdown(report))
-  }
-}

package/src/swe-arena/run-supervisor.mjs

@@ -1,297 +0,0 @@
-/**
- * Thin driver over the REAL loops pi extension. It constructs a minimal
- * ExtensionAPI, calls the extension's default export to register the real tools
- * (spawn_supervisor / supervisor_status / supervisor_watch / compute_watch), then
- * invokes the REAL spawn_supervisor tool and polls the on-disk run dir until the
- * supervisor settles. No supervisor logic is reimplemented — this only replaces
- * pi's chat operator as the caller of the already-registered tool.
- *
- * usage: node --import tsx run-supervisor.mjs <extPath> <cwd> <paramsJsonPath>
- */
-import { existsSync, readFileSync, readdirSync } from 'node:fs'
-import { join } from 'node:path'
-import { pathToFileURL } from 'node:url'
-
-const [extPath, cwd, paramsPath] = process.argv.slice(2)
-if (!extPath || !cwd || !paramsPath) {
-  console.error('usage: node --import tsx run-supervisor.mjs <extPath> <cwd> <paramsJsonPath>')
-  process.exit(2)
-}
-const params = JSON.parse(readFileSync(paramsPath, 'utf8'))
-
-const tools = new Map()
-const pi = {
-  on() {},
-  registerTool(t) { tools.set(t.name, t) },
-}
-
-const mod = await import(pathToFileURL(extPath).href)
-mod.default(pi)
-console.log(`[driver] registered tools: ${[...tools.keys()].join(', ')}`)
-
-const ctx = { cwd }
-const spawn = tools.get('spawn_supervisor')
-const cancel = tools.get('supervisor_cancel')
-if (!spawn || !cancel) {
-  console.error('[driver] extension must register spawn_supervisor and supervisor_cancel')
-  process.exit(2)
-}
-
-const positiveNumberEnv = (name, fallback) => {
-  const value = Number(process.env[name] ?? fallback)
-  if (!Number.isFinite(value) || value <= 0) throw new Error(`${name} must be a positive number`)
-  return value
-}
-const deadlineMs = positiveNumberEnv('DRIVER_DEADLINE_MS', 25 * 60 * 1000)
-const pollMs = positiveNumberEnv('DRIVER_POLL_MS', 4000)
-const cancelTimeoutMs = positiveNumberEnv('DRIVER_CANCEL_TIMEOUT_MS', 30_000)
-const settlementTimeoutMs = positiveNumberEnv('DRIVER_SETTLEMENT_TIMEOUT_MS', cancelTimeoutMs)
-const t0 = Date.now()
-const supervisorRoot = join(cwd, '.loops', 'supervisor')
-const readJson = (p) => { try { return JSON.parse(readFileSync(p, 'utf8')) } catch { return undefined } }
-const tail = (p, n) => {
-  if (!existsSync(p)) return []
-  const lines = readFileSync(p, 'utf8').split('\n').filter(Boolean)
-  return lines.slice(-n)
-}
-const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
-const listSupervisorIds = () => {
-  try {
-    return readdirSync(supervisorRoot, { withFileTypes: true })
-      .filter((entry) => entry.isDirectory() && /^sup-\d+-[a-z0-9]+$/i.test(entry.name))
-      .map((entry) => entry.name)
-      .sort()
-  } catch {
-    return []
-  }
-}
-const idsBeforeSpawn = new Set(listSupervisorIds())
-const runDirFor = (id) => join(supervisorRoot, id)
-
-const workerSettlement = (id) => {
-  const runDir = runDirFor(id)
-  const status = readJson(join(runDir, 'state.json'))?.status ?? 'missing'
-  const journalPath = join(runDir, 'journal.jsonl')
-  const lines = tail(journalPath, Number.MAX_SAFE_INTEGER)
-  let invalidLines = 0
-  const events = lines.flatMap((line) => {
-    try { return [JSON.parse(line)] } catch { invalidLines += 1; return [] }
-  })
-  const rootSpawned = events.some((event) => event.kind === 'spawned' && event.id === id && !event.parent)
-  const spawnedEvents = events.filter((event) => event.kind === 'spawned' && event.parent)
-  let missingIds = 0
-  const workerIds = new Set()
-  for (const event of spawnedEvents) {
-    if (typeof event.id === 'string' && event.id.length > 0) workerIds.add(event.id)
-    else missingIds += 1
-  }
-  const terminalIds = new Set(events.flatMap((event) =>
-    (event.kind === 'settled' || event.kind === 'cancelled') && typeof event.id === 'string' ? [event.id] : [],
-  ))
-  const unsettled = [...workerIds].filter((workerId) => !terminalIds.has(workerId))
-  return {
-    status,
-    journalPresent: existsSync(journalPath),
-    rootSpawned,
-    invalidLines,
-    workers: workerIds.size,
-    terminal: workerIds.size - unsettled.length,
-    missingIds,
-    unsettled,
-  }
-}
-
-const hasTerminalWorkerEvidence = (settlement) =>
-  settlement.journalPresent &&
-  settlement.rootSpawned &&
-  settlement.invalidLines === 0 &&
-  settlement.missingIds === 0 &&
-  settlement.unsettled.length === 0
-
-const settlementProblem = (settlement) =>
-  `status=${settlement.status}, journal=${settlement.journalPresent ? 'present' : 'missing'}, root=${settlement.rootSpawned ? 'present' : 'missing'}, ` +
-  `workers=${settlement.workers}, terminal=${settlement.terminal}, unsettled=${settlement.unsettled.length}, ` +
-  `missingIds=${settlement.missingIds}, invalidLines=${settlement.invalidLines}`
-
-const withTimeout = (promise, timeoutMs, message) => new Promise((resolve, reject) => {
-  const timer = setTimeout(() => reject(new Error(message)), timeoutMs)
-  promise.then(
-    (value) => { clearTimeout(timer); resolve(value) },
-    (err) => { clearTimeout(timer); reject(err) },
-  )
-})
-
-const cancellations = new Map()
-const cancelAndSettle = (id, reason, deadline = Date.now() + cancelTimeoutMs) => {
-  if (cancellations.has(id)) return cancellations.get(id)
-  const cancellation = (async () => {
-    console.error(`[driver] cancelling ${id}: ${reason}`)
-    const result = await withTimeout(
-      Promise.resolve(cancel.execute(`cancel-${id}`, { id }, undefined, () => {}, ctx)),
-      Math.max(1, deadline - Date.now()),
-      `supervisor cancellation did not settle within ${cancelTimeoutMs}ms`,
-    )
-    const message = result?.content?.[0]?.text ?? ''
-    while (true) {
-      const settlement = workerSettlement(id)
-      const terminalStatus = ['completed', 'failed', 'cancelled'].includes(settlement.status)
-      if (terminalStatus && hasTerminalWorkerEvidence(settlement)) {
-        console.error(`[driver] cancellation settled: workers=${settlement.workers}, terminal=${settlement.terminal}; ${message}`)
-        return settlement
-      }
-      if (Date.now() >= deadline) {
-        throw new Error(`supervisor cancellation left ${settlementProblem(settlement)}`)
-      }
-      await sleep(Math.min(pollMs, 250, Math.max(1, deadline - Date.now())))
-    }
-  })()
-  cancellations.set(id, cancellation)
-  return cancellation
-}
-
-let parsedId
-let spawnSettled = false
-let receivedSignal
-let signalExitPromise
-const startedIds = () => {
-  const ids = listSupervisorIds().filter((id) => !idsBeforeSpawn.has(id))
-  if (parsedId && !ids.includes(parsedId)) ids.push(parsedId)
-  return ids.sort()
-}
-const waitForStartedIds = async (deadline) => {
-  while (true) {
-    const ids = startedIds()
-    if (ids.length > 0 || spawnSettled) return ids
-    if (Date.now() >= deadline) {
-      throw new Error(`spawn did not expose a supervisor id within ${cancelTimeoutMs}ms`)
-    }
-    await sleep(Math.min(pollMs, 50, Math.max(1, deadline - Date.now())))
-  }
-}
-const cancelStartedRuns = async (reason, deadline) => {
-  const ids = await waitForStartedIds(deadline)
-  await Promise.all(ids.map((id) => cancelAndSettle(id, reason, deadline)))
-  return ids
-}
-const finishSignalExit = () => {
-  if (!receivedSignal) return Promise.resolve()
-  signalExitPromise ??= (async () => {
-    const { signal, code } = receivedSignal
-    try {
-      await cancelStartedRuns(`driver received ${signal}`, Date.now() + cancelTimeoutMs)
-      process.exit(code)
-    } catch (err) {
-      console.error(`[driver] cancellation failed: ${err instanceof Error ? err.message : String(err)}`)
-      process.exit(4)
-    }
-  })()
-  return signalExitPromise
-}
-
-for (const [signal, code] of [['SIGINT', 130], ['SIGTERM', 143]]) {
-  process.once(signal, () => {
-    if (receivedSignal) return
-    receivedSignal = { signal, code }
-    void finishSignalExit()
-  })
-}
-
-let res
-let spawnError
-try {
-  res = await spawn.execute('call-1', params, undefined, () => {}, ctx)
-  const responseText = res?.content?.[0]?.text ?? ''
-  parsedId = responseText.match(/supervisor (sup-\d+-[a-z0-9]+)/i)?.[1]
-} catch (err) {
-  spawnError = err
-} finally {
-  spawnSettled = true
-}
-
-if (receivedSignal) await finishSignalExit()
-
-const text = res?.content?.[0]?.text ?? ''
-if (res) console.log(`[driver] spawn_supervisor returned: ${text}`)
-if (spawnError || !parsedId) {
-  const ids = startedIds()
-  if (ids.length > 0) {
-    console.error(`[driver] spawn did not return a usable id; cancelling recovered run(s): ${ids.join(', ')}`)
-    try {
-      const deadline = Date.now() + cancelTimeoutMs
-      await Promise.all(ids.map((id) => cancelAndSettle(id, 'spawn did not return a usable supervisor id', deadline)))
-    } catch (err) {
-      console.error(`[driver] recovered-run cancellation failed: ${err instanceof Error ? err.message : String(err)}`)
-      process.exit(4)
-    }
-  }
-  if (spawnError) {
-    console.error(`[driver] spawn_supervisor failed: ${spawnError instanceof Error ? spawnError.message : String(spawnError)}`)
-  } else {
-    console.error('[driver] could not parse supervisor id')
-  }
-  process.exit(1)
-}
-
-const id = parsedId
-const runDir = runDirFor(id)
-const waitForCompletedSettlement = async () => {
-  const deadline = Date.now() + settlementTimeoutMs
-  while (true) {
-    const settlement = workerSettlement(id)
-    if (settlement.status !== 'completed' || hasTerminalWorkerEvidence(settlement)) return settlement
-    if (Date.now() >= deadline) {
-      throw new Error(`completed supervisor lacks terminal worker evidence after ${settlementTimeoutMs}ms: ${settlementProblem(settlement)}`)
-    }
-    await sleep(Math.min(pollMs, 250, Math.max(1, deadline - Date.now())))
-  }
-}
-
-let last = ''
-while (true) {
-  const st = readJson(join(runDir, 'state.json'))
-  const status = st?.status ?? 'pending'
-  const progress = st?.progress ?? ''
-  const journal = tail(join(runDir, 'journal.jsonl'), 40)
-  const spawned = journal.filter((l) => l.includes('"kind":"spawned"')).length
-  const settled = journal.filter((l) => l.includes('"kind":"settled"')).length
-  const stamp = `${((Date.now() - t0) / 1000).toFixed(0)}s status=${status} progress="${progress}" journal(spawned=${spawned},settled=${settled})`
-  if (stamp !== last) { console.log(`[watch] ${stamp}`); last = stamp }
-  if (st && ['completed', 'failed', 'cancelled'].includes(st.status)) {
-    const terminalStatus = st.status
-    let settlement = workerSettlement(id)
-    if (st.status === 'completed') {
-      try {
-        settlement = await waitForCompletedSettlement()
-      } catch (err) {
-        console.error(`[driver] refusing completed supervisor: ${err instanceof Error ? err.message : String(err)}`)
-        process.exit(4)
-      }
-      if (settlement.status !== 'completed') continue
-    } else if (!hasTerminalWorkerEvidence(settlement)) {
-      try {
-        settlement = await cancelAndSettle(
-          id,
-          `supervisor entered ${terminalStatus} before its workers settled`,
-        )
-      } catch (err) {
-        console.error(`[driver] terminal supervisor cleanup failed: ${err instanceof Error ? err.message : String(err)}`)
-        process.exit(4)
-      }
-    }
-    console.log(`\n[done] id=${id} status=${terminalStatus} verdict=${st.verdict ?? ''} result=${JSON.stringify(st.result ?? {})} error=${st.error ?? ''}`)
-    console.log(`[done] worker settlement: workers=${settlement.workers}, terminal=${settlement.terminal}`)
-    console.log(`[done] runDir=${runDir}`)
-    process.exit(terminalStatus === 'completed' ? 0 : 1)
-  }
-  if (Date.now() - t0 > deadlineMs) {
-    console.error(`[driver] deadline exceeded (${deadlineMs}ms) — supervisor still ${status}. runDir=${runDir}`)
-    try {
-      await cancelAndSettle(id, 'driver deadline exceeded')
-    } catch (err) {
-      console.error(`[driver] cancellation failed: ${err instanceof Error ? err.message : String(err)}`)
-      process.exit(4)
-    }
-    process.exit(3)
-  }
-  await sleep(pollMs)
-}