@tangle-network/agent-bench 0.1.0 → 0.3.5

package/src/pier-trial-supervisor.mjs

@@ -0,0 +1,352 @@
+import { execFileSync, spawn } from 'node:child_process'
+import {
+  closeSync,
+  createReadStream,
+  existsSync,
+  fsyncSync,
+  openSync,
+  readFileSync,
+  readdirSync,
+  renameSync,
+  writeFileSync,
+} from 'node:fs'
+import { basename, isAbsolute, join, resolve } from 'node:path'
+
+const identityFile = 'identity.json'
+const terminalFile = 'terminal.json'
+const stopFile = 'stop-requested'
+
+function fail(message) {
+  throw new Error(message)
+}
+
+function readJson(path, label) {
+  let value
+  try {
+    value = JSON.parse(readFileSync(path, 'utf8'))
+  } catch (error) {
+    throw new Error(`${label} is not valid JSON`, { cause: error })
+  }
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    fail(`${label} must be an object`)
+  }
+  return value
+}
+
+function syncDirectory(directory) {
+  const fd = openSync(directory, 'r')
+  try {
+    fsyncSync(fd)
+  } finally {
+    closeSync(fd)
+  }
+}
+
+function writeJsonAtomic(directory, name, value) {
+  const target = join(directory, name)
+  const temporary = join(directory, `.${name}.${process.pid}.tmp`)
+  const fd = openSync(temporary, 'wx', 0o600)
+  try {
+    writeFileSync(fd, `${JSON.stringify(value)}\n`, 'utf8')
+    fsyncSync(fd)
+  } finally {
+    closeSync(fd)
+  }
+  renameSync(temporary, target)
+  syncDirectory(directory)
+}
+
+async function readControlInput() {
+  const chunks = []
+  for await (const chunk of createReadStream('', { fd: 3 })) {
+    chunks.push(Buffer.from(chunk))
+  }
+  const bytes = Buffer.concat(chunks)
+  if (bytes.byteLength === 0) fail('Pier supervisor received no launch request')
+  return JSON.parse(bytes.toString('utf8'))
+}
+
+function processIdentity(pid) {
+  const text = readFileSync(`/proc/${pid}/stat`, 'utf8')
+  const closing = text.lastIndexOf(')')
+  if (closing < 0) fail(`cannot parse process identity for ${pid}`)
+  const fields = text.slice(closing + 2).trim().split(/\s+/)
+  const processGroupId = Number(fields[2])
+  const sessionId = Number(fields[3])
+  const startTicks = fields[19]
+  if (
+    !Number.isSafeInteger(processGroupId) ||
+    !Number.isSafeInteger(sessionId) ||
+    !/^\d+$/.test(startTicks ?? '')
+  ) {
+    fail(`cannot parse process identity for ${pid}`)
+  }
+  return { pid, processGroupId, sessionId, startTicks }
+}
+
+function processMatches(identity) {
+  try {
+    const current = processIdentity(identity.pid)
+    return (
+      current.startTicks === identity.startTicks &&
+      current.processGroupId === identity.processGroupId &&
+      current.sessionId === identity.sessionId
+    )
+  } catch (error) {
+    if (error?.code === 'ENOENT' || error?.code === 'ESRCH') return false
+    throw error
+  }
+}
+
+function sessionMembers(sessionId) {
+  const members = []
+  for (const entry of readdirSync('/proc', { withFileTypes: true })) {
+    if (!entry.isDirectory() || !/^\d+$/.test(entry.name)) continue
+    try {
+      const identity = processIdentity(Number(entry.name))
+      if (identity.sessionId === sessionId) members.push(identity)
+    } catch (error) {
+      if (error?.code !== 'ENOENT') throw error
+    }
+  }
+  return members
+}
+
+async function signalSessionUntilEmpty(identity, signal, timeoutMs) {
+  const deadline = Date.now() + timeoutMs
+  while (true) {
+    const members = sessionMembers(identity.sessionId)
+    if (members.length === 0) return true
+    for (const member of members) {
+      try {
+        process.kill(member.pid, signal)
+      } catch (error) {
+        if (error?.code !== 'ESRCH') throw error
+      }
+    }
+    if (Date.now() >= deadline) return false
+    await new Promise((resolveWait) => setTimeout(resolveWait, 20))
+  }
+}
+
+function sanitizeProject(value) {
+  let project = value.toLowerCase()
+  if (!/^[a-z0-9]/.test(project)) project = `0${project}`
+  return project.replace(/[^a-z0-9_-]/g, '-')
+}
+
+function trialProjects(jobsDirectory, jobName) {
+  const jobRoot = join(jobsDirectory, jobName)
+  if (!existsSync(jobRoot)) return []
+  return readdirSync(jobRoot, { withFileTypes: true })
+    .filter((entry) => entry.isDirectory() && !entry.isSymbolicLink())
+    .map((entry) => sanitizeProject(entry.name))
+}
+
+function matchingContainers(dockerCommand, projects) {
+  if (projects.length === 0) return []
+  const output = execFileSync(
+    dockerCommand,
+    ['ps', '-a', '--format', '{{.ID}}\t{{.Label "com.docker.compose.project"}}'],
+    { encoding: 'utf8', timeout: 30_000, maxBuffer: 4 * 1024 * 1024 },
+  )
+  const matches = []
+  for (const line of output.split('\n')) {
+    if (!line) continue
+    const tab = line.indexOf('\t')
+    if (tab < 1) continue
+    const id = line.slice(0, tab)
+    const project = line.slice(tab + 1)
+    if (
+      projects.some(
+        (expected) =>
+          project === expected || project.startsWith(`${expected}__verifier__`),
+      )
+    ) {
+      matches.push(id)
+    }
+  }
+  return matches
+}
+
+function removeTrialContainers(config) {
+  const projects = trialProjects(config.jobsDirectory, config.jobName)
+  const containers = matchingContainers(config.dockerCommand, projects)
+  if (containers.length > 0) {
+    try {
+      execFileSync(config.dockerCommand, ['rm', '-f', ...containers], {
+        encoding: 'utf8',
+        timeout: 30_000,
+        maxBuffer: 4 * 1024 * 1024,
+      })
+    } catch (error) {
+      if (matchingContainers(config.dockerCommand, projects).length > 0) throw error
+    }
+  }
+  const remaining = matchingContainers(config.dockerCommand, projects)
+  if (remaining.length > 0) {
+    fail(`Pier task containers survived removal: ${remaining.join(', ')}`)
+  }
+  return containers.length
+}
+
+async function waitForExit(exited, timeoutMs) {
+  return await Promise.race([
+    exited.then(() => true),
+    new Promise((resolveTimeout) => setTimeout(() => resolveTimeout(false), timeoutMs)),
+  ])
+}
+
+const directory = resolve(process.argv[2] ?? '')
+if (!isAbsolute(directory) || basename(directory) === '') {
+  fail('Pier supervisor control directory must be absolute')
+}
+
+let config
+let child
+let childIdentity
+let stopRequested = existsSync(join(directory, stopFile))
+let stopping
+
+async function stopChild() {
+  if (!child) return
+  if (!childIdentity) {
+    try {
+      child.process.kill('SIGKILL')
+    } catch (error) {
+      if (error?.code !== 'ESRCH') throw error
+    }
+    if (!(await waitForExit(child.exited, 5_000))) {
+      fail('Pier process with unreadable identity survived SIGKILL')
+    }
+    return
+  }
+  if (!(await signalSessionUntilEmpty(childIdentity, 'SIGTERM', 2_000))) {
+    if (!(await signalSessionUntilEmpty(childIdentity, 'SIGKILL', 5_000))) {
+      fail(`Pier process session ${childIdentity.sessionId} survived SIGKILL`)
+    }
+  }
+  if (!(await waitForExit(child.exited, 5_000))) {
+    fail(`Pier process session ${childIdentity.sessionId} retained child processes`)
+  }
+}
+
+function requestStop() {
+  stopRequested = true
+  stopping ??= stopChild()
+}
+
+process.on('SIGTERM', requestStop)
+process.on('SIGINT', requestStop)
+
+try {
+  config = await readControlInput()
+  if (
+    !config ||
+    typeof config !== 'object' ||
+    typeof config.command !== 'string' ||
+    !Array.isArray(config.args) ||
+    typeof config.cwd !== 'string' ||
+    !config.env ||
+    typeof config.env !== 'object' ||
+    typeof config.jobsDirectory !== 'string' ||
+    typeof config.jobName !== 'string' ||
+    typeof config.dockerCommand !== 'string'
+  ) {
+    fail('Pier supervisor launch request is malformed')
+  }
+
+  const identity = readJson(join(directory, identityFile), 'Pier trial identity')
+  if (stopRequested || existsSync(join(directory, stopFile))) {
+    const removedContainers = removeTrialContainers(config)
+    writeJsonAtomic(directory, terminalFile, {
+      schemaVersion: 1,
+      kind: 'pier-trial-terminal',
+      status: 'stopped',
+      processExited: true,
+      containersRemoved: true,
+      removedContainers,
+    })
+    process.exitCode = 0
+  } else {
+    const spawned = spawn(config.command, config.args, {
+      cwd: config.cwd,
+      env: config.env,
+      detached: true,
+      stdio: 'ignore',
+    })
+    if (spawned.pid === undefined) fail('Pier process started without a pid')
+    const exited = new Promise((resolveExit) => {
+      let settled = false
+      spawned.once('error', (error) => {
+        if (settled) return
+        settled = true
+        resolveExit({ code: null, signal: null, error })
+      })
+      spawned.once('close', (code, signal) => {
+        if (settled) return
+        settled = true
+        resolveExit({ code, signal })
+      })
+    })
+    child = { process: spawned, exited }
+    childIdentity = processIdentity(spawned.pid)
+    writeJsonAtomic(directory, identityFile, {
+      ...identity,
+      state: 'running',
+      pier: childIdentity,
+    })
+
+    if (stopRequested || existsSync(join(directory, stopFile))) {
+      stopRequested = true
+      stopping = stopChild()
+    }
+    const exit = await exited
+    if (stopping) await stopping
+    else await stopChild()
+    const removedContainers = removeTrialContainers(config)
+    const status = stopRequested ? 'stopped' : exit.code === 0 ? 'completed' : 'failed'
+    writeJsonAtomic(directory, terminalFile, {
+      schemaVersion: 1,
+      kind: 'pier-trial-terminal',
+      status,
+      processExited: true,
+      containersRemoved: true,
+      removedContainers,
+      exitCode: exit.code,
+      signal: exit.signal,
+      ...(exit.error ? { error: 'Pier process failed to start' } : {}),
+      ...(status === 'failed' && !exit.error
+        ? { error: `Pier process exited ${exit.signal ?? exit.code ?? 'without status'}` }
+        : {}),
+    })
+  }
+} catch (error) {
+  const cleanupErrors = []
+  try {
+    requestStop()
+    if (stopping) await stopping
+  } catch (cleanup) {
+    cleanupErrors.push(cleanup)
+  }
+  if (config) {
+    try {
+      removeTrialContainers(config)
+    } catch (cleanup) {
+      cleanupErrors.push(cleanup)
+    }
+  }
+  writeJsonAtomic(directory, terminalFile, {
+    schemaVersion: 1,
+    kind: 'pier-trial-terminal',
+    status: 'failed',
+    processExited: !childIdentity || !processMatches(childIdentity),
+    containersRemoved: cleanupErrors.length === 0,
+    error: `${error instanceof Error ? error.message : String(error)}${
+      cleanupErrors.length > 0
+        ? `; cleanup failed: ${cleanupErrors.map((item) => item?.message ?? String(item)).join('; ')}`
+        : ''
+    }`,
+  })
+  process.exitCode = 1
+}

package/src/resolve-client.ts

@@ -8,6 +8,10 @@
  *                              domains whose worker is a completion, or where box egress is blocked.
  *   - `router` + searchProvider → OFF-BOX `router-tools`: the same off-box loop with a live
  *                              `web_search` tool (the capability axis research benches need).
+ *   - `bridge`               → OFF-BOX: a local cli-bridge fronting a harness CLI
+ *                              (opencode/kimi-code/…) as the leaf executor. Same resumable
+ *                              `bridgeExecutor` the supervisor uses; harness+model ride the
+ *                              bridge `model` id (`${harness}/${model}`).
  *   - anything else (`sandbox`/a BackendType) → IN-BOX: a real `Sandbox`. The in-box backend
  *                              TYPE (opencode/codex/…) is set separately on the `AgentRunSpec`;
  *                              this only decides off-box-vs-in-box transport for `runLoop`.
@@ -15,7 +19,12 @@
  * Centralizing it here means a new entrypoint gets the full off-box/in-box matrix for free and
  * the mapping can't drift between callers.
  */
-import { createExecutor, inlineSandboxClient, type SandboxClient } from '@tangle-network/agent-runtime/loops'
+import {
+  createExecutor,
+  inlineSandboxClient,
+  resolveSandboxClient,
+  type SandboxClient,
+} from '@tangle-network/agent-runtime/loops'
 import { Sandbox } from '@tangle-network/sandbox'
 import { makeSearchExecutor, webSearchTool } from './search-tool'
 
@@ -29,8 +38,12 @@ export interface ResolveBenchClientOptions {
    *  with a live `web_search` tool backed by this provider (`you`/`exa`/…). */
   searchProvider?: string
   sandboxBaseUrl?: string
-  /** In-box sandbox timeout (ms). */
+  /** In-box sandbox timeout (ms). Also the per-turn deadline for the `bridge` backend. */
   timeoutMs?: number
+  /** `bridge` backend: cli-bridge base URL. Defaults to `http://127.0.0.1:3355`. */
+  bridgeUrl?: string
+  /** `bridge` backend: bearer the bridge requires. Falls back to `routerKey` when unset. */
+  bridgeBearer?: string
 }
 
 export function resolveBenchClient(opts: ResolveBenchClientOptions): SandboxClient {
@@ -50,6 +63,16 @@ export function resolveBenchClient(opts: ResolveBenchClientOptions): SandboxClie
     }
     return inlineSandboxClient(createExecutor({ backend: 'router', routerBaseUrl, routerKey, model }))
   }
+  if (backend === 'bridge') {
+    // bench's bearer fallback (`?? routerKey`) resolves first, then the shared
+    // resolver core wires the bridge seam — no re-implemented createExecutor branch.
+    const bridgeBearer = opts.bridgeBearer ?? routerKey
+    if (!bridgeBearer) throw new Error("resolveBenchClient: backend 'bridge' needs bridgeBearer or routerKey")
+    return resolveSandboxClient({
+      backend: 'bridge',
+      bridge: { url: opts.bridgeUrl, bearer: bridgeBearer, model, timeoutMs: opts.timeoutMs },
+    })
+  }
   return new Sandbox({
     baseUrl: opts.sandboxBaseUrl ?? 'https://sandbox.tangle.tools',
     apiKey: routerKey,

package/src/run-benchmarks-cli.mts

@@ -0,0 +1,72 @@
+/**
+ * Run a SUBSET of the benchmark registry over a matrix of agent cells, end to end:
+ *
+ *   export TANGLE_API_KEY=...                          # router + each adapter judge's creds
+ *   BENCHMARKS=humaneval,swe-bench \
+ *   CELLS=opencode/glm-4.6,codex/gpt-5,deepseek-v4-flash \
+ *   N=20 CONCURRENCY=6 tsx src/run-benchmarks-cli.mts
+ *
+ * A cell is `harness/model` (in-box: that coding harness runs the model) or a bare `model`
+ * (off-box: a router completion is the worker). The number is the adapter's OWN deterministic
+ * judge — pick benchmarks whose `adapter.judge` is a runnable checker. Output is a leaderboard:
+ * resolve-rate and mean graded score per (benchmark × cell), with any unavailable benchmark
+ * (missing Docker/venv/dataset) listed, never silently dropped.
+ */
+import { printBenchmarksReport, runBenchmarks, type BenchCell } from './run-benchmarks'
+
+const must = (k: string): string => {
+  const v = process.env[k]
+  if (!v) throw new Error(`env ${k} is required`)
+  return v
+}
+
+/** `harness/model` → in-box cell; bare `model` → off-box router cell. For cli-bridge, the
+ *  full model id is already `harness/model`, so do not split it. */
+function parseCell(spec: string): BenchCell {
+  const s = spec.trim()
+  if (process.env.BACKEND === 'bridge') {
+    return { label: s, model: s, backend: 'bridge', ...(process.env.SEARCH_PROVIDER ? { searchProvider: process.env.SEARCH_PROVIDER } : {}) }
+  }
+  const slash = s.indexOf('/')
+  if (slash > 0) {
+    const harness = s.slice(0, slash)
+    const model = s.slice(slash + 1)
+    return { label: s, harness, model, backend: process.env.BACKEND ?? 'sandbox', ...(process.env.SEARCH_PROVIDER ? { searchProvider: process.env.SEARCH_PROVIDER } : {}) }
+  }
+  return { label: s, model: s, backend: process.env.BACKEND ?? 'router', ...(process.env.SEARCH_PROVIDER ? { searchProvider: process.env.SEARCH_PROVIDER } : {}) }
+}
+
+async function main(): Promise<void> {
+  const benchmarks = (process.env.BENCHMARKS ?? '').split(',').map((s) => s.trim()).filter(Boolean)
+  if (benchmarks.length === 0) throw new Error('env BENCHMARKS is required (comma-separated registry keys, e.g. humaneval,swe-bench)')
+  const cells = (process.env.CELLS ?? process.env.WORKER_MODEL ?? 'deepseek-v4-flash')
+    .split(',').map((s) => s.trim()).filter(Boolean).map(parseCell)
+
+  const report = await runBenchmarks({
+    benchmarks,
+    cells,
+    routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',
+    routerKey: must('TANGLE_API_KEY'),
+    ...(process.env.BRIDGE_URL ? { bridgeUrl: process.env.BRIDGE_URL } : {}),
+    ...(process.env.BRIDGE_BEARER ?? process.env.CLI_BRIDGE_BEARER ? { bridgeBearer: process.env.BRIDGE_BEARER ?? process.env.CLI_BRIDGE_BEARER } : {}),
+    ...(process.env.SANDBOX_BASE ? { sandboxBaseUrl: process.env.SANDBOX_BASE } : {}),
+    n: Number(process.env.N ?? 10),
+    ...(process.env.IDS ? { ids: process.env.IDS.split(',') } : {}),
+    ...(process.env.SPLIT ? { split: process.env.SPLIT } : {}),
+    ...(process.env.REPS ? { reps: Number(process.env.REPS) } : {}),
+    ...(process.env.LOOP_ATTEMPTS ? { loopAttempts: Number(process.env.LOOP_ATTEMPTS) } : {}),
+    concurrency: Number(process.env.CONCURRENCY ?? 4),
+    ...(process.env.TIMEOUT_MS ? { timeoutMs: Number(process.env.TIMEOUT_MS) } : {}),
+    ...(process.env.VERIFY_JUDGE === '0' ? { verifyJudge: false } : {}),
+    onResult: (r) =>
+      console.log(`  [${r.ok ? (r.resolved ? 'PASS' : 'fail') : 'ERR '}] ${r.benchmark} ${r.cell} ${r.taskId} score=${r.score.toFixed(2)} ${(r.wallMs / 1000).toFixed(1)}s${r.detail ? ` (${r.detail})` : ''}`),
+  })
+
+  console.log(`\n=== benchmark matrix: ${report.benchmarks.join(', ')} × ${report.cells.length} cell(s) ===`)
+  console.log(printBenchmarksReport(report))
+}
+
+main().catch((e) => {
+  console.error(e instanceof Error ? (e.stack ?? e.message) : String(e))
+  process.exit(1)
+})

package/src/run-benchmarks-report.ts

@@ -0,0 +1,66 @@
+/**
+ * run-benchmarks-report — render a `runBenchmarks` result as a rich, multi-axis leaderboard (markdown +
+ * SVG + HTML), with the BENCHMARKS as the axes. `runBenchmarks` (the registry matrix runner) already
+ * returns a fixed-width CLI table; this bridges its per-task results into the domain-agnostic `leaderboard`
+ * engine so the same run also yields a publishable board — every harness×model×persona cell scored on every
+ * benchmark, the vals.ai-style surface.
+ *
+ * No new aggregation logic: it maps each `BenchCellTaskResult` into the universal `RunRecord` shape the
+ * engine reads (cell → profile, benchmark → axis, score → score), then calls the shared renderers.
+ * Errored shots (`ok:false`) are dropped, exactly as the runner excludes them from its own denominator.
+ */
+import type { RunRecord } from '@tangle-network/agent-eval'
+import {
+  type Leaderboard,
+  leaderboard,
+  renderLeaderboardHtml,
+  renderLeaderboardMarkdown,
+  renderLeaderboardSvg,
+} from '@tangle-network/agent-runtime/loops'
+import type { BenchCellTaskResult, RunBenchmarksReport } from './run-benchmarks'
+
+/** Project the runner's per-task results into `RunRecord`s the leaderboard engine reads. Cost/tokens are
+ *  left at 0 — `runBenchmarks` does not meter them per task, so the board shows them as not-captured rather
+ *  than inventing a number. */
+function toRecords(perTask: readonly BenchCellTaskResult[]): RunRecord[] {
+  return perTask
+    .filter((t) => t.ok)
+    .map(
+      (t) =>
+        ({
+          runId: `${t.benchmark}:${t.cell}:${t.taskId}:${t.rep}`,
+          experimentId: t.benchmark,
+          candidateId: t.cell,
+          seed: t.rep,
+          // The cell label is `harness/model` (or a persona id); carry it as the model + harness so the
+          // leaderboard's default profile key resolves to the cell.
+          model: t.cell,
+          promptHash: '',
+          configHash: '',
+          commitSha: '',
+          wallMs: t.wallMs,
+          costUsd: 0,
+          tokenUsage: { input: 0, output: 0 },
+          outcome: { holdoutScore: t.score, raw: { resolved: t.resolved ? 1 : 0 } },
+          splitTag: 'holdout',
+          scenarioId: t.benchmark,
+          agentProfile: { harness: t.cell.split('/')[0] ?? t.cell, model: t.cell },
+        }) as unknown as RunRecord,
+    )
+}
+
+/** Build the rich leaderboard from a `runBenchmarks` result — benchmarks are the axes, cells are the rows. */
+export function benchmarksLeaderboard(
+  report: RunBenchmarksReport,
+  opts: { title?: string; meta?: Record<string, string> } = {},
+): Leaderboard {
+  return leaderboard(toRecords(report.perTask), {
+    title: opts.title ?? 'Benchmark matrix — cell × benchmark',
+    // The cell (harness/model/persona) is the row label; one column per benchmark.
+    profileKeyOf: (r) => r.candidateId,
+    groupOf: (r) => r.scenarioId ?? r.experimentId,
+    ...(opts.meta ? { meta: opts.meta } : {}),
+  })
+}
+
+export { renderLeaderboardHtml, renderLeaderboardMarkdown, renderLeaderboardSvg }