@tangle-network/agent-bench 0.11.2 → 0.13.0

package/src/swe-arena/factory-command-container.ts

@@ -1,284 +0,0 @@
-import { randomUUID } from 'node:crypto'
-import { mkdir, mkdtemp, realpath, rm, stat } from 'node:fs/promises'
-import { tmpdir } from 'node:os'
-import { delimiter, isAbsolute, join } from 'node:path'
-
-import { run, type RunResult } from './proc'
-
-const WORKSPACE_PATH = '/workspace'
-const STATE_PATH = '/factory-state'
-const IMAGE_DIGEST_RE = /^.+@sha256:[0-9a-f]{64}$/
-const CONTAINER_PATH =
-  `${STATE_PATH}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin`
-const CONTROL_TIMEOUT_MS = 30_000
-
-export type FactoryCommandNetwork = 'enabled' | 'none'
-
-export interface RunFactoryCommandOptions {
-  image: string
-  network: FactoryCommandNetwork
-  timeoutMs: number
-  signal?: AbortSignal
-  maxBuffer?: number
-}
-
-function hostPath(baseEnv: NodeJS.ProcessEnv): string {
-  const path = (baseEnv.PATH ?? '')
-    .split(delimiter)
-    .filter((entry) => entry.length > 0 && isAbsolute(entry))
-    .join(delimiter)
-  if (!path) throw new Error('factory container control process requires an absolute PATH')
-  return path
-}
-
-async function createControlEnvironment(): Promise<{
-  env: NodeJS.ProcessEnv
-  dispose: () => Promise<void>
-}> {
-  const root = await mkdtemp(join(tmpdir(), 'factory-container-control-'))
-  const home = join(root, 'home')
-  const dockerConfig = join(root, 'docker')
-  const xdgConfig = join(root, 'xdg')
-  const temp = join(root, 'tmp')
-  try {
-    await Promise.all(
-      [home, dockerConfig, xdgConfig, temp].map((dir) =>
-        mkdir(dir, { recursive: true, mode: 0o700 }),
-      ),
-    )
-    let disposal: Promise<void> | undefined
-    return {
-      env: {
-        PATH: hostPath(process.env),
-        HOME: home,
-        DOCKER_CONFIG: dockerConfig,
-        XDG_CONFIG_HOME: xdgConfig,
-        TMPDIR: temp,
-        TMP: temp,
-        TEMP: temp,
-        LANG: 'C',
-        LC_ALL: 'C',
-        TZ: 'UTC',
-      },
-      dispose: () => (disposal ??= rm(root, { recursive: true, force: true })),
-    }
-  } catch (error) {
-    await rm(root, { recursive: true, force: true })
-    throw error
-  }
-}
-
-function currentUser(): { uid: number; gid: number } {
-  const uid = process.getuid?.()
-  const gid = process.getgid?.()
-  if (
-    typeof uid !== 'number' ||
-    typeof gid !== 'number' ||
-    !Number.isSafeInteger(uid) ||
-    !Number.isSafeInteger(gid)
-  ) {
-    throw new Error('factory command containers require a POSIX uid and gid')
-  }
-  if (uid === 0) {
-    throw new Error('factory command containers refuse to run candidate commands as root')
-  }
-  return { uid, gid }
-}
-
-function containerEnvironment(): string[] {
-  return [
-    `PATH=${CONTAINER_PATH}`,
-    `HOME=${STATE_PATH}/home`,
-    `XDG_CONFIG_HOME=${STATE_PATH}/xdg/config`,
-    `XDG_DATA_HOME=${STATE_PATH}/xdg/data`,
-    `XDG_CACHE_HOME=${STATE_PATH}/xdg/cache`,
-    `XDG_STATE_HOME=${STATE_PATH}/xdg/state`,
-    `XDG_RUNTIME_DIR=${STATE_PATH}/xdg/runtime`,
-    'TMPDIR=/tmp',
-    'TMP=/tmp',
-    'TEMP=/tmp',
-    'LANG=C',
-    'LC_ALL=C',
-    'TZ=UTC',
-    'CI=false',
-    `COREPACK_HOME=${STATE_PATH}/corepack`,
-    'COREPACK_DEFAULT_TO_LATEST=0',
-    'COREPACK_ENABLE_DOWNLOAD_PROMPT=0',
-    `PNPM_HOME=${STATE_PATH}/pnpm-home`,
-    `npm_config_store_dir=${STATE_PATH}/pnpm-store`,
-    `NPM_CONFIG_CACHE=${STATE_PATH}/npm-cache`,
-    `NPM_CONFIG_USERCONFIG=${STATE_PATH}/npm/user.npmrc`,
-    `NPM_CONFIG_GLOBALCONFIG=${STATE_PATH}/npm/global.npmrc`,
-    'GIT_CONFIG_NOSYSTEM=1',
-    `GIT_CONFIG_GLOBAL=${STATE_PATH}/git/global.config`,
-    `GIT_CONFIG_SYSTEM=${STATE_PATH}/git/system.config`,
-    'GIT_TERMINAL_PROMPT=0',
-    'GCM_INTERACTIVE=Never',
-  ]
-}
-
-const COMMAND_WRAPPER = `
-set -eu
-mkdir -p \
-  "$HOME" \
-  "$XDG_CONFIG_HOME" \
-  "$XDG_DATA_HOME" \
-  "$XDG_CACHE_HOME" \
-  "$XDG_STATE_HOME" \
-  "$XDG_RUNTIME_DIR" \
-  "$COREPACK_HOME" \
-  "$PNPM_HOME" \
-  "$npm_config_store_dir" \
-  "$NPM_CONFIG_CACHE" \
-  "${STATE_PATH}/npm" \
-  "${STATE_PATH}/git" \
-  "${STATE_PATH}/bin"
-touch \
-  "$NPM_CONFIG_USERCONFIG" \
-  "$NPM_CONFIG_GLOBALCONFIG" \
-  "$GIT_CONFIG_GLOBAL" \
-  "$GIT_CONFIG_SYSTEM"
-corepack enable --install-directory "${STATE_PATH}/bin" >/dev/null
-exec /bin/bash --noprofile --norc -c "$1"
-`
-
-function dockerArgs(input: {
-  name: string
-  workspace: string
-  command: string
-  image: string
-  network: FactoryCommandNetwork
-  uid: number
-  gid: number
-}): string[] {
-  const user = `${input.uid}:${input.gid}`
-  return [
-    'run',
-    '--rm',
-    '--name',
-    input.name,
-    '--hostname',
-    'factory-command',
-    '--pull=missing',
-    '--read-only',
-    '--init',
-    '--cap-drop=ALL',
-    '--security-opt=no-new-privileges',
-    '--pids-limit=512',
-    '--memory=4g',
-    '--memory-swap=4g',
-    '--cpus=2',
-    '--ulimit',
-    'nofile=4096:4096',
-    '--ipc=private',
-    '--network',
-    input.network === 'none' ? 'none' : 'bridge',
-    '--user',
-    user,
-    '--workdir',
-    WORKSPACE_PATH,
-    '--mount',
-    `type=bind,src=${input.workspace},dst=${WORKSPACE_PATH},bind-propagation=rprivate`,
-    '--tmpfs',
-    `/tmp:rw,nosuid,nodev,mode=1777,size=512m,uid=${input.uid},gid=${input.gid}`,
-    '--tmpfs',
-    `${STATE_PATH}:rw,nosuid,nodev,mode=700,size=2048m,uid=${input.uid},gid=${input.gid}`,
-    '--entrypoint=/usr/bin/env',
-    input.image,
-    '-i',
-    ...containerEnvironment(),
-    '/bin/bash',
-    '--noprofile',
-    '--norc',
-    '-c',
-    COMMAND_WRAPPER,
-    'factory-command',
-    input.command,
-  ]
-}
-
-function isAlreadyRemoved(result: RunResult): boolean {
-  return /no such container/iu.test(result.stderr || result.stdout)
-}
-
-/**
- * Run one repository-controlled command in a fresh container.
- *
- * The candidate sees one writable bind mount: its disposable workspace.
- * Process ids, root files, HOME, XDG state, Corepack, npm, and pnpm caches are
- * private to this invocation and disappear when the container exits.
- */
-export async function runFactoryCommand(
-  workspace: string,
-  command: string,
-  options: RunFactoryCommandOptions,
-): Promise<RunResult> {
-  if (!command.trim()) throw new Error('factory command must not be empty')
-  if (!IMAGE_DIGEST_RE.test(options.image)) {
-    throw new Error(`factory command image must be pinned by sha256 digest: ${options.image}`)
-  }
-  if (!Number.isSafeInteger(options.timeoutMs) || options.timeoutMs <= 0) {
-    throw new Error(`factory command timeout must be a positive integer: ${options.timeoutMs}`)
-  }
-  const workspacePath = await realpath(workspace)
-  if (!(await stat(workspacePath)).isDirectory()) {
-    throw new Error(`factory command workspace is not a directory: ${workspacePath}`)
-  }
-  if (workspacePath.includes(',')) {
-    throw new Error(`factory command workspace path contains an unsupported comma: ${workspacePath}`)
-  }
-  const { uid, gid } = currentUser()
-  const name = `factory-command-${process.pid}-${randomUUID()}`
-  const control = await createControlEnvironment()
-  let commandResult: RunResult | undefined
-  let commandError: unknown
-  try {
-    commandResult = await run(
-      'docker',
-      dockerArgs({
-        name,
-        workspace: workspacePath,
-        command,
-        image: options.image,
-        network: options.network,
-        uid,
-        gid,
-      }),
-      {
-        env: control.env,
-        timeoutMs: options.timeoutMs,
-        killGraceMs: 2_000,
-        ...(options.signal ? { signal: options.signal } : {}),
-        ...(options.maxBuffer === undefined ? {} : { maxBuffer: options.maxBuffer }),
-      },
-    )
-  } catch (error) {
-    commandError = error
-  }
-
-  let cleanupError: unknown
-  try {
-    const cleanup = await run('docker', ['rm', '-f', name], {
-      env: control.env,
-      timeoutMs: CONTROL_TIMEOUT_MS,
-    })
-    if (cleanup.code !== 0 && !isAlreadyRemoved(cleanup)) {
-      cleanupError = new Error(
-        `factory command container cleanup failed (${cleanup.code}): ${(cleanup.stderr || cleanup.stdout).slice(-1000)}`,
-      )
-    }
-  } catch (error) {
-    cleanupError = error
-  }
-  await control.dispose()
-
-  if (commandError !== undefined && cleanupError !== undefined) {
-    throw new AggregateError(
-      [commandError, cleanupError],
-      'factory command and container cleanup both failed',
-    )
-  }
-  if (commandError !== undefined) throw commandError
-  if (cleanupError !== undefined) throw cleanupError
-  return commandResult!
-}

package/src/swe-arena/factory-judge-child.mts

@@ -1,228 +0,0 @@
-/**
- * Factory judge child — the factory-bench equivalent of `judge-child.mts`.
- * One candidate patch, one hidden-test verdict, printed as a single
- * `JUDGE_RESULT {...}` line so serialized-judge.ts (queue, retry, SIGKILL
- * ceiling) carries over unchanged. The judge workspace is a `git archive`
- * export of the instance's base commit from the local mirror. Setup and judge
- * commands run in separate containers, and the PR's added test files are
- * overlaid from the merge commit only after setup completes.
- *
- * Steps: export base tree → apply candidate patch → run `setup_cmds` in a
- * fresh container → overlay judge tests from `judge_ref` → run `judge_cmds`
- * in a second network-disabled container →
- * parse vitest's pass/fail counts → JUDGE_RESULT with partial credit.
- *
- * Verdict semantics:
- *  - `resolved` = every calibrated judge test passes (passed === total, 0 failed).
- *  - `score` = passed / judgeTestTotal, where the denominator is the CALIBRATED
- *    total from the manifest's resolved_criterion — never vitest's reported
- *    total, which shrinks when a judge file fails to collect.
- *  - An unappliable patch is the candidate's failure → resolved:false, score 0.
- *  - A setup/parse failure is an INFRA failure → exit 1 with no JUDGE_RESULT
- *    line, so serialized-judge retries and then records `resolved: null`
- *    (inconclusive), never a fabricated verdict.
- *
- * usage: node --import tsx src/swe-arena/factory-judge-child.mts <instanceDir> <patchPath>
- */
-
-import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
-import { tmpdir } from 'node:os'
-import { dirname, join } from 'node:path'
-import { pathToFileURL } from 'node:url'
-import { applyPatchWithFallback } from './calibrate.ts'
-import { runFactoryCommand } from './factory-command-container.ts'
-import { loadFactoryInstance, type LoadedFactoryInstance } from './fixtures.ts'
-import { runOk, shq } from './proc.ts'
-
-export interface VitestSummary {
-  passed: number
-  failed: number
-  /** vitest's own reported total (collected tests only — informational). */
-  reportedTotal: number
-}
-
-const stripAnsi = (s: string): string => s.replace(/\u001b\[[0-9;]*m/g, '')
-
-/**
- * Parse the `Tests  …` summary line of a vitest run (v3 and v4 formats):
- *   "      Tests  30 passed (30)"
- *   "      Tests  17 failed | 13 passed (30)"
- *   "      Tests  no tests"        (collection failure — 0 passed is a verdict)
- * Returns undefined when no summary line exists (infra failure, not a verdict).
- */
-export function parseVitestSummary(output: string): VitestSummary | undefined {
-  for (const rawLine of stripAnsi(output).split('\n')) {
-    const m = /^\s*Tests\s{2,}(.+)$/.exec(rawLine)
-    if (!m) continue
-    const body = m[1].trim()
-    if (body.startsWith('no tests')) return { passed: 0, failed: 0, reportedTotal: 0 }
-    const passed = /(\d+) passed/.exec(body)
-    const failed = /(\d+) failed/.exec(body)
-    const total = /\((\d+)\)/.exec(body)
-    if (!passed && !failed) continue
-    return {
-      passed: passed ? Number(passed[1]) : 0,
-      failed: failed ? Number(failed[1]) : 0,
-      reportedTotal: total ? Number(total[1]) : 0,
-    }
-  }
-  return undefined
-}
-
-/** The JUDGE_RESULT row this child prints (superset of the SWE child's fields). */
-export interface FactoryJudgeResult {
-  iid: string
-  resolved: boolean
-  /** passed / judgeTestTotal — partial credit for the staircase. */
-  score: number
-  passed: number
-  total: number
-  secs: number
-  patch_bytes: number
-  error?: string
-}
-
-/** Parse one JUDGE_RESULT line back out of child stdout (calibrate + tests). */
-export function parseFactoryJudgeResult(stdout: string): FactoryJudgeResult | undefined {
-  const line = stdout.split('\n').find((l) => l.includes('JUDGE_RESULT'))
-  if (!line) return undefined
-  try {
-    const parsed = JSON.parse(line.slice(line.indexOf('JUDGE_RESULT') + 'JUDGE_RESULT'.length).trim())
-    if (typeof parsed !== 'object' || parsed === null || typeof parsed.resolved !== 'boolean') return undefined
-    return parsed as FactoryJudgeResult
-  } catch {
-    return undefined
-  }
-}
-
-/** `git archive <ref> | tar -x` — a bare tree, no .git, no refs, no future history. */
-export async function exportBaseTree(mirror: string, ref: string, dest: string): Promise<void> {
-  await mkdir(dest, { recursive: true })
-  await runOk('bash', ['-c', `git -C ${shq(mirror)} archive ${shq(ref)} | tar -x -C ${shq(dest)}`])
-}
-
-/** Overlay each judge test from the merge commit (`git show <judge_ref>:<path>`). */
-export async function overlayJudgeTests(inst: LoadedFactoryInstance, ws: string): Promise<void> {
-  for (const testPath of inst.judge_tests) {
-    const show = await runOk('git', ['-C', inst.repo_local_mirror, 'show', `${inst.judge_ref}:${testPath}`])
-    await mkdir(dirname(join(ws, testPath)), { recursive: true })
-    await writeFile(join(ws, testPath), show.stdout)
-  }
-}
-
-export interface FactoryJudgeOutcome {
-  result: FactoryJudgeResult
-  /** Raw judge_cmds output, for post-mortem. */
-  output: string
-}
-
-/**
- * Build the judge workspace and run the judge over one patch. Throws on infra
- * failure (export/overlay/setup/parse); returns a verdict otherwise.
- */
-export async function judgeFactoryPatch(
-  inst: LoadedFactoryInstance,
-  patch: string,
-  opts: { workDir?: string; keepWorkspace?: boolean } = {},
-): Promise<FactoryJudgeOutcome> {
-  const t0 = Date.now()
-  const ws = opts.workDir ?? (await mkdtemp(join(tmpdir(), `factory-judge-${inst.id.replace(/[^a-zA-Z0-9.-]/g, '-')}-`)))
-  const cmdTimeoutMs = inst.timeout_s * 1000
-  try {
-    await exportBaseTree(inst.repo_local_mirror, inst.base_commit, ws)
-    // A git repo (fresh, historyless) so `git apply` and repo tooling behave;
-    // judge-side only, so its refs are irrelevant to leak prevention.
-    await runOk('git', ['-C', ws, 'init', '-q'])
-
-    if (patch.trim().length > 0) {
-      const patchFile = join(ws, '.factory-candidate.patch')
-      await writeFile(patchFile, patch)
-      const applyRc = await applyPatchWithFallback(ws, patchFile)
-      await rm(patchFile, { force: true })
-      if (applyRc !== 0) {
-        return {
-          result: {
-            iid: inst.id,
-            resolved: false,
-            score: 0,
-            passed: 0,
-            total: inst.judgeTestTotal,
-            secs: Math.round((Date.now() - t0) / 1000),
-            patch_bytes: patch.length,
-            error: `apply-failed rc=${applyRc}`,
-          },
-          output: '',
-        }
-      }
-    }
-
-    for (const cmd of inst.setup_cmds) {
-      const res = await runFactoryCommand(ws, cmd, {
-        image: inst.command_image,
-        network: 'enabled',
-        timeoutMs: cmdTimeoutMs,
-      })
-      if (res.code !== 0) {
-        throw new Error(
-          `setup_cmd failed (rc=${res.code}${res.timedOut ? ', timeout' : ''}): ${cmd}\n${(res.stderr || res.stdout).slice(-2000)}`,
-        )
-      }
-    }
-
-    await overlayJudgeTests(inst, ws)
-
-    let passed = 0
-    let failed = 0
-    let output = ''
-    for (const cmd of inst.judge_cmds) {
-      const res = await runFactoryCommand(ws, cmd, {
-        image: inst.command_image,
-        network: 'none',
-        timeoutMs: cmdTimeoutMs,
-      })
-      const combined = res.stdout + res.stderr
-      output += combined
-      if (res.timedOut) throw new Error(`judge_cmd timed out after ${cmdTimeoutMs}ms: ${cmd}`)
-      const summary = parseVitestSummary(combined)
-      if (!summary) {
-        throw new Error(`judge_cmd produced no parseable test summary (rc=${res.code}): ${cmd}\n${combined.slice(-2000)}`)
-      }
-      passed += summary.passed
-      failed += summary.failed
-    }
-
-    const total = inst.judgeTestTotal
-    return {
-      result: {
-        iid: inst.id,
-        resolved: failed === 0 && passed === total,
-        score: total > 0 ? Number((passed / total).toFixed(4)) : 0,
-        passed,
-        total,
-        secs: Math.round((Date.now() - t0) / 1000),
-        patch_bytes: patch.length,
-      },
-      output,
-    }
-  } finally {
-    if (!opts.keepWorkspace) await rm(ws, { recursive: true, force: true })
-  }
-}
-
-// ---------------------------------------------------------------------------
-// CLI (guarded so calibrate/tests can import the helpers above).
-// ---------------------------------------------------------------------------
-
-const isMain = process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href
-
-if (isMain) {
-  const [, , instanceDir, patchPath] = process.argv
-  if (!instanceDir || !patchPath) {
-    console.error('usage: node --import tsx factory-judge-child.mts <instanceDir> <patchPath>')
-    process.exit(2)
-  }
-  const inst = loadFactoryInstance(instanceDir)
-  const patch = await readFile(patchPath, 'utf8')
-  const { result } = await judgeFactoryPatch(inst, patch)
-  console.log('JUDGE_RESULT ' + JSON.stringify(result))
-}