@tangle-network/agent-bench 0.3.6 → 0.3.8

package/src/swe-arena/factory-command-container.ts

@@ -0,0 +1,284 @@
+import { randomUUID } from 'node:crypto'
+import { mkdir, mkdtemp, realpath, rm, stat } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { delimiter, isAbsolute, join } from 'node:path'
+
+import { run, type RunResult } from './proc'
+
+const WORKSPACE_PATH = '/workspace'
+const STATE_PATH = '/factory-state'
+const IMAGE_DIGEST_RE = /^.+@sha256:[0-9a-f]{64}$/
+const CONTAINER_PATH =
+  `${STATE_PATH}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin`
+const CONTROL_TIMEOUT_MS = 30_000
+
+export type FactoryCommandNetwork = 'enabled' | 'none'
+
+export interface RunFactoryCommandOptions {
+  image: string
+  network: FactoryCommandNetwork
+  timeoutMs: number
+  signal?: AbortSignal
+  maxBuffer?: number
+}
+
+function hostPath(baseEnv: NodeJS.ProcessEnv): string {
+  const path = (baseEnv.PATH ?? '')
+    .split(delimiter)
+    .filter((entry) => entry.length > 0 && isAbsolute(entry))
+    .join(delimiter)
+  if (!path) throw new Error('factory container control process requires an absolute PATH')
+  return path
+}
+
+async function createControlEnvironment(): Promise<{
+  env: NodeJS.ProcessEnv
+  dispose: () => Promise<void>
+}> {
+  const root = await mkdtemp(join(tmpdir(), 'factory-container-control-'))
+  const home = join(root, 'home')
+  const dockerConfig = join(root, 'docker')
+  const xdgConfig = join(root, 'xdg')
+  const temp = join(root, 'tmp')
+  try {
+    await Promise.all(
+      [home, dockerConfig, xdgConfig, temp].map((dir) =>
+        mkdir(dir, { recursive: true, mode: 0o700 }),
+      ),
+    )
+    let disposal: Promise<void> | undefined
+    return {
+      env: {
+        PATH: hostPath(process.env),
+        HOME: home,
+        DOCKER_CONFIG: dockerConfig,
+        XDG_CONFIG_HOME: xdgConfig,
+        TMPDIR: temp,
+        TMP: temp,
+        TEMP: temp,
+        LANG: 'C',
+        LC_ALL: 'C',
+        TZ: 'UTC',
+      },
+      dispose: () => (disposal ??= rm(root, { recursive: true, force: true })),
+    }
+  } catch (error) {
+    await rm(root, { recursive: true, force: true })
+    throw error
+  }
+}
+
+function currentUser(): { uid: number; gid: number } {
+  const uid = process.getuid?.()
+  const gid = process.getgid?.()
+  if (
+    typeof uid !== 'number' ||
+    typeof gid !== 'number' ||
+    !Number.isSafeInteger(uid) ||
+    !Number.isSafeInteger(gid)
+  ) {
+    throw new Error('factory command containers require a POSIX uid and gid')
+  }
+  if (uid === 0) {
+    throw new Error('factory command containers refuse to run candidate commands as root')
+  }
+  return { uid, gid }
+}
+
+function containerEnvironment(): string[] {
+  return [
+    `PATH=${CONTAINER_PATH}`,
+    `HOME=${STATE_PATH}/home`,
+    `XDG_CONFIG_HOME=${STATE_PATH}/xdg/config`,
+    `XDG_DATA_HOME=${STATE_PATH}/xdg/data`,
+    `XDG_CACHE_HOME=${STATE_PATH}/xdg/cache`,
+    `XDG_STATE_HOME=${STATE_PATH}/xdg/state`,
+    `XDG_RUNTIME_DIR=${STATE_PATH}/xdg/runtime`,
+    'TMPDIR=/tmp',
+    'TMP=/tmp',
+    'TEMP=/tmp',
+    'LANG=C',
+    'LC_ALL=C',
+    'TZ=UTC',
+    'CI=false',
+    `COREPACK_HOME=${STATE_PATH}/corepack`,
+    'COREPACK_DEFAULT_TO_LATEST=0',
+    'COREPACK_ENABLE_DOWNLOAD_PROMPT=0',
+    `PNPM_HOME=${STATE_PATH}/pnpm-home`,
+    `npm_config_store_dir=${STATE_PATH}/pnpm-store`,
+    `NPM_CONFIG_CACHE=${STATE_PATH}/npm-cache`,
+    `NPM_CONFIG_USERCONFIG=${STATE_PATH}/npm/user.npmrc`,
+    `NPM_CONFIG_GLOBALCONFIG=${STATE_PATH}/npm/global.npmrc`,
+    'GIT_CONFIG_NOSYSTEM=1',
+    `GIT_CONFIG_GLOBAL=${STATE_PATH}/git/global.config`,
+    `GIT_CONFIG_SYSTEM=${STATE_PATH}/git/system.config`,
+    'GIT_TERMINAL_PROMPT=0',
+    'GCM_INTERACTIVE=Never',
+  ]
+}
+
+const COMMAND_WRAPPER = `
+set -eu
+mkdir -p \
+  "$HOME" \
+  "$XDG_CONFIG_HOME" \
+  "$XDG_DATA_HOME" \
+  "$XDG_CACHE_HOME" \
+  "$XDG_STATE_HOME" \
+  "$XDG_RUNTIME_DIR" \
+  "$COREPACK_HOME" \
+  "$PNPM_HOME" \
+  "$npm_config_store_dir" \
+  "$NPM_CONFIG_CACHE" \
+  "${STATE_PATH}/npm" \
+  "${STATE_PATH}/git" \
+  "${STATE_PATH}/bin"
+touch \
+  "$NPM_CONFIG_USERCONFIG" \
+  "$NPM_CONFIG_GLOBALCONFIG" \
+  "$GIT_CONFIG_GLOBAL" \
+  "$GIT_CONFIG_SYSTEM"
+corepack enable --install-directory "${STATE_PATH}/bin" >/dev/null
+exec /bin/bash --noprofile --norc -c "$1"
+`
+
+function dockerArgs(input: {
+  name: string
+  workspace: string
+  command: string
+  image: string
+  network: FactoryCommandNetwork
+  uid: number
+  gid: number
+}): string[] {
+  const user = `${input.uid}:${input.gid}`
+  return [
+    'run',
+    '--rm',
+    '--name',
+    input.name,
+    '--hostname',
+    'factory-command',
+    '--pull=missing',
+    '--read-only',
+    '--init',
+    '--cap-drop=ALL',
+    '--security-opt=no-new-privileges',
+    '--pids-limit=512',
+    '--memory=4g',
+    '--memory-swap=4g',
+    '--cpus=2',
+    '--ulimit',
+    'nofile=4096:4096',
+    '--ipc=private',
+    '--network',
+    input.network === 'none' ? 'none' : 'bridge',
+    '--user',
+    user,
+    '--workdir',
+    WORKSPACE_PATH,
+    '--mount',
+    `type=bind,src=${input.workspace},dst=${WORKSPACE_PATH},bind-propagation=rprivate`,
+    '--tmpfs',
+    `/tmp:rw,nosuid,nodev,mode=1777,size=512m,uid=${input.uid},gid=${input.gid}`,
+    '--tmpfs',
+    `${STATE_PATH}:rw,nosuid,nodev,mode=700,size=2048m,uid=${input.uid},gid=${input.gid}`,
+    '--entrypoint=/usr/bin/env',
+    input.image,
+    '-i',
+    ...containerEnvironment(),
+    '/bin/bash',
+    '--noprofile',
+    '--norc',
+    '-c',
+    COMMAND_WRAPPER,
+    'factory-command',
+    input.command,
+  ]
+}
+
+function isAlreadyRemoved(result: RunResult): boolean {
+  return /no such container/iu.test(result.stderr || result.stdout)
+}
+
+/**
+ * Run one repository-controlled command in a fresh container.
+ *
+ * The candidate sees one writable bind mount: its disposable workspace.
+ * Process ids, root files, HOME, XDG state, Corepack, npm, and pnpm caches are
+ * private to this invocation and disappear when the container exits.
+ */
+export async function runFactoryCommand(
+  workspace: string,
+  command: string,
+  options: RunFactoryCommandOptions,
+): Promise<RunResult> {
+  if (!command.trim()) throw new Error('factory command must not be empty')
+  if (!IMAGE_DIGEST_RE.test(options.image)) {
+    throw new Error(`factory command image must be pinned by sha256 digest: ${options.image}`)
+  }
+  if (!Number.isSafeInteger(options.timeoutMs) || options.timeoutMs <= 0) {
+    throw new Error(`factory command timeout must be a positive integer: ${options.timeoutMs}`)
+  }
+  const workspacePath = await realpath(workspace)
+  if (!(await stat(workspacePath)).isDirectory()) {
+    throw new Error(`factory command workspace is not a directory: ${workspacePath}`)
+  }
+  if (workspacePath.includes(',')) {
+    throw new Error(`factory command workspace path contains an unsupported comma: ${workspacePath}`)
+  }
+  const { uid, gid } = currentUser()
+  const name = `factory-command-${process.pid}-${randomUUID()}`
+  const control = await createControlEnvironment()
+  let commandResult: RunResult | undefined
+  let commandError: unknown
+  try {
+    commandResult = await run(
+      'docker',
+      dockerArgs({
+        name,
+        workspace: workspacePath,
+        command,
+        image: options.image,
+        network: options.network,
+        uid,
+        gid,
+      }),
+      {
+        env: control.env,
+        timeoutMs: options.timeoutMs,
+        killGraceMs: 2_000,
+        ...(options.signal ? { signal: options.signal } : {}),
+        ...(options.maxBuffer === undefined ? {} : { maxBuffer: options.maxBuffer }),
+      },
+    )
+  } catch (error) {
+    commandError = error
+  }
+
+  let cleanupError: unknown
+  try {
+    const cleanup = await run('docker', ['rm', '-f', name], {
+      env: control.env,
+      timeoutMs: CONTROL_TIMEOUT_MS,
+    })
+    if (cleanup.code !== 0 && !isAlreadyRemoved(cleanup)) {
+      cleanupError = new Error(
+        `factory command container cleanup failed (${cleanup.code}): ${(cleanup.stderr || cleanup.stdout).slice(-1000)}`,
+      )
+    }
+  } catch (error) {
+    cleanupError = error
+  }
+  await control.dispose()
+
+  if (commandError !== undefined && cleanupError !== undefined) {
+    throw new AggregateError(
+      [commandError, cleanupError],
+      'factory command and container cleanup both failed',
+    )
+  }
+  if (commandError !== undefined) throw commandError
+  if (cleanupError !== undefined) throw cleanupError
+  return commandResult!
+}

package/src/swe-arena/factory-judge-child.mts

@@ -0,0 +1,228 @@
+/**
+ * Factory judge child — the factory-bench equivalent of `judge-child.mts`.
+ * One candidate patch, one hidden-test verdict, printed as a single
+ * `JUDGE_RESULT {...}` line so serialized-judge.ts (queue, retry, SIGKILL
+ * ceiling) carries over unchanged. The judge workspace is a `git archive`
+ * export of the instance's base commit from the local mirror. Setup and judge
+ * commands run in separate containers, and the PR's added test files are
+ * overlaid from the merge commit only after setup completes.
+ *
+ * Steps: export base tree → apply candidate patch → run `setup_cmds` in a
+ * fresh container → overlay judge tests from `judge_ref` → run `judge_cmds`
+ * in a second network-disabled container →
+ * parse vitest's pass/fail counts → JUDGE_RESULT with partial credit.
+ *
+ * Verdict semantics:
+ *  - `resolved` = every calibrated judge test passes (passed === total, 0 failed).
+ *  - `score` = passed / judgeTestTotal, where the denominator is the CALIBRATED
+ *    total from the manifest's resolved_criterion — never vitest's reported
+ *    total, which shrinks when a judge file fails to collect.
+ *  - An unappliable patch is the candidate's failure → resolved:false, score 0.
+ *  - A setup/parse failure is an INFRA failure → exit 1 with no JUDGE_RESULT
+ *    line, so serialized-judge retries and then records `resolved: null`
+ *    (inconclusive), never a fabricated verdict.
+ *
+ * usage: node --import tsx src/swe-arena/factory-judge-child.mts <instanceDir> <patchPath>
+ */
+
+import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { dirname, join } from 'node:path'
+import { pathToFileURL } from 'node:url'
+import { applyPatchWithFallback } from './calibrate.ts'
+import { runFactoryCommand } from './factory-command-container.ts'
+import { loadFactoryInstance, type LoadedFactoryInstance } from './fixtures.ts'
+import { runOk, shq } from './proc.ts'
+
+export interface VitestSummary {
+  passed: number
+  failed: number
+  /** vitest's own reported total (collected tests only — informational). */
+  reportedTotal: number
+}
+
+const stripAnsi = (s: string): string => s.replace(/\u001b\[[0-9;]*m/g, '')
+
+/**
+ * Parse the `Tests  …` summary line of a vitest run (v3 and v4 formats):
+ *   "      Tests  30 passed (30)"
+ *   "      Tests  17 failed | 13 passed (30)"
+ *   "      Tests  no tests"        (collection failure — 0 passed is a verdict)
+ * Returns undefined when no summary line exists (infra failure, not a verdict).
+ */
+export function parseVitestSummary(output: string): VitestSummary | undefined {
+  for (const rawLine of stripAnsi(output).split('\n')) {
+    const m = /^\s*Tests\s{2,}(.+)$/.exec(rawLine)
+    if (!m) continue
+    const body = m[1].trim()
+    if (body.startsWith('no tests')) return { passed: 0, failed: 0, reportedTotal: 0 }
+    const passed = /(\d+) passed/.exec(body)
+    const failed = /(\d+) failed/.exec(body)
+    const total = /\((\d+)\)/.exec(body)
+    if (!passed && !failed) continue
+    return {
+      passed: passed ? Number(passed[1]) : 0,
+      failed: failed ? Number(failed[1]) : 0,
+      reportedTotal: total ? Number(total[1]) : 0,
+    }
+  }
+  return undefined
+}
+
+/** The JUDGE_RESULT row this child prints (superset of the SWE child's fields). */
+export interface FactoryJudgeResult {
+  iid: string
+  resolved: boolean
+  /** passed / judgeTestTotal — partial credit for the staircase. */
+  score: number
+  passed: number
+  total: number
+  secs: number
+  patch_bytes: number
+  error?: string
+}
+
+/** Parse one JUDGE_RESULT line back out of child stdout (calibrate + tests). */
+export function parseFactoryJudgeResult(stdout: string): FactoryJudgeResult | undefined {
+  const line = stdout.split('\n').find((l) => l.includes('JUDGE_RESULT'))
+  if (!line) return undefined
+  try {
+    const parsed = JSON.parse(line.slice(line.indexOf('JUDGE_RESULT') + 'JUDGE_RESULT'.length).trim())
+    if (typeof parsed !== 'object' || parsed === null || typeof parsed.resolved !== 'boolean') return undefined
+    return parsed as FactoryJudgeResult
+  } catch {
+    return undefined
+  }
+}
+
+/** `git archive <ref> | tar -x` — a bare tree, no .git, no refs, no future history. */
+export async function exportBaseTree(mirror: string, ref: string, dest: string): Promise<void> {
+  await mkdir(dest, { recursive: true })
+  await runOk('bash', ['-c', `git -C ${shq(mirror)} archive ${shq(ref)} | tar -x -C ${shq(dest)}`])
+}
+
+/** Overlay each judge test from the merge commit (`git show <judge_ref>:<path>`). */
+export async function overlayJudgeTests(inst: LoadedFactoryInstance, ws: string): Promise<void> {
+  for (const testPath of inst.judge_tests) {
+    const show = await runOk('git', ['-C', inst.repo_local_mirror, 'show', `${inst.judge_ref}:${testPath}`])
+    await mkdir(dirname(join(ws, testPath)), { recursive: true })
+    await writeFile(join(ws, testPath), show.stdout)
+  }
+}
+
+export interface FactoryJudgeOutcome {
+  result: FactoryJudgeResult
+  /** Raw judge_cmds output, for post-mortem. */
+  output: string
+}
+
+/**
+ * Build the judge workspace and run the judge over one patch. Throws on infra
+ * failure (export/overlay/setup/parse); returns a verdict otherwise.
+ */
+export async function judgeFactoryPatch(
+  inst: LoadedFactoryInstance,
+  patch: string,
+  opts: { workDir?: string; keepWorkspace?: boolean } = {},
+): Promise<FactoryJudgeOutcome> {
+  const t0 = Date.now()
+  const ws = opts.workDir ?? (await mkdtemp(join(tmpdir(), `factory-judge-${inst.id.replace(/[^a-zA-Z0-9.-]/g, '-')}-`)))
+  const cmdTimeoutMs = inst.timeout_s * 1000
+  try {
+    await exportBaseTree(inst.repo_local_mirror, inst.base_commit, ws)
+    // A git repo (fresh, historyless) so `git apply` and repo tooling behave;
+    // judge-side only, so its refs are irrelevant to leak prevention.
+    await runOk('git', ['-C', ws, 'init', '-q'])
+
+    if (patch.trim().length > 0) {
+      const patchFile = join(ws, '.factory-candidate.patch')
+      await writeFile(patchFile, patch)
+      const applyRc = await applyPatchWithFallback(ws, patchFile)
+      await rm(patchFile, { force: true })
+      if (applyRc !== 0) {
+        return {
+          result: {
+            iid: inst.id,
+            resolved: false,
+            score: 0,
+            passed: 0,
+            total: inst.judgeTestTotal,
+            secs: Math.round((Date.now() - t0) / 1000),
+            patch_bytes: patch.length,
+            error: `apply-failed rc=${applyRc}`,
+          },
+          output: '',
+        }
+      }
+    }
+
+    for (const cmd of inst.setup_cmds) {
+      const res = await runFactoryCommand(ws, cmd, {
+        image: inst.command_image,
+        network: 'enabled',
+        timeoutMs: cmdTimeoutMs,
+      })
+      if (res.code !== 0) {
+        throw new Error(
+          `setup_cmd failed (rc=${res.code}${res.timedOut ? ', timeout' : ''}): ${cmd}\n${(res.stderr || res.stdout).slice(-2000)}`,
+        )
+      }
+    }
+
+    await overlayJudgeTests(inst, ws)
+
+    let passed = 0
+    let failed = 0
+    let output = ''
+    for (const cmd of inst.judge_cmds) {
+      const res = await runFactoryCommand(ws, cmd, {
+        image: inst.command_image,
+        network: 'none',
+        timeoutMs: cmdTimeoutMs,
+      })
+      const combined = res.stdout + res.stderr
+      output += combined
+      if (res.timedOut) throw new Error(`judge_cmd timed out after ${cmdTimeoutMs}ms: ${cmd}`)
+      const summary = parseVitestSummary(combined)
+      if (!summary) {
+        throw new Error(`judge_cmd produced no parseable test summary (rc=${res.code}): ${cmd}\n${combined.slice(-2000)}`)
+      }
+      passed += summary.passed
+      failed += summary.failed
+    }
+
+    const total = inst.judgeTestTotal
+    return {
+      result: {
+        iid: inst.id,
+        resolved: failed === 0 && passed === total,
+        score: total > 0 ? Number((passed / total).toFixed(4)) : 0,
+        passed,
+        total,
+        secs: Math.round((Date.now() - t0) / 1000),
+        patch_bytes: patch.length,
+      },
+      output,
+    }
+  } finally {
+    if (!opts.keepWorkspace) await rm(ws, { recursive: true, force: true })
+  }
+}
+
+// ---------------------------------------------------------------------------
+// CLI (guarded so calibrate/tests can import the helpers above).
+// ---------------------------------------------------------------------------
+
+const isMain = process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href
+
+if (isMain) {
+  const [, , instanceDir, patchPath] = process.argv
+  if (!instanceDir || !patchPath) {
+    console.error('usage: node --import tsx factory-judge-child.mts <instanceDir> <patchPath>')
+    process.exit(2)
+  }
+  const inst = loadFactoryInstance(instanceDir)
+  const patch = await readFile(patchPath, 'utf8')
+  const { result } = await judgeFactoryPatch(inst, patch)
+  console.log('JUDGE_RESULT ' + JSON.stringify(result))
+}