@tangle-network/agent-bench 0.1.0 → 0.3.5

package/src/pier-result-grader.test.mts

@@ -0,0 +1,62 @@
+import assert from 'node:assert/strict'
+import { createHash } from 'node:crypto'
+import { readFile } from 'node:fs/promises'
+import test from 'node:test'
+
+import type { AgentCandidateArtifactRef } from '@tangle-network/agent-interface'
+
+import { createPierResultGrader } from './pier-result-grader'
+
+const digest = (bytes: Uint8Array): `sha256:${string}` =>
+  `sha256:${createHash('sha256').update(bytes).digest('hex')}`
+
+const artifact: AgentCandidateArtifactRef = {
+  locator: { kind: 's3', bucket: 'pier-graders', key: 'pier-result-grader.mjs' },
+  sha256: `sha256:${'a'.repeat(64)}`,
+  byteLength: 1,
+}
+
+test('executes admitted Pier grader bytes over exact official result evidence', async () => {
+  const implementation = await readFile(new URL('./pier-result-grader.mjs', import.meta.url))
+  const resultBytes = Buffer.from(
+    JSON.stringify({ verifier_result: { rewards: { reward: 1, patch_applied: 1 } } }),
+  )
+  const grader = createPierResultGrader({ name: 'pier-result', version: '1', artifact })
+  const output = await grader.run({
+    executionId: 'execution-1',
+    termination: { kind: 'exit', exitCode: 0 },
+    outcome: { evidence: { digest: `sha256:${'b'.repeat(64)}` } } as never,
+    implementation: { byteLength: implementation.byteLength, bytes: implementation },
+    signal: new AbortController().signal,
+    officialResult: { value: JSON.parse(resultBytes.toString('utf8')), bytes: resultBytes },
+  })
+
+  assert.deepEqual(output.evaluation, {
+    score: 1,
+    passed: true,
+    dimensions: { patch_applied: 1, reward: 1 },
+    raw: {},
+  })
+  assert.deepEqual(Buffer.from(output.evidence), resultBytes)
+  assert.equal(output.binding.implementationDigest, digest(implementation))
+  assert.equal(output.binding.outputDigest, digest(resultBytes))
+})
+
+test('fails closed for missing official rewards and pre-aborted execution', async () => {
+  const implementation = await readFile(new URL('./pier-result-grader.mjs', import.meta.url))
+  const grader = createPierResultGrader({ name: 'pier-result', version: '1', artifact })
+  const base = {
+    executionId: 'execution-1',
+    termination: { kind: 'exit', exitCode: 0 } as const,
+    outcome: { evidence: { digest: `sha256:${'b'.repeat(64)}` } } as never,
+    implementation: { byteLength: implementation.byteLength, bytes: implementation },
+    officialResult: { value: {}, bytes: Buffer.from('{}') },
+  }
+  await assert.rejects(
+    grader.run({ ...base, signal: new AbortController().signal }),
+    /omitted verifier_result.rewards/,
+  )
+  const controller = new AbortController()
+  controller.abort(new Error('deadline'))
+  await assert.rejects(grader.run({ ...base, signal: controller.signal }), /deadline/)
+})

package/src/pier-result-grader.ts

@@ -0,0 +1,108 @@
+/** Execute the exact admitted Pier result parser bytes in a fresh Node process. */
+import { spawn } from 'node:child_process'
+import { createHash } from 'node:crypto'
+import { chmod, mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+
+import type { BenchmarkEvaluation } from '@tangle-network/agent-eval'
+
+import type { PierCandidateGraderPort } from './pier-agent'
+
+const sha256 = (bytes: Uint8Array): `sha256:${string}` =>
+  `sha256:${createHash('sha256').update(bytes).digest('hex')}`
+
+export function createPierResultGrader(
+  descriptor: Pick<PierCandidateGraderPort, 'name' | 'version' | 'artifact'>,
+): PierCandidateGraderPort {
+  return {
+    ...descriptor,
+    run: async (input) => {
+      input.signal.throwIfAborted()
+      const implementation = Uint8Array.from(input.implementation.bytes)
+      if (implementation.byteLength !== input.implementation.byteLength) {
+        throw new Error('Pier grader implementation length changed before execution')
+      }
+      const evidence = Uint8Array.from(input.officialResult.bytes)
+      const evaluation = await executeGrader(implementation, evidence, input.signal)
+      input.signal.throwIfAborted()
+      return {
+        evaluation,
+        evidence,
+        binding: {
+          implementationDigest: sha256(implementation),
+          taskOutcomeDigest: input.outcome.evidence.digest,
+          outputDigest: sha256(evidence),
+        },
+      }
+    },
+  }
+}
+
+async function executeGrader(
+  implementation: Uint8Array,
+  resultBytes: Uint8Array,
+  signal: AbortSignal,
+): Promise<BenchmarkEvaluation> {
+  const directory = await mkdtemp(join(tmpdir(), 'pier-result-grader-'))
+  const script = join(directory, 'grader.mjs')
+  try {
+    await writeFile(script, implementation, { flag: 'wx', mode: 0o600 })
+    await chmod(script, 0o500)
+    const stdout = await runNode(script, resultBytes, signal)
+    const parsed = JSON.parse(stdout.toString('utf8')) as BenchmarkEvaluation
+    if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
+      throw new Error('Pier result grader returned a non-object evaluation')
+    }
+    return parsed
+  } finally {
+    await rm(directory, { recursive: true, force: true })
+  }
+}
+
+async function runNode(
+  script: string,
+  input: Uint8Array,
+  signal: AbortSignal,
+): Promise<Buffer> {
+  signal.throwIfAborted()
+  return await new Promise((resolveResult, reject) => {
+    const child = spawn(process.execPath, ['--no-warnings', script], {
+      env: {
+        PATH: process.env.PATH,
+        HOME: process.env.HOME,
+        LANG: 'C',
+        LC_ALL: 'C',
+      },
+      stdio: ['pipe', 'pipe', 'pipe'],
+      signal,
+    })
+    const stdout: Buffer[] = []
+    const stderr: Buffer[] = []
+    let bytes = 0
+    const collect = (target: Buffer[]) => (chunk: Buffer) => {
+      bytes += chunk.byteLength
+      if (bytes > 1024 * 1024) {
+        child.kill('SIGKILL')
+        reject(new Error('Pier result grader exceeded its output limit'))
+        return
+      }
+      target.push(Buffer.from(chunk))
+    }
+    child.stdout.on('data', collect(stdout))
+    child.stderr.on('data', collect(stderr))
+    child.once('error', reject)
+    child.once('close', (code, childSignal) => {
+      if (code !== 0) {
+        reject(
+          new Error(
+            `Pier result grader failed (${childSignal ?? code}): ${Buffer.concat(stderr).toString('utf8').trim()}`,
+          ),
+        )
+        return
+      }
+      resolveResult(Buffer.concat(stdout))
+    })
+    child.stdin.end(input)
+  })
+}

package/src/pier-task-outcome.test.mts

@@ -0,0 +1,117 @@
+import assert from 'node:assert/strict'
+import { execFileSync } from 'node:child_process'
+import { chmodSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import test from 'node:test'
+
+import {
+  captureAgentCandidateWorkspaceFiles,
+  createAgentCandidateWorkspacePort,
+} from '@tangle-network/agent-runtime/candidate-execution'
+
+import { capturePierTaskOutcome } from './pier-task-outcome'
+
+test('Pier task outcome reconstructs the exact tree and deterministic archive from a patch', async () => {
+  const root = mkdtempSync(join(tmpdir(), 'pier-task-outcome-test-'))
+  try {
+    mkdirSync(join(root, 'src'))
+    writeFileSync(join(root, 'src/status.txt'), 'not-ready\n')
+    chmodSync(join(root, 'src/status.txt'), 0o644)
+    git(root, ['init', '-b', 'main'])
+    git(root, ['config', 'user.email', 'fixture@example.com'])
+    git(root, ['config', 'user.name', 'Fixture'])
+    git(root, ['add', '-A'])
+    git(root, ['commit', '-m', 'base'])
+    const baseCommit = git(root, ['rev-parse', 'HEAD'])
+    const baseTree = git(root, ['rev-parse', 'HEAD^{tree}'])
+    writeFileSync(join(root, 'src/status.txt'), 'ready\n')
+    const patch = execFileSync('git', ['-C', root, 'diff', '--binary'])
+    git(root, ['restore', '.'])
+
+    const captured = await capturePierTaskOutcome({ repositoryRoot: root, baseCommit, baseTree, patch })
+    assert.notEqual(captured.resultTree, baseTree)
+    assert.deepEqual(Buffer.from(captured.gitDiff), patch)
+    const expected = await captureAgentCandidateWorkspaceFiles([
+      { path: 'src/status.txt', mode: 0o644, bytes: Buffer.from('ready\n') },
+    ])
+    assert.deepEqual(captured.afterState, expected.snapshot.material)
+    assert.deepEqual(Buffer.from(captured.archive), Buffer.from(expected.archive))
+    const destination = join(root, 'materialized')
+    await createAgentCandidateWorkspacePort().materialize({
+      role: 'candidate',
+      archive: captured.archive,
+      snapshot: expected.snapshot,
+      destination,
+    })
+    assert.equal(readFileSync(join(destination, 'src/status.txt'), 'utf8'), 'ready\n')
+  } finally {
+    rmSync(root, { recursive: true, force: true })
+  }
+})
+
+test('Pier task outcome preserves the signed base for an empty patch', async () => {
+  const root = mkdtempSync(join(tmpdir(), 'pier-task-outcome-test-'))
+  try {
+    writeFileSync(join(root, 'README.md'), 'base\n')
+    git(root, ['init', '-b', 'main'])
+    git(root, ['config', 'user.email', 'fixture@example.com'])
+    git(root, ['config', 'user.name', 'Fixture'])
+    git(root, ['add', '-A'])
+    git(root, ['commit', '-m', 'base'])
+    const baseCommit = git(root, ['rev-parse', 'HEAD'])
+    const baseTree = git(root, ['rev-parse', 'HEAD^{tree}'])
+
+    const captured = await capturePierTaskOutcome({
+      repositoryRoot: root,
+      baseCommit,
+      baseTree,
+      patch: Buffer.alloc(0),
+    })
+    assert.equal(captured.resultTree, baseTree)
+    assert.equal(captured.gitDiff.byteLength, 0)
+  } finally {
+    rmSync(root, { recursive: true, force: true })
+  }
+})
+
+test('Pier task outcome returns large workspace bytes for the runtime capture path', async () => {
+  const root = mkdtempSync(join(tmpdir(), 'pier-task-outcome-test-'))
+  try {
+    const payload = Buffer.alloc(1024 * 1024 + 1, 7)
+    writeFileSync(join(root, 'large.bin'), payload)
+    git(root, ['init', '-b', 'main'])
+    git(root, ['config', 'user.email', 'fixture@example.com'])
+    git(root, ['config', 'user.name', 'Fixture'])
+    git(root, ['add', '-A'])
+    git(root, ['commit', '-m', 'base'])
+    const baseCommit = git(root, ['rev-parse', 'HEAD'])
+    const baseTree = git(root, ['rev-parse', 'HEAD^{tree}'])
+    const captured = await capturePierTaskOutcome({
+      repositoryRoot: root,
+      baseCommit,
+      baseTree,
+      patch: Buffer.alloc(0),
+    })
+
+    assert.ok(captured.archive.byteLength > 1024 * 1024)
+    assert.equal(captured.afterState.files[0]?.byteLength, payload.byteLength)
+  } finally {
+    rmSync(root, { recursive: true, force: true })
+  }
+})
+
+function git(root: string, args: string[]): string {
+  return execFileSync('git', ['-C', root, ...args], {
+    encoding: 'utf8',
+    env: {
+      ...process.env,
+      GIT_CONFIG_NOSYSTEM: '1',
+      GIT_CONFIG_GLOBAL: '/dev/null',
+      GIT_CONFIG_SYSTEM: '/dev/null',
+      GIT_TERMINAL_PROMPT: '0',
+      GIT_NO_REPLACE_OBJECTS: '1',
+      LC_ALL: 'C',
+    },
+  }).trim()
+}

package/src/pier-task-outcome.ts

@@ -0,0 +1,240 @@
+/** Reconstruct a runtime task outcome from Pier's evaluator-captured binary patch. */
+import { spawn } from 'node:child_process'
+import { mkdir, mkdtemp, realpath, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join, resolve } from 'node:path'
+
+import {
+  type AgentCandidateExecutorTaskOutcomeCapture,
+  captureAgentCandidateWorkspaceFiles,
+} from '@tangle-network/agent-runtime/candidate-execution'
+
+interface GitResult {
+  readonly stdout: Buffer
+  readonly stderr: Buffer
+}
+
+/** Apply the official patch in an isolated Git object directory and capture its exact tree. */
+export async function capturePierTaskOutcome(input: {
+  readonly repositoryRoot: string
+  readonly baseCommit: string
+  readonly baseTree: string
+  readonly patch: Uint8Array
+  readonly signal?: AbortSignal
+}): Promise<Extract<AgentCandidateExecutorTaskOutcomeCapture, { kind: 'workspace' }>> {
+  input.signal?.throwIfAborted()
+  const repositoryRoot = resolve(input.repositoryRoot)
+  const head = (await git(repositoryRoot, ['rev-parse', 'HEAD'], undefined, {}, input.signal)).stdout
+    .toString('utf8')
+    .trim()
+  const headTree = (
+    await git(repositoryRoot, ['rev-parse', 'HEAD^{tree}'], undefined, {}, input.signal)
+  ).stdout
+    .toString('utf8')
+    .trim()
+  if (head !== input.baseCommit || headTree !== input.baseTree) {
+    throw new Error('Pier task outcome repository drifted from its signed base')
+  }
+  const gitDir = resolve(
+    (
+      await git(repositoryRoot, ['rev-parse', '--absolute-git-dir'], undefined, {}, input.signal)
+    ).stdout
+      .toString('utf8')
+      .trim(),
+  )
+  if ((await realpath(gitDir)) !== gitDir || gitDir.includes(':')) {
+    throw new Error('Pier task outcome Git directory is unsupported')
+  }
+  const replacements = (
+    await git(
+      repositoryRoot,
+      ['for-each-ref', '--format=%(refname)', 'refs/replace'],
+      undefined,
+      {},
+      input.signal,
+    )
+  ).stdout
+    .toString('utf8')
+    .trim()
+  if (replacements) throw new Error('Pier task outcome repository contains Git replace refs')
+
+  const temporary = await mkdtemp(join(tmpdir(), 'pier-task-outcome-'))
+  try {
+    const objectDirectory = join(temporary, 'objects')
+    const indexFile = join(temporary, 'index')
+    await mkdir(objectDirectory)
+    const environment = {
+      GIT_INDEX_FILE: indexFile,
+      GIT_OBJECT_DIRECTORY: objectDirectory,
+      GIT_ALTERNATE_OBJECT_DIRECTORIES: join(gitDir, 'objects'),
+    }
+    await git(repositoryRoot, ['read-tree', input.baseTree], undefined, environment, input.signal)
+    const patch = Uint8Array.from(input.patch)
+    if (patch.byteLength > 0) {
+      await git(
+        repositoryRoot,
+        ['apply', '--cached', '--binary', '--whitespace=nowarn', '-'],
+        patch,
+        environment,
+        input.signal,
+      )
+    }
+    const resultTree = (
+      await git(repositoryRoot, ['write-tree'], undefined, environment, input.signal)
+    ).stdout
+      .toString('utf8')
+      .trim()
+    const entries = parseTreeEntries(
+      (
+        await git(
+          repositoryRoot,
+          ['ls-tree', '-rz', '--full-tree', resultTree],
+          undefined,
+          environment,
+          input.signal,
+        )
+      ).stdout,
+    )
+    if (entries.length === 0) throw new Error('Pier task outcome tree cannot be empty')
+    const files = await Promise.all(
+      entries.map(async (entry) => {
+        if (entry.type !== 'blob' || (entry.mode !== '100644' && entry.mode !== '100755')) {
+          throw new Error(`Pier task outcome contains a non-regular entry: ${entry.path}`)
+        }
+        return {
+          path: safeGitPath(entry.path),
+          mode: entry.mode === '100755' ? (0o755 as const) : (0o644 as const),
+          bytes: (
+            await git(
+              repositoryRoot,
+              ['cat-file', 'blob', entry.object],
+              undefined,
+              environment,
+              input.signal,
+            )
+          ).stdout,
+        }
+      }),
+    )
+    // The executor outcome must still return exact bytes; the runtime verifies
+    // and persists its final task references after this capture completes.
+    const captured = await captureAgentCandidateWorkspaceFiles(files, {
+      limits: { maxEmbeddedArtifactBytes: 512 * 1024 * 1024 },
+    })
+    return {
+      kind: 'workspace',
+      resultTree,
+      afterState: captured.snapshot.material,
+      archive: captured.archive,
+      gitDiff: patch,
+    }
+  } finally {
+    await rm(temporary, { recursive: true, force: true })
+  }
+}
+
+function parseTreeEntries(bytes: Uint8Array): Array<{
+  readonly mode: string
+  readonly type: string
+  readonly object: string
+  readonly path: string
+}> {
+  const raw = Buffer.from(bytes)
+  const decoded = raw.toString('utf8')
+  if (!Buffer.from(decoded, 'utf8').equals(raw)) {
+    throw new Error('Pier task outcome contains a non-UTF-8 path')
+  }
+  return decoded
+    .split('\0')
+    .filter(Boolean)
+    .map((row) => {
+      const tab = row.indexOf('\t')
+      const [mode, type, object] = row.slice(0, tab).split(' ')
+      const path = row.slice(tab + 1)
+      if (tab < 1 || !mode || !type || !object || !path) {
+        throw new Error('Pier task outcome contains a malformed Git entry')
+      }
+      return { mode, type, object, path }
+    })
+}
+
+function safeGitPath(path: string): string {
+  const parts = path.split('/')
+  if (
+    !path ||
+    path.startsWith('/') ||
+    path.includes('\\') ||
+    path.includes('\0') ||
+    parts.some((part) => !part || part === '.' || part === '..') ||
+    parts[0]?.toLowerCase() === '.git'
+  ) {
+    throw new Error(`Pier task outcome contains an unsafe Git path: ${path}`)
+  }
+  return path
+}
+
+async function git(
+  repositoryRoot: string,
+  args: readonly string[],
+  input: Uint8Array | undefined,
+  extraEnvironment: Readonly<Record<string, string>>,
+  signal: AbortSignal | undefined,
+): Promise<GitResult> {
+  signal?.throwIfAborted()
+  const environment = Object.fromEntries(
+    Object.entries(process.env).filter(([name]) => !name.startsWith('GIT_')),
+  ) as Record<string, string>
+  Object.assign(environment, {
+    GIT_CONFIG_NOSYSTEM: '1',
+    GIT_CONFIG_GLOBAL: '/dev/null',
+    GIT_CONFIG_SYSTEM: '/dev/null',
+    GIT_TERMINAL_PROMPT: '0',
+    GIT_NO_REPLACE_OBJECTS: '1',
+    LC_ALL: 'C',
+    ...extraEnvironment,
+  })
+  return await new Promise((resolveResult, reject) => {
+    const child = spawn(
+      'git',
+      [
+        '-c',
+        'core.hooksPath=/dev/null',
+        '-c',
+        'protocol.file.allow=never',
+        '-C',
+        repositoryRoot,
+        ...args,
+      ],
+      { env: environment, stdio: ['pipe', 'pipe', 'pipe'], signal },
+    )
+    const stdout: Buffer[] = []
+    const stderr: Buffer[] = []
+    let outputBytes = 0
+    const collect = (target: Buffer[]) => (chunk: Buffer) => {
+      outputBytes += chunk.byteLength
+      if (outputBytes > 64 * 1024 * 1024) {
+        child.kill('SIGKILL')
+        reject(new Error(`git ${args[0] ?? '<command>'} exceeded the output limit`))
+        return
+      }
+      target.push(Buffer.from(chunk))
+    }
+    child.stdout.on('data', collect(stdout))
+    child.stderr.on('data', collect(stderr))
+    child.once('error', reject)
+    child.once('close', (code, childSignal) => {
+      const out = Buffer.concat(stdout)
+      const err = Buffer.concat(stderr)
+      if (code !== 0) {
+        reject(
+          new Error(
+            `git ${args[0] ?? '<command>'} failed (${childSignal ?? code}): ${err.toString('utf8').trim()}`,
+          ),
+        )
+        return
+      }
+      resolveResult({ stdout: out, stderr: err })
+    })
+    child.stdin.end(input)
+  })
+}