@tangle-network/agent-bench 0.11.3 → 0.13.0

package/src/swe-arena/score-split.test.mts

@@ -1,123 +0,0 @@
-/**
- * Gen-5 public/private score split: determinism (seeded by runId), persisted
- * resume stability, and the never-surfaced invariant's checkable pieces.
- */
-
-import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
-import { tmpdir } from 'node:os'
-import { join } from 'node:path'
-import { afterEach, beforeEach, describe, expect, it } from 'vitest'
-import {
-  leaksPrivateInstance,
-  loadOrCreateScoreSplit,
-  publicOnly,
-  SCORE_SPLIT_FILENAME,
-  splitInstances,
-  subScores,
-} from './score-split.mts'
-
-const SIX = [
-  'astropy__astropy-13033',
-  'django__django-11532',
-  'matplotlib__matplotlib-20826',
-  'pydata__xarray-4687',
-  'pytest-dev__pytest-6197',
-  'sphinx-doc__sphinx-9658',
-]
-
-describe('splitInstances', () => {
-  it('is deterministic: same (runId, instances, publicCount) → identical split', () => {
-    const a = splitInstances('r4-abc123', SIX, 4)
-    const b = splitInstances('r4-abc123', SIX, 4)
-    expect(a).toEqual(b)
-  })
-
-  it('is instance-ORDER-insensitive (ranking is content-derived)', () => {
-    const a = splitInstances('r4-abc123', SIX, 4)
-    const b = splitInstances('r4-abc123', [...SIX].reverse(), 4)
-    expect(a).toEqual(b)
-  })
-
-  it('respects publicCount and partitions exactly (4 public / 2 private of 6)', () => {
-    const split = splitInstances('r4-abc123', SIX, 4)
-    expect(split.publicInstances).toHaveLength(4)
-    expect(split.privateInstances).toHaveLength(2)
-    expect([...split.publicInstances, ...split.privateInstances].sort()).toEqual([...SIX].sort())
-    expect(split.publicInstances.filter((i) => split.privateInstances.includes(i))).toEqual([])
-  })
-
-  it('is SEEDED by runId: different runIds produce different splits', () => {
-    const splits = new Set(
-      Array.from({ length: 10 }, (_, i) => splitInstances(`r4-run${i}`, SIX, 4).privateInstances.join('|')),
-    )
-    expect(splits.size).toBeGreaterThan(1)
-  })
-
-  it('rejects publicCount outside (0, n) and duplicate instance ids', () => {
-    expect(() => splitInstances('r', SIX, 0)).toThrow(/publicCount/)
-    expect(() => splitInstances('r', SIX, 6)).toThrow(/publicCount/)
-    expect(() => splitInstances('r', SIX, 2.5)).toThrow(/publicCount/)
-    expect(() => splitInstances('r', ['a', 'a', 'b'], 1)).toThrow(/duplicate/)
-  })
-})
-
-describe('loadOrCreateScoreSplit (resume stability)', () => {
-  let outDir: string
-  beforeEach(async () => {
-    outDir = await mkdtemp(join(tmpdir(), 'split-'))
-  })
-  afterEach(async () => {
-    await rm(outDir, { recursive: true, force: true })
-  })
-
-  it('persists on first computation and RELOADS verbatim under a NEW runId (a resume cannot rotate private into view)', async () => {
-    const first = await loadOrCreateScoreSplit({ outDir, runId: 'r4-first', instances: SIX, publicCount: 4 })
-    expect(JSON.parse(await readFile(join(outDir, SCORE_SPLIT_FILENAME), 'utf8'))).toEqual(first)
-    const resumed = await loadOrCreateScoreSplit({ outDir, runId: 'r4-DIFFERENT', instances: SIX, publicCount: 4 })
-    expect(resumed).toEqual(first)
-    expect(resumed.seededBy).toBe('r4-first')
-  })
-
-  it('fails loud when the persisted split covers different instances or a different publicCount', async () => {
-    await loadOrCreateScoreSplit({ outDir, runId: 'r4-first', instances: SIX, publicCount: 4 })
-    await expect(
-      loadOrCreateScoreSplit({ outDir, runId: 'r4-x', instances: [...SIX.slice(0, 5), 'other__iid-1'], publicCount: 4 }),
-    ).rejects.toThrow(/refusing to silently re-split/)
-    await expect(
-      loadOrCreateScoreSplit({ outDir, runId: 'r4-x', instances: SIX, publicCount: 3 }),
-    ).rejects.toThrow(/refusing to silently re-split/)
-  })
-
-  it('fails loud on an unknown schema in the persisted file', async () => {
-    await writeFile(join(outDir, SCORE_SPLIT_FILENAME), JSON.stringify({ schema: 'nope' }))
-    await expect(
-      loadOrCreateScoreSplit({ outDir, runId: 'r', instances: SIX, publicCount: 4 }),
-    ).rejects.toThrow(/unknown schema/)
-  })
-})
-
-describe('never-surfaced invariant helpers', () => {
-  const split = splitInstances('r4-abc123', SIX, 4)
-
-  it('leaksPrivateInstance names every private id present in a text', () => {
-    expect(leaksPrivateInstance('nothing to see', split)).toEqual([])
-    const leakText = `evidence for ${split.privateInstances[0]} here`
-    expect(leaksPrivateInstance(leakText, split)).toEqual([split.privateInstances[0]])
-  })
-
-  it('publicOnly drops private-instance records and is identity without a split', () => {
-    const records = SIX.map((iid) => ({ iid, x: 1 }))
-    const filtered = publicOnly(records, (r) => r.iid, split)
-    expect(filtered.map((r) => r.iid).sort()).toEqual(split.publicInstances)
-    expect(publicOnly(records, (r) => r.iid, null)).toEqual(records)
-  })
-
-  it('subScores counts fail-closed AND-verdicts per half (missing verdict = 0)', () => {
-    const verdicts: Record<string, boolean> = {}
-    verdicts[split.publicInstances[0]!] = true
-    verdicts[split.publicInstances[1]!] = false
-    verdicts[split.privateInstances[0]!] = true
-    // privateInstances[1] has NO verdict — counts 0.
-    expect(subScores(verdicts, split)).toEqual({ publicResolvedCount: 1, privateResolvedCount: 1 })
-  })
-})

package/src/swe-arena/scratch-worktree-serialization.test.mts

@@ -1,72 +0,0 @@
-import { mkdtemp, rm } from 'node:fs/promises'
-import { tmpdir } from 'node:os'
-import { join } from 'node:path'
-import { afterEach, describe, expect, it, vi } from 'vitest'
-
-const gitCalls = vi.hoisted(() => ({
-  activeByRepository: new Map<string, number>(),
-  maximumByRepository: new Map<string, number>(),
-  active: 0,
-  maximum: 0,
-}))
-
-vi.mock('./proc', () => {
-  const recordGitCall = async (_binary: string, arguments_: string[]) => {
-    const repository = arguments_[1] ?? ''
-    const activeForRepository = (gitCalls.activeByRepository.get(repository) ?? 0) + 1
-    gitCalls.activeByRepository.set(repository, activeForRepository)
-    gitCalls.maximumByRepository.set(
-      repository,
-      Math.max(gitCalls.maximumByRepository.get(repository) ?? 0, activeForRepository),
-    )
-    gitCalls.active += 1
-    gitCalls.maximum = Math.max(gitCalls.maximum, gitCalls.active)
-    await new Promise((resolve) => setTimeout(resolve, 5))
-    gitCalls.active -= 1
-    gitCalls.activeByRepository.set(repository, activeForRepository - 1)
-    return { code: 0, stdout: '', stderr: '' }
-  }
-  return {
-    run: vi.fn(recordGitCall),
-    runOk: vi.fn(recordGitCall),
-  }
-})
-
-import {
-  createDetachedWorktree,
-  pruneDetachedWorktrees,
-  removeDetachedWorktree,
-} from './scratch-worktree.ts'
-
-describe('scratch worktree mutation serialization', () => {
-  const roots: string[] = []
-
-  afterEach(async () => {
-    gitCalls.activeByRepository.clear()
-    gitCalls.maximumByRepository.clear()
-    gitCalls.active = 0
-    gitCalls.maximum = 0
-    await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })))
-  })
-
-  it('serializes metadata changes per repository without blocking separate repositories', async () => {
-    const output = await mkdtemp(join(tmpdir(), 'scratch-worktree-serialization-'))
-    roots.push(output)
-    const operations = ['repository-a', 'repository-b'].flatMap((repository) =>
-      Array.from({ length: 4 }, (_, index) => {
-        const destination = join(output, `${repository}-candidate-${index}`)
-        return [
-          createDetachedWorktree(repository, 'commit', destination),
-          removeDetachedWorktree(repository, destination),
-          pruneDetachedWorktrees(repository),
-        ]
-      }),
-    )
-
-    await Promise.all(operations.flat())
-
-    expect(gitCalls.maximumByRepository.get('repository-a')).toBe(1)
-    expect(gitCalls.maximumByRepository.get('repository-b')).toBe(1)
-    expect(gitCalls.maximum).toBe(2)
-  })
-})

package/src/swe-arena/scratch-worktree.test.mts

@@ -1,56 +0,0 @@
-import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
-import { tmpdir } from 'node:os'
-import { join } from 'node:path'
-import { afterEach, describe, expect, it } from 'vitest'
-import { runOk } from './proc.ts'
-import {
-  createDetachedWorktree,
-  pruneDetachedWorktrees,
-  removeDetachedWorktree,
-} from './scratch-worktree.ts'
-
-describe('scratch worktrees', () => {
-  const roots: string[] = []
-
-  afterEach(async () => {
-    await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })))
-  })
-
-  it('creates and removes a parallel batch without pruning live metadata', async () => {
-    const repository = await mkdtemp(join(tmpdir(), 'scratch-worktree-repo-'))
-    const output = await mkdtemp(join(tmpdir(), 'scratch-worktree-out-'))
-    roots.push(repository, output)
-    await runOk('git', ['init', '-q', '-b', 'main', repository])
-    await runOk('git', ['-C', repository, 'config', 'core.hooksPath', '/dev/null'])
-    await runOk('git', ['-C', repository, 'config', 'user.email', 'test@example.com'])
-    await runOk('git', ['-C', repository, 'config', 'user.name', 'Test'])
-    await writeFile(join(repository, 'seed.txt'), 'seed\n')
-    await runOk('git', ['-C', repository, 'add', 'seed.txt'])
-    await runOk('git', ['-C', repository, 'commit', '-q', '-m', 'seed'])
-    const commit = (
-      await runOk('git', ['-C', repository, 'rev-parse', 'HEAD'])
-    ).stdout.trim()
-    for (let batch = 0; batch < 5; batch += 1) {
-      const worktrees = Array.from(
-        { length: 8 },
-        (_, index) => join(output, `batch-${batch}-candidate-${index}`),
-      )
-      await pruneDetachedWorktrees(repository)
-      await Promise.all(
-        worktrees.map((worktree) => createDetachedWorktree(repository, commit, worktree)),
-      )
-      expect(
-        await Promise.all(
-          worktrees.map((worktree) => readFile(join(worktree, 'seed.txt'), 'utf8')),
-        ),
-      ).toEqual(Array.from({ length: 8 }, () => 'seed\n'))
-      await Promise.all(
-        worktrees.map((worktree) => removeDetachedWorktree(repository, worktree)),
-      )
-    }
-    const listed = (
-      await runOk('git', ['-C', repository, 'worktree', 'list', '--porcelain'])
-    ).stdout
-    expect(listed.match(/^worktree /gmu)).toHaveLength(1)
-  }, 60_000)
-})

package/src/swe-arena/scratch-worktree.ts

@@ -1,64 +0,0 @@
-import { rm } from 'node:fs/promises'
-import { run, runOk } from './proc'
-
-// Git worktree subcommands mutate shared metadata under .git/worktrees and are
-// not safe to overlap, even when each command targets a different destination.
-const repositoryMutationQueues = new Map<string, Promise<void>>()
-
-async function mutateWorktrees<T>(
-  repository: string,
-  operation: () => Promise<T>,
-): Promise<T> {
-  const predecessor = repositoryMutationQueues.get(repository) ?? Promise.resolve()
-  const result = predecessor.then(operation, operation)
-  const settled = result.then(
-    () => undefined,
-    () => undefined,
-  )
-  repositoryMutationQueues.set(repository, settled)
-  try {
-    return await result
-  } finally {
-    if (repositoryMutationQueues.get(repository) === settled) {
-      repositoryMutationQueues.delete(repository)
-    }
-  }
-}
-
-export async function createDetachedWorktree(
-  repository: string,
-  commit: string,
-  destination: string,
-): Promise<void> {
-  await mutateWorktrees(repository, async () => {
-    await run('git', ['-C', repository, 'worktree', 'remove', '--force', '--', destination])
-    await rm(destination, { recursive: true, force: true })
-    await runOk('git', ['-C', repository, 'worktree', 'add', '--detach', destination, commit])
-  })
-}
-
-export async function pruneDetachedWorktrees(repository: string): Promise<void> {
-  await mutateWorktrees(repository, async () => {
-    await runOk('git', ['-C', repository, 'worktree', 'prune'])
-  })
-}
-
-export async function removeDetachedWorktree(
-  repository: string,
-  destination: string,
-): Promise<void> {
-  await mutateWorktrees(repository, async () => {
-    const result = await run('git', [
-      '-C',
-      repository,
-      'worktree',
-      'remove',
-      '--force',
-      '--',
-      destination,
-    ])
-    if (result.code !== 0) {
-      await rm(destination, { recursive: true, force: true })
-    }
-  })
-}