@tangle-network/agent-bench 0.1.0 → 0.3.6

package/src/swe-structural-policy.test.ts

@@ -0,0 +1,124 @@
+import assert from 'node:assert/strict'
+import { describe, it } from 'node:test'
+import { assertNoHiddenLeak } from './swe-jail'
+import {
+  assertExactCompletedWorkerSessions,
+  continuationDisposition,
+  continuationStateNotice,
+  preferLaterCandidate,
+  resolveExperimentArm,
+  resolveExperimentTemperature,
+  shouldAcceptContinuation,
+  shouldRunContinuation,
+} from './swe-structural-policy'
+
+describe('structural experiment arms', () => {
+  it('derives exactly two independent sessions without repair', () => {
+    assert.deepEqual(resolveExperimentArm('independent-2'), {
+      arm: 'independent-2',
+      k: 2,
+      repairs: 0,
+      alwaysRunContinuation: false,
+      persistent: false,
+      workerSessions: 2,
+    })
+  })
+
+  it('derives exactly one initial plus one persistent continuation session', () => {
+    const preset = resolveExperimentArm('persistent-refine-2')
+    assert.deepEqual(preset, {
+      arm: 'persistent-refine-2',
+      k: 1,
+      repairs: 1,
+      alwaysRunContinuation: true,
+      persistent: true,
+      workerSessions: 2,
+    })
+    assert.equal(shouldRunContinuation({ round: 1, preset }), true)
+    assert.equal(shouldRunContinuation({ round: 2, preset }), false)
+  })
+
+  it('rejects every untyped arm instead of accepting free-form K/repair settings', () => {
+    assert.throws(() => resolveExperimentArm('system'), /independent-2\|persistent-refine-2/)
+    assert.throws(() => resolveExperimentArm(''), /independent-2\|persistent-refine-2/)
+  })
+
+  it('uses TEMPERATURE without repurposing Node TEMP as a model knob', () => {
+    assert.equal(resolveExperimentTemperature({}), 0.8)
+    assert.equal(resolveExperimentTemperature({ TEMPERATURE: '0.35' }), 0.35)
+    assert.equal(resolveExperimentTemperature({ TEMP: '/tmp', TEMPERATURE: '0.35' }), 0.35)
+    assert.throws(
+      () => resolveExperimentTemperature({ TEMP: '0.8' }),
+      /TEMP is the operating-system temporary-directory root.*TEMPERATURE/,
+    )
+    assert.throws(() => resolveExperimentTemperature({ TEMPERATURE: 'hot' }), /finite number/)
+  })
+})
+
+describe('shared later-on-visible-tie policy', () => {
+  it('prefers session two on a visible tie in both arm shapes', () => {
+    assert.equal(preferLaterCandidate(0, 0), true)
+    assert.equal(preferLaterCandidate(1, 1), true)
+    assert.equal(shouldAcceptContinuation(0, 0), true)
+    assert.equal(shouldAcceptContinuation(1, 1), true)
+  })
+
+  it('prefers a visible improvement and rejects a visible regression', () => {
+    assert.equal(preferLaterCandidate(1, 0), true)
+    assert.equal(preferLaterCandidate(0, 1), false)
+    assert.equal(shouldAcceptContinuation(1, 0), true)
+    assert.equal(shouldAcceptContinuation(0, 1), false)
+  })
+
+  it('never labels an identical second patch as a refinement', () => {
+    assert.deepEqual(continuationDisposition(true, false), {
+      finalFrom: 'session:2-identical',
+      stop: 'continuation-accepted-identical',
+    })
+    assert.equal(continuationDisposition(true, true).finalFrom, 'session:2-refinement')
+  })
+
+  it('inherits parent state without echoing parent patch bytes into session two', () => {
+    const parent = 'diff --git a/a.py b/a.py\n+gold-like-added-line\n'
+    const notice = continuationStateNotice(parent)
+    assert.match(notice, /already applied to this checkout/)
+    assert.equal(notice.includes('gold-like-added-line'), false)
+    assert.doesNotThrow(() => assertNoHiddenLeak(['+gold-like-added-line'], [{ role: 'user', content: notice }]))
+  })
+})
+
+describe('exact completed worker compute', () => {
+  const complete = { calls: 1, completions: 1, attemptError: null }
+
+  it('admits exactly two completed model-backed sessions', () => {
+    assert.doesNotThrow(() =>
+      assertExactCompletedWorkerSessions({
+        started: 2,
+        completed: 2,
+        sessions: [complete, complete],
+        context: 'row',
+      }),
+    )
+  })
+
+  it('rejects an attempted-but-failed or zero-call session', () => {
+    assert.throws(() =>
+      assertExactCompletedWorkerSessions({
+        started: 2,
+        completed: 1,
+        sessions: [complete, { calls: 0, completions: 0, attemptError: 'timeout' }],
+        context: 'row',
+      }),
+      /expected exactly 2 started and completed/,
+    )
+    assert.throws(() =>
+      assertExactCompletedWorkerSessions({
+        started: 2,
+        completed: 2,
+        sessions: [complete, { calls: 0, completions: 1, attemptError: null }],
+        context: 'row',
+      }),
+      /session 2 did not complete successfully/,
+    )
+  })
+})

package/src/swe-structural-policy.ts

@@ -0,0 +1,132 @@
+import { isAbsolute } from 'node:path'
+
+export type ExperimentArm = 'independent-2' | 'persistent-refine-2'
+
+export interface ExperimentArmPreset {
+  arm: ExperimentArm
+  k: 1 | 2
+  repairs: 0 | 1
+  alwaysRunContinuation: boolean
+  persistent: boolean
+  workerSessions: 2
+}
+
+export interface WorkerSessionReceipt {
+  calls: number
+  completions: number
+  attemptError: string | null
+}
+
+const PRESETS: Record<ExperimentArm, ExperimentArmPreset> = {
+  'independent-2': {
+    arm: 'independent-2',
+    k: 2,
+    repairs: 0,
+    alwaysRunContinuation: false,
+    persistent: false,
+    workerSessions: 2,
+  },
+  'persistent-refine-2': {
+    arm: 'persistent-refine-2',
+    k: 1,
+    repairs: 1,
+    alwaysRunContinuation: true,
+    persistent: true,
+    workerSessions: 2,
+  },
+}
+
+export function resolveExperimentArm(value: string): ExperimentArmPreset {
+  const preset = PRESETS[value as ExperimentArm]
+  if (!preset) {
+    throw new Error(
+      `EXPERIMENT_ARM must be independent-2|persistent-refine-2, got "${value}"`,
+    )
+  }
+  return preset
+}
+
+/** Keep model sampling configuration disjoint from Node's TEMP-controlled filesystem root. */
+export function resolveExperimentTemperature(
+  env: Readonly<Record<string, string | undefined>>,
+): number {
+  if (env.TEMP?.trim() && !isAbsolute(env.TEMP)) {
+    throw new Error(
+      `TEMP is the operating-system temporary-directory root and must be absolute, got "${env.TEMP}"; ` +
+        'use TEMPERATURE for model sampling',
+    )
+  }
+  const value = Number(env.TEMPERATURE ?? 0.8)
+  if (!Number.isFinite(value)) {
+    throw new Error(`TEMPERATURE must be a finite number, got "${env.TEMPERATURE}"`)
+  }
+  return value
+}
+
+/** Both arms use one selection rule: lower visible severity wins and a tie prefers session two. */
+export function preferLaterCandidate(currentSeverity: number, laterSeverity: number): boolean {
+  return laterSeverity <= currentSeverity
+}
+
+/** The persistent arm always starts its one continuation, including after a visible pass. */
+export function shouldRunContinuation(input: {
+  round: number
+  preset: ExperimentArmPreset
+}): boolean {
+  return input.preset.alwaysRunContinuation && input.preset.repairs === 1 && input.round === 1
+}
+
+/** A continuation replaces its parent under the same later-on-tie rule as independent selection. */
+export function shouldAcceptContinuation(currentSeverity: number, continuationSeverity: number): boolean {
+  return preferLaterCandidate(currentSeverity, continuationSeverity)
+}
+
+export function continuationDisposition(accepted: boolean, changedFromParent: boolean): {
+  finalFrom: 'session:2-refinement' | 'session:2-identical' | 'session:1-visible-better'
+  stop: 'continuation-accepted-changed' | 'continuation-accepted-identical' | 'continuation-rejected-visible-regression'
+} {
+  if (!accepted) {
+    return { finalFrom: 'session:1-visible-better', stop: 'continuation-rejected-visible-regression' }
+  }
+  return changedFromParent
+    ? { finalFrom: 'session:2-refinement', stop: 'continuation-accepted-changed' }
+    : { finalFrom: 'session:2-identical', stop: 'continuation-accepted-identical' }
+}
+
+/** Describe inherited workspace state without echoing model-authored patch bytes back through the
+ * outbound prompt. The checkout already contains the parent state; repeating its diff can collide
+ * with hidden-patch leak marks when session one independently finds the correct line. */
+export function continuationStateNotice(parentDiff: string): string {
+  return parentDiff.trim()
+    ? '--- PREVIOUS FIX STATE ---\nSession one changes are already applied to this checkout. Inspect the current files directly.\n\n'
+    : '--- NO FIX APPLIED YET (session one produced no change) ---\n\n'
+}
+
+/** Admit only two genuinely completed model-backed sessions, never two attempted dispatches. */
+export function assertExactCompletedWorkerSessions(input: {
+  started: number
+  completed: number
+  sessions: readonly WorkerSessionReceipt[]
+  context: string
+}): void {
+  if (input.started !== 2 || input.completed !== 2 || input.sessions.length !== 2) {
+    throw new Error(
+      `${input.context}: expected exactly 2 started and completed worker sessions ` +
+        `(started=${input.started}, completed=${input.completed}, receipts=${input.sessions.length})`,
+    )
+  }
+  input.sessions.forEach((session, index) => {
+    if (
+      session.attemptError !== null ||
+      !Number.isInteger(session.calls) ||
+      session.calls < 1 ||
+      !Number.isInteger(session.completions) ||
+      session.completions < 1
+    ) {
+      throw new Error(
+        `${input.context}: session ${index + 1} did not complete successfully ` +
+          `(error=${session.attemptError ?? 'none'}, calls=${session.calls}, completions=${session.completions})`,
+      )
+    }
+  })
+}

package/src/swe-structural-provenance.test.ts

@@ -0,0 +1,93 @@
+import assert from 'node:assert/strict'
+import { describe, it } from 'node:test'
+import {
+  assertFingerprintsEqual,
+  createExecutionReceipt,
+  createFingerprints,
+  diffChanged,
+  diffFingerprint,
+  fingerprint,
+  runtimeImplementationFingerprint,
+} from './swe-structural-provenance'
+
+const inputs = {
+  source: { files: ['runner', 'environment'] },
+  config: { arm: 'independent-2', k: 2 },
+  commonConfig: { model: 'worker' },
+  repro: { script: 'raise SystemExit(1)' },
+  prompt: 'prompt',
+  tools: [{ name: 'run' }],
+  task: { id: 'repo__repo-1' },
+}
+
+describe('structural provenance', () => {
+  it('hashes objects independent of key insertion order', () => {
+    assert.equal(fingerprint({ b: 2, a: 1 }), fingerprint({ a: 1, b: 2 }))
+  })
+
+  it('rejects a changed resume input by its named fingerprint', () => {
+    const expected = createFingerprints(inputs)
+    const actual = createFingerprints({ ...inputs, prompt: 'changed prompt' })
+    assert.throws(() => assertFingerprintsEqual(actual, expected, 'resume row'), /prompt fingerprint mismatch/)
+  })
+
+  it('records identical and changed continuation patches exactly', () => {
+    const parent = 'diff --git a/a.py b/a.py\n+x\n'
+    assert.match(diffFingerprint(parent), /^sha256:[0-9a-f]{64}$/)
+    assert.equal(diffChanged(parent, parent), false)
+    assert.equal(diffChanged(parent, `${parent}+y\n`), true)
+  })
+
+  it('binds actual runtime callables, run settings, scorer version, and immutable image identity', () => {
+    const runtime = runtimeImplementationFingerprint({
+      runAgentic: function runAgenticA() { return 'a' },
+      refine: { name: 'refine', driver: function refineA() { return 'a' } },
+    })
+    const changedRuntime = runtimeImplementationFingerprint({
+      runAgentic: function runAgenticB() { return 'b' },
+      refine: { name: 'refine', driver: function refineA() { return 'a' } },
+    })
+    assert.notEqual(changedRuntime, runtime)
+
+    const shared = {
+      runTool: { timeoutS: 120, outputLimit: 10_000 },
+      runtimeImplementationFingerprint: runtime,
+      runtimeTreeFingerprint: fingerprint({ runtime: 'tree-a' }),
+      officialScorer: {
+        package: 'swebench' as const,
+        version: '4.0.0',
+        cacheLevel: 'instance' as const,
+        namespacePolicy: 'phase-a-image' as const,
+      },
+    }
+    const receipt = createExecutionReceipt(shared, {
+      tag: 'sweb.eval.example:latest',
+      namespace: 'swebench',
+      identity: { id: 'sha256:image-a', repoDigests: ['sweb.eval.example@sha256:digest-a'] },
+    })
+    const movedTag = createExecutionReceipt(shared, {
+      tag: 'sweb.eval.example:latest',
+      namespace: 'swebench',
+      identity: { id: 'sha256:image-b', repoDigests: ['sweb.eval.example@sha256:digest-b'] },
+    })
+    const movedRuntimeTree = createExecutionReceipt(
+      { ...shared, runtimeTreeFingerprint: fingerprint({ runtime: 'tree-b' }) },
+      {
+        tag: 'sweb.eval.example:latest',
+        namespace: 'swebench',
+        identity: { id: 'sha256:image-a', repoDigests: ['sweb.eval.example@sha256:digest-a'] },
+      },
+    )
+    const movedScorer = createExecutionReceipt(
+      { ...shared, officialScorer: { ...shared.officialScorer, version: '4.0.1' } },
+      {
+        tag: 'sweb.eval.example:latest',
+        namespace: 'swebench',
+        identity: { id: 'sha256:image-a', repoDigests: ['sweb.eval.example@sha256:digest-a'] },
+      },
+    )
+    assert.notEqual(fingerprint(receipt), fingerprint(movedTag))
+    assert.notEqual(fingerprint(receipt), fingerprint(movedRuntimeTree))
+    assert.notEqual(fingerprint(receipt), fingerprint(movedScorer))
+  })
+})

package/src/swe-structural-provenance.ts

@@ -0,0 +1,138 @@
+import { createHash } from 'node:crypto'
+
+export const FINGERPRINT_SCHEMA = 'swe-structural-v2' as const
+
+export interface Fingerprints {
+  schema: typeof FINGERPRINT_SCHEMA
+  source: string
+  config: string
+  commonConfig: string
+  repro: string
+  prompt: string
+  tools: string
+  task: string
+  composite: string
+}
+
+export interface FingerprintInputs {
+  source: unknown
+  config: unknown
+  commonConfig: unknown
+  repro: unknown
+  prompt: unknown
+  tools: unknown
+  task: unknown
+}
+
+export interface SharedExecutionReceipt {
+  runTool: {
+    timeoutS: number
+    outputLimit: number
+  }
+  runtimeImplementationFingerprint: string
+  runtimeTreeFingerprint: string
+  officialScorer: {
+    package: 'swebench'
+    version: string
+    cacheLevel: 'instance'
+    namespacePolicy: 'phase-a-image'
+  }
+}
+
+export interface ExecutionReceipt extends SharedExecutionReceipt {
+  image: {
+    tag: string
+    namespace: 'swebench' | 'none'
+    id: string
+    repoDigests: string[]
+  }
+}
+
+function canonicalize(value: unknown): unknown {
+  if (Array.isArray(value)) return value.map(canonicalize)
+  if (value && typeof value === 'object') {
+    return Object.fromEntries(
+      Object.entries(value as Record<string, unknown>)
+        .sort(([a], [b]) => a.localeCompare(b))
+        .map(([key, child]) => [key, canonicalize(child)]),
+    )
+  }
+  return value
+}
+
+export function canonicalJson(value: unknown): string {
+  return JSON.stringify(canonicalize(value))
+}
+
+export function fingerprint(value: unknown): string {
+  return `sha256:${createHash('sha256').update(canonicalJson(value)).digest('hex')}`
+}
+
+export function createFingerprints(inputs: FingerprintInputs): Fingerprints {
+  const parts = {
+    schema: FINGERPRINT_SCHEMA,
+    source: fingerprint(inputs.source),
+    config: fingerprint(inputs.config),
+    commonConfig: fingerprint(inputs.commonConfig),
+    repro: fingerprint(inputs.repro),
+    prompt: fingerprint(inputs.prompt),
+    tools: fingerprint(inputs.tools),
+    task: fingerprint(inputs.task),
+  }
+  return { ...parts, composite: fingerprint(parts) }
+}
+
+export function assertFingerprintsEqual(
+  actual: Fingerprints | undefined,
+  expected: Fingerprints,
+  context: string,
+): void {
+  if (!actual) throw new Error(`${context}: missing ${FINGERPRINT_SCHEMA} fingerprints`)
+  for (const key of Object.keys(expected) as Array<keyof Fingerprints>) {
+    if (actual[key] !== expected[key]) {
+      throw new Error(`${context}: ${key} fingerprint mismatch (${actual[key]} != ${expected[key]})`)
+    }
+  }
+}
+
+export function diffFingerprint(diff: string): string {
+  return fingerprint({ diff })
+}
+
+export function diffChanged(parent: string, candidate: string): boolean {
+  return diffFingerprint(parent) !== diffFingerprint(candidate)
+}
+
+/** Hash the actual imported callables, not merely the source file expected to contain them. */
+export function runtimeImplementationFingerprint(input: {
+  runAgentic: unknown
+  refine: { name: string; driver: unknown }
+}): string {
+  if (typeof input.runAgentic !== 'function' || typeof input.refine.driver !== 'function') {
+    throw new Error('runtime implementation receipt requires callable runAgentic and refine.driver')
+  }
+  return fingerprint({
+    runAgentic: Function.prototype.toString.call(input.runAgentic),
+    refine: {
+      name: input.refine.name,
+      driver: Function.prototype.toString.call(input.refine.driver),
+    },
+  })
+}
+
+export function createExecutionReceipt(
+  shared: SharedExecutionReceipt,
+  image: { tag: string; namespace: 'swebench' | 'none'; identity: { id: string; repoDigests: string[] } },
+): ExecutionReceipt {
+  return {
+    ...shared,
+    runTool: { ...shared.runTool },
+    officialScorer: { ...shared.officialScorer },
+    image: {
+      tag: image.tag,
+      namespace: image.namespace,
+      id: image.identity.id,
+      repoDigests: [...image.identity.repoDigests],
+    },
+  }
+}