@tangle-network/agent-bench 0.3.8 → 0.4.1

package/src/smoke-structural-rollout.mts

@@ -199,14 +199,25 @@ async function runTask(t: HumanEvalTask): Promise<TaskRow> {
       `${t.taskId}: ${result.selection.length} receipts vs ${scored.length} scored candidates`,
     )
   }
-  for (const r of result.selection) {
+  const receipts = result.selection.map((receipt) => {
+    if (receipt.score === undefined || receipt.reason === undefined) {
+      throw new Error(`${t.taskId}: receipt #${receipt.candidateIndex} is missing score or reason`)
+    }
+    return {
+      candidateIndex: receipt.candidateIndex,
+      selected: receipt.selected,
+      score: receipt.score,
+      reason: receipt.reason,
+    }
+  })
+  for (const r of receipts) {
     const rec = scored[r.candidateIndex]
     if (!rec || Math.abs(r.score - visibleCheckScore(rec.outcome)) > 1e-9) {
       throw new Error(`${t.taskId}: receipt #${r.candidateIndex} score ${r.score} does not match the recorded outcome`)
     }
   }
 
-  const sampleCount = result.selection.filter((r) => r.reason.startsWith('sample')).length
+  const sampleCount = receipts.filter((r) => r.reason.startsWith('sample')).length
   const samples = scored.slice(0, sampleCount)
   if (samples.length === 0) throw new Error(`${t.taskId}: no sample candidates settled`)
 
@@ -223,7 +234,7 @@ async function runTask(t: HumanEvalTask): Promise<TaskRow> {
   const sampleHidden = await Promise.all(samples.map((s) => grade(s.candidate)))
   const selectedIdx = selectBestIndex(samples.map((s) => s.outcome))
   const selectedHidden = sampleHidden[selectedIdx] as number
-  const winner = result.selection.find((r) => r.selected)
+  const winner = receipts.find((r) => r.selected)
   if (!winner) throw new Error(`${t.taskId}: no receipt marked selected`)
   const finalIdx = winner.candidateIndex
   const finalHidden = await grade((scored[finalIdx] as ScoredCandidate).candidate)
@@ -241,12 +252,7 @@ async function runTask(t: HumanEvalTask): Promise<TaskRow> {
     finalIdx,
     finalHidden,
     selectedVisible: visibleCheckScore((samples[selectedIdx] as ScoredCandidate).outcome),
-    receipts: result.selection.map((r) => ({
-      candidateIndex: r.candidateIndex,
-      selected: r.selected,
-      score: r.score,
-      reason: r.reason,
-    })),
+    receipts,
     tokens: result.tokens,
     usd: result.usd,
     ms: result.ms,

package/src/swe-arena/activation.mts

@@ -26,7 +26,6 @@ import { run } from './proc.ts'
 export const ACTIVATION_PREDICATE_RELPATH = '.improve/activation.json'
 
 export interface ActivationPredicate {
-  version: 'v1'
   /** One sentence: which mechanism this proves fired. */
   description: string
   kind: 'grep' | 'script'
@@ -53,7 +52,6 @@ export function parseActivationPredicate(raw: string): ParsedPredicate {
     return { ok: false, error: 'must be a JSON object' }
   }
   const p = value as Record<string, unknown>
-  if (p.version !== 'v1') return { ok: false, error: `version must be "v1", got ${JSON.stringify(p.version)}` }
   if (typeof p.description !== 'string' || p.description.trim().length === 0) {
     return { ok: false, error: 'description must be a non-empty string' }
   }
@@ -90,13 +88,12 @@ export function activationPredicateInstruction(): string {
     '',
     'Template (kind "grep" — a RegExp tested over the run artifacts):',
     '  {',
-    '    "version": "v1",',
     '    "description": "patchRiskWarnings emitted in at least one settle",',
     '    "kind": "grep",',
     '    "pattern": "patchRiskWarnings|patch-risk",',
     '    "files": ["journal.jsonl", "workers/"]',
     '  }',
-    'Or kind "script": {"version":"v1","description":"...","kind":"script","script":"grep -rq NEW_SECTION ws/.loops"}',
+    'Or kind "script": {"description":"...","kind":"script","script":"grep -rq NEW_SECTION ws/.loops"}',
     '(exit 0 in any run dir = fired).',
     'Pick a pattern that can ONLY appear when your mechanism ran — not one that matches the diff itself.',
   ].join('\n')

package/src/swe-arena/activation.test.mts

@@ -26,7 +26,6 @@ describe('parseActivationPredicate', () => {
   it('accepts a valid grep predicate (with optional files filters)', () => {
     const parsed = parseActivationPredicate(
       JSON.stringify({
-        version: 'v1',
         description: 'patchRiskWarnings emitted in at least one settle',
         kind: 'grep',
         pattern: 'patchRiskWarnings|patch-risk',
@@ -38,7 +37,7 @@ describe('parseActivationPredicate', () => {
 
   it('accepts a valid script predicate', () => {
     const parsed = parseActivationPredicate(
-      JSON.stringify({ version: 'v1', description: 'x', kind: 'script', script: 'grep -rq X .' }),
+      JSON.stringify({ description: 'x', kind: 'script', script: 'grep -rq X .' }),
     )
     expect(parsed.ok).toBe(true)
   })
@@ -46,13 +45,12 @@ describe('parseActivationPredicate', () => {
   it.each([
     ['not json', 'not valid JSON'],
     ['[]', 'JSON object'],
-    [JSON.stringify({ version: 'v2', description: 'x', kind: 'grep', pattern: 'a' }), 'version'],
-    [JSON.stringify({ version: 'v1', description: '', kind: 'grep', pattern: 'a' }), 'description'],
-    [JSON.stringify({ version: 'v1', description: 'x', kind: 'grep' }), 'pattern'],
-    [JSON.stringify({ version: 'v1', description: 'x', kind: 'grep', pattern: '(' }), 'RegExp'],
-    [JSON.stringify({ version: 'v1', description: 'x', kind: 'grep', pattern: 'a', files: [1] }), 'files'],
-    [JSON.stringify({ version: 'v1', description: 'x', kind: 'script' }), 'script'],
-    [JSON.stringify({ version: 'v1', description: 'x', kind: 'sql' }), 'kind'],
+    [JSON.stringify({ description: '', kind: 'grep', pattern: 'a' }), 'description'],
+    [JSON.stringify({ description: 'x', kind: 'grep' }), 'pattern'],
+    [JSON.stringify({ description: 'x', kind: 'grep', pattern: '(' }), 'RegExp'],
+    [JSON.stringify({ description: 'x', kind: 'grep', pattern: 'a', files: [1] }), 'files'],
+    [JSON.stringify({ description: 'x', kind: 'script' }), 'script'],
+    [JSON.stringify({ description: 'x', kind: 'sql' }), 'kind'],
   ])('rejects %s', (raw, want) => {
     const parsed = parseActivationPredicate(raw)
     expect(parsed.ok).toBe(false)
@@ -84,7 +82,6 @@ describe('runActivationPredicate', () => {
   })
 
   const grep = (pattern: string, files?: string[]) => ({
-    version: 'v1' as const,
     description: 'd',
     kind: 'grep' as const,
     pattern,
@@ -127,7 +124,7 @@ describe('runActivationPredicate', () => {
   })
 
   it('script kind fires on rc=0 in any run dir, not-fired otherwise', async () => {
-    const script = (s: string) => ({ version: 'v1' as const, description: 'd', kind: 'script' as const, script: s })
+    const script = (s: string) => ({ description: 'd', kind: 'script' as const, script: s })
     const hit = await runActivationPredicate(script('grep -q patchRiskWarnings driver.log'), [runA, runB])
     expect(hit.fired).toBe(true)
     expect(hit.evidence[0]).toContain(runB)
@@ -261,11 +258,11 @@ describe('activation-predicate prefilter', () => {
           await mkdir(join(args.worktreePath, '.improve'), { recursive: true })
           await writeFile(
             join(args.worktreePath, ACTIVATION_PREDICATE_RELPATH),
-            JSON.stringify({ version: 'v1', description: 'd', kind: 'grep', pattern: 'x' }),
+            JSON.stringify({ description: 'd', kind: 'grep', pattern: 'x' }),
           )
         } else if (proposer.name === 'invalid-predicate') {
           await mkdir(join(args.worktreePath, '.improve'), { recursive: true })
-          await writeFile(join(args.worktreePath, ACTIVATION_PREDICATE_RELPATH), '{"version":"v1"}')
+          await writeFile(join(args.worktreePath, ACTIVATION_PREDICATE_RELPATH), '{}')
         }
         return { applied: true, summary: `${proposer.name} edit` }
       },