@tangle-network/agent-bench 0.4.3 → 0.4.4

package/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 0.4.4
+
+- Consume Runtime 0.107.2 and Sandbox 0.15.0 with the current Eval, Interface, and Knowledge packages.
+- Publish fake Pier terminal results atomically so the full test suite cannot observe partial JSON.
+
 ## 0.4.3
 
 - Consume Runtime 0.107.1 with Eval 0.133.0, Interface 0.36.0, and Knowledge 6.1.2.

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.4.3",
+  "version": "0.4.4",
   "type": "module",
-  "description": "The unified benchmark suite for agent-runtime agents: 31 adapters (commit0, enterpriseops-gym, ragbench, crag, nomiracl, open-rag-bench, t2-ragbench, tau3-banking, bfcl, finresearchbench, …) behind one resolveAdapter registry, each with a real judge or fail-loud unsupported scorer. Score any profile/skill/prompt change against them. Map: bench/HARNESS.md.",
+  "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/tangle-network/agent-runtime.git",
@@ -28,8 +28,8 @@
     "@tangle-network/agent-eval": "0.133.0",
     "@tangle-network/agent-interface": "0.36.0",
     "@tangle-network/agent-knowledge": "6.1.2",
-    "@tangle-network/sandbox": "0.14.0",
-    "@tangle-network/agent-runtime": "0.107.1"
+    "@tangle-network/sandbox": "0.15.0",
+    "@tangle-network/agent-runtime": "0.107.2"
   },
   "devDependencies": {
     "@arethetypeswrong/cli": "0.18.5",

package/src/pier-trial-controller.test.mts

@@ -85,18 +85,20 @@ test('the Pier result wait does not add time beyond an expired deadline', async
     await chmod(fakeDocker, 0o700)
     await writeFile(
       supervisorPath,
-      `import { createReadStream } from 'node:fs'
-import { writeFileSync } from 'node:fs'
+      `import { createReadStream, renameSync, writeFileSync } from 'node:fs'
 import path from 'node:path'
 const directory = process.argv[2]
 process.on('SIGTERM', () => {
-  writeFileSync(path.join(directory, 'terminal.json'), JSON.stringify({
+  const target = path.join(directory, 'terminal.json')
+  const temporary = path.join(directory, '.terminal.json.tmp')
+  writeFileSync(temporary, JSON.stringify({
     schemaVersion: 1,
     kind: 'pier-trial-terminal',
     status: 'stopped',
     processExited: true,
     containersRemoved: true,
   }))
+  renameSync(temporary, target)
   process.exit(0)
 })
 for await (const _chunk of createReadStream('', { fd: 3 })) {}
@@ -156,7 +158,7 @@ test('the supervisor receives only launch data and no inherited evaluator enviro
     await chmod(fakeDocker, 0o700)
     await writeFile(
       supervisorPath,
-      `import { createReadStream, writeFileSync } from 'node:fs'
+      `import { createReadStream, renameSync, writeFileSync } from 'node:fs'
 import path from 'node:path'
 const chunks = []
 for await (const chunk of createReadStream('', { fd: 3 })) chunks.push(Buffer.from(chunk))
@@ -165,7 +167,9 @@ const isolated = process.env.PIER_SUPERVISOR_MODE === 'explicit' &&
   process.env.PIER_PARENT_SECRET === undefined &&
   launch.env.PIER_PARENT_SECRET === undefined &&
   launch.env.PIER_CHILD_SECRET === 'child-only'
-writeFileSync(path.join(process.argv[2], 'terminal.json'), JSON.stringify({
+const target = path.join(process.argv[2], 'terminal.json')
+const temporary = path.join(process.argv[2], '.terminal.json.tmp')
+writeFileSync(temporary, JSON.stringify({
   schemaVersion: 1,
   kind: 'pier-trial-terminal',
   status: isolated ? 'completed' : 'failed',
@@ -173,6 +177,7 @@ writeFileSync(path.join(process.argv[2], 'terminal.json'), JSON.stringify({
   containersRemoved: true,
   ...(isolated ? {} : { error: 'supervisor inherited evaluator environment' }),
 }))
+renameSync(temporary, target)
 `,
     )
     const controller = new FilePierCandidateTrialController({
@@ -224,10 +229,12 @@ test('terminal acknowledgements reject unknown fields', async () => {
     await chmod(fakeDocker, 0o700)
     await writeFile(
       supervisorPath,
-      `import { createReadStream, writeFileSync } from 'node:fs'
+      `import { createReadStream, renameSync, writeFileSync } from 'node:fs'
 import path from 'node:path'
 for await (const _chunk of createReadStream('', { fd: 3 })) {}
-writeFileSync(path.join(process.argv[2], 'terminal.json'), JSON.stringify({
+const target = path.join(process.argv[2], 'terminal.json')
+const temporary = path.join(process.argv[2], '.terminal.json.tmp')
+writeFileSync(temporary, JSON.stringify({
   schemaVersion: 1,
   kind: 'pier-trial-terminal',
   status: 'completed',
@@ -235,6 +242,7 @@ writeFileSync(path.join(process.argv[2], 'terminal.json'), JSON.stringify({
   containersRemoved: true,
   containerRemoved: true,
 }))
+renameSync(temporary, target)
 `,
     )
     const controller = new FilePierCandidateTrialController({