@tangle-network/agent-bench 0.9.1 → 0.9.3

package/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## 0.9.3
+
+Requires Runtime 0.201.0, Interface 2.5.0, Eval 0.176, and Knowledge 15 for scoped candidate knowledge.
+Publishing waits until dependency archives can be downloaded and integrity-checked, including registry propagation delays.
+
+## 0.9.2
+
+Requires Interface 2.4.0 and Runtime 0.200.0 for the shared execution contracts.
+
 ## 0.9.1
 
 Omni agent engines use the configured optimizer model through the metered Anthropic endpoint.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.9.1",
+  "version": "0.9.3",
   "type": "module",
   "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.",
   "repository": {
@@ -25,11 +25,11 @@
     }
   },
   "dependencies": {
-    "@tangle-network/agent-eval": ">=0.175.0 <0.176.0",
-    "@tangle-network/agent-interface": "^2.3.0",
-    "@tangle-network/agent-knowledge": "^14.0.3",
+    "@tangle-network/agent-eval": ">=0.176.0 <0.177.0",
+    "@tangle-network/agent-interface": "^2.5.0",
+    "@tangle-network/agent-knowledge": "^15.0.0",
     "@tangle-network/sandbox": ">=0.36.4 <0.38.0",
-    "@tangle-network/agent-runtime": "^0.199.0"
+    "@tangle-network/agent-runtime": "^0.201.0"
   },
   "devDependencies": {
     "@arethetypeswrong/cli": "0.18.5",

package/scripts/wait-for-published-dependencies.mjs

@@ -118,32 +118,44 @@ export async function waitForPublishedDependencies(
   }
 }
 
-async function probeNpm(dependency, { timeoutMs }) {
+export async function probeNpm(dependency, { timeoutMs, run = execFileAsync }) {
+  const scratch = await mkdtemp(path.join(tmpdir(), 'agent-bench-dependency-probe-'))
   try {
-    const { stdout } = await execFileAsync(
+    // A fresh cache makes npm retrieve and integrity-check the archive, even when
+    // registry metadata has propagated before the archive becomes available.
+    const { stdout } = await run(
       'npm',
       [
-        'view',
+        'pack',
         `${dependency.name}@${dependency.spec}`,
-        'version',
         '--json',
+        '--ignore-scripts',
         '--prefer-online',
+        '--cache',
+        path.join(scratch, 'cache'),
+        '--pack-destination',
+        scratch,
       ],
       {
         env: { ...process.env, npm_config_color: 'false' },
-        maxBuffer: 1024 * 1024,
+        maxBuffer: 10 * 1024 * 1024,
         timeout: Math.min(30_000, timeoutMs),
       },
     )
-    const version = JSON.parse(stdout)
+    const packed = JSON.parse(stdout)
     const available =
-      (typeof version === 'string' && version.length > 0) ||
-      (Array.isArray(version) && version.length > 0)
+      Array.isArray(packed) &&
+      packed.length === 1 &&
+      packed[0].name === dependency.name &&
+      typeof packed[0].version === 'string' &&
+      packed[0].version.length > 0
     return available
       ? { available: true }
-      : { available: false, detail: 'npm returned no matching version' }
+      : { available: false, detail: 'npm returned no matching package archive' }
   } catch (error) {
     return { available: false, detail: summarizeCommandError(error) }
+  } finally {
+    await rm(scratch, { recursive: true, force: true })
   }
 }
 

package/scripts/wait-for-published-dependencies.test.mjs

@@ -1,8 +1,11 @@
 import assert from 'node:assert/strict'
+import { access } from 'node:fs/promises'
+import path from 'node:path'
 import test from 'node:test'
 
 import {
   collectRequiredTangleDependencies,
+  probeNpm,
   waitForPublishedDependencies,
 } from './wait-for-published-dependencies.mjs'
 
@@ -94,3 +97,70 @@ test('fails at the deadline and names every dependency still missing', async ()
   )
   assert.equal(currentTime, 100)
 })
+
+test('waits for the exact package archive after metadata is published', async () => {
+  const dependency = { name: '@tangle-network/agent-runtime', spec: '0.200.0' }
+  let currentTime = 0
+  let archiveAttempts = 0
+  const scratchDirectories = []
+  const result = await waitForPublishedDependencies([dependency], {
+    timeoutMs: 100,
+    intervalMs: 25,
+    now: () => currentTime,
+    sleep: async (milliseconds) => {
+      currentTime += milliseconds
+    },
+    report: () => {},
+    probe: (requested, options) => probeNpm(requested, {
+      ...options,
+      run: async (command, args, executionOptions) => {
+        assert.equal(command, 'npm')
+        // Metadata is already visible, but its archive is not installable yet.
+        if (args[0] === 'view') return { stdout: JSON.stringify('0.200.0') }
+        assert.equal(args[0], 'pack')
+        assert.equal(args[1], '@tangle-network/agent-runtime@0.200.0')
+        assert.ok(args.includes('--ignore-scripts'))
+        const scratch = args[args.indexOf('--pack-destination') + 1]
+        assert.equal(args[args.indexOf('--cache') + 1], path.join(scratch, 'cache'))
+        scratchDirectories.push(scratch)
+        assert.ok(executionOptions.timeout <= 100 - currentTime)
+        archiveAttempts++
+        if (archiveAttempts === 1) {
+          throw Object.assign(new Error('archive unavailable'), {
+            stderr: 'npm error E404 GET agent-runtime-0.200.0.tgz',
+          })
+        }
+        return { stdout: JSON.stringify([{ name: dependency.name, version: '0.200.0' }]) }
+      },
+    }),
+  })
+  assert.deepEqual(result, { attempts: 2, elapsedMs: 25 })
+  assert.equal(archiveAttempts, 2)
+  assert.equal(new Set(scratchDirectories).size, 2)
+  for (const scratch of scratchDirectories) {
+    await assert.rejects(access(scratch), { code: 'ENOENT' })
+  }
+})
+
+test('reports a permanently invalid package archive at the existing deadline', async () => {
+  let currentTime = 0
+  await assert.rejects(waitForPublishedDependencies([
+    { name: '@tangle-network/agent-runtime', spec: '0.200.0' },
+  ], {
+    timeoutMs: 50,
+    intervalMs: 25,
+    now: () => currentTime,
+    sleep: async (milliseconds) => {
+      currentTime += milliseconds
+    },
+    report: () => {},
+    probe: (dependency, options) => probeNpm(dependency, {
+      ...options,
+      run: async () => {
+        throw Object.assign(new Error('archive integrity mismatch'), {
+          stderr: 'npm error code EINTEGRITY',
+        })
+      },
+    }),
+  }), /Timed out after 50ms and 2 attempts[\s\S]*agent-runtime@0\.200\.0: npm error code EINTEGRITY/)
+})