@tangle-network/agent-bench 0.9.2 → 0.9.4

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 0.9.4
+
+Requires Runtime 0.202.0 so Bench consumers receive explicit failed-execution settlement and retained failure recovery.
+Supports an optional Node test concurrency limit for hosts with constrained memory.
+
+## 0.9.3
+
+Requires Runtime 0.201.0, Interface 2.5.0, Eval 0.176, and Knowledge 15 for scoped candidate knowledge.
+Publishing waits until dependency archives can be downloaded and integrity-checked, including registry propagation delays.
+
 ## 0.9.2
 
 Requires Interface 2.4.0 and Runtime 0.200.0 for the shared execution contracts.

package/HARNESS.md

@@ -37,6 +37,14 @@ pnpm verify:bench
 pnpm verify:official-optimizers
 ```
 
+Set `AGENT_BENCH_PACKAGE_TEST_CONCURRENCY=1` to run Node source test files sequentially on memory-constrained hosts.
+The value must be a positive safe integer; leaving it unset preserves Node's default concurrency.
+Vitest uses the worker limit in `bench/vitest.config.ts`.
+
+Packing resolves the Runtime dependency from `workspace:^` to a version range.
+A new Runtime compatibility line does not update previously published Bench packages.
+Check the packed dependency range and release Bench when consumers must receive that Runtime version.
+
 `verify:official-optimizers` exercises the official Optimize Anything bridge, engine identities, equal input budgets, resume compatibility, candidate callbacks, accounting, and package provenance. Its deterministic candidate improvement is deliberately a fixture. It does **not** reproduce the published GEPA or Omni benchmark numbers.
 
 ### Bounded benchmark matrix

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.9.2",
+  "version": "0.9.4",
   "type": "module",
   "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.",
   "repository": {
@@ -25,11 +25,11 @@
     }
   },
   "dependencies": {
-    "@tangle-network/agent-eval": ">=0.175.0 <0.176.0",
-    "@tangle-network/agent-interface": "^2.4.0",
-    "@tangle-network/agent-knowledge": "^14.0.3",
+    "@tangle-network/agent-eval": ">=0.176.0 <0.177.0",
+    "@tangle-network/agent-interface": "^2.5.0",
+    "@tangle-network/agent-knowledge": "^15.0.0",
     "@tangle-network/sandbox": ">=0.36.4 <0.38.0",
-    "@tangle-network/agent-runtime": "^0.200.0"
+    "@tangle-network/agent-runtime": "^0.202.0"
   },
   "devDependencies": {
     "@arethetypeswrong/cli": "0.18.5",

package/scripts/run-package-tests.mjs

@@ -8,6 +8,21 @@ const execFileAsync = promisify(execFile)
 const benchDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..')
 const sourceDir = path.join(benchDir, 'src')
 
+export function packageNodeTestArgs(files, env = process.env) {
+  const raw = env.AGENT_BENCH_PACKAGE_TEST_CONCURRENCY
+  const concurrency = raw === undefined ? undefined : Number(raw)
+  if (raw !== undefined && (!/^[1-9]\d*$/.test(raw) || !Number.isSafeInteger(concurrency))) {
+    throw new Error('AGENT_BENCH_PACKAGE_TEST_CONCURRENCY must be a positive safe integer')
+  }
+  return [
+    '--test',
+    ...(concurrency === undefined ? [] : [`--test-concurrency=${concurrency}`]),
+    '--import',
+    'tsx',
+    ...files,
+  ]
+}
+
 export function resolvePackageTestTimeoutMs(env = process.env) {
   const raw = env.AGENT_BENCH_PACKAGE_TEST_TIMEOUT_MS
   if (raw === undefined) return undefined
@@ -74,7 +89,7 @@ async function main() {
   if (nodeTests.length > 0) {
     await run(
       process.execPath,
-      ['--test', '--import', 'tsx', ...nodeTests.map((file) => path.relative(benchDir, file))],
+      packageNodeTestArgs(nodeTests.map((file) => path.relative(benchDir, file))),
       {
         ...process.env,
         TSX_TSCONFIG_PATH: 'tsconfig.public.json',

package/scripts/run-package-tests.test.mjs

@@ -1,6 +1,20 @@
 import assert from 'node:assert/strict'
 import { test } from 'node:test'
-import { resolvePackageTestTimeoutMs, run } from './run-package-tests.mjs'
+import { packageNodeTestArgs, resolvePackageTestTimeoutMs, run } from './run-package-tests.mjs'
+
+test('package test concurrency reaches Node without changing selected files', () => {
+  const files = ['src/first.test.mts', 'src/second.test.ts']
+  assert.deepEqual(packageNodeTestArgs(files, {}), ['--test', '--import', 'tsx', ...files])
+  assert.deepEqual(packageNodeTestArgs(files, { AGENT_BENCH_PACKAGE_TEST_CONCURRENCY: '1' }), [
+    '--test', '--test-concurrency=1', '--import', 'tsx', ...files,
+  ])
+  for (const value of ['', '0', '-1', '1.5', 'Infinity', '9007199254740992']) {
+    assert.throws(
+      () => packageNodeTestArgs(files, { AGENT_BENCH_PACKAGE_TEST_CONCURRENCY: value }),
+      /must be a positive safe integer/,
+    )
+  }
+})
 
 test('package test timeout is optional and caller-controlled', () => {
   assert.equal(resolvePackageTestTimeoutMs({}), undefined)

package/scripts/wait-for-published-dependencies.mjs

@@ -118,32 +118,44 @@ export async function waitForPublishedDependencies(
   }
 }
 
-async function probeNpm(dependency, { timeoutMs }) {
+export async function probeNpm(dependency, { timeoutMs, run = execFileAsync }) {
+  const scratch = await mkdtemp(path.join(tmpdir(), 'agent-bench-dependency-probe-'))
   try {
-    const { stdout } = await execFileAsync(
+    // A fresh cache makes npm retrieve and integrity-check the archive, even when
+    // registry metadata has propagated before the archive becomes available.
+    const { stdout } = await run(
       'npm',
       [
-        'view',
+        'pack',
         `${dependency.name}@${dependency.spec}`,
-        'version',
         '--json',
+        '--ignore-scripts',
         '--prefer-online',
+        '--cache',
+        path.join(scratch, 'cache'),
+        '--pack-destination',
+        scratch,
       ],
       {
         env: { ...process.env, npm_config_color: 'false' },
-        maxBuffer: 1024 * 1024,
+        maxBuffer: 10 * 1024 * 1024,
         timeout: Math.min(30_000, timeoutMs),
       },
     )
-    const version = JSON.parse(stdout)
+    const packed = JSON.parse(stdout)
     const available =
-      (typeof version === 'string' && version.length > 0) ||
-      (Array.isArray(version) && version.length > 0)
+      Array.isArray(packed) &&
+      packed.length === 1 &&
+      packed[0].name === dependency.name &&
+      typeof packed[0].version === 'string' &&
+      packed[0].version.length > 0
     return available
       ? { available: true }
-      : { available: false, detail: 'npm returned no matching version' }
+      : { available: false, detail: 'npm returned no matching package archive' }
   } catch (error) {
     return { available: false, detail: summarizeCommandError(error) }
+  } finally {
+    await rm(scratch, { recursive: true, force: true })
   }
 }
 

package/scripts/wait-for-published-dependencies.test.mjs

@@ -1,8 +1,11 @@
 import assert from 'node:assert/strict'
+import { access } from 'node:fs/promises'
+import path from 'node:path'
 import test from 'node:test'
 
 import {
   collectRequiredTangleDependencies,
+  probeNpm,
   waitForPublishedDependencies,
 } from './wait-for-published-dependencies.mjs'
 
@@ -94,3 +97,70 @@ test('fails at the deadline and names every dependency still missing', async ()
   )
   assert.equal(currentTime, 100)
 })
+
+test('waits for the exact package archive after metadata is published', async () => {
+  const dependency = { name: '@tangle-network/agent-runtime', spec: '0.200.0' }
+  let currentTime = 0
+  let archiveAttempts = 0
+  const scratchDirectories = []
+  const result = await waitForPublishedDependencies([dependency], {
+    timeoutMs: 100,
+    intervalMs: 25,
+    now: () => currentTime,
+    sleep: async (milliseconds) => {
+      currentTime += milliseconds
+    },
+    report: () => {},
+    probe: (requested, options) => probeNpm(requested, {
+      ...options,
+      run: async (command, args, executionOptions) => {
+        assert.equal(command, 'npm')
+        // Metadata is already visible, but its archive is not installable yet.
+        if (args[0] === 'view') return { stdout: JSON.stringify('0.200.0') }
+        assert.equal(args[0], 'pack')
+        assert.equal(args[1], '@tangle-network/agent-runtime@0.200.0')
+        assert.ok(args.includes('--ignore-scripts'))
+        const scratch = args[args.indexOf('--pack-destination') + 1]
+        assert.equal(args[args.indexOf('--cache') + 1], path.join(scratch, 'cache'))
+        scratchDirectories.push(scratch)
+        assert.ok(executionOptions.timeout <= 100 - currentTime)
+        archiveAttempts++
+        if (archiveAttempts === 1) {
+          throw Object.assign(new Error('archive unavailable'), {
+            stderr: 'npm error E404 GET agent-runtime-0.200.0.tgz',
+          })
+        }
+        return { stdout: JSON.stringify([{ name: dependency.name, version: '0.200.0' }]) }
+      },
+    }),
+  })
+  assert.deepEqual(result, { attempts: 2, elapsedMs: 25 })
+  assert.equal(archiveAttempts, 2)
+  assert.equal(new Set(scratchDirectories).size, 2)
+  for (const scratch of scratchDirectories) {
+    await assert.rejects(access(scratch), { code: 'ENOENT' })
+  }
+})
+
+test('reports a permanently invalid package archive at the existing deadline', async () => {
+  let currentTime = 0
+  await assert.rejects(waitForPublishedDependencies([
+    { name: '@tangle-network/agent-runtime', spec: '0.200.0' },
+  ], {
+    timeoutMs: 50,
+    intervalMs: 25,
+    now: () => currentTime,
+    sleep: async (milliseconds) => {
+      currentTime += milliseconds
+    },
+    report: () => {},
+    probe: (dependency, options) => probeNpm(dependency, {
+      ...options,
+      run: async () => {
+        throw Object.assign(new Error('archive integrity mismatch'), {
+          stderr: 'npm error code EINTEGRITY',
+        })
+      },
+    }),
+  }), /Timed out after 50ms and 2 attempts[\s\S]*agent-runtime@0\.200\.0: npm error code EINTEGRITY/)
+})