@tangle-network/agent-bench 0.8.12 → 0.8.15

package/dist/{router-turn-C2wMiDoo.js → router-turn-uTYO6KQ1.js}

@@ -9,16 +9,18 @@ import "@tangle-network/agent-interface";
 */
 async function runBenchRouterTurn(config, input) {
 	if (!config.profile.model?.default) throw new Error("runBenchRouterTurn: profile.model.default is required");
+	const factory = createExecutor({
+		backend: "router",
+		routerBaseUrl: config.routerBaseUrl,
+		routerKey: config.routerKey,
+		...config.tools ? { tools: config.tools } : {}
+	});
+	const turnInput = typeof input === "string" ? { prompt: input } : { providerOptions: { messages: input.messages.map((message) => ({ ...message })) } };
 	const turn = await collectAgentTurn(streamAgentTurn({
 		kind: "executor",
-		factory: createExecutor({
-			backend: "router",
-			routerBaseUrl: config.routerBaseUrl,
-			routerKey: config.routerKey,
-			...config.tools ? { tools: config.tools } : {}
-		}),
+		factory,
 		profile: config.profile
-	}, input, {
+	}, turnInput, {
 		...config.timeoutMs === void 0 ? {} : { timeoutMs: config.timeoutMs },
 		...config.signal ? { signal: config.signal } : {}
 	}));
@@ -28,4 +30,4 @@ async function runBenchRouterTurn(config, input) {
 //#endregion
 export { runBenchRouterTurn as t };
 
-//# sourceMappingURL=router-turn-C2wMiDoo.js.map
+//# sourceMappingURL=router-turn-uTYO6KQ1.js.map

package/dist/router-turn-uTYO6KQ1.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"router-turn-uTYO6KQ1.js","names":[],"sources":["../src/router-turn.ts"],"sourcesContent":["import {\n  type AgentProfile,\n  agentProfileSchema,\n  type ReasoningEffort,\n} from '@tangle-network/agent-interface'\nimport {\n  collectAgentTurn,\n  createExecutor,\n  streamAgentTurn,\n  type AgentTurnInput,\n  type CollectedAgentTurn,\n  type ToolSpec,\n} from '@tangle-network/agent-runtime/kernel'\n\n/** Bench-local target shorthand; Runtime still executes only the exact profile below. */\nexport interface BenchRouterTarget {\n  routerBaseUrl: string\n  routerKey: string\n  profile: AgentProfile\n}\n\nexport interface BenchRouterTurnConfig extends BenchRouterTarget {\n  tools?: ReadonlyArray<ToolSpec>\n  timeoutMs?: number\n  signal?: AbortSignal\n}\n\nexport interface BenchProfileSettings {\n  systemPrompt?: string\n  temperature?: number\n  maxTokens?: number\n  retry?: {\n    maxAttempts?: number\n    initialBackoffMs?: number\n    maxBackoffMs?: number\n    jitter?: number\n    retryStatuses?: ReadonlyArray<number>\n    requestTimeoutMs?: number\n  }\n  maxTurns?: number\n  seed?: number\n  reasoningEffort?: ReasoningEffort\n  extraBody?: Readonly<Record<string, unknown>>\n  toolChoice?: 'auto' | 'required' | 'none'\n}\n\n/** Author an exact direct-Router profile for a benchmark. This is profile construction only;\n * execution still accepts no model or generation fields outside the returned AgentProfile. */\nexport function benchRouterProfile(\n  name: string,\n  model: string,\n  settings: BenchProfileSettings = {},\n): AgentProfile {\n  return withBenchProfile(\n    {\n      name,\n      harness: 'cli-base',\n      model: { provider: 'tangle-router', default: model },\n    },\n    settings,\n  )\n}\n\n/** Derive another exact profile while preserving all untouched canonical axes. */\nexport function withBenchProfile(\n  base: AgentProfile,\n  settings: BenchProfileSettings & { name?: string },\n): AgentProfile {\n  const metadata = {\n    ...(base.model?.metadata ?? {}),\n    ...(settings.temperature !== undefined ? { temperature: settings.temperature } : {}),\n    ...(settings.maxTokens !== undefined ? { maxTokens: settings.maxTokens } : {}),\n    ...(settings.retry !== undefined ? { retry: settings.retry } : {}),\n    ...(settings.maxTurns !== undefined ? { maxTurns: settings.maxTurns } : {}),\n    ...(settings.seed !== undefined ? { seed: settings.seed } : {}),\n    ...(settings.extraBody !== undefined ? { extraBody: settings.extraBody } : {}),\n    ...(settings.toolChoice !== undefined ? { toolChoice: settings.toolChoice } : {}),\n  }\n  return agentProfileSchema.parse({\n    ...base,\n    ...(settings.name ? { name: settings.name } : {}),\n    model: {\n      ...base.model,\n      ...(settings.reasoningEffort !== undefined\n        ? { reasoningEffort: settings.reasoningEffort }\n        : {}),\n      ...(Object.keys(metadata).length > 0 ? { metadata } : {}),\n    },\n    ...(settings.systemPrompt !== undefined\n      ? { prompt: { ...base.prompt, systemPrompt: settings.systemPrompt } }\n      : {}),\n  })\n}\n\nexport function benchProfileModel(profile: AgentProfile): string {\n  const model = profile.model?.default\n  if (typeof model !== 'string' || model.length === 0 || model === 'runtime-selected') {\n    throw new Error('benchmark AgentProfile.model.default must be concrete')\n  }\n  return model\n}\n\n/**\n * The benchmark-side entry to Runtime's canonical one-turn path.\n * It is only an ergonomic composition: Runtime still parses the exact profile,\n * materializes the executor, records identity/usage/result events, and refuses\n * profile axes the direct Router backend cannot carry.\n */\nexport async function runBenchRouterTurn(\n  config: BenchRouterTurnConfig,\n  input: string | { readonly messages: ReadonlyArray<Readonly<Record<string, unknown>>> },\n): Promise<CollectedAgentTurn> {\n  if (!config.profile.model?.default) {\n    throw new Error('runBenchRouterTurn: profile.model.default is required')\n  }\n  const factory = createExecutor({\n    backend: 'router',\n    routerBaseUrl: config.routerBaseUrl,\n    routerKey: config.routerKey,\n    ...(config.tools ? { tools: config.tools } : {}),\n  })\n  const turnInput: AgentTurnInput =\n    typeof input === 'string'\n      ? { prompt: input }\n      : {\n          providerOptions: {\n            messages: input.messages.map((message) => ({ ...message })),\n          },\n        }\n  const turn = await collectAgentTurn(\n    streamAgentTurn(\n      { kind: 'executor', factory, profile: config.profile },\n      turnInput,\n      {\n        ...(config.timeoutMs === undefined ? {} : { timeoutMs: config.timeoutMs }),\n        ...(config.signal ? { signal: config.signal } : {}),\n      },\n    ),\n  )\n  if (turn.status !== 'completed') {\n    throw new Error(turn.error?.message ?? `Router turn ended with status ${turn.status}`)\n  }\n  return turn\n}\n"],"mappings":";;;;;;;;;AA4GA,eAAsB,mBACpB,QACA,OAC6B;CAC7B,IAAI,CAAC,OAAO,QAAQ,OAAO,SACzB,MAAM,IAAI,MAAM,uDAAuD;CAEzE,MAAM,UAAU,eAAe;EAC7B,SAAS;EACT,eAAe,OAAO;EACtB,WAAW,OAAO;EAClB,GAAI,OAAO,QAAQ,EAAE,OAAO,OAAO,MAAM,IAAI,CAAC;CAChD,CAAC;CACD,MAAM,YACJ,OAAO,UAAU,WACb,EAAE,QAAQ,MAAM,IAChB,EACE,iBAAiB,EACf,UAAU,MAAM,SAAS,KAAK,aAAa,EAAE,GAAG,QAAQ,EAAE,EAC5D,EACF;CACN,MAAM,OAAO,MAAM,iBACjB,gBACE;EAAE,MAAM;EAAY;EAAS,SAAS,OAAO;CAAQ,GACrD,WACA;EACE,GAAI,OAAO,cAAc,KAAA,IAAY,CAAC,IAAI,EAAE,WAAW,OAAO,UAAU;EACxE,GAAI,OAAO,SAAS,EAAE,QAAQ,OAAO,OAAO,IAAI,CAAC;CACnD,CACF,CACF;CACA,IAAI,KAAK,WAAW,aAClB,MAAM,IAAI,MAAM,KAAK,OAAO,WAAW,iCAAiC,KAAK,QAAQ;CAEvF,OAAO;AACT"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.8.12",
+  "version": "0.8.15",
   "type": "module",
   "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.",
   "repository": {
@@ -25,11 +25,11 @@
     }
   },
   "dependencies": {
-    "@tangle-network/agent-eval": "0.145.21",
-    "@tangle-network/agent-interface": "1.0.0",
-    "@tangle-network/agent-knowledge": "8.0.5",
-    "@tangle-network/sandbox": "0.27.1",
-    "@tangle-network/agent-runtime": "0.137.0"
+    "@tangle-network/agent-eval": ">=0.148.0 <0.149.0",
+    "@tangle-network/agent-interface": "^1.1.0",
+    "@tangle-network/agent-knowledge": "^8.0.9",
+    "@tangle-network/sandbox": ">=0.27.1 <0.28.0",
+    "@tangle-network/agent-runtime": "^0.141.1"
   },
   "devDependencies": {
     "@arethetypeswrong/cli": "0.18.5",
@@ -73,7 +73,8 @@
     "typecheck:public": "tsc -p tsconfig.public.json",
     "verify:package": "pnpm run verify:package:static && node scripts/verify-packed-consumer.mjs",
     "verify:package:local-runtime": "pnpm run verify:package:static && node scripts/verify-packed-consumer.mjs --local-runtime",
-    "verify:package:static": "pnpm run build && publint && attw --pack --profile esm-only .",
+    "verify:package:static": "pnpm run build && publint && attw --pack --profile esm-only . && node ../scripts/check-published-ranges.mjs bench && node ../scripts/check-api-surface.mjs bench",
+    "api:surface": "node ../scripts/check-api-surface.mjs --write bench",
     "wait:published-dependencies": "node scripts/wait-for-published-dependencies.mjs",
     "verify:pier": "tsx scripts/verify-pier-pair.mts"
   }

package/src/agent-graphs-improve.mts

@@ -226,7 +226,7 @@ export async function callAuthor(
   const turn = await collectAgentTurn(
     streamAgentTurn(
       { kind: 'executor', factory, profile, agentRunName: profile.name ?? 'agent-graphs-author' },
-      prompt,
+      { prompt },
       timeoutMs === undefined ? {} : { timeoutMs },
     ),
   )

package/src/atom-mcp-e2e.mts

@@ -103,7 +103,7 @@ async function bridgeChat(opts: {
   const turn = await collectAgentTurn(
     streamAgentTurn(
       { kind: 'executor', factory, profile },
-      opts.messages.map((message) => message.content).join('\n\n'),
+      { prompt: opts.messages.map((message) => message.content).join('\n\n') },
     ),
   )
   if (turn.status !== 'completed') {

package/src/benchmarks/appworld.ts

@@ -440,7 +440,7 @@ export function appworldToolLoopClient(cfg: {
             const loop = await collectAgentTurn(
               streamAgentTurn(
                 { kind: 'executor', factory, profile },
-                `Task: ${instruction}`,
+                { prompt: `Task: ${instruction}` },
                 { signal },
               ),
             )

package/src/commit0-gate.mts

@@ -330,7 +330,7 @@ async function runShotLocal(task: BenchTask, attempt: number, cfg: ShotCfg, stee
     const turn = await collectAgentTurn(
       streamAgentTurn(
         { kind: 'executor', factory, profile: workerProfile(cfg, `commit0-local-${attempt}`) },
-        prompt,
+        { prompt },
         cfg.timeoutMs > 0 ? { timeoutMs: cfg.timeoutMs } : {},
       ),
     )

package/src/humaneval-repair-gate.mts

@@ -89,7 +89,7 @@ async function repairAttempt(cfg: BenchRouterTarget, task: HumanEvalTask, k: num
     },
   })
   const r = await collectAgentTurn(
-    streamAgentTurn({ kind: 'executor', factory, profile }, basePrompt(task)),
+    streamAgentTurn({ kind: 'executor', factory, profile }, { prompt: basePrompt(task) }),
   )
   if (r.status !== 'completed') {
     throw new Error(r.error?.message ?? `repair turn ended with status ${r.status}`)

package/src/mcp-mount-probe.mts

@@ -70,7 +70,7 @@ async function bridgeChat(messages: Array<{ role: string; content: string }>, mc
   const turn = await collectAgentTurn(
     streamAgentTurn(
       { kind: 'executor', factory, profile },
-      messages.map((message) => message.content).join('\n\n'),
+      { prompt: messages.map((message) => message.content).join('\n\n') },
     ),
   )
   if (turn.status !== 'completed') {

package/src/quant-arena/quant-loop.mts

@@ -249,7 +249,7 @@ async function profileShot(opts: {
   const turn = await collectAgentTurn(
     streamAgentTurn(
       { kind: 'executor', factory, profile: opts.profile },
-      opts.prompt,
+      { prompt: opts.prompt },
       { timeoutMs: opts.timeoutMs },
     ),
   )

package/src/router-turn.ts

@@ -7,6 +7,7 @@ import {
   collectAgentTurn,
   createExecutor,
   streamAgentTurn,
+  type AgentTurnInput,
   type CollectedAgentTurn,
   type ToolSpec,
 } from '@tangle-network/agent-runtime/kernel'
@@ -118,10 +119,18 @@ export async function runBenchRouterTurn(
     routerKey: config.routerKey,
     ...(config.tools ? { tools: config.tools } : {}),
   })
+  const turnInput: AgentTurnInput =
+    typeof input === 'string'
+      ? { prompt: input }
+      : {
+          providerOptions: {
+            messages: input.messages.map((message) => ({ ...message })),
+          },
+        }
   const turn = await collectAgentTurn(
     streamAgentTurn(
       { kind: 'executor', factory, profile: config.profile },
-      input,
+      turnInput,
       {
         ...(config.timeoutMs === undefined ? {} : { timeoutMs: config.timeoutMs }),
         ...(config.signal ? { signal: config.signal } : {}),

package/src/run-benchmarks.ts

@@ -185,13 +185,17 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB
     ...(timeoutMs ? { timeoutMs } : {}),
   })
   const harness = cell.harness ?? (cell.profile?.metadata?.backendType as string | undefined) ?? 'opencode'
-  const profile: AgentProfile =
-    cell.profile ?? {
-      name: cell.label,
-      harness: harness as AgentProfile['harness'],
-      model: { provider: 'tangle-router', default: cell.model },
-      metadata: { backendType: harness },
-    }
+  // The cell's harness and model are the identity the box must run, so they override whatever a
+  // supplied profile declared rather than being dropped when one is supplied. `buildBackendOptions`
+  // now sends this provider/model pair on the create request and refuses a conflicting override,
+  // so the two must be derived from the same profile.
+  const profileProvider = cell.profile?.model?.provider ?? 'tangle-router'
+  const profile: AgentProfile = {
+    ...(cell.profile ?? { name: cell.label }),
+    harness: harness as AgentProfile['harness'],
+    model: { ...cell.profile?.model, provider: profileProvider, default: cell.model },
+    metadata: { ...cell.profile?.metadata, backendType: harness },
+  }
   // Unique per shot: the same (adapter, task) runs concurrently across cells and reps, so the box
   // name and runId must not collide.
   const uniq = Math.random().toString(36).slice(2, 8)
@@ -202,7 +206,7 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB
     sandboxOverrides: {
       name: `bench-${adapter.name}-${task.id}-${uniq}`.replace(/[^a-zA-Z0-9_.-]/g, '_').slice(0, 60),
       environment: 'universal',
-      backend: { type: harness as never, model: { provider: 'openai', model: cell.model, baseUrl: routerBaseUrl } },
+      backend: { type: harness as never, model: { provider: profileProvider, model: cell.model, baseUrl: routerBaseUrl } },
     },
   }
   const deliverable: Deliverable<string> = {

package/src/swe-arena/arms.ts

@@ -525,7 +525,7 @@ export async function runSoloArm(spec: SoloArmSpec, ctx: ArmRunContext): Promise
   const turn = await collectAgentTurn(
     streamAgentTurn(
       { kind: 'executor', factory, profile: spec.profile },
-      prompt,
+      { prompt },
       { timeoutMs, ...(ctx.signal ? { signal: ctx.signal } : {}) },
     ),
   )

package/dist/router-turn-C2wMiDoo.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"router-turn-C2wMiDoo.js","names":[],"sources":["../src/router-turn.ts"],"sourcesContent":["import {\n  type AgentProfile,\n  agentProfileSchema,\n  type ReasoningEffort,\n} from '@tangle-network/agent-interface'\nimport {\n  collectAgentTurn,\n  createExecutor,\n  streamAgentTurn,\n  type CollectedAgentTurn,\n  type ToolSpec,\n} from '@tangle-network/agent-runtime/kernel'\n\n/** Bench-local target shorthand; Runtime still executes only the exact profile below. */\nexport interface BenchRouterTarget {\n  routerBaseUrl: string\n  routerKey: string\n  profile: AgentProfile\n}\n\nexport interface BenchRouterTurnConfig extends BenchRouterTarget {\n  tools?: ReadonlyArray<ToolSpec>\n  timeoutMs?: number\n  signal?: AbortSignal\n}\n\nexport interface BenchProfileSettings {\n  systemPrompt?: string\n  temperature?: number\n  maxTokens?: number\n  retry?: {\n    maxAttempts?: number\n    initialBackoffMs?: number\n    maxBackoffMs?: number\n    jitter?: number\n    retryStatuses?: ReadonlyArray<number>\n    requestTimeoutMs?: number\n  }\n  maxTurns?: number\n  seed?: number\n  reasoningEffort?: ReasoningEffort\n  extraBody?: Readonly<Record<string, unknown>>\n  toolChoice?: 'auto' | 'required' | 'none'\n}\n\n/** Author an exact direct-Router profile for a benchmark. This is profile construction only;\n * execution still accepts no model or generation fields outside the returned AgentProfile. */\nexport function benchRouterProfile(\n  name: string,\n  model: string,\n  settings: BenchProfileSettings = {},\n): AgentProfile {\n  return withBenchProfile(\n    {\n      name,\n      harness: 'cli-base',\n      model: { provider: 'tangle-router', default: model },\n    },\n    settings,\n  )\n}\n\n/** Derive another exact profile while preserving all untouched canonical axes. */\nexport function withBenchProfile(\n  base: AgentProfile,\n  settings: BenchProfileSettings & { name?: string },\n): AgentProfile {\n  const metadata = {\n    ...(base.model?.metadata ?? {}),\n    ...(settings.temperature !== undefined ? { temperature: settings.temperature } : {}),\n    ...(settings.maxTokens !== undefined ? { maxTokens: settings.maxTokens } : {}),\n    ...(settings.retry !== undefined ? { retry: settings.retry } : {}),\n    ...(settings.maxTurns !== undefined ? { maxTurns: settings.maxTurns } : {}),\n    ...(settings.seed !== undefined ? { seed: settings.seed } : {}),\n    ...(settings.extraBody !== undefined ? { extraBody: settings.extraBody } : {}),\n    ...(settings.toolChoice !== undefined ? { toolChoice: settings.toolChoice } : {}),\n  }\n  return agentProfileSchema.parse({\n    ...base,\n    ...(settings.name ? { name: settings.name } : {}),\n    model: {\n      ...base.model,\n      ...(settings.reasoningEffort !== undefined\n        ? { reasoningEffort: settings.reasoningEffort }\n        : {}),\n      ...(Object.keys(metadata).length > 0 ? { metadata } : {}),\n    },\n    ...(settings.systemPrompt !== undefined\n      ? { prompt: { ...base.prompt, systemPrompt: settings.systemPrompt } }\n      : {}),\n  })\n}\n\nexport function benchProfileModel(profile: AgentProfile): string {\n  const model = profile.model?.default\n  if (typeof model !== 'string' || model.length === 0 || model === 'runtime-selected') {\n    throw new Error('benchmark AgentProfile.model.default must be concrete')\n  }\n  return model\n}\n\n/**\n * The benchmark-side entry to Runtime's canonical one-turn path.\n * It is only an ergonomic composition: Runtime still parses the exact profile,\n * materializes the executor, records identity/usage/result events, and refuses\n * profile axes the direct Router backend cannot carry.\n */\nexport async function runBenchRouterTurn(\n  config: BenchRouterTurnConfig,\n  input: string | { readonly messages: ReadonlyArray<Readonly<Record<string, unknown>>> },\n): Promise<CollectedAgentTurn> {\n  if (!config.profile.model?.default) {\n    throw new Error('runBenchRouterTurn: profile.model.default is required')\n  }\n  const factory = createExecutor({\n    backend: 'router',\n    routerBaseUrl: config.routerBaseUrl,\n    routerKey: config.routerKey,\n    ...(config.tools ? { tools: config.tools } : {}),\n  })\n  const turn = await collectAgentTurn(\n    streamAgentTurn(\n      { kind: 'executor', factory, profile: config.profile },\n      input,\n      {\n        ...(config.timeoutMs === undefined ? {} : { timeoutMs: config.timeoutMs }),\n        ...(config.signal ? { signal: config.signal } : {}),\n      },\n    ),\n  )\n  if (turn.status !== 'completed') {\n    throw new Error(turn.error?.message ?? `Router turn ended with status ${turn.status}`)\n  }\n  return turn\n}\n"],"mappings":";;;;;;;;;AA2GA,eAAsB,mBACpB,QACA,OAC6B;CAC7B,IAAI,CAAC,OAAO,QAAQ,OAAO,SACzB,MAAM,IAAI,MAAM,uDAAuD;CAQzE,MAAM,OAAO,MAAM,iBACjB,gBACE;EAAE,MAAM;EAAY,SARR,eAAe;GAC7B,SAAS;GACT,eAAe,OAAO;GACtB,WAAW,OAAO;GAClB,GAAI,OAAO,QAAQ,EAAE,OAAO,OAAO,MAAM,IAAI,CAAC;EAChD,CAG8B;EAAG,SAAS,OAAO;CAAQ,GACrD,OACA;EACE,GAAI,OAAO,cAAc,KAAA,IAAY,CAAC,IAAI,EAAE,WAAW,OAAO,UAAU;EACxE,GAAI,OAAO,SAAS,EAAE,QAAQ,OAAO,OAAO,IAAI,CAAC;CACnD,CACF,CACF;CACA,IAAI,KAAK,WAAW,aAClB,MAAM,IAAI,MAAM,KAAK,OAAO,WAAW,iCAAiC,KAAK,QAAQ;CAEvF,OAAO;AACT"}