@tangle-network/agent-bench 0.1.0

package/src/search-bench/parametric-check.mts

@@ -0,0 +1,63 @@
+/**
+ * Discrimination check: how many fresh-docs tasks does a model get WRONG from
+ * memory (no tools, no search)? A high parametric fail-rate is the prerequisite
+ * for the whole comparison — if the model already knows the answer, no search
+ * backend can help. This is a pure router chat-completion per task, scored by
+ * the same deterministic oracle the sandbox run uses. No sandbox, no deploy dep.
+ *
+ * Run:
+ *   dotenvx run -f ~/company/devops/secrets/.env.keys -f ~/company/devops/secrets/agent-state.env -- \
+ *     env MODEL=gpt-4.1 CONCURRENCY=4 pnpm exec tsx src/search-bench/parametric-check.mts
+ */
+import { writeFileSync } from 'node:fs'
+import { runPool } from '../run-pool'
+import { routerChatWithUsage } from '@tangle-network/agent-runtime/loops'
+import { freshTasks } from './tasks-fresh'
+import { scoreTask, taskToPrompt } from './tasks'
+
+async function main(): Promise<void> {
+  const model = process.env.MODEL ?? 'deepseek-v4-flash'
+  const routerKey = process.env.TANGLE_API_KEY
+  if (!routerKey) throw new Error('missing TANGLE_API_KEY')
+  const cfg = {
+    routerBaseUrl: process.env.ROUTER_BASE_URL ?? 'https://router.tangle.tools/v1',
+    routerKey,
+    model,
+  }
+  const conc = Number(process.env.CONCURRENCY ?? 4)
+
+  const outcomes = await runPool(freshTasks, conc, async (task) => {
+    try {
+      const res = await routerChatWithUsage(cfg, [{ role: 'user', content: taskToPrompt(task) }])
+      const { score } = scoreTask(task, res.content)
+      return { id: task.id, score: score as 0 | 1 | null, cost: res.costUsd, err: undefined as string | undefined }
+    } catch (err) {
+      return { id: task.id, score: null as 0 | 1 | null, cost: undefined, err: err instanceof Error ? err.message : String(err) }
+    }
+  })
+  const rows = outcomes.map((o) => o.value!).filter(Boolean)
+
+  const scored = rows.filter((r) => r.score !== null)
+  const passed = scored.filter((r) => r.score === 1).length
+  const failed = scored.length - passed
+  console.log(`\n=== Parametric (no-search) discrimination · ${model} · n=${scored.length} ===`)
+  for (const r of rows.sort((a, b) => (a.score ?? -1) - (b.score ?? -1))) {
+    console.log(`  ${r.score === null ? 'ERR ' : r.score === 1 ? 'KNEW' : 'MISS'}  ${r.id}${r.err ? ` (${r.err})` : ''}`)
+  }
+  console.log(
+    `\nparametric FAIL (search-correctable headroom): ${failed}/${scored.length} = ${((failed / scored.length) * 100).toFixed(0)}%` +
+      ` · model already knew: ${passed}/${scored.length}`,
+  )
+  if (process.env.PARAM_OUT) {
+    writeFileSync(
+      process.env.PARAM_OUT,
+      rows.map((r) => JSON.stringify({ id: r.id, model, knewParametrically: r.score === 1, score: r.score })).join('\n') + '\n',
+    )
+    console.log(`wrote per-task baseline → ${process.env.PARAM_OUT}`)
+  }
+}
+
+main().catch((e) => {
+  console.error(e)
+  process.exit(1)
+})

package/src/search-bench/profiles.ts

@@ -0,0 +1,98 @@
+/**
+ * Per-arm AgentProfiles for the coding-harness web-search comparison.
+ *
+ * Three arms, one variable (the search backend the agent can reach):
+ *   - `native`   — the harness's built-in web search/fetch, untouched. No MCP.
+ *   - `provider` — native search DISABLED + a router-backed search MCP added
+ *     (provider-pinned: you / exa / …). The agent's only web access is the MCP.
+ *
+ * The disable directive is the harness-agnostic `tools` map landed in
+ * agent-dev-container#1810 (claude `--disallowed-tools`, codex
+ * `-c tools.web_search=false`, opencode `tools.{websearch,webfetch}`). We set
+ * both canonical (`web_search`/`web_fetch`) and opencode-native
+ * (`websearch`/`webfetch`) keys so a single profile is correct on every harness
+ * pre- and post-deploy, plus the opencode `permission.webfetch=deny` belt.
+ *
+ * The provider MCP mirrors the SDK's official `buildTangleRouterSearchProfile`
+ * shape — a `transport:'http'` server at the router's `/v1/search/mcp` endpoint,
+ * provider pinned via the `?provider=` query param.
+ */
+import type { AgentProfile } from '@tangle-network/sandbox'
+
+export type SearchArm = 'native' | 'off' | { provider: string }
+
+const routerSearchMcpUrl = (provider: string, routerBaseUrl: string): string => {
+  // routerBaseUrl is typically https://router.tangle.tools/v1 — the search MCP
+  // lives at /v1/search/mcp, so trim a trailing /v1 then re-append the path.
+  const root = routerBaseUrl.replace(/\/+$/, '').replace(/\/v1$/, '')
+  return `${root}/v1/search/mcp?provider=${encodeURIComponent(provider)}`
+}
+
+/** Tool keys that turn OFF a harness's native web search/fetch across harnesses. */
+const nativeWebToolsDisabled: Record<string, boolean> = {
+  web_search: false,
+  web_fetch: false,
+  websearch: false,
+  webfetch: false,
+}
+
+export interface BuildArmProfileArgs {
+  arm: SearchArm
+  /** Router base URL (…/v1). Used to derive the search-MCP endpoint. */
+  routerBaseUrl: string
+  /** Bearer for the router search MCP. Required for a provider arm. Valid for the
+   *  `bridge` (local) backend; in `sandbox` mode the box egress proxy rejects
+   *  foreign router credentials (403), so the provider arm needs the box-side
+   *  credential flow before sandbox-backed provider runs are trustworthy. */
+  tangleApiKey: string
+  /** Optional name/metadata to merge. */
+  name?: string
+  metadata?: Record<string, unknown>
+}
+
+/**
+ * Build the AgentProfile fragment (tools / permission / mcp) for one search arm.
+ * Returned as a partial profile to be spread into `sandboxAgentRun({ profile })`.
+ */
+export function buildArmProfile(args: BuildArmProfileArgs): AgentProfile {
+  const { arm, routerBaseUrl, tangleApiKey } = args
+  const base: AgentProfile = {
+    name: args.name ?? 'search-bench-worker',
+    ...(args.metadata ? { metadata: args.metadata } : {}),
+  } as AgentProfile
+
+  if (arm === 'native') {
+    // Native web tools stay on (harness default). No search MCP. For codex,
+    // whose web_search ships off, explicitly enable it so the native arm is real.
+    return { ...base, tools: { web_search: true } } as AgentProfile
+  }
+
+  if (arm === 'off') {
+    // No web access at all — the parametric floor (search contributes nothing).
+    return { ...base, tools: { ...nativeWebToolsDisabled }, permission: { webfetch: 'deny' } } as AgentProfile
+  }
+
+  if (!tangleApiKey) {
+    throw new Error(`buildArmProfile: provider arm "${arm.provider}" requires a tangleApiKey for the search MCP`)
+  }
+  return {
+    ...base,
+    tools: { ...nativeWebToolsDisabled },
+    permission: { webfetch: 'deny' },
+    mcp: {
+      tangle_search: {
+        transport: 'http',
+        url: routerSearchMcpUrl(arm.provider, routerBaseUrl),
+        headers: { Authorization: `Bearer ${tangleApiKey}` },
+        enabled: true,
+      },
+    },
+  } as AgentProfile
+}
+
+/** Stable condition label for the corpus: `<harness>:<arm>`. */
+export function armLabel(arm: SearchArm): string {
+  if (arm === 'native') return 'native'
+  if (arm === 'off') return 'off'
+  return arm.provider
+}

package/src/search-bench/run.mts

@@ -0,0 +1,287 @@
+/**
+ * Coding-harness × web-search-backend benchmark runner.
+ *
+ * For each (task × harness × arm) cell: spin up a live sandbox, run the coding
+ * task once through the harness with that arm's AgentProfile (native search /
+ * provider-MCP+native-disabled), score the answer with the task's deterministic
+ * oracle, and record score + cost + tokens + latency + tool-call count +
+ * citations. Cells are concurrency-bounded and fault-isolated; an infra-errored
+ * cell is recorded as such and excluded from rates (never a silent zero).
+ *
+ * Run (opencode, you.com vs native, seed tasks):
+ *   dotenvx run -f ~/company/devops/secrets/.env.keys -f ~/company/devops/secrets/agent-state.env -- \
+ *     env HARNESSES=opencode ARMS=native,you MODEL=opencode/deepseek/deepseek-v4-flash \
+ *     OUT=/tmp/search-bench.jsonl pnpm exec tsx src/search-bench/run.mts
+ */
+import { appendFileSync, mkdirSync, writeFileSync } from 'node:fs'
+import { dirname } from 'node:path'
+import { extractLlmCallEvent, openSandboxRun } from '@tangle-network/agent-runtime/loops'
+import { Sandbox, type SandboxEvent } from '@tangle-network/sandbox'
+import { answerOutput, sandboxAgentRun, type WorkerBackendType } from '../sandbox-run'
+import { type BridgeCfg, runBridgeCell } from './bridge'
+import { type SearchArm, armLabel, buildArmProfile } from './profiles'
+import { freshTasks } from './tasks-fresh'
+import { type SearchTask, scoreTask, seedTasks, taskToPrompt } from './tasks'
+
+export interface SearchCellResult {
+  taskId: string
+  domain: string
+  harness: string
+  /** Search arm: 'native' | 'off' | provider id (e.g. 'you', 'exa'). */
+  arm: string
+  model: string
+  /** Deterministic oracle outcome. null ⇒ infra error (excluded from rates). */
+  score: 0 | 1 | null
+  reasons: string[]
+  costUsd?: number
+  tokensIn?: number
+  tokensOut?: number
+  wallMs: number
+  /** Distinct tool calls observed (evidence the agent actually searched). */
+  toolCalls: number
+  /** Tool names used — distinguishes native (`webfetch`) from the provider MCP. */
+  toolNames: string[]
+  /** Distinct http(s) URLs cited in the answer. */
+  citations: string[]
+  /** Final answer text (truncated in the export; full in the raw JSONL). */
+  answer: string
+  infraError?: string
+  ts: string
+}
+
+const urlRe = /https?:\/\/[^\s)\]}"'<>]+/gi
+
+function extractCitations(answer: string): string[] {
+  const seen = new Set<string>()
+  for (const m of answer.match(urlRe) ?? []) seen.add(m.replace(/[.,;]+$/, ''))
+  return [...seen]
+}
+
+/** Sum token usage + cost across the run's llm_call events (the kernel's ledger). */
+function tally(events: SandboxEvent[]): { costUsd?: number; tokensIn?: number; tokensOut?: number } {
+  let costUsd = 0
+  let tokensIn = 0
+  let tokensOut = 0
+  let any = false
+  for (const ev of events) {
+    const call = extractLlmCallEvent(ev as never, 'search-bench')
+    if (!call) continue
+    any = true
+    costUsd += call.costUsd ?? 0
+    tokensIn += call.tokensIn ?? 0
+    tokensOut += call.tokensOut ?? 0
+  }
+  return any ? { costUsd, tokensIn, tokensOut } : {}
+}
+
+/**
+ * Distinct tool calls + the tool names used. opencode emits a tool call as
+ * repeated `message.part.updated` events sharing a `data.part.callID` with
+ * `data.part.type==='tool'` and `data.part.tool` the name (e.g. `webfetch`,
+ * or an MCP tool like `tangle_search_web_search`). We dedupe by callID so a
+ * single call counts once, and surface the names so the export can show which
+ * search backend the agent actually used (native vs the provider MCP).
+ */
+function extractTools(events: SandboxEvent[]): { count: number; names: string[] } {
+  const calls = new Map<string, string>()
+  for (const ev of events) {
+    const part = (ev as { data?: { part?: Record<string, unknown> } }).data?.part
+    if (!part || typeof part !== 'object' || part.type !== 'tool') continue
+    const callId = typeof part.callID === 'string' ? part.callID : `${calls.size}`
+    const name = typeof part.tool === 'string' ? part.tool : 'unknown'
+    calls.set(callId, name)
+  }
+  return { count: calls.size, names: [...new Set(calls.values())] }
+}
+
+export interface RunCfg {
+  tasks: SearchTask[]
+  harnesses: WorkerBackendType[]
+  arms: SearchArm[]
+  model: string
+  routerBaseUrl: string
+  tangleApiKey: string
+  sandboxBaseUrl: string
+  sandboxKey: string
+  outPath: string
+  /** In-box model provider — `openai` (default) or `openai-compat` for cheap models. */
+  provider?: string
+  /** Execution surface: `sandbox` (live box) or `bridge` (local cli-bridge). */
+  backend?: 'sandbox' | 'bridge'
+  /** cli-bridge wiring, required when `backend === 'bridge'`. */
+  bridge?: BridgeCfg
+  concurrency?: number
+  timeoutMs?: number
+}
+
+async function runCell(
+  cfg: RunCfg,
+  client: Sandbox,
+  task: SearchTask,
+  harness: WorkerBackendType,
+  arm: SearchArm,
+): Promise<SearchCellResult> {
+  const startedAt = Date.now()
+  const armId = armLabel(arm)
+  const base = {
+    taskId: task.id,
+    domain: task.domain,
+    harness,
+    arm: armId,
+    model: cfg.model,
+    ts: new Date(startedAt).toISOString(),
+  }
+  const controller = new AbortController()
+  const timer = setTimeout(() => controller.abort(), cfg.timeoutMs ?? 300_000)
+  try {
+    const agentRun = sandboxAgentRun({
+      model: cfg.model,
+      routerBaseUrl: cfg.routerBaseUrl,
+      backendType: harness,
+      ...(cfg.provider ? { provider: cfg.provider } : {}),
+      profile: buildArmProfile({
+        arm,
+        routerBaseUrl: cfg.routerBaseUrl,
+        tangleApiKey: cfg.tangleApiKey,
+        name: `search-bench-${harness}-${armId}`,
+        metadata: { harness, arm: armId, taskId: task.id },
+      }),
+    })
+    const run = await openSandboxRun<string>(
+      client,
+      { agentRun, signal: controller.signal },
+      { kind: 'events', fromEvents: (events) => answerOutput.parse(events as never) },
+    )
+    let turn: Awaited<ReturnType<typeof run.start>>
+    try {
+      turn = await run.start(taskToPrompt(task))
+    } finally {
+      await run.close().catch(() => {})
+    }
+    if (process.env.DUMP_EVENTS) {
+      writeFileSync(process.env.DUMP_EVENTS, JSON.stringify(turn.events, null, 2))
+    }
+    const answer = turn.out ?? ''
+    const { score, reasons } = scoreTask(task, answer)
+    const tools = extractTools(turn.events)
+    return {
+      ...base,
+      score,
+      reasons,
+      ...tally(turn.events),
+      wallMs: Date.now() - startedAt,
+      toolCalls: tools.count,
+      toolNames: tools.names,
+      citations: extractCitations(answer),
+      answer,
+    }
+  } catch (err) {
+    return {
+      ...base,
+      score: null,
+      reasons: [],
+      wallMs: Date.now() - startedAt,
+      toolCalls: 0,
+      toolNames: [],
+      citations: [],
+      answer: '',
+      infraError: err instanceof Error ? err.message : String(err),
+    }
+  } finally {
+    clearTimeout(timer)
+  }
+}
+
+export async function runSearchBench(cfg: RunCfg): Promise<SearchCellResult[]> {
+  const useBridge = cfg.backend === 'bridge'
+  if (useBridge && !cfg.bridge) throw new Error('backend=bridge requires cfg.bridge')
+  const client = useBridge ? null : new Sandbox({ baseUrl: cfg.sandboxBaseUrl, apiKey: cfg.sandboxKey })
+  const runOne = (task: SearchTask, harness: WorkerBackendType, arm: SearchArm): Promise<SearchCellResult> =>
+    useBridge ? runBridgeCell(cfg.bridge!, task, harness, arm) : runCell(cfg, client!, task, harness, arm)
+  const cells: Array<{ task: SearchTask; harness: WorkerBackendType; arm: SearchArm }> = []
+  for (const task of cfg.tasks)
+    for (const harness of cfg.harnesses) for (const arm of cfg.arms) cells.push({ task, harness, arm })
+
+  mkdirSync(dirname(cfg.outPath), { recursive: true })
+  writeFileSync(cfg.outPath, '')
+  const results: SearchCellResult[] = []
+  const conc = cfg.concurrency ?? 3
+  let next = 0
+  const worker = async (): Promise<void> => {
+    while (next < cells.length) {
+      const i = next++
+      const { task, harness, arm } = cells[i]
+      const r = await runOne(task, harness, arm)
+      results.push(r)
+      appendFileSync(cfg.outPath, `${JSON.stringify(r)}\n`)
+      const mark = r.score === null ? 'ERR' : r.score === 1 ? 'PASS' : 'fail'
+      console.error(
+        `[${i + 1}/${cells.length}] ${harness}:${armLabel(arm)} ${task.id} → ${mark}` +
+          (r.infraError ? ` (${r.infraError})` : ` tools=${r.toolCalls}[${r.toolNames.join(',')}] cites=${r.citations.length} ${Math.round(r.wallMs / 1000)}s`),
+      )
+    }
+  }
+  await Promise.all(Array.from({ length: Math.min(conc, cells.length) }, () => worker()))
+  return results
+}
+
+// ── CLI ──────────────────────────────────────────────────────────────────────
+function env(name: string, fallback?: string): string {
+  const v = process.env[name] ?? fallback
+  if (v === undefined) throw new Error(`missing env ${name}`)
+  return v
+}
+
+async function main(): Promise<void> {
+  const harnesses = env('HARNESSES', 'opencode').split(',').map((s) => s.trim()) as WorkerBackendType[]
+  const arms: SearchArm[] = env('ARMS', 'native,you')
+    .split(',')
+    .map((s) => s.trim())
+    .map((s) => (s === 'native' || s === 'off' ? s : { provider: s }))
+  const tangleApiKey = env('TANGLE_API_KEY')
+  const backend = env('BACKEND', 'sandbox') as 'sandbox' | 'bridge'
+  let bridge: BridgeCfg | undefined
+  if (backend === 'bridge') {
+    bridge = {
+      bridgeUrl: env('BRIDGE_URL', 'http://127.0.0.1:3355'),
+      bridgeBearer: env('BRIDGE_BEARER'),
+      tangleApiKey,
+      routerSearchMcp: env('ROUTER_SEARCH_MCP', 'https://router.tangle.tools/v1/search/mcp'),
+      bridgeModels: JSON.parse(
+        env('BRIDGE_MODELS', '{"claude-code":"claude-code/sonnet","opencode":"opencode/zai-coding-plan/glm-5.1"}'),
+      ) as Record<string, string>,
+      timeoutMs: Number(env('TIMEOUT_MS', '300000')),
+    }
+  }
+  const taskSet = (process.env.TASK_SET ?? 'fresh') === 'seed' ? seedTasks : freshTasks
+  const onlyIds = (process.env.TASK_IDS ?? '').split(',').map((s) => s.trim()).filter(Boolean)
+  const tasks = onlyIds.length ? taskSet.filter((t) => onlyIds.includes(t.id)) : taskSet
+  if (tasks.length === 0) throw new Error(`no tasks matched TASK_IDS=${process.env.TASK_IDS} in TASK_SET=${process.env.TASK_SET ?? 'fresh'}`)
+  const results = await runSearchBench({
+    tasks,
+    harnesses,
+    arms,
+    backend,
+    ...(bridge ? { bridge } : {}),
+    model: env('MODEL', 'opencode/deepseek/deepseek-v4-flash'),
+    routerBaseUrl: env('ROUTER_BASE_URL', 'https://router.tangle.tools/v1'),
+    tangleApiKey,
+    sandboxBaseUrl: env('SANDBOX_BASE_URL', 'https://sandbox.tangle.tools'),
+    sandboxKey: env('SANDBOX_KEY', tangleApiKey),
+    provider: env('PROVIDER', 'openai'),
+    outPath: env('OUT', '/tmp/search-bench.jsonl'),
+    concurrency: Number(env('CONCURRENCY', '3')),
+    timeoutMs: Number(env('TIMEOUT_MS', '300000')),
+  })
+  const scored = results.filter((r) => r.score !== null)
+  console.error(
+    `\nDone: ${results.length} cells, ${scored.length} scored, ${results.length - scored.length} infra-errored → ${env('OUT', '/tmp/search-bench.jsonl')}`,
+  )
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  main().catch((e) => {
+    console.error(e)
+    process.exit(1)
+  })
+}