@tangle-network/agent-bench 0.1.0

package/src/research-shot.ts

@@ -0,0 +1,134 @@
+/**
+ * One research rollout as a reusable primitive: 2-step RAG — (1) provider-pinned web
+ * search via the router's proven `/v1/search?provider=<id>` + `web_fetch` of the top-K
+ * result pages, (2) answer with that evidence via `routerChatWithUsage` (no tools on the
+ * answer call → `content` always present, so a search arm differs from the parametric
+ * control ONLY by the evidence). Pure router HTTP (bearer `TANGLE_API_KEY`).
+ *
+ * Shared by the off-sandbox RAG leaderboard (`research-gate.mts`) and the router-backed
+ * loop executor (`router-executor.ts`), so both score the identical retrieve→answer body
+ * — the only difference is who drives the rounds (a flat best-of-k pool vs the real
+ * `runLoop` kernel with analyst steering).
+ */
+import { routerChatWithUsage } from '@tangle-network/agent-runtime/loops'
+
+export interface ShotCfg {
+  model: string
+  /** search provider id: 'default'/'off'/'none' = no search (parametric control); else a router provider. */
+  search: string
+  maxResults: number
+  /** how many top search URLs to web_fetch full page content for (0 = snippets only). */
+  fetchTopK: number
+  temperature: number
+  routerBaseUrl: string
+  routerKey: string
+  timeoutMs: number
+}
+
+export interface Shot {
+  taskId: string
+  attempt: number
+  answer: string
+  ok: boolean
+  detail?: string
+  wallMs: number
+  /** count of search hits retrieved (0 ⇒ no search happened / it failed). */
+  searches: number
+}
+
+/** Fetch a URL's extracted page text via the router web_fetch MCP tool. Returns '' on any failure. */
+export async function fetchPage(url: string, cfg: ShotCfg): Promise<string> {
+  try {
+    const res = await fetch(`${cfg.routerBaseUrl}/search/mcp?provider=${encodeURIComponent(cfg.search)}`, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json', authorization: `Bearer ${cfg.routerKey}` },
+      body: JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'tools/call', params: { name: 'web_fetch', arguments: { url } } }),
+      ...(cfg.timeoutMs ? { signal: AbortSignal.timeout(Math.min(cfg.timeoutMs, 60_000)) } : {}),
+    })
+    if (!res.ok) return ''
+    const body = (await res.json()) as { result?: { content?: Array<{ text?: string }> } }
+    const text = body.result?.content?.[0]?.text ?? ''
+    // The tool returns a JSON string {url,title,content}; pull `content` if parseable, else the raw text.
+    try {
+      const parsed = JSON.parse(text) as { content?: string }
+      return (parsed.content ?? text).slice(0, 2500)
+    } catch {
+      return text.slice(0, 2500)
+    }
+  } catch {
+    return ''
+  }
+}
+
+/**
+ * One research rollout, 2-step RAG against `prompt` (the task question, possibly with a
+ * steer appended): (1) provider-pinned web search + web_fetch of the top-K pages, (2)
+ * answer with that evidence. The search query is the clean question (first non-empty
+ * line) — appended worker-contract / steer boilerplate pollutes the query and returns 0
+ * hits — while the ANSWER sees the full `prompt` so a steer round can act on it. The
+ * COMMIT prompt stops the model deferring ("may I search?"), which otherwise scores 0.
+ * Fault-isolated: a flaky call → a NO-ANSWER `Shot`, never a throw.
+ */
+export async function runResearchShot(prompt: string, taskId: string, attempt: number, cfg: ShotCfg): Promise<Shot> {
+  const startedAt = Date.now()
+  const useSearch = cfg.search !== 'default' && cfg.search !== 'off' && cfg.search !== 'none'
+  let searches = 0
+  try {
+    // 1) Provider-pinned web search (proven /v1/search). The control arm skips this.
+    let context = ''
+    if (useSearch) {
+      const query = (prompt.split('\n').find((l) => l.trim().length > 0) ?? prompt).slice(0, 300)
+      const sres = await fetch(`${cfg.routerBaseUrl}/search?provider=${encodeURIComponent(cfg.search)}`, {
+        method: 'POST',
+        headers: { 'content-type': 'application/json', authorization: `Bearer ${cfg.routerKey}` },
+        body: JSON.stringify({ query, count: cfg.maxResults }),
+        ...(cfg.timeoutMs ? { signal: AbortSignal.timeout(cfg.timeoutMs) } : {}),
+      })
+      if (!sres.ok) {
+        // Surface, never silently degrade to parametric — a failed search must be visible.
+        console.warn(`  [search FAIL ${taskId}#${attempt}] HTTP ${sres.status}: ${(await sres.text()).slice(0, 140)}`)
+      } else {
+        const sb = (await sres.json()) as { data?: Array<{ title?: string; url?: string; snippet?: string }> }
+        const hits = sb.data ?? []
+        searches = hits.length
+        // Fetch the full page text of the top-K results (snippets rarely carry exact figures).
+        const fetched = await Promise.all(hits.slice(0, cfg.fetchTopK).map((h) => (h.url ? fetchPage(h.url, cfg) : Promise.resolve(''))))
+        context = hits
+          .map((h, i) => `[${i + 1}] ${h.title ?? ''}\n${h.snippet ?? ''}\n${h.url ?? ''}${fetched[i] ? `\nPAGE CONTENT:\n${fetched[i]}` : ''}`)
+          .join('\n\n')
+      }
+    }
+
+    // 2) Answer — no tools (content always present), COMMIT (no deferral), via the shared router primitive.
+    const commit =
+      'You have no further tools and cannot ask questions or request more research. ' +
+      'Output a SINGLE, FINAL answer to the question, leading with the value in the exact units and precision requested ' +
+      '(e.g. "Answer: -47.9 billion USD"). ' +
+      (useSearch
+        ? 'Use the WEB SEARCH RESULTS below (snippets + fetched page content) as your primary evidence; cite the source. '
+        : 'Answer from your own knowledge. ') +
+      'If you are not fully certain, still COMMIT to your single best estimate — never refuse, defer, or reply with a question.'
+    const userContent = useSearch && context ? `${prompt}\n\n=== WEB SEARCH RESULTS (provider: ${cfg.search}) ===\n${context}` : prompt
+    const { content } = await routerChatWithUsage(
+      { routerBaseUrl: cfg.routerBaseUrl, routerKey: cfg.routerKey, model: cfg.model },
+      [
+        { role: 'system', content: commit },
+        { role: 'user', content: userContent },
+      ],
+      { temperature: cfg.temperature, ...(cfg.timeoutMs ? { signal: AbortSignal.timeout(cfg.timeoutMs) } : {}) },
+    )
+    const answer = content.trim()
+    const ok = answer.length > 0
+    return { taskId, attempt, answer, ok, searches, wallMs: Date.now() - startedAt, ...(ok ? {} : { detail: `empty answer (searches=${searches})` }) }
+  } catch (err) {
+    return {
+      taskId,
+      attempt,
+      answer: '',
+      ok: false,
+      searches,
+      wallMs: Date.now() - startedAt,
+      detail: `rollout error: ${(err instanceof Error ? err.message : String(err)).slice(0, 200)}`,
+    }
+  }
+}

package/src/resolve-client.ts

@@ -0,0 +1,58 @@
+/**
+ * The ONE harness → worker-client mapping for the bench layer. Every bench entrypoint
+ * (rsi.ts, run.ts, …) derives the `SandboxClient` the kernel's `runLoop` drives from a single
+ * selector instead of hand-rolling the branch:
+ *
+ *   - `router`               → OFF-BOX: a router chat-completion as the leaf executor, presented
+ *                              as a SandboxClient (no sandbox dependency). For deployable-checker
+ *                              domains whose worker is a completion, or where box egress is blocked.
+ *   - `router` + searchProvider → OFF-BOX `router-tools`: the same off-box loop with a live
+ *                              `web_search` tool (the capability axis research benches need).
+ *   - anything else (`sandbox`/a BackendType) → IN-BOX: a real `Sandbox`. The in-box backend
+ *                              TYPE (opencode/codex/…) is set separately on the `AgentRunSpec`;
+ *                              this only decides off-box-vs-in-box transport for `runLoop`.
+ *
+ * Centralizing it here means a new entrypoint gets the full off-box/in-box matrix for free and
+ * the mapping can't drift between callers.
+ */
+import { createExecutor, inlineSandboxClient, type SandboxClient } from '@tangle-network/agent-runtime/loops'
+import { Sandbox } from '@tangle-network/sandbox'
+import { makeSearchExecutor, webSearchTool } from './search-tool'
+
+export interface ResolveBenchClientOptions {
+  /** The selector (the `BACKEND` env value): `router` = off-box; anything else = in-box Sandbox. */
+  backend: string
+  routerBaseUrl: string
+  routerKey: string
+  model: string
+  /** When set on the `router` backend, the off-box worker becomes a `router-tools` agentic loop
+   *  with a live `web_search` tool backed by this provider (`you`/`exa`/…). */
+  searchProvider?: string
+  sandboxBaseUrl?: string
+  /** In-box sandbox timeout (ms). */
+  timeoutMs?: number
+}
+
+export function resolveBenchClient(opts: ResolveBenchClientOptions): SandboxClient {
+  const { backend, routerBaseUrl, routerKey, model, searchProvider } = opts
+  if (backend === 'router') {
+    if (searchProvider) {
+      return inlineSandboxClient(
+        createExecutor({
+          backend: 'router-tools',
+          routerBaseUrl,
+          routerKey,
+          model,
+          tools: [webSearchTool],
+          executeToolCall: makeSearchExecutor({ routerBaseUrl, routerKey, provider: searchProvider }),
+        }),
+      )
+    }
+    return inlineSandboxClient(createExecutor({ backend: 'router', routerBaseUrl, routerKey, model }))
+  }
+  return new Sandbox({
+    baseUrl: opts.sandboxBaseUrl ?? 'https://sandbox.tangle.tools',
+    apiKey: routerKey,
+    timeoutMs: opts.timeoutMs ?? 1_200_000,
+  } as never)
+}

package/src/router-executor.ts

@@ -0,0 +1,51 @@
+/**
+ * Router-backed research executor — the "router" cost-dial backend (experiment.ts:
+ * "backend = the injected SandboxClient (router / bridge / sandbox)"). Each run is
+ * ONE research shot (router web-search + answer, off-sandbox); its answer is the
+ * terminal `finalText` the kernel's `answerOutput` parses.
+ *
+ * Why off-sandbox: research is retrieval, not in-box code execution — it never
+ * needed a box, and a real sandbox box reaches only the router (egress allowlist,
+ * ops-board #976), so it cannot web-search natively anyway. Driving the loop with
+ * this executor instead of a box gives the REAL `runLoop` kernel full `rounds` +
+ * analyst steering (the depth regime), search working, no sandbox dependency.
+ *
+ * This is a BYO `Executor` over `runResearchShot`; `inlineSandboxClient` supplies
+ * the one shared box-shell so this file owns only the research-shot specifics
+ * (no hand-rolled create/streamPrompt/delete). A fresh executor per round ⇒ no
+ * fork/live session; statefulness is the across-round steer, not a box.
+ */
+import {
+  type ExecutorFactory,
+  type ExecutorResult,
+  inlineSandboxClient,
+  type SandboxClient,
+} from '@tangle-network/agent-runtime/loops'
+import { runResearchShot, type ShotCfg } from './research-shot'
+
+export function routerSandboxClient(cfg: ShotCfg): SandboxClient {
+  let seq = 0
+  const factory: ExecutorFactory<unknown> = () => {
+    const id = `router-research-${seq++}`
+    let artifact: ExecutorResult<unknown> | undefined
+    return {
+      runtime: 'router',
+      async execute(task): Promise<ExecutorResult<unknown>> {
+        const started = Date.now()
+        const shot = await runResearchShot(String(task), id, 0, cfg)
+        artifact = {
+          outRef: `router-research:${id}`,
+          out: { content: shot.answer },
+          spent: { iterations: 1, tokens: { input: 0, output: 0 }, usd: 0, ms: Date.now() - started },
+        }
+        return artifact
+      },
+      teardown: () => Promise.resolve({ destroyed: true }),
+      resultArtifact() {
+        if (!artifact) throw new Error('routerSandboxClient: resultArtifact() read before execute()')
+        return artifact
+      },
+    }
+  }
+  return inlineSandboxClient(factory)
+}

package/src/run-pool.ts

@@ -0,0 +1,48 @@
+/**
+ * Bounded-concurrency pool — the ONE pool every batch runner uses.
+ *
+ * Every batch-* command, the finsearch loop, and terminal-compare hand-rolled the
+ * same shape: a shared `next` cursor, `Math.min(concurrency, items.length)` drain
+ * workers, and `Promise.all` over them. That boilerplate lives here once.
+ * Aggregation and console output stay at each call site — the pool owns scheduling,
+ * nothing else. Item-agnostic on purpose: a pool shouldn't know about BenchTask
+ * (terminal-compare pools over task-id strings), so it's generic in the item type.
+ *
+ * A worker that throws yields `{ ok:false, error }` for THAT item; the batch never
+ * aborts because one item failed. Results are returned in item order (indexed by
+ * the item's position), not completion order.
+ */
+
+export interface PoolOutcome<T, R> {
+  index: number
+  item: T
+  ok: boolean
+  value?: R
+  error?: string
+}
+
+export async function runPool<T, R>(
+  items: T[],
+  concurrency: number,
+  worker: (item: T, index: number) => Promise<R>,
+  opts?: { onResult?: (o: PoolOutcome<T, R>) => void },
+): Promise<PoolOutcome<T, R>[]> {
+  const results: PoolOutcome<T, R>[] = new Array(items.length)
+  let next = 0
+  const drain = async (): Promise<void> => {
+    while (next < items.length) {
+      const index = next++
+      const item = items[index] as T
+      let outcome: PoolOutcome<T, R>
+      try {
+        outcome = { index, item, ok: true, value: await worker(item, index) }
+      } catch (err) {
+        outcome = { index, item, ok: false, error: err instanceof Error ? err.message : String(err) }
+      }
+      results[index] = outcome
+      opts?.onResult?.(outcome)
+    }
+  }
+  await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, drain))
+  return results
+}

package/src/runtime-hook-recorder.ts

@@ -0,0 +1,137 @@
+export type BenchRuntimeHookPhase = 'before' | 'after' | 'error' | 'event'
+
+export interface BenchRuntimeHookEvent<Payload = unknown> {
+  id: string
+  runId: string
+  scenarioId?: string
+  target: string
+  phase: BenchRuntimeHookPhase
+  timestamp: number
+  stepIndex?: number
+  parentId?: string
+  payload?: Payload
+  metadata?: Record<string, unknown>
+}
+
+export interface BenchRuntimeDecisionEvidenceRef {
+  source: string
+  id: string
+  detail?: string
+  metadata?: Record<string, unknown>
+}
+
+export interface BenchRuntimeDecisionPoint {
+  id: string
+  runId: string
+  scenarioId?: string
+  stepIndex: number
+  kind: string
+  candidateActions: string[]
+  context?: string
+  evidence: BenchRuntimeDecisionEvidenceRef[]
+  metadata?: Record<string, unknown>
+}
+
+export interface BenchRuntimeHooks {
+  onEvent?: (
+    event: BenchRuntimeHookEvent,
+    context: { signal?: AbortSignal },
+  ) => void | Promise<void>
+  onDecisionPoint?: (
+    point: BenchRuntimeDecisionPoint,
+    context: { signal?: AbortSignal },
+  ) => void | Promise<void>
+}
+
+export interface RuntimeHookRecorder {
+  readonly events: BenchRuntimeHookEvent[]
+  readonly decisionPoints: BenchRuntimeDecisionPoint[]
+  readonly hooks: BenchRuntimeHooks
+}
+
+const MAX_STRING_LENGTH = 12_000
+const MAX_CONTEXT_LENGTH = 20_000
+const MAX_EVIDENCE_DETAIL_LENGTH = 2_000
+const MAX_CANDIDATE_ACTIONS = 50
+const MAX_EVIDENCE_REFS = 50
+const MAX_METADATA_DEPTH = 4
+const MAX_METADATA_KEYS = 100
+const SENSITIVE_KEY_RE = /(?:authorization|api[_-]?key|token|secret|password|cookie|credential|bearer)/i
+const SENSITIVE_VALUE_RES = [
+  /\bBearer\s+[A-Za-z0-9._~+/=-]+/gi,
+  /\b(?:sk|gh[pousr])_[A-Za-z0-9_]{20,}\b/g,
+  /\b(?:sk|ghp|gho|ghu|ghs|ghr)-[A-Za-z0-9_-]{20,}\b/g,
+]
+const SENSITIVE_ASSIGNMENT_RE =
+  /\b(api[_-]?key|token|secret|password|cookie)\s*[:=]\s*["']?[^"'\s,;}]+/gi
+
+function sanitizeString(value: string, maxLength: number): string {
+  let sanitized = value
+  for (const pattern of SENSITIVE_VALUE_RES) {
+    sanitized = sanitized.replace(pattern, '[REDACTED]')
+  }
+  sanitized = sanitized.replace(SENSITIVE_ASSIGNMENT_RE, (_match, key: string) => `${key}=[REDACTED]`)
+  if (sanitized.length <= maxLength) return sanitized
+  return sanitized.slice(0, maxLength)
+}
+
+function sanitizeMetadata(value: unknown, depth = 0): unknown {
+  if (value == null) return value
+  if (typeof value === 'string') return sanitizeString(value, MAX_STRING_LENGTH)
+  if (typeof value === 'number' || typeof value === 'boolean') return value
+  if (Array.isArray(value)) {
+    if (depth >= MAX_METADATA_DEPTH) return '[MaxDepth]'
+    return value.slice(0, MAX_METADATA_KEYS).map((item) => sanitizeMetadata(item, depth + 1))
+  }
+  if (typeof value !== 'object') return undefined
+  if (depth >= MAX_METADATA_DEPTH) return '[MaxDepth]'
+
+  const sanitized: Record<string, unknown> = {}
+  for (const [key, nested] of Object.entries(value).slice(0, MAX_METADATA_KEYS)) {
+    sanitized[key] = SENSITIVE_KEY_RE.test(key) ? '[REDACTED]' : sanitizeMetadata(nested, depth + 1)
+  }
+  return sanitized
+}
+
+function sanitizeMetadataRecord(metadata: Record<string, unknown> | undefined): Record<string, unknown> | undefined {
+  if (!metadata) return undefined
+  const sanitized = sanitizeMetadata(metadata)
+  if (!sanitized || typeof sanitized !== 'object' || Array.isArray(sanitized)) return undefined
+  return sanitized as Record<string, unknown>
+}
+
+function snapshotDecisionPoint(point: BenchRuntimeDecisionPoint): BenchRuntimeDecisionPoint {
+  return {
+    id: point.id,
+    runId: point.runId,
+    scenarioId: point.scenarioId,
+    stepIndex: point.stepIndex,
+    kind: point.kind,
+    candidateActions: point.candidateActions.slice(0, MAX_CANDIDATE_ACTIONS).map((action) => sanitizeString(action, MAX_STRING_LENGTH)),
+    context: typeof point.context === 'string' ? sanitizeString(point.context, MAX_CONTEXT_LENGTH) : undefined,
+    evidence: point.evidence.slice(0, MAX_EVIDENCE_REFS).map((ref) => ({
+      source: sanitizeString(ref.source, MAX_STRING_LENGTH),
+      id: sanitizeString(ref.id, MAX_STRING_LENGTH),
+      detail: typeof ref.detail === 'string' ? sanitizeString(ref.detail, MAX_EVIDENCE_DETAIL_LENGTH) : undefined,
+      metadata: sanitizeMetadataRecord(ref.metadata),
+    })),
+    metadata: sanitizeMetadataRecord(point.metadata),
+  }
+}
+
+export function createRuntimeHookRecorder(): RuntimeHookRecorder {
+  const events: BenchRuntimeHookEvent[] = []
+  const decisionPoints: BenchRuntimeDecisionPoint[] = []
+  return {
+    events,
+    decisionPoints,
+    hooks: {
+      onEvent: (event) => {
+        events.push(event)
+      },
+      onDecisionPoint: (point) => {
+        decisionPoints.push(snapshotDecisionPoint(point))
+      },
+    },
+  }
+}

package/src/sandbox-run.ts

@@ -0,0 +1,125 @@
+/**
+ * Shared sandbox-rollout helpers for the bench harnesses.
+ *
+ * The worker plumbing every sandbox-backed bench needs, independent of how the
+ * loop is driven: build the standard `AgentRunSpec` (`sandboxAgentRun`), parse
+ * the agent's final answer from the event stream (`answerOutput`), name the
+ * cost-dial backend (`WorkerBackendType`), and run a single-model "review the
+ * prior attempt" analyst (`llmAnalyst`/`AnalystFn`). These are pure profile /
+ * backend / parsing plumbing — no experiment shell, no topology arms.
+ */
+
+import {
+  type AgentProfile,
+  type AgentRunSpec,
+  type OutputAdapter,
+  routerChatWithUsage,
+} from '@tangle-network/agent-runtime/loops'
+// `BackendType` is the sandbox SDK's harness union — its canonical home. agent-runtime consumes
+// it from there too; it is not re-exported from the loops barrel.
+import type { BackendType } from '@tangle-network/sandbox'
+
+/** Parse the agent's final answer from the event stream (harness-agnostic).
+ *  The default deliverable; a benchmark whose artifact is a file overrides via
+ *  its own `OutputAdapter` that reads from the run. */
+export const answerOutput: OutputAdapter<string> = {
+  parse(events) {
+    let answer = ''
+    for (const ev of events) {
+      const d = (ev as { data?: Record<string, unknown> })?.data
+      const t = d?.finalText ?? d?.text ?? d?.result
+      if (typeof t === 'string' && t.length > 0) answer = t
+    }
+    return answer
+  },
+}
+
+/** What an analyst sees of each prior attempt: its output, its verdict, and its
+ *  raw trace events. The events are the trace an analyst reads. */
+export type SteerHistory = ReadonlyArray<{
+  output?: string
+  verdict?: { valid?: boolean; score?: number; notes?: string }
+  events?: readonly unknown[]
+}>
+
+/**
+ * The investigation: read the prior attempt's trace, return targeted feedback for
+ * the next one. It observes BEHAVIOR (output, trace), never the judge's verdict —
+ * the selector != judge firewall.
+ */
+export type AnalystFn = (history: SteerHistory, task?: string) => Promise<string>
+
+/** Simple analyst: ONE model call reads the public task plus a bounded view of the
+ *  last attempt (its output + a tail of its trace events) and returns a concrete
+ *  correction. Selector != judge firewall: it NEVER reads the held-out judge's
+ *  verdict or failure detail — that would be a non-deployable oracle gradient
+ *  toward the reference answer. A deployable steerer must locate the fault from the
+ *  task and the agent's own behavior alone. */
+export const llmAnalyst = (cfg: { routerBaseUrl: string; routerKey: string; model: string }): AnalystFn =>
+  async (history, task) => {
+    const last = history.at(-1)
+    const traceTail = (last?.events ?? [])
+      .slice(-12)
+      .map((e) => (typeof e === 'string' ? e : JSON.stringify(e)))
+      .join('\n')
+      .slice(-2000)
+    const { content } = await routerChatWithUsage(cfg, [
+      {
+        role: 'system',
+        content:
+          "You review an AI agent's previous attempt at a task. From the task, the attempt's output, and its execution trace ALONE, judge whether it correctly and completely solved the task. If you find a specific fault — a wrong value, a guessed API signature, a missing step, a misread requirement — name it and give the concrete correction in 1-3 sentences. Reply exactly 'no change needed' if the attempt looks correct and complete.",
+      },
+      {
+        role: 'user',
+        content: `Task:\n${task ?? '(task unavailable)'}\n\nPrevious answer:\n${last?.output ?? '(none)'}\n\nTrace tail:\n${traceTail}`,
+      },
+    ])
+    return content
+  }
+
+/** Cost-dial backend = the SDK's canonical `BackendType` (single source of truth; no local
+ *  literal copy that drifts from the harness set). `hermes` = the inference-router agent (the
+ *  cheap "router llm-call" dial); the rest are agent CLIs. The ONLY knob that changes which
+ *  agent runs — no per-backend worker. */
+export type WorkerBackendType = BackendType
+
+/** Build the standard sandbox `AgentRunSpec` for a benchmark — the worker the
+ *  kernel injects. `backendType` is the cost dial. Model auth is the BOX'S OWN
+ *  provisioned credential: `backend.model` pins provider/model/baseUrl only, and
+ *  the platform generates the in-box provider config keyed to
+ *  `{env:OPENCODE_MODEL_API_KEY}`. Never pass an external router key into the
+ *  box — the egress proxy rejects foreign credentials (403, empty output). */
+export function sandboxAgentRun(opts: {
+  model: string
+  routerBaseUrl: string
+  backendType?: WorkerBackendType
+  /** In-box model provider. Default `openai` (registered models like gpt-4.1).
+   *  Cheap router models (deepseek/kimi/glm) are not in opencode's `openai`
+   *  registry and 404 in-box — pass `openai-compat` (generic passthrough). */
+  provider?: string
+  name?: string
+  taskToPrompt?: (task: string) => string
+  /** Extra box-level env (e.g. `TANGLE_SEARCH_DEFAULT_PROVIDER` to pin the in-box
+   *  agent's web-search provider, provider keys like EXA_API_KEY). Allowlisted
+   *  keys only reach the spawned CLI. Must NOT carry router/model credentials. */
+  env?: Record<string, string>
+  /** The developer's AgentProfile — the one knob for "which agent" (prompt / model /
+   *  tools / mcp). Spread through verbatim; the backend cost-dial is tagged into
+   *  metadata. Omitted ⇒ a minimal worker profile. */
+  profile?: AgentProfile
+}): AgentRunSpec<string> {
+  const backendType = opts.backendType ?? 'opencode'
+  const name = opts.profile?.name ?? opts.name ?? `${backendType}-worker`
+  return {
+    profile: { ...opts.profile, name, metadata: { ...opts.profile?.metadata, backendType } },
+    name,
+    taskToPrompt: opts.taskToPrompt ?? ((t) => t),
+    sandboxOverrides: {
+      ...(opts.env ? { env: opts.env } : {}),
+      backend: {
+        type: backendType,
+        model: { provider: opts.provider ?? 'openai', model: opts.model, baseUrl: opts.routerBaseUrl },
+      },
+    },
+  }
+}

package/src/search-bench/bridge.ts

@@ -0,0 +1,124 @@
+/**
+ * cli-bridge cell executor — the unblocked path to the head-to-head numbers
+ * while the sandbox sidecar image is pending #1810.
+ *
+ * Same arms, same deterministic oracle, same export. The only difference from
+ * the sandbox path is HOW the harness runs: a single OpenAI-compatible chat call
+ * to the local cli-bridge (`/v1/chat/completions`) with an `agent_profile` that
+ * (a) disables native web tools via `metadata.disallowedTools` and (b) adds the
+ * provider search MCP via `mcp` — both PROVEN to work on the bridge. Native arm
+ * leaves the harness untouched.
+ *
+ * The bridge model id IS the harness selector (e.g. `claude-code/sonnet`,
+ * `opencode/zai-coding-plan/glm-5.1`), so `harness` here is just the label.
+ */
+import { createExecutor } from '@tangle-network/agent-runtime/loops'
+import type { SearchArm } from './profiles'
+import { armLabel } from './profiles'
+import type { SearchCellResult } from './run.mts'
+import { type SearchTask, scoreTask, taskToPrompt } from './tasks'
+
+const nativeWebDisallowed = ['WebSearch', 'WebFetch', 'web_search', 'web_fetch', 'websearch', 'webfetch', 'fetch']
+
+/** Build the cli-bridge `agent_profile` for one arm (bridge dialect: disable via
+ *  `metadata.disallowedTools`, search MCP via `mcp.<name>.transport:'http'`). */
+function bridgeProfile(arm: SearchArm, routerSearchMcp: string, tangleApiKey: string, label: string): Record<string, unknown> {
+  if (arm === 'native') return { name: `search-bench-${label}` }
+  const base = { name: `search-bench-${label}`, metadata: { disallowedTools: nativeWebDisallowed } }
+  if (arm === 'off') return base
+  return {
+    ...base,
+    mcp: {
+      tangle_search: {
+        transport: 'http',
+        url: `${routerSearchMcp}?provider=${encodeURIComponent(arm.provider)}`,
+        headers: { Authorization: `Bearer ${tangleApiKey}` },
+        enabled: true,
+      },
+    },
+  }
+}
+
+const urlRe = /https?:\/\/[^\s)\]}"'<>]+/gi
+function citationsOf(answer: string): string[] {
+  return [...new Set((answer.match(urlRe) ?? []).map((u) => u.replace(/[.,;]+$/, '')))]
+}
+
+export interface BridgeCfg {
+  bridgeUrl: string
+  bridgeBearer: string
+  tangleApiKey: string
+  /** Router search-MCP endpoint, e.g. https://router.tangle.tools/v1/search/mcp */
+  routerSearchMcp: string
+  /** harness label → cli-bridge model id (the harness selector). */
+  bridgeModels: Record<string, string>
+  timeoutMs?: number
+}
+
+export async function runBridgeCell(
+  cfg: BridgeCfg,
+  task: SearchTask,
+  harness: string,
+  arm: SearchArm,
+): Promise<SearchCellResult> {
+  const startedAt = Date.now()
+  const armId = armLabel(arm)
+  const base = {
+    taskId: task.id,
+    domain: task.domain,
+    harness,
+    arm: armId,
+    model: cfg.bridgeModels[harness] ?? harness,
+    ts: new Date(startedAt).toISOString(),
+  }
+  const controller = new AbortController()
+  const timer = setTimeout(() => controller.abort(), cfg.timeoutMs ?? 300_000)
+  try {
+    // One harness turn through the unified bridge executor — same backend the
+    // loop path uses; this cell scorer just adds oracle scoring + citations.
+    const exec = createExecutor({
+      backend: 'bridge',
+      bridgeUrl: cfg.bridgeUrl,
+      bridgeBearer: cfg.bridgeBearer,
+      model: cfg.bridgeModels[harness] ?? harness,
+      agentProfile: bridgeProfile(arm, cfg.routerSearchMcp, cfg.tangleApiKey, `${harness}-${armId}`),
+      timeoutMs: cfg.timeoutMs ?? 300_000,
+    })({ profile: { name: `${harness}-${armId}` }, harness: null }, { signal: controller.signal, seams: {} })
+    // bridgeExecutor is one-shot (async execute resolves an ExecutorResult).
+    const artifact = (await exec.execute(taskToPrompt(task), controller.signal)) as {
+      out: unknown
+      spent: { tokens: { input: number; output: number }; usd: number }
+    }
+    const out = artifact.out as { content?: string; toolCalls?: string[] }
+    const answer = out.content ?? ''
+    const names = out.toolCalls ?? []
+    const { score, reasons } = scoreTask(task, answer)
+    return {
+      ...base,
+      score,
+      reasons,
+      ...(artifact.spent.usd ? { costUsd: artifact.spent.usd } : {}),
+      ...(artifact.spent.tokens.input ? { tokensIn: artifact.spent.tokens.input } : {}),
+      ...(artifact.spent.tokens.output ? { tokensOut: artifact.spent.tokens.output } : {}),
+      wallMs: Date.now() - startedAt,
+      toolCalls: names.length,
+      toolNames: [...new Set(names)],
+      citations: citationsOf(answer),
+      answer,
+    }
+  } catch (err) {
+    return {
+      ...base,
+      score: null,
+      reasons: [],
+      wallMs: Date.now() - startedAt,
+      toolCalls: 0,
+      toolNames: [],
+      citations: [],
+      answer: '',
+      infraError: err instanceof Error ? err.message : String(err),
+    }
+  } finally {
+    clearTimeout(timer)
+  }
+}