@tangle-network/agent-bench 0.1.0

package/src/profile-coordinates.ts

@@ -0,0 +1,134 @@
+/**
+ * The AgentProfile genome as independently-optimizable coordinates. Every field the harness lets
+ * you define — prompt, skills, subagents, hooks, tools, mcp — is a coordinate: a way to inject a
+ * set of named candidates into a profile while holding every OTHER field fixed. This is the one
+ * abstraction behind "improve any part of the agent, freeze the rest, combine freely":
+ *
+ *   compose(baseProfile, selected) = { ...baseProfile, <thisField>: inject(selected) }
+ *
+ * Freeze a coordinate ⇒ never select it (it stays as-is in the base profile). Optimize one ⇒
+ * vary its selection. Combine ⇒ compose several coordinates' composers in sequence. Because a
+ * subagent is ITSELF an AgentProfile, the same coordinates apply recursively to any node in the
+ * supervisor flow (root driver, worker, sub-worker) — you point a coordinate at that node's
+ * base profile.
+ */
+import { readFileSync, readdirSync } from 'node:fs'
+import { join } from 'node:path'
+import type { AgentProfile } from '@tangle-network/sandbox'
+import { defineInlineResource } from '@tangle-network/sandbox'
+
+export interface ProfileCoordinate {
+  /** Coordinate id (the COORDINATE= knob value). */
+  readonly name: string
+  /** Candidate names the optimizer screens (one per independently-testable unit). */
+  candidates(): readonly string[]
+  /** Inject the selected candidates into THIS field of the profile, holding all others fixed.
+   *  Empty selection ⇒ the base profile unchanged (the frozen/baseline arm). */
+  compose(base: AgentProfile, selected: readonly string[]): AgentProfile
+}
+
+const here = (p: string) => join(import.meta.dirname, p)
+
+// ── skills: SKILL.md packages materialized to disk (resources.skills) ───────────────
+function skillsCoordinate(dir = here(process.env.SKILLS_DIR ?? 'coding-skills')): ProfileCoordinate {
+  const files = () => readdirSync(dir).filter((f) => f.endsWith('.md')).sort()
+  return {
+    name: 'skills',
+    candidates: () => files().map((f) => f.replace(/\.md$/, '')),
+    compose: (base, selected) => {
+      if (!selected.length) return base
+      const refs = selected.map((n) => defineInlineResource(n, readFileSync(join(dir, `${n}.md`), 'utf8')))
+      return { ...base, resources: { ...base.resources, skills: [...(base.resources?.skills ?? []), ...refs] } }
+    },
+  }
+}
+
+// ── hooks: shell commands the harness fires on lifecycle events (enforced, not advisory) ──
+const hookDefs: Record<string, { event: string; command: string; matcher?: string }> = {
+  'lint-before-edit': { event: 'PreToolUse', matcher: 'Edit|Write', command: 'ruff check . 2>/dev/null || true' },
+  'tests-after-edit': { event: 'PostToolUse', matcher: 'Edit|Write', command: 'python -m pytest -q 2>/dev/null | tail -5 || true' },
+  'no-print-debugging': { event: 'PreToolUse', matcher: 'Edit|Write', command: 'true' },
+}
+function hooksCoordinate(): ProfileCoordinate {
+  return {
+    name: 'hooks',
+    candidates: () => Object.keys(hookDefs),
+    compose: (base, selected) => {
+      if (!selected.length) return base
+      const hooks: Record<string, { command: string; matcher?: string }[]> = { ...(base.hooks ?? {}) }
+      for (const n of selected) {
+        const d = hookDefs[n]
+        if (!d) throw new Error(`unknown hook ${n}`)
+        hooks[d.event] = [...(hooks[d.event] ?? []), { command: d.command, ...(d.matcher ? { matcher: d.matcher } : {}) }]
+      }
+      return { ...base, hooks: hooks as AgentProfile['hooks'] }
+    },
+  }
+}
+
+// ── tools: enable/disable named harness tools ───────────────────────────────────────
+const toolCandidates = ['webfetch', 'websearch', 'bash', 'edit', 'read', 'grep']
+function toolsCoordinate(): ProfileCoordinate {
+  return {
+    name: 'tools',
+    candidates: () => toolCandidates,
+    compose: (base, selected) => {
+      if (!selected.length) return base
+      const tools: Record<string, boolean> = { ...(base.tools ?? {}) }
+      for (const n of selected) tools[n] = true
+      return { ...base, tools }
+    },
+  }
+}
+
+// ── prompt: extra instruction lines appended to the active system prompt ─────────────
+const instructionDefs: Record<string, string> = {
+  'be-surgical': 'Make the smallest change that satisfies the task; do not touch unrelated code.',
+  'check-examples': 'Before finalizing, re-read the examples in the docstring and confirm your output matches them exactly.',
+  'edge-cases': 'Enumerate boundary inputs (empty, zero, negative, max) and make sure your solution handles each.',
+}
+function promptCoordinate(): ProfileCoordinate {
+  return {
+    name: 'prompt',
+    candidates: () => Object.keys(instructionDefs),
+    compose: (base, selected) => {
+      if (!selected.length) return base
+      const instructions = [...(base.prompt?.instructions ?? []), ...selected.map((n) => instructionDefs[n]!)]
+      return { ...base, prompt: { ...base.prompt, instructions } }
+    },
+  }
+}
+
+// ── subagents: helper agents the root can delegate to (each is itself a mini-profile) ──
+const subagentDefs: Record<string, { description: string; prompt: string; tools?: Record<string, boolean> }> = {
+  reviewer: { description: 'Reviews a proposed change for bugs before it is finalized.', prompt: 'You are a strict code reviewer. Find bugs, edge cases, and contract violations in the proposed change. Be concise.' },
+  tester: { description: 'Writes and runs a focused test for the change.', prompt: 'You write the minimal test that would catch a regression in this change, run it, and report pass/fail.' },
+}
+function subagentsCoordinate(): ProfileCoordinate {
+  return {
+    name: 'subagents',
+    candidates: () => Object.keys(subagentDefs),
+    compose: (base, selected) => {
+      if (!selected.length) return base
+      const subagents = { ...(base.subagents ?? {}) }
+      for (const n of selected) subagents[n] = subagentDefs[n]!
+      return { ...base, subagents: subagents as AgentProfile['subagents'] }
+    },
+  }
+}
+
+const REGISTRY: Record<string, () => ProfileCoordinate> = {
+  skills: () => skillsCoordinate(),
+  hooks: hooksCoordinate,
+  tools: toolsCoordinate,
+  prompt: promptCoordinate,
+  subagents: subagentsCoordinate,
+}
+
+export function getCoordinate(name: string): ProfileCoordinate {
+  const make = REGISTRY[name]
+  if (!make) throw new Error(`unknown coordinate ${name} (have: ${Object.keys(REGISTRY).join(', ')})`)
+  return make()
+}
+
+export const coordinateNames = (): string[] => Object.keys(REGISTRY)

package/src/profiles.ts

@@ -0,0 +1,128 @@
+/**
+ * The profile directory — roles are DATA, not code. One primitive (a profiled agent run as
+ * `Agent.act` in an executor, orchestrated by the Supervisor); worker / analyst / driver are the
+ * SAME agent, differing only by the profile below. This is the full unification: there is no
+ * "operator type" or "analyst type" — there are profiles + the operator toolbox they use to manage
+ * each other (in-process via the Scope, in a sandbox via the same verbs exposed as MCP tools).
+ *
+ * The directory is OPTIMIZED over time by the agent-eval RSI loop (`runImprovementLoop`): `gepaProposer`
+ * evolves the prompts, a `skillOptProposer` evolves `skills`, Pareto-tracked + holdout-gated.
+ * The trace-analyst's findings are the optimizer's input — the loop improves the profile that the
+ * findings say is weak.
+ */
+
+/** The operator toolbox — the verbs a DRIVER profile uses to lead the workers it drives. In-process
+ *  these are `Scope` methods; in a sandbox they are the SAME verbs exposed as MCP tools (Scope-as-MCP).
+ *  `analyze`/`define_analyst`/`run_analyst` make the trace-analyst a first-class, definable sub-agent —
+ *  the driver triggers analysts as a tool (or spawns one), never via raw bash. */
+export const OPERATOR_TOOLS = [
+  'list_analysts', // the trace analysts already available
+  'define_analyst', // author + register a NEW trace analyst (a profile) on the fly
+  'run_analyst', // run an analyst over a worker's trace → findings (selector≠judge: trace, not score)
+  'observe_worker', // a worker's in-flight trace, or its last finished episode/shot
+  'spawn_worker', // start a worker (or a sub-analyst) — drive many; parallelize when independent
+  'steer_worker', // send a live worker a message down: instruction, course-correction, or continuation (interrupt? for forceful)
+  'stop', // declare the task complete (verified) or abandon a line
+] as const
+
+export type OperatorTool = (typeof OPERATOR_TOOLS)[number]
+
+/** One role in the directory. The unified run-config: who the agent is (`systemPrompt`/`skills`), what
+ *  it can do (`tools` + `mcp`), and which lane it serves (`role`). The same shape the sandbox executor
+ *  runs and the optimizer mutates. */
+export interface RoleProfile {
+  readonly id: string
+  readonly role: 'worker' | 'analyst' | 'driver'
+  /** Default model snapshot; the optimizer may pin/sweep it. */
+  readonly model: string
+  readonly systemPrompt: string
+  /** Reusable capabilities the skillopt driver evolves; the driver may PROPOSE new ones from findings. */
+  readonly skills: string[]
+  /** Tool names available in-box. Workers get the artifact's tools; drivers also get the operator toolbox. */
+  readonly tools: string[]
+  /** MCP servers wired in-box (the operator toolbox is one, when the driver runs in a sandbox). */
+  readonly mcp?: string[]
+}
+
+/** WORKER: does the task over the artifact. Its tools are the artifact's (the surface supplies them);
+ *  its dynamic-workflow / fanout capability comes from the harness it runs (pi/codex/claude-code) +
+ *  its skills — which is exactly why those live on the profile. */
+export const workerProfile: RoleProfile = {
+  id: 'worker/default',
+  role: 'worker',
+  model: 'deepseek-v4-flash',
+  systemPrompt:
+    'You are a worker. Use the available tools to bring the artifact to the required final state. ' +
+    'Address every distinct change the request implies; after each tool result, check what remains ' +
+    'and continue; verify each value you set actually took. Use your harness’s parallelism / dynamic ' +
+    'workflow / sub-agents when the sub-tasks are genuinely independent. Reply DONE only once every ' +
+    'required change is made and verified.',
+  skills: [],
+  tools: ['<artifact tools>'],
+}
+
+/** ANALYST: reads a trace, emits findings. Firewall — judges from observed tool RESULTS, never a score
+ *  (selector≠judge). A driver can `define_analyst` to spawn specialized variants of this. */
+export const analystProfile: RoleProfile = {
+  id: 'analyst/trace',
+  role: 'analyst',
+  model: 'deepseek-v4-flash',
+  systemPrompt:
+    'You audit a worker’s trajectory. From ONLY the task and the worker’s tool-call trace (calls + ' +
+    'their RESULTS), list every required change that does NOT yet appear done or verified. Judge from ' +
+    'observed results, never from intent, and never from a grader/score. Be specific. If everything ' +
+    'required appears done and verified, reply exactly COMPLETE.',
+  skills: [],
+  tools: ['read_trace'],
+}
+
+/** DRIVER: the lead steerer. A sandbox agent that reviews the workers it drives, investigates their
+ *  traces (using/defining analysts, fanning out when useful), takes the lead or dispatches, and
+ *  parallelizes intelligently. This is the profile you sketched — formalized. Topology is ITS choice. */
+export const driverProfile: RoleProfile = {
+  id: 'driver/operator',
+  role: 'driver',
+  model: 'deepseek-v4-flash',
+  systemPrompt: [
+    'You are the OPERATOR. You lead one or more worker agents to fully complete a task over a shared',
+    'artifact. You may delegate, investigate, and — when it is faster or more reliable — do the work',
+    'yourself. You judge from traces, never from a grader/score (selector ≠ judge).',
+    '',
+    'TOOLS:',
+    '- list_analysts / run_analyst(id, worker): the trace analysts available — run one over a worker’s',
+    '  trajectory to get findings.',
+    '- define_analyst(profile): if no existing analyst fits, AUTHOR a new one and run it. Specialized',
+    '  analysts are cheap; make them when a worker’s failure mode needs a focused lens.',
+    '- observe_worker(worker): the worker’s IN-FLIGHT trace if it is still running, else its last',
+    '  finished episode/shot.',
+    '- spawn_worker(profile, task) / steer_worker(worker, instruction, interrupt?) / stop.',
+    '- the artifact’s own tools (read/edit/run) — use them to inspect the workspace and to contribute',
+    '  decisive work yourself.',
+    '',
+    'EACH CYCLE:',
+    '1. Review the workers you are driving — analyze their in-flight traces, or their last finished',
+    '   shot. Investigate deeply: run analysts, define a new analyst if needed, and fan out / spawn',
+    '   sub-analysts in PARALLEL to inspect traces faster when that genuinely helps.',
+    '2. Identify everything you can — incomplete work, errors, wrong approaches, policy violations,',
+    '   stalls, repeated mistakes. Be concrete about what remains.',
+    '3. If a finding is a reusable capability, PROPOSE it as a skill (so the optimizer can fold it into',
+    '   the profile directory) rather than re-deriving it every run.',
+    '4. Decide and act. If you are confident on the issue and dispatching would be slower or less',
+    '   reliable, TAKE THE LEAD: do the decisive turn yourself (directly, via sub-agents, or a dynamic',
+    '   workflow). Otherwise STEER the worker with a specific, verifiable instruction.',
+    '5. Parallelize as much as you intelligently can — NOT frivolously. Fan out only when the sub-tasks',
+    '   are genuinely independent; go deep on a single line when the work is sequential.',
+    '',
+    'STOP only when every required change is made AND confirmed in the workers’ tool results.',
+  ].join('\n'),
+  skills: ['trace-review', 'fanout-when-independent', 'take-the-lead-when-confident'],
+  tools: [...OPERATOR_TOOLS, '<artifact tools>'],
+  mcp: ['coordination'], // Scope-as-MCP: the coordination verbs, in a sandbox
+}
+
+/** The directory — the population the RSI loop optimizes (gepa over prompts, skillopt over skills). */
+export const profileDirectory: Record<string, RoleProfile> = {
+  [workerProfile.id]: workerProfile,
+  [analystProfile.id]: analystProfile,
+  [driverProfile.id]: driverProfile,
+}

package/src/refine-loop.test.mts

@@ -0,0 +1,106 @@
+/**
+ * Self-checking test for runRefineLoop (bench is a tsx-script package — no vitest).
+ *   tsx src/refine-loop.test.mts   (non-zero exit on failure)
+ *
+ * Regressions it defends — the behaviors the 7 forked loops all relied on, now
+ * owned by ONE atom:
+ *  - no judge → run all k rounds; prompt carries prior rounds (history) forward;
+ *  - judge wired → stop on the first valid round (default decide), no extra shots;
+ *  - a custom `decide` overrides;
+ *  - the execution Ctx is created once and threaded to every prompt + shot;
+ *  - teardown ALWAYS runs (finally), even when a shot throws.
+ */
+import assert from 'node:assert/strict'
+import { runRefineLoop } from './refine-loop.ts'
+
+async function run(): Promise<void> {
+  // (1) no judge → all k rounds; carry-forward via history
+  const prompts: string[] = []
+  const r1 = await runRefineLoop<string>({
+    rounds: 3,
+    prompt: (round, history) => {
+      const p = round === 1 ? 'base' : `base+prev:${history[history.length - 1]?.artifact}`
+      prompts.push(p)
+      return p
+    },
+    runShot: async (_p, round) => ({ artifact: `r${round}` }),
+  })
+  assert.equal(r1.rounds.length, 3, 'runs all k with no judge')
+  assert.equal(r1.blind.artifact, 'r1')
+  assert.equal(r1.final.artifact, 'r3')
+  assert.equal(r1.resolved, false, 'no judge → not resolved')
+  assert.equal(prompts[1], 'base+prev:r1', 'round 2 carries round 1 artifact')
+  assert.equal(prompts[2], 'base+prev:r2', 'round 3 carries round 2 artifact')
+
+  // (2) judge → stop on the first valid round, no extra shots
+  let shots = 0
+  const r2 = await runRefineLoop<string>({
+    rounds: 5,
+    prompt: () => 'p',
+    runShot: async () => {
+      shots += 1
+      return { artifact: shots === 2 ? 'good' : 'bad' }
+    },
+    judge: async (a) => ({ valid: a === 'good', score: a === 'good' ? 1 : 0 }),
+  })
+  assert.equal(r2.rounds.length, 2, 'stops at the first valid round')
+  assert.equal(r2.resolved, true)
+  assert.equal(shots, 2, 'no extra shots after stop')
+
+  // (3) a custom decide overrides the default
+  const r3 = await runRefineLoop<number>({
+    rounds: 9,
+    prompt: () => 'p',
+    runShot: async (_p, round) => ({ artifact: round }),
+    decide: (h) => h.length >= 2,
+  })
+  assert.equal(r3.rounds.length, 2, 'custom decide stops at 2')
+
+  // (4) Ctx created once + threaded to prompt and shot
+  let tornDown = 0
+  const seen: string[] = []
+  await runRefineLoop<string, { id: string }>({
+    rounds: 2,
+    setup: async () => ({ id: 'ctx-1' }),
+    prompt: (_r, _h, ctx) => {
+      seen.push(`prompt:${ctx.id}`)
+      return 'p'
+    },
+    runShot: async (_p, _r, ctx) => {
+      seen.push(`shot:${ctx.id}`)
+      return { artifact: 'a' }
+    },
+    teardown: async (ctx) => {
+      assert.equal(ctx.id, 'ctx-1')
+      tornDown += 1
+    },
+  })
+  assert.equal(tornDown, 1, 'teardown runs exactly once')
+  assert.ok(seen.includes('prompt:ctx-1') && seen.includes('shot:ctx-1'), 'ctx threaded to prompt + shot')
+
+  // (5) teardown runs even when a shot throws (finally)
+  let tornDown2 = 0
+  await assert.rejects(
+    runRefineLoop<string, { id: string }>({
+      rounds: 3,
+      setup: async () => ({ id: 'c' }),
+      prompt: () => 'p',
+      runShot: async (_p, round) => {
+        if (round === 2) throw new Error('boom')
+        return { artifact: 'a' }
+      },
+      teardown: async () => {
+        tornDown2 += 1
+      },
+    }),
+    /boom/,
+  )
+  assert.equal(tornDown2, 1, 'teardown runs on throw')
+
+  console.log('refine-loop.test: all assertions passed')
+}
+
+run().catch((e) => {
+  console.error('refine-loop.test FAILED:', e instanceof Error ? e.message : e)
+  process.exit(1)
+})

package/src/refine-loop.ts

@@ -0,0 +1,106 @@
+/**
+ * runRefineLoop — the ONE shared k-shot loop (see docs/architecture.md §1, §12).
+ *
+ * It replaces the seven hand-rolled `for (round 1..k) { shot → judge → decide →
+ * carry-forward }` copies across the workers (~700 LOC of duplicated skeleton).
+ * The worker is an OPAQUE `runShot`; this loop owns iteration, carry-forward
+ * steering, optional per-round judging + early stop, and round bookkeeping.
+ *
+ * Two carry-forward channels, both first-class (the extraction found both in the
+ * wild and they must stay pluggable):
+ *   - EXECUTION CONTEXT (`Ctx`): created once by `setup`, threaded to every shot,
+ *     torn down by `teardown`. This is how filesystem/session state carries — a
+ *     cloned repo whose edits persist (SWE refine), a shared sandbox box whose
+ *     session persists (sandbox research), or a scratch dir.
+ *   - PROMPT (`prompt(round, history, ctx)`): how textual state carries — round 1
+ *     is the blind prompt; rounds 2+ fold prior rounds' artifacts + a directive.
+ *
+ * Judging is OPTIONAL: workers that run all k rounds and let the orchestrator
+ * judge omit `judge`/`decide` (the loop runs to budget); a loop that should stop
+ * on the first valid answer wires `judge` (default `decide` = stop-on-valid).
+ *
+ * This is the inference-timescale instance of the spine's atom: `runShot` is the
+ * worker Agent's `act→Output`; `prompt` is the driver Agent's `act→steer`. It is
+ * deliberately corpus-agnostic — callers map `RefineLoopResult` to a RunRecord.
+ */
+
+/** Minimal per-round verdict — `valid` gates early-stop, `score` is informational. */
+export interface RoundVerdict {
+  valid: boolean
+  score?: number
+}
+
+export interface RoundRecord<Artifact> {
+  /** 1-based round index. */
+  round: number
+  prompt: string
+  artifact: Artifact
+  /** Present only when a `judge` is wired. */
+  verdict?: RoundVerdict
+  /** Non-fatal note (e.g. a liveness backstop fired this round). */
+  note?: string
+}
+
+export interface RefineLoopSpec<Artifact, Ctx = void> {
+  /** Max shots. Always ≥ 1. */
+  rounds: number
+  /** Build the per-task execution context once (clone repo / create box / mkdtemp). */
+  setup?: () => Promise<Ctx>
+  /** Round `r`'s prompt. `r === 1` is the blind prompt; `r > 1` carries `history`. */
+  prompt: (round: number, history: ReadonlyArray<RoundRecord<Artifact>>, ctx: Ctx) => string
+  /** Run ONE shot — the opaque worker (local spawn / sandbox stream / router chat). */
+  runShot: (
+    prompt: string,
+    round: number,
+    ctx: Ctx,
+  ) => Promise<{ artifact: Artifact; note?: string }>
+  /** Optional per-round judge. Omit to run all `rounds` (orchestrator judges later). */
+  judge?: (artifact: Artifact, round: number) => Promise<RoundVerdict>
+  /** Stop after this round? Default: stop once a round's verdict is valid (else run to budget). */
+  decide?: (history: ReadonlyArray<RoundRecord<Artifact>>) => boolean
+  /** Always runs (even on throw) when `setup` ran. */
+  teardown?: (ctx: Ctx) => Promise<void>
+}
+
+export interface RefineLoopResult<Artifact> {
+  /** Every executed round in order. */
+  rounds: RoundRecord<Artifact>[]
+  /** Round 1 — the blind artifact. */
+  blind: RoundRecord<Artifact>
+  /** The last executed round. */
+  final: RoundRecord<Artifact>
+  /** True iff any round was judged valid (only meaningful when `judge` is wired). */
+  resolved: boolean
+}
+
+const defaultDecide = <Artifact>(history: ReadonlyArray<RoundRecord<Artifact>>): boolean =>
+  history[history.length - 1]?.verdict?.valid === true
+
+export async function runRefineLoop<Artifact, Ctx = void>(
+  spec: RefineLoopSpec<Artifact, Ctx>,
+): Promise<RefineLoopResult<Artifact>> {
+  const rounds = Math.max(1, spec.rounds)
+  const decide = spec.decide ?? defaultDecide
+  let ctxSet = false
+  let ctx = undefined as Ctx
+  try {
+    if (spec.setup) {
+      ctx = await spec.setup()
+      ctxSet = true
+    }
+    const history: RoundRecord<Artifact>[] = []
+    for (let r = 1; r <= rounds; r += 1) {
+      const prompt = spec.prompt(r, history, ctx)
+      const { artifact, note } = await spec.runShot(prompt, r, ctx)
+      const verdict = spec.judge ? await spec.judge(artifact, r) : undefined
+      history.push({ round: r, prompt, artifact, verdict, note })
+      if (decide(history)) break
+    }
+    // The loop runs ≥ 1 round, so history is non-empty.
+    const blind = history[0] as RoundRecord<Artifact>
+    const final = history[history.length - 1] as RoundRecord<Artifact>
+    return { rounds: history, blind, final, resolved: history.some((h) => h.verdict?.valid === true) }
+  } finally {
+    if (ctxSet && spec.teardown) await spec.teardown(ctx)
+  }
+}

package/src/research-gate.mts

@@ -0,0 +1,132 @@
+/**
+ * Research leaderboard — model × web-search-provider × multi-shot, on the research
+ * benches (finsearchcomp / frames / hotpotqa / simpleqa). This is the OFF-SANDBOX
+ * RAG baseline: per shot, (1) provider-pinned web search via the router's proven
+ * `/v1/search?provider=<id>` + `web_fetch` of the top-K result pages, (2) answer with
+ * the evidence in context (no tools → `content` always present, so every arm differs
+ * ONLY by the search provider — a clean controlled A/B). `SEARCH=default` skips search
+ * (parametric control). Pure router HTTP (bearer `TANGLE_API_KEY`) — never touches the
+ * sandbox, so it never contends with sandbox-bound gates.
+ *
+ * The retrieve→answer body is the shared `runResearchShot` (research-shot.ts). Reuses
+ * `runPool` (bounded concurrency), `appendRunRecord` (the durable corpus), and the bench's own
+ * `adapter.judge`; nothing is reinvented. The AGENTIC HARNESS regime (opencode/pi multi-turn in
+ * a box) runs through the gate (`runGate`) with a sandbox-backed executor; this
+ * file is the flat, non-agentic search-RAG baseline.
+ *
+ * Each shot's answer is graded by the bench judge; writes one corpus RunRecord/task
+ * tagged `search:<provider>` + `model` so the leaderboard slices by arm. Fault-isolated
+ * (a flaky call → a NO-ANSWER attempt, never a throw).
+ *
+ *   dotenvx run -f ~/company/devops/secrets/.env.keys -f ~/company/devops/secrets/agent-state.env -- \
+ *     env BENCH=simpleqa SIMPLEQA_FIXTURES=1 MODEL=gpt-4o-mini SEARCH=you N=10 K=1 \
+ *     CONCURRENCY=3 JUDGE_MODEL=gpt-4o-mini CORPUS=/tmp/research-you.jsonl tsx src/research-gate.mts
+ *   tsx src/corpus-report.mts <armA.jsonl> <armB.jsonl>   # paired-bootstrap across arms
+ */
+
+import { ADAPTERS } from './adapters'
+import { type AttemptRecord, appendRunRecord, type RunRecord } from './corpus'
+import { runResearchShot, type ShotCfg } from './research-shot'
+import { runPool } from './run-pool'
+
+function must(name: string): string {
+  const v = process.env[name]
+  if (!v) throw new Error(`env ${name} is required`)
+  return v
+}
+
+async function main(): Promise<void> {
+  const benchName = process.env.BENCH ?? 'finsearchcomp'
+  const makeAdapter = ADAPTERS[benchName]
+  if (!makeAdapter) throw new Error(`unknown BENCH=${benchName} (have: ${Object.keys(ADAPTERS).join(', ')})`)
+
+  const n = Number(process.env.N ?? 6)
+  const k = Number(process.env.K ?? 3)
+  const model = process.env.MODEL ?? process.env.WORKER_MODEL ?? 'deepseek-v4-flash'
+  const search = process.env.SEARCH ?? 'default'
+  const cfg: ShotCfg = {
+    model,
+    search,
+    maxResults: Number(process.env.SEARCH_MAX_RESULTS ?? 5),
+    fetchTopK: Number(process.env.FETCH_TOP_K ?? 3),
+    temperature: Number(process.env.TEMPERATURE ?? 0.7),
+    routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',
+    routerKey: must('TANGLE_API_KEY'), // worker (router) + bench judge both need it
+    timeoutMs: process.env.SHOT_TIMEOUT_MS ? Number(process.env.SHOT_TIMEOUT_MS) : 600_000,
+  }
+  const concurrency = Number(process.env.CONCURRENCY ?? 3)
+  const corpusPath = process.env.CORPUS ?? `/tmp/research-${benchName}-${model.replace(/[^a-z0-9]/gi, '_')}-${search}.jsonl`
+  if (!Number.isInteger(n) || n < 1) throw new Error(`N must be a positive integer, got ${process.env.N}`)
+  if (!Number.isInteger(k) || k < 1) throw new Error(`K must be a positive integer, got ${process.env.K}`)
+
+  const adapter = makeAdapter()
+  console.log(`=== research gate (router-RAG) · bench=${benchName} · model=${model} · search=${search} · N=${n} K=${k} conc=${concurrency} ===`)
+  await adapter.preflight()
+  const tasks = await adapter.loadTasks({ limit: n })
+  console.log(`loaded ${tasks.length} task(s)`)
+
+  // Phase 1 — rollouts, concurrent (the shared runPool), off-sandbox. Fault-isolated.
+  const units = tasks.flatMap((task) => Array.from({ length: k }, (_, attempt) => ({ task, attempt })))
+  console.log(`\n▶ phase 1: ${units.length} rollouts (conc=${concurrency}) · search=${search}`)
+  const shots = await runPool(units, concurrency, async (u) => {
+    const s = await runResearchShot(u.task.prompt, u.task.id, u.attempt, cfg)
+    console.log(`  rollout ${u.task.id}#${u.attempt}: ${s.ok ? `answer ${s.answer.length}B · ${s.searches} search(es)` : `NO ANSWER (${s.detail})`} (${(s.wallMs / 1000) | 0}s)`)
+    return s
+  })
+  const shotOf = (id: string, i: number) => shots.find((o) => o.value?.taskId === id && o.value?.attempt === i)?.value
+
+  // Phase 2 — judge via the bench's OWN judge; write one RunRecord/task (the shared corpus).
+  console.log(`\n▶ phase 2: judging via ${adapter.name} judge → ${corpusPath}`)
+  let scoredTasks = 0
+  for (const task of tasks) {
+    const attempts: AttemptRecord[] = []
+    for (let i = 0; i < k; i += 1) {
+      const s = shotOf(task.id, i)
+      let sc: { score: number; resolved: boolean } | undefined
+      if (s?.ok) {
+        try {
+          const v = await adapter.judge(task, s.answer)
+          sc = { score: v.score, resolved: v.resolved }
+          console.log(`  judge ${task.id}#${i}: score=${(v.score * 100).toFixed(1)}% resolved=${v.resolved}`)
+        } catch (err) {
+          console.log(`  judge ${task.id}#${i}: ERROR ${(err instanceof Error ? err.message : String(err)).slice(0, 160)}`)
+        }
+      } else {
+        console.log(`  judge ${task.id}#${i}: skipped (no answer)`)
+      }
+      attempts.push({
+        round: i,
+        prompt: `research:${search}`,
+        output: s?.answer ?? '',
+        ...(sc ? { valid: sc.resolved, score: sc.score } : {}),
+        wallMs: s?.wallMs ?? 0,
+        eventCount: s?.searches ?? 0,
+        eventTypes: { 'web_search.requests': s?.searches ?? 0 },
+        traceTail: (s?.answer ?? '').slice(-600),
+      })
+    }
+    if (attempts.some((a) => a.score !== undefined)) scoredTasks += 1
+    const record: RunRecord = {
+      ts: new Date().toISOString(),
+      benchmark: adapter.name,
+      instanceId: task.id,
+      condition: `search:${search}`,
+      model,
+      blindResolved: attempts[0]?.valid === true,
+      resolved: attempts.some((a) => a.valid === true),
+      attempts,
+      infraError: false,
+    }
+    await appendRunRecord(corpusPath, record)
+  }
+
+  console.log(
+    `\n=== wrote ${tasks.length} task(s) (${scoredTasks} with ≥1 scored attempt) → ${corpusPath} ===\n` +
+      `  arm = model:${model} × search:${search} · compare arms: tsx src/corpus-report.mts <armA.jsonl> <armB.jsonl>`,
+  )
+}
+
+main().catch((err) => {
+  console.error(`research-gate: ${err instanceof Error ? err.message : String(err)}`)
+  process.exit(1)
+})