@tangle-network/agent-bench 0.1.0

package/src/examples/strategy-demo.mts

@@ -0,0 +1,119 @@
+/**
+ * Strategy demo — the optimization suite in three layers, on a toy Environment (no gym).
+ *
+ * The whole idea in one file: you implement an `Environment` (5 hooks: open/tools/call/
+ * score/close), and you get optimization STRATEGIES — sample (best-of-N), refine
+ * (iterate-with-feedback), and any you author — compared and scored by your own check,
+ * for free. This uses a trivial "counter" environment so it runs with just a router key
+ * (no benchmark dataset, no sandbox).
+ *
+ *   dotenvx run -f …/.env.keys -- env WORKER_MODEL=gpt-4o-mini tsx src/examples/strategy-demo.mts
+ *
+ * The three layers shown below:
+ *   1. just run it     — runBenchmark(env, …) compares the default strategies, free.
+ *   2. pick strategies — pass [sample, refine, adaptiveRefine].
+ *   3. author your own — defineStrategy(name, body) in ~10 lines, no Supervisor ceremony.
+ */
+import { adaptiveRefine, type AgenticTask, type ArtifactHandle, defineStrategy, type Environment, printBenchmarkReport, refine, runBenchmark, sample } from '@tangle-network/agent-runtime/loops'
+
+// ── 1. Implement an Environment (the only thing a new domain writes) ──────────────
+// A toy: the agent must drive a counter to exactly the target using the increment tool.
+// score = how close it got. This is the seam every real benchmark (EOPS, a coding repo,
+// a browser task) implements the same way — open a checkable artifact, expose tools,
+// score it. Here the "artifact" is just an in-memory counter.
+
+const target = 5
+const counters = new Map<string, { count: number }>()
+
+const counterEnv: Environment = {
+  name: 'counter',
+  async open(_task) {
+    const id = `counter-${Math.random().toString(36).slice(2, 8)}`
+    counters.set(id, { count: 0 })
+    return { id, surface: 'counter' } satisfies ArtifactHandle
+  },
+  async tools() {
+    return [
+      { type: 'function', function: { name: 'increment', description: 'Add 1 to the counter.', parameters: { type: 'object', properties: {} } } },
+      { type: 'function', function: { name: 'read_count', description: 'Read the current counter value.', parameters: { type: 'object', properties: {} } } },
+    ]
+  },
+  async call(handle, name) {
+    const c = counters.get(handle.id)
+    if (!c) return 'ERROR: no such counter'
+    if (name === 'increment') {
+      c.count += 1
+      return `count is now ${c.count}`
+    }
+    if (name === 'read_count') return `count is ${c.count}`
+    return `ERROR: unknown tool ${name}`
+  },
+  // The deployable CHECK: exact hits toward the target. score = passes/total.
+  async score(_task, handle) {
+    const c = counters.get(handle.id)
+    const count = c?.count ?? 0
+    return { passes: Math.min(count, target), total: target, errored: 0 }
+  },
+  async close(handle) {
+    counters.delete(handle.id)
+  },
+}
+
+const task: AgenticTask = {
+  id: 'counter-to-5',
+  systemPrompt: 'You operate a counter with tools.',
+  userPrompt: `Use the increment tool to bring the counter to exactly ${target}. Use read_count to verify before you finish. Reply DONE when the count equals ${target}.`,
+}
+
+// ── 3. Author your OWN strategy in ~10 lines — the lego (no Supervisor ceremony) ──
+// "doubleCheck": one attempt, then critique twice (extra steering passes) before stopping.
+// A strategy body composes two steps: shot() (one worker attempt) + critique() (the
+// firewalled analyst → a steer). That's it. This is the skillifiable unit.
+const doubleCheck = defineStrategy('doubleCheck', async ({ surface, task: t, budget, shot, critique }) => {
+  const handle = await surface.open(t)
+  const progression: number[] = []
+  let messages: Record<string, unknown>[] | undefined
+  let steer: string | undefined
+  let completions = 0
+  try {
+    for (let i = 0; i < budget; i += 1) {
+      const out = await shot({ handle, messages, steer })
+      if (!out) break
+      completions += out.completions
+      progression.push(out.score)
+      if (out.score >= 1) break
+      messages = out.messages
+      const findings = await critique(out.messages)
+      completions += 1
+      if (!findings) break
+      steer = `Not done yet. ${findings}`
+    }
+    const score = progression.length ? Math.max(...progression) : 0
+    return { score, resolved: score >= 1, completions, progression, shots: progression.length }
+  } finally {
+    await surface.close(handle)
+  }
+})
+
+async function main(): Promise<void> {
+  const worker = {
+    routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',
+    routerKey: process.env.TANGLE_API_KEY ?? '',
+    model: process.env.WORKER_MODEL ?? 'deepseek-v4-flash',
+    innerTurns: 6,
+  }
+  if (!worker.routerKey) throw new Error('set TANGLE_API_KEY (the worker calls the router)')
+
+  console.log('Layer 1 — just run it (default strategies, scored by the env\'s own check):')
+  printBenchmarkReport(await runBenchmark({ environment: counterEnv, tasks: [task], worker, budget: 3 }))
+
+  console.log('\nLayer 2+3 — pick the built-ins AND your own authored strategy:')
+  printBenchmarkReport(
+    await runBenchmark({ environment: counterEnv, tasks: [task], worker, budget: 3, strategies: [sample, refine, adaptiveRefine, doubleCheck] }),
+  )
+}
+
+main().catch((e) => {
+  console.error(e instanceof Error ? (e.stack ?? e.message) : String(e))
+  process.exit(1)
+})

package/src/fleet.mts

@@ -0,0 +1,121 @@
+/**
+ * The whole vision, end to end, runnable from a laptop: a thin local driver
+ * fans out N workers to CLOUD sandboxes, observes each worker's trace, reports
+ * what to fix, and writes durable learnings to a corpus the NEXT run reads back.
+ *
+ * Local process = the driver (orchestrate + observe). All agent work runs in the
+ * cloud (Tangle sandbox SDK). Scale is the API key, not the laptop.
+ *
+ *   dotenvx run -f ~/company/devops/secrets/.env.keys -f ~/company/devops/secrets/agent-state.env -- \
+ *     env BACKEND=opencode MODEL=gpt-4.1 N=2 CORPUS=/tmp/fleet-corpus.jsonl \
+ *     pnpm exec tsx src/fleet.mts
+ *
+ * Run it twice: the second run injects the first run's learnings into the workers.
+ */
+import { createChatClient } from '@tangle-network/agent-eval'
+import { FileCorpus, observe, openSandboxRun, renderReport } from '@tangle-network/agent-runtime/loops'
+import { Sandbox } from '@tangle-network/sandbox'
+import { answerOutput, sandboxAgentRun, type WorkerBackendType } from './sandbox-run'
+
+function env(name: string, fallback?: string): string {
+  const v = process.env[name] ?? fallback
+  if (v === undefined) throw new Error(`missing env ${name}`)
+  return v
+}
+
+// The fleet's work: each worker gets one subtask. Swap for any real decomposition.
+const subtasks = [
+  'Write a Python function `is_prime(n)` and three asserts proving it. Run them and report PASS/FAIL.',
+  'Write a Python function `fib(n)` (iterative) and three asserts proving it. Run them and report PASS/FAIL.',
+  'Write a Python function `rev_words(s)` that reverses word order, with asserts. Run them and report PASS/FAIL.',
+]
+
+interface WorkerResult {
+  id: string
+  task: string
+  output: string
+  events: unknown[]
+  wallMs: number
+  error?: string
+}
+
+async function runWorker(
+  client: Sandbox,
+  cfg: { backendType: WorkerBackendType; model: string; routerBaseUrl: string },
+  id: string,
+  task: string,
+  priorLearnings: string,
+): Promise<WorkerResult> {
+  const startedAt = Date.now()
+  const prompt = priorLearnings ? `${priorLearnings}\n\n---\n\n${task}` : task
+  const controller = new AbortController()
+  const timer = setTimeout(() => controller.abort(), Number(process.env.TIMEOUT_MS ?? 240_000))
+  try {
+    const agentRun = sandboxAgentRun({ ...cfg, name: id })
+    const run = await openSandboxRun<string>(
+      client,
+      { agentRun, signal: controller.signal },
+      { kind: 'events', fromEvents: (events) => answerOutput.parse(events as never) },
+    )
+    try {
+      const turn = await run.start(prompt)
+      return { id, task, output: (turn.out ?? '').trim(), events: turn.events, wallMs: Date.now() - startedAt }
+    } finally {
+      await run.close().catch(() => {})
+    }
+  } catch (err) {
+    return { id, task, output: '', events: [], wallMs: Date.now() - startedAt, error: err instanceof Error ? err.message : String(err) }
+  } finally {
+    clearTimeout(timer)
+  }
+}
+
+async function main(): Promise<void> {
+  const routerKey = env('TANGLE_API_KEY')
+  const cfg = {
+    backendType: env('BACKEND', 'opencode') as WorkerBackendType,
+    model: env('MODEL', 'gpt-4.1'),
+    routerBaseUrl: env('ROUTER_BASE_URL', 'https://router.tangle.tools/v1'),
+  }
+  const n = Math.min(Number(env('N', '2')), subtasks.length)
+  const corpus = new FileCorpus(env('CORPUS', '/tmp/fleet-corpus.jsonl'))
+  const observerModel = env('OBSERVER_MODEL', 'gpt-4.1')
+  const chat = createChatClient({ transport: 'router', apiKey: routerKey, baseUrl: cfg.routerBaseUrl, defaultModel: observerModel })
+  const client = new Sandbox({ baseUrl: env('SANDBOX_BASE_URL', 'https://sandbox.tangle.tools'), apiKey: routerKey })
+
+  // ── continuous: read what prior runs LEARNED, inject it into this run's workers
+  const prior = await corpus.query({ tags: ['audience:agent'], limit: 8 })
+  const priorLearnings = prior.length
+    ? `PRIOR LEARNINGS (from earlier runs — apply them):\n${prior.map((r) => `- ${r.claim}`).join('\n')}`
+    : ''
+  console.error(`\n=== FLEET · ${n} workers · ${cfg.backendType}/${cfg.model} · cloud ===`)
+  console.error(prior.length ? `carrying ${prior.length} prior learning(s) into the workers\n` : 'first run — no prior learnings yet\n')
+
+  // ── fan out N workers to cloud sandboxes, in parallel
+  const tasks = subtasks.slice(0, n)
+  const workers = await Promise.all(
+    tasks.map((task, i) => runWorker(client, cfg, `worker-${i + 1}`, task, priorLearnings)),
+  )
+
+  // ── observe each worker's trace → findings → operator report + durable learnings
+  let totalLearned = 0
+  for (const w of workers) {
+    console.error(`\n── ${w.id} (${Math.round(w.wallMs / 1000)}s)${w.error ? ` — ERROR: ${w.error}` : ''}`)
+    if (w.error) continue
+    const ob = await observe(
+      { task: w.task, output: w.output, trace: w.events, outcome: w.output ? 'passed' : 'unknown', runId: w.id },
+      { chat, model: observerModel, corpus, tags: [cfg.backendType, 'fleet'] },
+    )
+    totalLearned += ob.learned.length
+    console.error(`  answer: ${w.output.slice(0, 120).replace(/\n/g, ' ')}`)
+    console.error(renderReport(ob.findings).split('\n').map((l) => `  ${l}`).join('\n'))
+    console.error(`  → ${ob.learned.length} new learning(s) saved to the corpus`)
+  }
+
+  console.error(`\n=== fleet done: ${workers.filter((w) => !w.error).length}/${n} workers ok · ${totalLearned} learnings banked → run again to apply them ===`)
+}
+
+main().catch((e) => {
+  console.error(e)
+  process.exit(1)
+})

package/src/gate-cli.mts

@@ -0,0 +1,101 @@
+/**
+ * Run the diverse-vs-blind gate THROUGH the recursive atom, end to end, in two lines:
+ *
+ *   export TANGLE_API_KEY=...        # router + the deployable judge's creds
+ *   BENCH=enterpriseops-gym EOPS_FIXTURES=1 N=20 K=4 tsx src/gate-cli.mts
+ *
+ * The arms are equal-k by construction (both open K children; blind = K identical copies,
+ * diverse = K distinct strategy directives). The deployable selector is the benchmark's OWN
+ * judge — pick a benchmark whose `adapter.judge` is a runnable checker (enterpriseops-gym,
+ * swe-bench, terminal-bench), NOT an LLM-judge bench (finsearchcomp), or the selector is not
+ * deployable and the result does not answer the gate.
+ *
+ * Output is a verdict-first report: the headline delta, the per-arm resolve table, the equal-k
+ * proof, and the per-task paired booleans (feed them to a paired-bootstrap + BH downstream — a
+ * single CLI run is the instrument, not the significance test).
+ */
+
+import type { AgentProfile } from '@tangle-network/agent-runtime/loops'
+import { resolveAdapter } from './adapters'
+import { runGate } from './gate'
+
+const must = (k: string): string => {
+  const v = process.env[k]
+  if (!v) throw new Error(`env ${k} is required`)
+  return v
+}
+
+/** Four content-free, genuinely distinct solve strategies — the diverse arm. K = this length;
+ *  the blind arm runs K identical copies, so the two arms are equal-k. Trim/extend to change K. */
+const defaultStrategies: ReadonlyArray<string> = [
+  'Restate the task in your own words, then solve it directly and concisely.',
+  'Decompose the task into ordered sub-steps; solve each, then assemble the final deliverable.',
+  'Identify the most likely failure mode for this task FIRST, then produce a solution that explicitly avoids it.',
+  'Produce a first solution, critique it against every stated requirement, then output the corrected final version only.',
+]
+
+async function main(): Promise<void> {
+  const benchKey = process.env.BENCH ?? 'enterpriseops-gym'
+  const adapter = resolveAdapter(benchKey)
+  const model = process.env.WORKER_MODEL ?? 'deepseek-v4-flash'
+  const routerBaseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
+  const routerKey = must('TANGLE_API_KEY')
+  const k = Number(process.env.K ?? defaultStrategies.length)
+  const strategies = defaultStrategies.slice(0, k)
+  if (strategies.length < 2) throw new Error('K must be >= 2')
+
+  const profile = {
+    name: 'gate-solver',
+    model: { default: model },
+    prompt: { systemPrompt: 'You are an expert agent. Produce the single best deliverable the task’s grader will accept.' },
+  } as unknown as AgentProfile
+
+  const report = await runGate({
+    adapter,
+    profile,
+    strategies,
+    routerBaseUrl,
+    routerKey,
+    model,
+    ...(process.env.TEMPERATURE ? { temperature: Number(process.env.TEMPERATURE) } : {}),
+    n: Number(process.env.N ?? 20),
+    ...(process.env.IDS ? { ids: process.env.IDS.split(',') } : {}),
+    ...(process.env.SPLIT ? { split: process.env.SPLIT } : {}),
+    ...(process.env.PER_CHILD_TOKENS ? { perChildTokens: Number(process.env.PER_CHILD_TOKENS) } : {}),
+  })
+
+  const pct = (x: number) => `${(x * 100).toFixed(1)}%`
+  const blind = report.arms.find((a) => a.label === 'blind')!
+  const diverse = report.arms.find((a) => a.label === 'diverse')!
+
+  console.log(`\n=== gate: ${report.benchmark} (k=${report.k}, n=${report.n}) ===`)
+  console.log(
+    `VERDICT: diverse ${report.deltaScorePp >= 0 ? '+' : ''}${report.deltaScorePp.toFixed(1)}pp graded-score ` +
+      `(binary ${report.deltaPp >= 0 ? '+' : ''}${report.deltaPp.toFixed(1)}pp) vs blind` +
+      ` — equal-k=${report.equalK.withinTolerance ? 'OK' : 'VIOLATED'} (token spread ${report.equalK.spread.tokens})`,
+  )
+  console.log('arm      resolved  errored  resolveRate  meanScore  tokens')
+  for (const a of [blind, diverse]) {
+    const tok = a.totalSpend.tokens.input + a.totalSpend.tokens.output
+    console.log(
+      `${a.label.padEnd(8)} ${String(a.resolved).padStart(8)} ${String(a.errored).padStart(8)} ` +
+        `${pct(a.resolveRate).padStart(11)}  ${pct(a.meanScore).padStart(9)}  ${tok}`,
+    )
+  }
+  for (const a of [blind, diverse]) {
+    if (a.errored > 0) console.log(`  ${a.label} first failure: ${a.sampleBlocker}`)
+  }
+  const erroredFrac = (blind.errored + diverse.errored) / (2 * report.n)
+  if (erroredFrac > 0.2) {
+    console.log(`\nWARNING: ${(erroredFrac * 100).toFixed(0)}% of runs ERRORED — this 0%/delta is NOT a clean gate result; fix the failure above first.`)
+  }
+  if (!report.equalK.withinTolerance) {
+    console.log('\nWARNING: arms are NOT at equal compute — the delta is confounded, not a gate result.')
+  }
+  console.log(`\npaired (id: blind|diverse): ${report.perTask.map((t) => `${t.id}:${t.blind ? 1 : 0}|${t.diverse ? 1 : 0}`).join('  ')}`)
+}
+
+main().catch((e) => {
+  console.error(e instanceof Error ? (e.stack ?? e.message) : String(e))
+  process.exit(1)
+})

package/src/gate.test.mts

@@ -0,0 +1,129 @@
+/**
+ * Offline gate-plumbing test. A deterministic stub `ExecutorRegistry` is injected so the gate
+ * path runs with NO network: it proves the bridge wiring end-to-end — the persona + `fanout`
+ * drives the `Supervisor`, the per-child deployable verdict drives selection, the paired metric is
+ * derived from the run's own trajectory, and the conserved pool yields equal-k across arms.
+ *
+ * The LIVE solve-and-grade path (`benchSolverRegistry` → router + `adapter.judge`) is exercised
+ * by a real gate run against a deployable-checker domain, not here.
+ *
+ *   tsx bench/src/gate.test.mts
+ */
+
+import assert from 'node:assert/strict'
+import type {
+  AgentSpec,
+  DefaultVerdict,
+  ExecutorContext,
+  ExecutorRegistry,
+  Executor,
+  ExecutorFactory,
+  ExecutorResult,
+} from '@tangle-network/agent-runtime/loops'
+import type { BenchmarkAdapter, BenchScore, BenchTask } from './benchmarks/types'
+import { runGate, type SolveTask } from './gate'
+
+/** A child whose verdict is decided purely by whether its prompt carries the STRONG marker — so a
+ *  diverse arm that injects a STRONG strategy beats a blind arm that never does. Fixed spend per
+ *  child (independent of prompt length) so both arms spend identically → equal-k is exact. */
+function stubLeaf(_spec: AgentSpec, ctx: ExecutorContext): Executor<unknown> {
+  let artifact: ExecutorResult<unknown> | undefined
+  return {
+    runtime: 'stub',
+    execute(task): Promise<ExecutorResult<unknown>> {
+      void ctx.signal
+      const t = task as SolveTask
+      const strong = t.prompt.includes('STRONG')
+      const verdict: DefaultVerdict = { valid: strong, score: strong ? 0.9 : 0.2 }
+      artifact = {
+        outRef: `stub:${t.instance.id}:${strong ? 1 : 0}`,
+        out: `cand:${t.instance.id}:${strong ? 'strong' : 'weak'}`,
+        verdict,
+        spent: { iterations: 1, tokens: { input: 100, output: 20 }, usd: 0, ms: 1 },
+      }
+      return Promise.resolve(artifact)
+    },
+    teardown: () => Promise.resolve({ destroyed: true }),
+    resultArtifact() {
+      if (!artifact) throw new Error('stubLeaf: resultArtifact before execute')
+      return artifact
+    },
+  }
+}
+
+const stubRegistry: ExecutorRegistry = {
+  register() {
+    throw new Error('stub: register unsupported')
+  },
+  resolve<Out>(_spec: AgentSpec) {
+    const factory: ExecutorFactory<Out> = (s, ctx) => stubLeaf(s, ctx) as Executor<Out>
+    return { succeeded: true as const, value: factory }
+  },
+}
+
+function stubAdapter(n: number): BenchmarkAdapter {
+  const tasks: BenchTask[] = Array.from({ length: n }, (_v, i) => ({
+    id: `t${i}`,
+    prompt: `solve instance ${i}`,
+  }))
+  return {
+    name: 'stub-bench',
+    preflight: () => Promise.resolve(),
+    loadTasks: (opts) =>
+      Promise.resolve(opts?.limit !== undefined ? tasks.slice(0, opts.limit) : tasks),
+    // judge is NOT exercised on the injected-registry path (the stub leaf sets verdicts directly);
+    // a real run reaches the adapter judge through benchSolverRegistry.
+    judge: (): Promise<BenchScore> => {
+      throw new Error('stub judge must not be called on the injected-registry path')
+    },
+    goldArtifact: () => Promise.resolve(undefined),
+  }
+}
+
+const profile = { name: 'stub-solver', model: { default: 'stub-model' } } as never
+
+const report = await runGate({
+  adapter: stubAdapter(5),
+  profile,
+  strategies: ['plain restate', 'use the STRONG verified approach', 'enumerate edge cases'],
+  routerBaseUrl: 'http://unused',
+  routerKey: 'unused',
+  model: 'stub-model',
+  solverRegistry: stubRegistry,
+})
+
+assert.equal(report.benchmark, 'stub-bench')
+assert.equal(report.k, 3, 'k = strategies.length')
+assert.equal(report.n, 5, 'n = loaded tasks')
+assert.equal(report.perTask.length, 5)
+
+const blind = report.arms.find((a) => a.label === 'blind')
+const diverse = report.arms.find((a) => a.label === 'diverse')
+assert.ok(blind && diverse, 'both arms present')
+
+// Blind = k identical copies of a prompt with no STRONG marker → every child weak → never resolves.
+assert.equal(blind.resolved, 0, 'blind resolves nothing (no STRONG strategy)')
+assert.equal(blind.errored, 0, 'no infra-errored blind runs')
+assert.equal(blind.resolveRate, 0)
+
+// Diverse injects a STRONG strategy → the deployable selector picks that child → resolves each task.
+assert.equal(diverse.resolved, 5, 'diverse resolves every task via the STRONG strategy child')
+assert.equal(diverse.resolveRate, 1)
+
+assert.equal(report.deltaPp, 100, 'diverse − blind = +100pp on this constructed signal')
+for (const row of report.perTask) {
+  assert.equal(row.blind, false, `${row.id}: blind not resolved`)
+  assert.equal(row.diverse, true, `${row.id}: diverse resolved`)
+}
+
+// Both arms opened k=3 children at a fixed per-child spend → identical conserved cost → equal-k.
+assert.equal(report.equalK.withinTolerance, true, 'arms are at equal compute (the equal-k proof)')
+assert.equal(blind.totalSpend.iterations, 15, 'blind: 5 tasks × 3 children')
+assert.equal(diverse.totalSpend.iterations, 15, 'diverse: 5 tasks × 3 children')
+assert.equal(
+  blind.totalSpend.tokens.input + blind.totalSpend.tokens.output,
+  diverse.totalSpend.tokens.input + diverse.totalSpend.tokens.output,
+  'identical token spend across arms',
+)
+
+console.log('gate.test: OK — gate runs through the Supervisor, deployable selection + equal-k verified')