@tangle-network/agent-bench 0.1.0

package/src/decoder-live.mts

@@ -0,0 +1,133 @@
+/**
+ * LIVE validation of the per-harness tool-part decoders against a REAL sandbox box.
+ *
+ * Spin a real box for HARNESS, make the harness call tools, capture the actual session part shapes,
+ * and prove the harness's decoder extracts the tool calls (with error status where the harness carries
+ * it). No mock. Exits non-zero on a decode miss so it can gate CI. The box is ALWAYS deleted.
+ *
+ * Run: HARNESS=opencode dotenvx run -f ~/company/devops/secrets/agent-state.env -- \
+ *        pnpm exec tsx bench/src/decoder-live.mts        (also: claude-code, codex, kimi-code, …)
+ */
+import { Sandbox } from '@tangle-network/sandbox'
+import { decodeToolPart } from '../../src/runtime/supervise/trace-source'
+
+function must(k: string): string {
+  const v = process.env[k]
+  if (!v) throw new Error(`missing env ${k}`)
+  return v
+}
+
+function* candidateParts(node: unknown, depth = 0): Generator<unknown> {
+  if (!node || typeof node !== 'object' || depth > 6) return
+  const o = node as Record<string, unknown>
+  yield o
+  if (o.part) yield o.part
+  if (Array.isArray(o.parts)) for (const p of o.parts) yield p
+  if (o.data) yield* candidateParts(o.data, depth + 1)
+  if (o.message) yield* candidateParts(o.message, depth + 1)
+}
+
+const HARNESS = process.env.HARNESS ?? 'opencode'
+
+async function main(): Promise<number> {
+  const client = new Sandbox({
+    baseUrl: process.env.SANDBOX_BASE_URL ?? 'https://sandbox.tangle.tools',
+    apiKey: must('TANGLE_API_KEY'),
+    timeoutMs: 600_000,
+  } as never)
+
+  console.error(`[live] creating ${HARNESS} box (${process.env.WORKER_MODEL ?? 'deepseek-v4-flash'})…`)
+  const box = (await client.create({
+    backend: {
+      type: HARNESS,
+      model: {
+        provider: process.env.WORKER_PROVIDER ?? 'openai',
+        model: process.env.WORKER_MODEL ?? 'deepseek-v4-flash',
+        baseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',
+      },
+      profile: { name: 'decoder-live' },
+    },
+  } as never)) as Record<string, (...a: never[]) => unknown> & { id?: string }
+
+  try {
+    console.error('[live] box', box.id, '— waiting for running…')
+    await box.waitFor('running' as never, { timeoutMs: 180_000 } as never)
+
+    // Force several distinct tool calls AND at least one tool error (to exercise error-status decode).
+    const prompt =
+      'Use your tools, one tool call at a time: (1) list the files in the current directory; ' +
+      '(2) run `echo hello-from-tool`; (3) run `false` (this command fails on purpose); ' +
+      '(4) create notes.txt containing "hi". A separate tool call for each. Then reply "done".'
+
+    const rawEvents: unknown[] = []
+    const ac = new AbortController()
+    const timer = setTimeout(() => ac.abort(), 240_000)
+    try {
+      for await (const ev of box.streamPrompt(prompt as never, {
+        signal: ac.signal,
+      } as never) as AsyncGenerator<unknown>) {
+        rawEvents.push(ev)
+      }
+    } finally {
+      clearTimeout(timer)
+    }
+
+    // Vocabulary: every distinct part `type` the harness emitted (so we learn its real schema).
+    const types = new Map<string, number>()
+    for (const ev of rawEvents)
+      for (const part of candidateParts(ev)) {
+        const t = (part as Record<string, unknown>)?.type
+        if (typeof t === 'string') types.set(t, (types.get(t) ?? 0) + 1)
+      }
+
+    // Decode tool calls via the HARNESS adapter + de-dup by callId.
+    const decoded: Array<{ toolName: string; status?: string; callId?: string }> = []
+    const seen = new Set<string>()
+    for (const ev of rawEvents)
+      for (const part of candidateParts(ev)) {
+        const step = decodeToolPart(part, HARNESS)
+        if (!step) continue
+        if (step.callId && seen.has(step.callId)) continue
+        if (step.callId) seen.add(step.callId)
+        decoded.push({ toolName: step.toolName, ...(step.status ? { status: step.status } : {}), ...(step.callId ? { callId: step.callId } : {}) })
+      }
+
+    // Surface any harness/model error events (so a no-tools run isn't mistaken for a decoder miss).
+    const errs = rawEvents.filter((e) => JSON.stringify(e).match(/"error"/i)).slice(0, 2)
+    if (errs.length) console.error(`[live] harness error events:`, errs.map((e) => JSON.stringify(e).slice(0, 300)))
+
+    console.error(`\n========== LIVE DECODER RESULT — harness=${HARNESS} ==========`)
+    console.error(`raw stream events: ${rawEvents.length}`)
+    console.error(`part-type vocabulary:`, JSON.stringify(Object.fromEntries(types)))
+    console.error(`decoded ${decoded.length} tool calls:`, JSON.stringify(decoded))
+    console.error(`with error status: ${decoded.filter((d) => d.status === 'error').length}`)
+    // Sample raw shapes the decoder did NOT match but that mention a tool (the schema to learn from).
+    console.error(`\n--- raw toolish parts NOT decoded (first 4) ---`)
+    let shown = 0
+    for (const ev of rawEvents) {
+      if (shown >= 4) break
+      for (const part of candidateParts(ev)) {
+        if (shown >= 4) break
+        const s = JSON.stringify(part)
+        if (s.match(/tool/i) && !decodeToolPart(part, HARNESS)) {
+          console.error(s.slice(0, 380))
+          shown++
+        }
+      }
+    }
+    return decoded.length > 0 ? 0 : 2
+  } finally {
+    await (box.delete as (...a: never[]) => Promise<void>)().catch(() => {})
+    console.error('[live] box deleted.')
+  }
+}
+
+main()
+  .then((code) => {
+    console.error(code === 0 ? `\n✅ ${HARNESS}: decoder extracts real tool calls.` : `\n❌ ${HARNESS}: decoder extracted ZERO — see raw shapes above.`)
+    process.exit(code)
+  })
+  .catch((e) => {
+    console.error('[live] FAILED:', e)
+    process.exit(1)
+  })

package/src/directives.ts

@@ -0,0 +1,84 @@
+/**
+ * The steer surface (docs/architecture.md §4–§5): the refine directives a driver
+ * injects on rounds 2+. They live HERE, not in the worker files — a worker is a
+ * SUBSTRATE that runs a prompt; the directive is the DRIVER's steer, and the thing
+ * GEPA optimizes. Centralizing them makes the optimization surface explicit and
+ * stops workers from owning steer text.
+ *
+ * Honest caveat (architecture.md §11, rung-0): these hand-written / early-GEPA
+ * directives are the *intrinsic verify-and-revise* family, which LOSES to
+ * compute-matched random@k on FinSearchComp. They are kept as the optimization
+ * SURFACE and the baseline the real trace-fed driver must beat — NOT as a
+ * recommended default. The end state is a driver that supplies the steer from the
+ * trace+analysis; until run.ts is unified onto that path, workers take a directive
+ * as an optional param defaulting to the relevant constant below.
+ */
+
+/**
+ * APPROACH-diversity strategy lenses — a DIFFERENT family from the verify-and-revise
+ * directives above. The caveat names why verify-and-revise loses: k identical-directive
+ * shots cluster on the SAME reasoning path, so they share a failure mode and a random
+ * pick among them gains nothing. These lenses make the k shots take DIFFERENT reasoning
+ * paths, so a deployable self-consistency selector can pick the consensus across
+ * genuinely independent attempts. That is the gate hypothesis runProgram's `parallel`
+ * is built to deploy: diverse@k (parallel of distinct lenses) vs random@k (blind
+ * sample of identical) at EQUAL k. Each lens is a PREFIX to the task; it changes HOW
+ * the model reasons, never the answer the judge scores. GEPA optimizes the shared base
+ * (`improve-prompt`); these lenses layer on top via `composeStrategies`.
+ */
+const DIVERSE_STRATEGY_LENSES: ReadonlyArray<string> = [
+  'Answer directly and decisively from what you already know. State the single best answer without hedging.',
+  'Decompose the question into the sub-facts it depends on. Establish each sub-fact explicitly, then compose them into the answer.',
+  'Reason from first principles. Ignore the most obvious or popular guess; derive the answer from underlying facts and relationships.',
+  'Name the most plausible WRONG answer and the trap that makes it tempting. Rule it out, then commit to the answer that survives.',
+]
+
+/** Compose the k diverse strategies for a run: each = an approach lens layered on the
+ *  (optionally GEPA-learned) shared base. Returns exactly `k` distinct strategy prefixes
+ *  (lenses cycle if k exceeds the lens count, but each carries its index so prompts differ). */
+export function composeStrategies(base: string, k: number): string[] {
+  const lenses = DIVERSE_STRATEGY_LENSES
+  return Array.from({ length: k }, (_, i) => {
+    const lens = lenses[i % lenses.length] as string
+    const tag = i < lenses.length ? '' : ` (variant ${Math.floor(i / lenses.length) + 1})`
+    return `${lens}${tag}\n\n${base}`
+  })
+}
+
+/** Research (local opencode, model-knowledge) refine directive — hand-written. */
+export const DEFAULT_RESEARCH_REFINE_DIRECTIVE =
+  'Double-check it for a specific factual or reasoning error. If it is correct, restate the SAME answer unchanged. Change it ONLY if you identify a concrete, specific error — do not change a correct answer. End with the FINAL ANSWER line.'
+
+/** Sandbox research (live web/market sources) refine directive — hand-written. */
+export const DEFAULT_SANDBOX_REFINE_DIRECTIVE =
+  'Double-check it: re-verify the figure against live sources and the requested units/precision/tolerance. If it is correct, restate the SAME final answer unchanged. Change it ONLY if you find a concrete error in the value or the source. End with the explicit final answer.'
+
+/** GEPA-learned sandbox refine directive (bycd31l10, +7.1pp held-out vs the
+ *  hand-written one on the GEPA run, n=8/noisy). Separates the verification note
+ *  from the verbatim-preserved final answer, fixing the blank-reply failure mode. */
+export const GEPA_LEARNED_DIRECTIVE =
+  'Double-check it: re-verify the fact/value against a reliable, citable source. Provide a brief Verification note naming the source you used (link or title); this note is not part of the final answer. Confirm the requested units/precision/tolerance exactly. If the prior answer is correct, copy the SAME final answer text verbatim with identical formatting—do not add or remove words. Change it ONLY if you find a concrete error in the value or in the cited source; in that case, briefly describe the specific error in the Verification note and provide the corrected value with the requested units/precision/tolerance. If you cannot verify, state that in the Verification note, but do not alter or omit the final answer. Always place the final answer as the last line of your reply, containing only the answer text.'
+
+/** OpenSCAD authoring directive (CAD design) — the GEPA-optimizable system prompt.
+ *  Minimal on purpose: states the contract (compile, source-only) but not HOW to
+ *  satisfy the geometric spec, leaving headroom for the optimizer. */
+export const DEFAULT_CAD_DIRECTIVE =
+  'You are an expert OpenSCAD engineer. Output ONLY valid OpenSCAD source — no prose, no markdown fences. The model MUST compile with `openscad -o out.stl model.scad`. Match the brief as closely as you can.'
+
+/** OpenSCAD authoring directive for the SANDBOX path (solveCadRefine) — a terser
+ *  variant of DEFAULT_CAD_DIRECTIVE (no "match the brief" tail). Kept distinct
+ *  rather than unified to preserve that path's exact prompt. */
+export const DEFAULT_CAD_SANDBOX_DIRECTIVE =
+  'You are an expert OpenSCAD engineer. Output ONLY valid OpenSCAD source — no prose, no markdown fences. The model must compile with `openscad -o out.stl model.scad`.'
+
+/** Blender bpy authoring directive (BlenderLLM/CADBench) — the GEPA-optimizable
+ *  system prompt. States the contract (runs headless, builds mesh(es) at origin,
+ *  no cameras/lights) but not how to satisfy the criteria. */
+export const DEFAULT_BLENDER_DIRECTIVE =
+  'You are an expert Blender Python (bpy) modeller. Output ONLY a complete bpy script — no prose, no markdown fences. The script must run under `blender --background --python` and build the described object as one or more mesh objects at the world origin. Do NOT add cameras, lights, or render calls; the harness adds those. Match the description.'
+
+/** build123d authoring directive (CADGenBench) — the GEPA-optimizable system
+ *  prompt. States the contract (build123d → output.step, valid watertight solid,
+ *  exact dimensions) but not how to satisfy it. */
+export const DEFAULT_BUILD123D_DIRECTIVE =
+  'You are an expert CAD engineer. Write a build123d (Python, OpenCascade BREP) script that builds the described part and saves it with `export_step(part, "output.step")`. Output ONLY the Python script — no prose, no markdown fences. The script must run, produce a single VALID watertight solid, and match the described geometry as precisely as possible (exact stated dimensions).'

package/src/diverse-gate.mjs

@@ -0,0 +1,112 @@
+// The beat-blind gate, as ONE command — the experiment that decides whether the
+// diversification/selection surface (the one PR #145 defers to as "tracked separately")
+// actually beats compute-matched best-of-N. Composes only LANDED pieces:
+//   1. random@k corpus  — k identical-directive shots/instance (the compute control)
+//   2. diverse@k corpus — k DIFFERENT strategy lenses/instance (DIVERSE=1; the bet)
+//   3. selector replay  — selfConsistencySelect@k over each (corpus-replay --selector)
+//   4. paired report    — bootstrap CI + Benjamini-Hochberg over both (corpus-report)
+//
+// The decomposition it yields:
+//   random@k           = more-compute, no picking          (control)
+//   selector@k (homog) = picking over IDENTICAL attempts   (#143: −8.2pp on the committed corpus)
+//   diverse-selector@k = picking over DIVERSE attempts      (THE bet: does approach-diversity
+//                        give self-consistency the signal identical attempts don't?)
+// Beat-blind iff diverse-selector@k > random@k at significant n.
+//
+//   node diverse-gate.mjs            run it (generates corpora — a real worker run)
+//   node diverse-gate.mjs --dry      print the plan only (no run; safe while another
+//                                    sandbox run is live — zero router/sandbox contention)
+//
+// Knobs (env): BENCH (default hotpotqa) · N (default 30) · K (default 4) ·
+//   RESEARCH=1 (local opencode, default) | SANDBOX=1 (prod sandbox web worker) · MODELS ·
+//   DIVERSE_BASE (compose #145's GEPA-learned directive as the lens base — follow-on).
+
+import { spawn } from 'node:child_process'
+
+const DRY = process.argv.includes('--dry')
+const BENCH = process.env.BENCH ?? 'hotpotqa'
+const N = process.env.N ?? '30'
+const K = process.env.K ?? '4'
+const RANDOM_CORPUS = process.env.RANDOM_CORPUS ?? '/tmp/dg-random.jsonl'
+const DIVERSE_CORPUS = process.env.DIVERSE_CORPUS ?? '/tmp/dg-diverse.jsonl'
+// Worker-mode env passes through to batch-oracle unchanged (RESEARCH=1 / SANDBOX=1 / MODELS / TANGLE_API_KEY).
+const passEnv = { ...process.env, BENCH, N, K }
+
+const steps = [
+  {
+    label: 'random@k corpus (control — identical directive)',
+    cmd: 'npx',
+    args: ['tsx', 'src/run.ts', 'batch-oracle', N],
+    env: { ...passEnv, CORPUS: RANDOM_CORPUS },
+  },
+  {
+    label: 'diverse@k corpus (the bet — k distinct strategy lenses)',
+    cmd: 'npx',
+    args: ['tsx', 'src/run.ts', 'batch-oracle', N],
+    env: { ...passEnv, CORPUS: DIVERSE_CORPUS, DIVERSE: '1' },
+  },
+  {
+    label: 'selector@k over the CONTROL corpus (homogeneous)',
+    cmd: 'npx',
+    args: ['tsx', 'src/corpus-replay.mts', RANDOM_CORPUS, '--selector'],
+    env: passEnv,
+  },
+  {
+    label: 'selector@k over the DIVERSE corpus (the beat-blind number)',
+    cmd: 'npx',
+    // The diverse corpus records carry condition="diverse@4"; corpus-replay's
+    // selector filter defaults to "random", so match the diverse condition here.
+    args: ['tsx', 'src/corpus-replay.mts', DIVERSE_CORPUS, '--selector', '--condition=diverse'],
+    env: passEnv,
+  },
+  {
+    label: 'paired bootstrap CI + Benjamini-Hochberg over both corpora',
+    cmd: 'npx',
+    args: ['tsx', 'src/corpus-report.mts', RANDOM_CORPUS, DIVERSE_CORPUS],
+    env: passEnv,
+  },
+]
+
+const shellPreview = (s) => {
+  const envStr = Object.entries(s.env)
+    .filter(([k]) => ['BENCH', 'N', 'K', 'CORPUS', 'DIVERSE', 'RESEARCH', 'SANDBOX', 'MODELS'].includes(k))
+    .map(([k, v]) => `${k}=${v}`)
+    .join(' ')
+  return `  ${envStr} ${s.cmd} ${s.args.join(' ')}`.replace(/\s+/g, ' ')
+}
+
+function runStep(s) {
+  return new Promise((resolve, reject) => {
+    console.log(`\n▶ ${s.label}`)
+    const child = spawn(s.cmd, s.args, { cwd: process.cwd(), env: s.env, stdio: 'inherit' })
+    child.on('error', reject)
+    // Fail loud: a non-zero step aborts the gate (no silent partial result).
+    child.on('exit', (code) => (code === 0 ? resolve() : reject(new Error(`${s.label} exited ${code}`))))
+  })
+}
+
+async function main() {
+  console.log(`=== beat-blind gate · BENCH=${BENCH} N=${N} K=${K} ${DRY ? '(DRY — plan only)' : ''} ===`)
+  if (DRY) {
+    console.log('plan (no run — zero sandbox/router contention):')
+    for (const s of steps) console.log(shellPreview(s))
+    console.log(
+      '\nbeat-blind iff diverse-selector@k > random@k at significant n.' +
+        '\nDIVERSE_BASE=<file> composes #145\'s GEPA-learned directive as the lens base (follow-on).',
+    )
+    return
+  }
+  for (const s of steps) await runStep(s)
+  console.log(
+    '\n=== read the gate ===\n' +
+      `  random@k (control)        — from ${RANDOM_CORPUS} replay\n` +
+      `  selector@k (homogeneous)  — same corpus, the pick (#143: −8.2pp on committed finsearch)\n` +
+      `  diverse-selector@k        — ${DIVERSE_CORPUS} replay: THE bet\n` +
+      '  beat-blind iff diverse-selector@k > random@k, significant per the BH report above.',
+  )
+}
+
+main().catch((err) => {
+  console.error(`diverse-gate: ${err instanceof Error ? err.message : String(err)}`)
+  process.exit(1)
+})

package/src/egress-probe.mts

@@ -0,0 +1,26 @@
+/**
+ * Sandbox->router egress probe: spins a box and makes one authed chat call to
+ * https://router.tangle.tools/v1 using the BOX-PROVISIONED key (OPENCODE_MODEL_API_KEY
+ * inside the box). That is the sanctioned flow: the egress proxy validates/injects
+ * credentials at the boundary and 403s foreign keys passed in from outside — a raw
+ * external key in this probe would report a false "blocked".
+ *   dotenvx run -f ~/company/devops/secrets/.env.keys -f ~/company/devops/secrets/agent-state.env -- npx tsx src/egress-probe.mts
+ */
+import { Sandbox } from '@tangle-network/sandbox'
+
+const key = process.env.SANDBOX_KEY ?? process.env.TANGLE_API_KEY
+if (!key) throw new Error('no SANDBOX_KEY/TANGLE_API_KEY')
+const client = new Sandbox({ baseUrl: process.env.SANDBOX_BASE_URL ?? 'https://sandbox.tangle.tools', apiKey: key, timeoutMs: 300_000 } as never)
+const box = await client.create({ name: 'egress-probe' } as never)
+console.log('sandbox up:', box.id)
+const out = (r: unknown) => ((r as { stdout?: string }).stdout ?? '').trim()
+try {
+  for (const h of ['router.tangle.tools', 'id.tangle.tools']) {
+    const r = await box.exec(`curl -s -o /dev/null -w '%{http_code}' --max-time 15 https://${h}/ || echo FAIL`)
+    console.log(h, '→', out(r))
+  }
+  const chat = await box.exec(`curl -s --max-time 30 -X POST https://router.tangle.tools/v1/chat/completions -H "Authorization: Bearer $OPENCODE_MODEL_API_KEY" -H 'Content-Type: application/json' -d '{"model":"deepseek-v4-flash","messages":[{"role":"user","content":"say ok"}],"max_tokens":5}' -o /tmp/b -w '%{http_code}'; echo ' |'; head -c 250 /tmp/b`)
+  console.log('chat (box-provisioned key):', out(chat) || 'EMPTY — egress broken')
+} finally {
+  await box.delete()
+}

package/src/eops-skills/address-every-subtask.md

@@ -0,0 +1,5 @@
+---
+name: address-every-subtask
+description: Decompose the request into every distinct sub-task and plan a tool call for each — partial completion is the dominant failure.
+---
+Before planning calls, list every distinct change the request implies (each user, each ticket, each field). Plan tool calls that complete ALL of them. Many requests bundle several independent actions; finishing only the first is the most common way to fail the final-state check.

package/src/eops-skills/exact-tools-and-args.md

@@ -0,0 +1,5 @@
+---
+name: exact-tools-and-args
+description: Use only the listed tools, with their exact names and argument shapes.
+---
+Call ONLY the tools listed as available, by their exact names. For each call, provide every required argument with the correct key names and value types the tool expects. A misspelled tool name or a missing/extra argument makes the call a no-op in the judge and the final state will be wrong.

package/src/eops-skills/full-sequence-to-goal.md

@@ -0,0 +1,5 @@
+---
+name: full-sequence-to-goal
+description: Plan the COMPLETE ordered sequence that brings the database to the required final state.
+---
+Think in terms of the required FINAL state, then plan the full ordered sequence of calls that gets there from the seeded start — including any reads needed to ground values, and in an order that respects dependencies (create before reference, set status before close). Do not stop at the first action.

package/src/eops-skills/ground-every-value.md

@@ -0,0 +1,5 @@
+---
+name: ground-every-value
+description: Never invent a field value, id, or relationship — derive each from what the task gives you.
+---
+Do not guess ids, names, statuses, or relationships. Use the exact values the task supplies; when a value must be looked up, plan the read call first and use its result. A hallucinated field value is a leading cause of a silently-wrong final state.

package/src/eops-skills/honor-the-policies.md

@@ -0,0 +1,5 @@
+---
+name: honor-the-policies
+description: Re-read the role's policies and constraints before planning, and make every call comply.
+---
+The role description contains policies (who may do what, required ordering, forbidden actions). Re-read them, then ensure every planned tool call complies — correct assignee, correct status transitions, no skipped approval step. A plan that reaches the data goal but violates a stated policy still fails.

package/src/examples/README.md

@@ -0,0 +1,58 @@
+# Examples — the optimization suite in three layers
+
+`strategy-demo.mts` is the smallest end-to-end demonstration of the optimization suite.
+It runs on a toy "counter" `Environment` so it needs only a router key — no benchmark
+dataset, no sandbox, no gym.
+
+```bash
+TANGLE_API_KEY=... WORKER_MODEL=gpt-4o-mini pnpm tsx src/examples/strategy-demo.mts
+```
+
+## The model
+
+You have a **task**, a deployable **check**, and a compute **budget**. A *strategy* is
+**how you spend the budget to beat the check**. You implement an `Environment` (5 hooks)
+and get the strategies compared, scored by your own check, for free.
+
+## The three layers (each is a few lines in the demo)
+
+1. **Just run it** — `runBenchmark({ environment, tasks, worker })` compares the default
+   strategies and reports the paired lift. Black box; no vocabulary needed.
+
+2. **Pick strategies** — pass `strategies: [sample, refine, adaptiveRefine]`. Named by
+   what they *do*:
+   - **`sample`** — N independent attempts, keep the best-verifying (best-of-N / resample).
+   - **`refine`** — attempt → a critic reads the trace → steer the next → repeat (iterate).
+   - **`adaptiveRefine`** — refine, but abandon-and-restart a line that stops improving
+     (branch-when-stuck).
+
+3. **Author your own** — `defineStrategy(name, body)`. A strategy body composes two steps
+   — `shot()` (one worker attempt over an artifact) and `critique()` (the firewalled
+   analyst reads the trace → a steer) — with **zero** Supervisor/Scope ceremony. The demo
+   authors `doubleCheck` inline in ~10 lines. This is the unit a skill (or an agent) emits.
+
+## The answer-shaped template — `math-demo.mts`
+
+`math-demo.mts` is the same suite on the **answer-shaped** domain template — the shape
+tax/legal/gtm products use. `createVerifierEnvironment({ name, check, extraTools,
+callExtra })` builds the whole `Environment` from one deterministic `check` (here: 3
+GSM8K-style problems, graded by exact numeric match), and `sampleThenRefine` joins the
+built-ins compared at equal budget.
+
+```bash
+TANGLE_API_KEY=... WORKER_MODEL=gpt-4o-mini pnpm tsx src/examples/math-demo.mts
+```
+
+## The hooks you customize (world-class-DX surface)
+
+- **the check / verifier** → `Environment.score` (your deployable success criterion)
+- **the critic / steerer** → `worker.analystInstruction` (the analyst prompt; GEPA tunes this)
+- **the worker** → the model (`worker.model`)
+- **the strategy** → `defineStrategy` (or drop to `runAgentic` / the Supervisor for novel topologies)
+
+## Where the real results live
+
+On a trivial task all strategies tie. The differences (e.g. refine/adaptiveRefine beating
+sample on stateful agentic work, +16.4pp on EnterpriseOps-Gym) show on real domains — see
+`bench/HARNESS.md` and `bench/src/agentic-run.mts` (the EOPS Environment), and
+`bench/src/eops-gepa.mts` (GEPA evolving the analyst/critic prompt against the check).

package/src/examples/math-demo.mts

@@ -0,0 +1,110 @@
+/**
+ * Math through the suite — the ANY-DOMAIN generalization proof. No tool surface, no gym,
+ * no sandbox: the domain is one `check` function (numeric answer match), built with
+ * `createVerifierEnvironment`. The same strategies (sample / refine / your own) compete
+ * on word problems exactly as they do on EOPS tickets or commit0 repos.
+ *
+ * This is the template for every answer-shaped product domain: tax (a computed return
+ * value), legal (a clause checklist), creative/gtm (a rubric scorer) — swap `check`.
+ *
+ *   WORKER_MODEL=gpt-4o-mini BUDGET=3 tsx src/examples/math-demo.mts
+ */
+import {
+  type AgenticTask,
+  createVerifierEnvironment,
+  printBenchmarkReport,
+  refine,
+  runBenchmark,
+  sample,
+  sampleThenRefine,
+} from '@tangle-network/agent-runtime/loops'
+
+// GSM8K-style problems; meta.answer is the ground truth the deployable check compares to.
+const problems: Array<{ q: string; answer: number }> = [
+  {
+    q: 'A bakery sells loaves for $4 each. On Monday it sold 23 loaves, on Tuesday twice as many as Monday, and on Wednesday 11 fewer than Tuesday. How many dollars did the bakery earn across the three days?',
+    answer: (23 + 46 + 35) * 4, // 416
+  },
+  {
+    q: 'Tickets cost $12 for adults and $7 for children. A group of 9 people paid $83. How many children were in the group?',
+    answer: 5,
+  },
+  {
+    q: 'A tank holds 2400 liters. Pump A fills 40 L/min and pump B drains 25 L/min. Both run together starting from an empty tank. After how many minutes is the tank exactly half full?',
+    answer: 80,
+  },
+]
+
+const tasks: AgenticTask[] = problems.map((p, i) => ({
+  id: `math-${i + 1}`,
+  systemPrompt:
+    'You are a careful mathematician. Work step by step, use the calculator tool for arithmetic, then submit ONLY the final number with submit_answer.',
+  userPrompt: p.q,
+  meta: { answer: p.answer },
+}))
+
+/** Last number in the submission — tolerant of "The answer is 416." */
+function extractNumber(answer: string): number | null {
+  const m = answer.replace(/,/g, '').match(/-?\d+(?:\.\d+)?(?!.*-?\d)/s)
+  return m ? Number(m[0]) : null
+}
+
+const mathEnv = createVerifierEnvironment({
+  name: 'math',
+  check: (task, answer) => {
+    const got = extractNumber(answer)
+    const want = (task.meta as { answer: number }).answer
+    return { passes: got !== null && Math.abs(got - want) < 1e-6 ? 1 : 0, total: 1, errored: 0 }
+  },
+  extraTools: [
+    {
+      type: 'function',
+      function: {
+        name: 'calculator',
+        description: 'Evaluate an arithmetic expression (numbers and + - * / ( ) only).',
+        parameters: {
+          type: 'object',
+          properties: { expression: { type: 'string' } },
+          required: ['expression'],
+        },
+      },
+    },
+  ],
+  callExtra: (_task, name, args) => {
+    if (name !== 'calculator') return `ERROR: unknown tool ${name}`
+    const expr = String(args.expression ?? '')
+    if (!/^[\d\s+\-*/().]+$/.test(expr)) return 'ERROR: arithmetic only'
+    try {
+      // The whitelist above admits only arithmetic — no identifiers can reach Function.
+      const value = new Function(`return (${expr})`)() as number
+      return Number.isFinite(value) ? String(value) : 'ERROR: not finite'
+    } catch (e) {
+      return `ERROR: ${e instanceof Error ? e.message : String(e)}`
+    }
+  },
+})
+
+async function main(): Promise<void> {
+  const routerKey = process.env.TANGLE_API_KEY
+  if (!routerKey) throw new Error('set TANGLE_API_KEY')
+  const report = await runBenchmark({
+    environment: mathEnv,
+    tasks,
+    worker: {
+      routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',
+      routerKey,
+      model: process.env.WORKER_MODEL ?? 'deepseek-v4-flash',
+      innerTurns: 6,
+      temperature: 0.6,
+    },
+    strategies: [sample, refine, sampleThenRefine],
+    budget: Number(process.env.BUDGET ?? 3),
+    concurrency: 3,
+  })
+  printBenchmarkReport(report)
+}
+
+main().catch((e) => {
+  console.error(e instanceof Error ? (e.stack ?? e.message) : String(e))
+  process.exit(1)
+})