@tangle-network/agent-bench 0.3.5 → 0.3.7

package/src/swe-arena/bootstrap-meta.mts

@@ -0,0 +1,188 @@
+/**
+ * Metadata bootstrap — regenerates every per-instance input the experiment
+ * used to hand-generate into its scratchpad (`task-meta.json` via
+ * `load_meta.py`) straight from the SWE-bench Verified dataset through the
+ * bench adapter. The HF dataset cache under ~/.cache survives host reboots;
+ * the scratchpad does not — this module removes that scratch dependency
+ * permanently (proven necessary: a reboot wiped /tmp and took task-meta.json,
+ * the verify scripts, and the round configs with it).
+ *
+ *   tsx src/swe-arena/bootstrap-meta.mts --task-meta <out.json> [iid...]
+ *   tsx src/swe-arena/bootstrap-meta.mts --problems <outDir> [iid...]
+ *   tsx src/swe-arena/bootstrap-meta.mts --calibrate <verifyDir> <workDir> [iid...]
+ *
+ * `iid...` defaults to the round-4 improvement set (defaultRound4Config).
+ *
+ * HONESTY FIREWALL: `--problems` writes ONLY problem statements — the mode a
+ * verify-script author is allowed to read. Gold patches surface ONLY on the
+ * `--calibrate` path, where calibrate.ts applies them MECHANICALLY (git apply
+ * / patch --fuzz=3) and the official judge grades them; no author-facing
+ * output ever contains patch/test_patch/FAIL_TO_PASS content.
+ */
+
+import { mkdir, writeFile } from 'node:fs/promises'
+import { join, resolve } from 'node:path'
+import { pathToFileURL } from 'node:url'
+import { createSweBenchAdapter } from '../benchmarks/swe-bench.ts'
+import { calibrateInstance, type CalibrationResult } from './calibrate.ts'
+import { defaultRound4Config } from './outer-loop.mts'
+import { loadInstanceImages } from './run-experiment.mts'
+import { createSerializedJudge } from './serialized-judge.ts'
+import type { SweInstance } from './types.ts'
+
+/** Validate one adapter task-metadata record into the SweInstance shape the
+ *  harness consumes. Throws on any missing/empty load-bearing field — a blank
+ *  problem statement or gold patch must never flow silently into a run. */
+export function assertSweInstance(id: string, metadata: Record<string, unknown> | undefined): SweInstance {
+  if (!metadata) throw new Error(`bootstrap-meta: ${id} has no metadata on its adapter task`)
+  const str = (key: keyof SweInstance): string => {
+    const v = metadata[key]
+    if (typeof v !== 'string' || v.length === 0) {
+      throw new Error(`bootstrap-meta: ${id} has a missing/empty ${String(key)}`)
+    }
+    return v
+  }
+  const optStr = (key: keyof SweInstance): string | null => {
+    const v = metadata[key]
+    return typeof v === 'string' && v.length > 0 ? v : null
+  }
+  const instance: SweInstance = {
+    instance_id: str('instance_id'),
+    repo: str('repo'),
+    base_commit: str('base_commit'),
+    problem_statement: str('problem_statement'),
+    patch: str('patch'),
+    test_patch: str('test_patch'),
+    FAIL_TO_PASS: str('FAIL_TO_PASS'),
+    PASS_TO_PASS: str('PASS_TO_PASS'),
+    version: optStr('version'),
+    environment_setup_commit: optStr('environment_setup_commit'),
+  }
+  if (instance.instance_id !== id) {
+    throw new Error(`bootstrap-meta: metadata instance_id ${instance.instance_id} != requested ${id}`)
+  }
+  return instance
+}
+
+/** One dataset load for all ids, via the adapter (single source of truth). */
+export async function loadSweInstances(ids: string[]): Promise<Map<string, SweInstance>> {
+  if (ids.length === 0) throw new Error('bootstrap-meta: empty instance list')
+  const adapter = createSweBenchAdapter()
+  const tasks = await adapter.loadTasks({ ids, split: 'test' })
+  const byId = new Map(tasks.map((t) => [t.id, t]))
+  const out = new Map<string, SweInstance>()
+  for (const id of ids) {
+    const task = byId.get(id)
+    if (!task) throw new Error(`bootstrap-meta: ${id} not found in SWE-bench_Verified`)
+    out.set(id, assertSweInstance(id, task.metadata as Record<string, unknown> | undefined))
+  }
+  return out
+}
+
+/** task-meta.json — the experiment's exact shape: `{ [iid]: SweInstance }`. */
+export async function writeTaskMeta(ids: string[], outPath: string): Promise<void> {
+  const instances = await loadSweInstances(ids)
+  await writeFile(outPath, JSON.stringify(Object.fromEntries(instances), null, 1) + '\n')
+}
+
+/** <outDir>/<iid>.problem.md — problem statements ONLY (author-safe). */
+export async function writeProblemStatements(ids: string[], outDir: string): Promise<string[]> {
+  const instances = await loadSweInstances(ids)
+  await mkdir(outDir, { recursive: true })
+  const written: string[] = []
+  for (const [id, inst] of instances) {
+    const path = join(outDir, `${id}.problem.md`)
+    await writeFile(path, inst.problem_statement)
+    written.push(path)
+  }
+  return written
+}
+
+/** Dual-calibrate each instance through the M2 path: repro base-fail/gold-pass
+ *  plus the official-judge gold gate. Gold patches come from the adapter and
+ *  are only ever applied mechanically. */
+export async function calibrateIds(
+  ids: string[],
+  opts: { verifyDir: string; workDir: string; instanceImagesPath?: string; keepWorkspaces?: boolean },
+): Promise<CalibrationResult[]> {
+  const instances = await loadSweInstances(ids)
+  const images = await loadInstanceImages(opts.instanceImagesPath)
+  const judge = createSerializedJudge()
+  await mkdir(opts.workDir, { recursive: true })
+  const results: CalibrationResult[] = []
+  for (const id of ids) {
+    const inst = instances.get(id)!
+    const entry = images[id]
+    if (!entry) throw new Error(`bootstrap-meta: ${id} has no image mapping (fixtures/instances.json)`)
+    // verifyCmd runs with cwd = the materialized WORKSPACE (and, downstream,
+    // worker clones) — the script path must be absolute or bash exits 127.
+    const result = await calibrateInstance({
+      instanceId: id,
+      image: entry.image,
+      baseCommit: entry.base_commit,
+      goldPatch: inst.patch,
+      verifyCmd: `bash ${resolve(opts.verifyDir, `${id}.sh`)}`,
+      workDir: opts.workDir,
+      judge,
+      ...(opts.keepWorkspaces !== undefined ? { keepWorkspaces: opts.keepWorkspaces } : {}),
+    })
+    results.push(result)
+    console.log(
+      `CALIBRATION ${id}: baseRc=${result.baseRc} goldApplyRc=${result.goldApplyRc} goldRc=${result.goldRc} ` +
+        `verifyCalibrated=${result.verifyCalibrated} goldOfficialResolved=${result.goldOfficialResolved} ` +
+        `experimentValid=${result.experimentValid}`,
+    )
+  }
+  return results
+}
+
+// ---------------------------------------------------------------------------
+// CLI.
+// ---------------------------------------------------------------------------
+
+const isMain = process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href
+
+if (isMain) {
+  const [mode, ...rest] = process.argv.slice(2)
+  const defaultIds = defaultRound4Config().instances
+  if (mode === '--task-meta') {
+    const [outPath, ...ids] = rest
+    if (!outPath) {
+      console.error('usage: bootstrap-meta.mts --task-meta <out.json> [iid...]')
+      process.exit(2)
+    }
+    await writeTaskMeta(ids.length > 0 ? ids : defaultIds, outPath)
+    console.log(`task-meta → ${outPath}`)
+  } else if (mode === '--problems') {
+    const [outDir, ...ids] = rest
+    if (!outDir) {
+      console.error('usage: bootstrap-meta.mts --problems <outDir> [iid...]')
+      process.exit(2)
+    }
+    for (const p of await writeProblemStatements(ids.length > 0 ? ids : defaultIds, outDir)) {
+      console.log(`problem statement → ${p}`)
+    }
+  } else if (mode === '--calibrate') {
+    const [verifyDir, workDir, ...ids] = rest
+    if (!verifyDir || !workDir) {
+      console.error('usage: bootstrap-meta.mts --calibrate <verifyDir> <workDir> [iid...]')
+      process.exit(2)
+    }
+    const results = await calibrateIds(ids.length > 0 ? ids : defaultIds, { verifyDir, workDir })
+    const summaryPath = join(workDir, 'calibration.json')
+    await writeFile(summaryPath, JSON.stringify(results, null, 2) + '\n')
+    console.log(`calibration summary → ${summaryPath}`)
+    const invalid = results.filter((r) => !r.experimentValid)
+    if (invalid.length > 0) {
+      console.error(`NOT experiment-valid: ${invalid.map((r) => r.iid).join(', ')}`)
+      process.exit(1)
+    }
+  } else {
+    console.error(
+      'usage: tsx src/swe-arena/bootstrap-meta.mts --task-meta <out.json> [iid...]\n' +
+        '       tsx src/swe-arena/bootstrap-meta.mts --problems <outDir> [iid...]\n' +
+        '       tsx src/swe-arena/bootstrap-meta.mts --calibrate <verifyDir> <workDir> [iid...]',
+    )
+    process.exit(2)
+  }
+}

package/src/swe-arena/bootstrap-meta.test.mts

@@ -0,0 +1,51 @@
+/**
+ * Unit tests for the metadata bootstrap's validation layer — pure, no dataset
+ * load. The adapter-backed loaders are exercised by the calibration CLI.
+ */
+
+import { describe, expect, it } from 'vitest'
+import { assertSweInstance } from './bootstrap-meta.mts'
+
+const full = {
+  instance_id: 'django__django-11532',
+  repo: 'django/django',
+  base_commit: 'a5308514fb4bc5086c9a16a8a24a945eeebb073c',
+  problem_statement: 'Email messages crash on non-ASCII domain…',
+  patch: 'diff --git a/django/core/mail/utils.py …',
+  test_patch: 'diff --git a/tests/mail/tests.py …',
+  FAIL_TO_PASS: '["test_unicode_dns (mail.tests.MailTests)"]',
+  PASS_TO_PASS: '["test_ascii (mail.tests.MailTests)"]',
+  version: '3.0',
+  environment_setup_commit: '419a78300f7cd27611196e1e464d50fd0385ff27',
+}
+
+describe('assertSweInstance', () => {
+  it('accepts a complete metadata record and returns the typed instance', () => {
+    const inst = assertSweInstance('django__django-11532', full)
+    expect(inst.instance_id).toBe('django__django-11532')
+    expect(inst.patch).toContain('diff --git')
+    expect(inst.version).toBe('3.0')
+  })
+
+  it('normalizes absent optional fields to null', () => {
+    const inst = assertSweInstance('django__django-11532', {
+      ...full,
+      version: undefined,
+      environment_setup_commit: '',
+    })
+    expect(inst.version).toBeNull()
+    expect(inst.environment_setup_commit).toBeNull()
+  })
+
+  it('throws on a missing/empty load-bearing field (never a silent blank)', () => {
+    expect(() => assertSweInstance('django__django-11532', { ...full, problem_statement: '' })).toThrow(
+      /problem_statement/,
+    )
+    expect(() => assertSweInstance('django__django-11532', { ...full, patch: undefined })).toThrow(/patch/)
+    expect(() => assertSweInstance('django__django-11532', undefined)).toThrow(/no metadata/)
+  })
+
+  it('throws on an instance_id mismatch', () => {
+    expect(() => assertSweInstance('astropy__astropy-13033', full)).toThrow(/instance_id/)
+  })
+})

package/src/swe-arena/calibrate.ts

@@ -0,0 +1,116 @@
+/**
+ * Dual calibration — an instance may enter an experiment only if BOTH gates
+ * hold, mirroring the experiment's `calibrate.sh` (repro gate) plus the
+ * gold-family judge rows M1 reconciles on (official-judge gate):
+ *
+ *  1. REPRO GATE (`verifyCalibrated`): on a pristine image-materialized
+ *     workspace the self-repro verify command must FAIL at base_commit and
+ *     PASS once the official gold patch is applied (git apply, then
+ *     `patch --fuzz=3` fallback — several Verified gold patches only apply
+ *     fuzzily to their own base). A verify that can't see the gold fix can't
+ *     grade an arm's fix.
+ *
+ *  2. OFFICIAL-JUDGE GOLD GATE (`goldOfficialResolved`): the official swebench
+ *     judge (via serialized-judge → adapter.judge) must resolve the gold patch
+ *     itself. psf__requests-2931/-2317 proved a judge can be blind on an
+ *     instance whose verify calibrates fine — those became the excluded
+ *     "gold-ungradeable" rows in the M1 denominator.
+ */
+
+import { rm, writeFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { materializeWorkspace } from './materialize'
+import { run, shq } from './proc'
+import type { SerializedJudge } from './serialized-judge'
+
+export interface CalibrateOptions {
+  instanceId: string
+  image: string
+  baseCommit: string
+  /** The official gold patch text (task-meta `patch`). */
+  goldPatch: string
+  /**
+   * Self-repro verify command, run via `bash -c` with cwd = the workspace
+   * (calibrate.sh ran `bash verify/<iid>.sh` from inside the tree).
+   */
+  verifyCmd: string
+  /** Scratch root; two throwaway workspaces are created and removed under it. */
+  workDir: string
+  /** Judge for the official gold gate. */
+  judge: SerializedJudge
+  /** Ceiling for one verify run (repro scripts self-limit at 180s; this is a backstop). */
+  verifyTimeoutMs?: number
+  /** Keep the calibration workspaces for post-mortem. Default: removed. */
+  keepWorkspaces?: boolean
+}
+
+export interface CalibrationResult {
+  iid: string
+  /** rc of verify on pristine base (must be nonzero). */
+  baseRc: number
+  /** rc of applying the gold patch (0 via git apply or the fuzz fallback). */
+  goldApplyRc: number
+  /** rc of verify with gold applied (must be zero). */
+  goldRc: number
+  /** base FAILS and gold PASSES. */
+  verifyCalibrated: boolean
+  /** Official judge resolves the gold patch. */
+  goldOfficialResolved: boolean
+  /** verifyCalibrated && goldOfficialResolved — the experiment admission bar. */
+  experimentValid: boolean
+}
+
+async function runVerify(verifyCmd: string, ws: string, timeoutMs: number): Promise<number> {
+  const res = await run('bash', ['-c', verifyCmd], { cwd: ws, timeoutMs })
+  return res.code
+}
+
+/**
+ * Apply a patch file with calibrate.sh's exact fallback chain:
+ * `git apply --whitespace=nowarn`, then `patch -p1 --fuzz=3` on failure.
+ * Returns the rc of the LAST attempt (0 = applied).
+ */
+export async function applyPatchWithFallback(ws: string, patchFile: string): Promise<number> {
+  const gitApply = await run('git', ['apply', '--whitespace=nowarn', patchFile], { cwd: ws })
+  if (gitApply.code === 0) return 0
+  const fuzz = await run('bash', ['-c', `patch -p1 --fuzz=3 < ${shq(patchFile)}`], { cwd: ws })
+  return fuzz.code
+}
+
+export async function calibrateInstance(opts: CalibrateOptions): Promise<CalibrationResult> {
+  const { instanceId, image, baseCommit, verifyCmd, judge } = opts
+  const verifyTimeoutMs = opts.verifyTimeoutMs ?? 600_000
+  const baseWs = join(opts.workDir, `cal-base-${instanceId}`)
+  const goldWs = join(opts.workDir, `cal-gold-${instanceId}`)
+  const goldPatchFile = join(opts.workDir, `${instanceId}.gold.patch`)
+
+  try {
+    await materializeWorkspace({ instanceId, image, baseCommit, dest: baseWs })
+    const baseRc = await runVerify(verifyCmd, baseWs, verifyTimeoutMs)
+
+    await materializeWorkspace({ instanceId, image, baseCommit, dest: goldWs })
+    await writeFile(goldPatchFile, opts.goldPatch)
+    const goldApplyRc = await applyPatchWithFallback(goldWs, goldPatchFile)
+    const goldRc = await runVerify(verifyCmd, goldWs, verifyTimeoutMs)
+
+    const verifyCalibrated = baseRc !== 0 && goldRc === 0
+
+    const goldVerdict = await judge.judge(instanceId, goldPatchFile, 'gold')
+    const goldOfficialResolved = goldVerdict.resolved === true
+
+    return {
+      iid: instanceId,
+      baseRc,
+      goldApplyRc,
+      goldRc,
+      verifyCalibrated,
+      goldOfficialResolved,
+      experimentValid: verifyCalibrated && goldOfficialResolved,
+    }
+  } finally {
+    if (!opts.keepWorkspaces) {
+      await rm(baseWs, { recursive: true, force: true })
+      await rm(goldWs, { recursive: true, force: true })
+    }
+  }
+}

package/src/swe-arena/capabilities.mts

@@ -0,0 +1,76 @@
+/**
+ * Substrate passthrough guard for the improve-loop options this harness
+ * depends on: `selfImprove` forwarding `premeasuredBaseline` and
+ * `budget.maxImprovementShots` into the loop (merged to agent-eval main).
+ *
+ * The remaining risk is a STALE INSTALL: the bench consumes agent-eval via a
+ * pnpm `file:` dependency, which snapshots the checkout at install time — a
+ * rebuilt-but-never-reinstalled substrate silently reverts to a bundle whose
+ * `selfImprove` DROPS both options. Silent drop is the worst failure mode
+ * here (a "premeasured" baseline would quietly re-run and re-spend; the depth
+ * dial would quietly pin to the lib default), so the guard FAILS LOUD instead
+ * of falling back.
+ *
+ * The probe reads the RESOLVED `@tangle-network/agent-eval/contract` module
+ * text (the bundle that contains the compiled `selfImprove`) and requires
+ * both option names. Verified against the pre-merge build: that bundle
+ * contained ZERO occurrences of either literal (selfImprove never named them;
+ * `runOptimization`'s own seam compiles into a different chunk), so the probe
+ * cannot false-positive on a stale substrate.
+ */
+
+import { readFileSync } from 'node:fs'
+import { createRequire } from 'node:module'
+import { fileURLToPath } from 'node:url'
+
+export interface ImproveLoopPassthroughCaps {
+  /** `selfImprove` forwards `premeasuredBaseline` into the loop. */
+  premeasuredBaseline: boolean
+  /** `selfImprove` forwards `budget.maxImprovementShots` into the loop. */
+  maxImprovementShots: boolean
+}
+
+/** Pure probe over the contract module text — unit-testable. */
+export function detectPassthroughCaps(contractModuleText: string): ImproveLoopPassthroughCaps {
+  return {
+    premeasuredBaseline: contractModuleText.includes('premeasuredBaseline'),
+    maxImprovementShots: contractModuleText.includes('maxImprovementShots'),
+  }
+}
+
+/** Resolve the installed agent-eval contract bundle's file path. */
+export function resolveContractModulePath(): string {
+  const require = createRequire(import.meta.url)
+  const url = import.meta.resolve?.('@tangle-network/agent-eval/contract')
+  if (typeof url === 'string' && url.startsWith('file:')) return fileURLToPath(url)
+  return require.resolve('@tangle-network/agent-eval/contract')
+}
+
+/** Fail-loud stale-install guard: throws unless the resolved substrate names
+ *  BOTH passthrough options in its contract bundle. An unreadable bundle also
+ *  throws — nothing here ever downgrades to a silent fallback. */
+export function assertSubstratePassthroughs(log: (msg: string) => void = () => {}): void {
+  let path: string
+  let caps: ImproveLoopPassthroughCaps
+  try {
+    path = resolveContractModulePath()
+    caps = detectPassthroughCaps(readFileSync(path, 'utf8'))
+  } catch (cause) {
+    throw new Error(
+      'substrate passthrough probe failed — cannot prove the installed agent-eval forwards ' +
+        `premeasuredBaseline/maxImprovementShots: ${(cause as Error).message}`,
+      { cause },
+    )
+  }
+  log(
+    `substrate caps (${path}): premeasuredBaseline=${caps.premeasuredBaseline} maxImprovementShots=${caps.maxImprovementShots}`,
+  )
+  const missing = (Object.keys(caps) as Array<keyof ImproveLoopPassthroughCaps>).filter((k) => !caps[k])
+  if (missing.length > 0) {
+    throw new Error(
+      `stale substrate install: the resolved agent-eval contract bundle (${path}) never names ` +
+        `${missing.join(' + ')}, so selfImprove would silently drop the option(s). ` +
+        'Rebuild the checkout (cd ~/code/agent-eval && git pull && pnpm build), then `pnpm install --force` in the bench.',
+    )
+  }
+}

package/src/swe-arena/capabilities.test.mts

@@ -0,0 +1,57 @@
+/**
+ * Substrate passthrough guard: the pure probe over module text plus the live
+ * contract check against the installed agent-eval bundle. The guard must pass
+ * cleanly on a substrate that threads premeasuredBaseline +
+ * maxImprovementShots and THROW LOUD (never fall back) on one that drops
+ * them — so a run against a stale install dies at t≈0, before it can
+ * silently re-spend its baseline.
+ */
+
+import { readFileSync } from 'node:fs'
+import { describe, expect, it } from 'vitest'
+import {
+  assertSubstratePassthroughs,
+  detectPassthroughCaps,
+  resolveContractModulePath,
+} from './capabilities.mts'
+
+describe('detectPassthroughCaps (pure)', () => {
+  it('an option name absent from the bundle reads as no capability', () => {
+    expect(detectPassthroughCaps('')).toEqual({ premeasuredBaseline: false, maxImprovementShots: false })
+    expect(detectPassthroughCaps('function selfImprove(opts) { return runSelfImprove(opts) }')).toEqual({
+      premeasuredBaseline: false,
+      maxImprovementShots: false,
+    })
+  })
+
+  it('the forwarding literals flip their capability independently', () => {
+    expect(detectPassthroughCaps('premeasuredBaseline: opts.premeasuredBaseline,')).toEqual({
+      premeasuredBaseline: true,
+      maxImprovementShots: false,
+    })
+    expect(detectPassthroughCaps('maxImprovementShots: budget.maxImprovementShots,')).toEqual({
+      premeasuredBaseline: false,
+      maxImprovementShots: true,
+    })
+  })
+})
+
+describe('live substrate guard', () => {
+  it('passes on a substrate that names both passthroughs, throws loud otherwise', () => {
+    const path = resolveContractModulePath()
+    expect(path).toMatch(/agent-eval/)
+    const caps = detectPassthroughCaps(readFileSync(path, 'utf8'))
+    // The guard's contract holds against whatever substrate is installed:
+    // both literals present ⇒ clean pass with the caps logged; anything less
+    // ⇒ a loud actionable error naming the stale bundle (never a silent
+    // fallback). A run against a stale install dies HERE, at t≈0.
+    if (caps.premeasuredBaseline && caps.maxImprovementShots) {
+      const logged: string[] = []
+      expect(() => assertSubstratePassthroughs((msg) => logged.push(msg))).not.toThrow()
+      expect(logged.join('\n')).toContain('premeasuredBaseline=true')
+      expect(logged.join('\n')).toContain('maxImprovementShots=true')
+    } else {
+      expect(() => assertSubstratePassthroughs()).toThrow(/stale substrate install/)
+    }
+  })
+})

package/src/swe-arena/capacity.ts

@@ -0,0 +1,194 @@
+/**
+ * Endpoint capacity gate — the typed port of `probe-capacity.sh`, generalized
+ * after a proven blind spot: the bash probe watched ONLY the z.ai coding
+ * endpoint while the supervisor BRAIN rides router.tangle.tools — three
+ * evolution rounds went infra-null because the gate said "capacity" while the
+ * router 503-stormed. Rule encoded here: gate every arm on the endpoint that
+ * arm actually calls; supervisor arms MUST include the router-path probe.
+ *
+ * Secrets discipline: probes spawn `dotenvx run … -- bash -c 'curl …'` from
+ * the secrets dir; the API key is referenced by NAME inside the child shell
+ * (single-quoted script, so it is never expanded — let alone logged — in this
+ * process).
+ */
+
+import { run } from './proc'
+import type { SecretsEnv } from './arms'
+
+export type CapacityProbe = (signal?: AbortSignal) => Promise<boolean>
+
+export interface EndpointCapacityGate {
+  /** Human label for status lines (e.g. 'z.ai-coding', 'router'). */
+  name: string
+  probe: CapacityProbe
+  /** Window passes when >= k of n probes succeed (bash: 3 of 4). */
+  kOfN: { k: number; n: number }
+  /** Consecutive passing windows required before opening. Default 1. */
+  steadyM?: number
+  /** Total wait budget; exceeded → gate reports no-capacity (orchestrate: 300 min). */
+  waitCeilingMs: number
+  /** Pause between probes inside a window (bash: 1s). */
+  probeIntervalMs?: number
+  /** Pause between windows while waiting (orchestrate: 30s). */
+  retryDelayMs?: number
+  onStatus?: (msg: string) => void
+}
+
+export function sleepWithSignal(ms: number, signal?: AbortSignal): Promise<void> {
+  signal?.throwIfAborted()
+  return new Promise((resolve, reject) => {
+    let timer: NodeJS.Timeout | undefined
+    const onAbort = () => {
+      if (timer) clearTimeout(timer)
+      signal?.removeEventListener('abort', onAbort)
+      reject(signal?.reason ?? new Error('capacity wait aborted'))
+    }
+    timer = setTimeout(() => {
+      signal?.removeEventListener('abort', onAbort)
+      resolve()
+    }, ms)
+    signal?.addEventListener('abort', onAbort, { once: true })
+  })
+}
+
+/** One k-of-n probe window. Exported for direct reuse (bash probe-capacity.sh body). */
+export async function probeWindow(
+  gate: EndpointCapacityGate,
+  signal?: AbortSignal,
+): Promise<{ ok: number; passed: boolean }> {
+  const { k, n } = gate.kOfN
+  let ok = 0
+  for (let i = 0; i < n; i++) {
+    signal?.throwIfAborted()
+    try {
+      if (await gate.probe(signal)) ok += 1
+    } catch {
+      // Endpoint failures count as a failed probe; caller cancellation does not.
+      signal?.throwIfAborted()
+    }
+    signal?.throwIfAborted()
+    if (i < n - 1) await sleepWithSignal(gate.probeIntervalMs ?? 1000, signal)
+  }
+  return { ok, passed: ok >= k }
+}
+
+/**
+ * Block until the endpoint shows steady capacity (steadyM consecutive passing
+ * k-of-n windows) or the ceiling elapses. Returns whether capacity was found —
+ * callers decide whether a closed gate skips the instance or aborts the run.
+ */
+export async function waitForCapacity(gate: EndpointCapacityGate, signal?: AbortSignal): Promise<boolean> {
+  signal?.throwIfAborted()
+  const steadyM = gate.steadyM ?? 1
+  const deadline = Date.now() + gate.waitCeilingMs
+  let consecutive = 0
+  for (;;) {
+    signal?.throwIfAborted()
+    const { ok, passed } = await probeWindow(gate, signal)
+    signal?.throwIfAborted()
+    gate.onStatus?.(`[${gate.name}] capacity: ${ok}/${gate.kOfN.n}${passed ? '' : ' (below k)'} steady=${passed ? consecutive + 1 : 0}/${steadyM}`)
+    if (passed) {
+      consecutive += 1
+      if (consecutive >= steadyM) return true
+    } else {
+      consecutive = 0
+    }
+    if (Date.now() >= deadline) return false
+    await sleepWithSignal(gate.retryDelayMs ?? 30_000, signal)
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Probe functions.
+// ---------------------------------------------------------------------------
+
+export interface HttpProbeSpec {
+  url: string
+  /** NAME of the env var holding the bearer key (resolved in the dotenvx child). */
+  apiKeyEnv: string
+  model: string
+  secrets: SecretsEnv
+  /** curl --max-time, seconds. Default 40 (probe-capacity.sh). */
+  maxTimeS?: number
+  /**
+   * max_tokens in the probe body. Default 8000 — glm-5.2 returns empty content
+   * below that (measured), and an empty-content 200 would be a lying probe.
+   */
+  maxTokens?: number
+}
+
+export const ZAI_CODING_ENDPOINT = 'https://api.z.ai/api/coding/paas/v4/chat/completions'
+export const ROUTER_ENDPOINT = 'https://router.tangle.tools/v1/chat/completions'
+
+/** Build the probe request body (probe-body.json semantics). */
+export function probeBody(model: string, maxTokens: number): string {
+  return JSON.stringify({
+    model,
+    messages: [{ role: 'user', content: 'Reply with the single word OK.' }],
+    max_tokens: maxTokens,
+    temperature: 0,
+  })
+}
+
+/**
+ * Generic chat-completions probe: true iff the endpoint returns HTTP 200
+ * within the time budget. The key stays inside the child shell.
+ */
+export function httpCapacityProbe(spec: HttpProbeSpec): CapacityProbe {
+  if (!/^[A-Z_][A-Z0-9_]*$/.test(spec.apiKeyEnv)) {
+    throw new Error(`invalid apiKeyEnv name: ${spec.apiKeyEnv}`)
+  }
+  const body = probeBody(spec.model, spec.maxTokens ?? 8000)
+  const maxTime = spec.maxTimeS ?? 40
+  return async (signal?: AbortSignal) => {
+    signal?.throwIfAborted()
+    // Body via stdin (--data @-) so the payload never sits on a command line.
+    // The HTTP code is marker-anchored because dotenvx writes its injection
+    // banner to the same stdout stream.
+    const script =
+      `curl -sS -o /dev/null -w "HTTP_CODE=%{http_code}" --max-time ${maxTime} ` +
+      `-X POST "$PROBE_URL" ` +
+      `-H "Authorization: Bearer $${spec.apiKeyEnv}" -H "Content-Type: application/json" ` +
+      `--data @-`
+    const argv = ['run', ...spec.secrets.envFiles.flatMap((f) => ['-f', f]), '--', 'bash', '-c', script]
+    const res = await run('dotenvx', argv, {
+      cwd: spec.secrets.secretsDir,
+      timeoutMs: (maxTime + 20) * 1000,
+      stdin: body,
+      env: { ...process.env, PROBE_URL: spec.url },
+      signal,
+    })
+    signal?.throwIfAborted()
+    return /HTTP_CODE=200\s*$/.test(res.stdout)
+  }
+}
+
+/** probe-capacity.sh's z.ai coding-plan probe (the WORKER path). */
+export function zaiCodingProbe(secrets: SecretsEnv, model = 'glm-5.2'): CapacityProbe {
+  return httpCapacityProbe({ url: ZAI_CODING_ENDPOINT, apiKeyEnv: 'ZAI_API_KEY', model, secrets })
+}
+
+/**
+ * Router-path probe (the BRAIN path — router.tangle.tools with TANGLE_API_KEY).
+ * Supervisor arms must gate on this; probing only z.ai is the proven blind spot.
+ */
+export function routerProbe(secrets: SecretsEnv, model = 'glm-5.2'): CapacityProbe {
+  return httpCapacityProbe({ url: ROUTER_ENDPOINT, apiKeyEnv: 'TANGLE_API_KEY', model, secrets })
+}
+
+/** The gates an arm must pass, by kind: solo → worker path; supervisor → BOTH paths. */
+export function gatesForArmKind(
+  kind: 'solo' | 'supervisor',
+  secrets: SecretsEnv,
+  opts: { waitCeilingMs?: number; model?: string; onStatus?: (msg: string) => void } = {},
+): EndpointCapacityGate[] {
+  const base = {
+    kOfN: { k: 3, n: 4 },
+    waitCeilingMs: opts.waitCeilingMs ?? 300 * 60_000,
+    ...(opts.onStatus ? { onStatus: opts.onStatus } : {}),
+  }
+  const worker: EndpointCapacityGate = { name: 'z.ai-coding', probe: zaiCodingProbe(secrets, opts.model), ...base }
+  if (kind === 'solo') return [worker]
+  const brain: EndpointCapacityGate = { name: 'router', probe: routerProbe(secrets, opts.model), ...base }
+  return [worker, brain]
+}