opencode-harness-kit 0.6.4 → 0.7.0

package/eval/ab.mjs

@@ -23,8 +23,8 @@
 // spawning a single agent. The grader has been wrong five times in this repo and
 // every fix so far would otherwise have cost another twenty runs.
 
-import { spawnSync } from "node:child_process"
-import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, statSync, symlinkSync, writeFileSync, cpSync } from "node:fs"
+import { spawn } from "node:child_process"
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync, cpSync } from "node:fs"
 import { createHash } from "node:crypto"
 import { homedir, tmpdir } from "node:os"
 import { join } from "node:path"
@@ -34,6 +34,7 @@ import { countFlag, numberFlag } from "./flags.mjs"
 import { gradeAnswer, gradeMutation, gradeMutationMany } from "./verify.mjs"
 import { checkPremises } from "./premises.mjs"
 import { parseJsonc, configDir } from "../src/profile.js"
+import { requestTokens } from "../src/budget.js"
 
 /**
  * The message of a caught value, whatever it turned out to be.
@@ -81,28 +82,112 @@ const suitePath = flag("--suite")
 // has been wrong five times in this repo and every fix would otherwise have
 // cost another twenty runs.
 const rescoreDir = flag("--rescore")
+// Where the streams go, and therefore whether a suite can be continued.
+//
+// Without it every run lands in a fresh temp directory, so an interrupted
+// suite has nothing to continue from: a 36-run suite stopped at run 13 pays
+// for those 13 again. With `--out` the verdict for each run is written beside
+// its stream, and a later invocation with the same `--out` skips what is
+// already there.
+const outFlag = flag("--out")
+// A ceiling on what the whole suite may spend, in context tokens - the same
+// number the guard counts. The free tier is one shared window and the suite is
+// the biggest single consumer of it; a cap is the difference between "the
+// suite stopped" and "the account is out until the window rolls". Counting
+// uncached input here made the cap 5.6 times too generous on the measured
+// streams, and 43 times on a single fully cached step.
+let suiteBudget = 0
+try {
+  const n = numberFlag("--budget")
+  if (n !== null) {
+    if (!(n > 0)) throw new Error("--budget must be more than 0 tokens, or the cap is removed")
+    suiteBudget = Math.floor(n)
+  }
+} catch (err) {
+  console.error(String(errorMessage(err)))
+  process.exit(1)
+}
 // The lane to run, for a suite that measures the kit's PROCESS rather than
 // its answers: a reviewer that does not edit, a verify that does not fix.
 const agent = flag("--agent")
 // One pathological run burned 4.4 minutes once. Without a ceiling a single bad
 // arm can hold the whole suite open indefinitely.
 //
-// A timeout of zero would REMOVE the ceiling - node only applies a timeout
-// when it is above zero - which is the opposite of what the flag says. And a
-// non-numeric one was NaN, which node rejects with a RangeError out of
-// spawnSync.
+// A timeout of zero would REMOVE the ceiling - a zero-delay timer fires at
+// once - which is the opposite of what the flag says. And a non-numeric one
+// was NaN, which a setTimeout treats as zero and kills every run instantly.
 let runTimeoutMs
 try {
-  const minutes = numberFlag("--timeout") ?? 600
-  if (minutes <= 0) throw new Error("--timeout must be more than 0, or the ceiling is removed")
-  runTimeoutMs = minutes * 1000
+  const seconds = numberFlag("--timeout") ?? 600
+  if (seconds <= 0) throw new Error("--timeout must be more than 0, or the ceiling is removed")
+  runTimeoutMs = seconds * 1000
 } catch (err) {
   console.error(String(errorMessage(err)))
   process.exit(1)
 }
 
+// A spent free route does not fail: it hangs, then the per-run ceiling kills
+// it, and the run is recorded as a timeout with no tool calls and no text.
+// At 600s a dead route costs ten minutes per attempt, and a 36-run suite
+// spends six hours measuring the provider's quota rather than the kit.
+//
+// Two in a row is not a slow task, it is a route that is not answering. Stop
+// and say so, with the raw streams kept so `--rescore` can run later.
+let failFast
+try {
+  failFast = countFlag("--fail-fast") ?? 2
+} catch (err) {
+  console.error(String(errorMessage(err)))
+  process.exit(1)
+}
+
+// The free tier does not always fail the process. `opencode run --format json`
+// prints an error event on stdout, then keeps retrying or exits 0 - and the
+// A/B treated that as an empty answer, kept going, and spent the suite
+// measuring the provider. The same text can appear on stderr. Either one is
+// the provider refusing the run, so the suite stops on the first one.
+const QUOTA_RE = /Free usage limit reached|FreeUsageLimitError|HTTP 429|"statusCode"\s*:\s*429/i
+
+/**
+ * The provider's own words when a run was refused for quota, or null.
+ *
+ * @param {string} stdout
+ * @param {string} stderr
+ * @returns {string | null}
+ */
+/**
+ * The first error event in a stream, or null.
+ *
+ * `opencode run --format json` reports a failure as a `{"type":"error"}` line
+ * and exits 0 or 1; the message is all the caller gets. The first one is the
+ * one that stopped the run.
+ *
+ * @param {string} stdout
+ * @returns {string | null}
+ */
+function streamError(stdout) {
+  for (const line of stdout.split("\n")) {
+    if (!line.includes('"type":"error"')) continue
+    try {
+      const ev = JSON.parse(line)
+      const msg = ev?.error?.data?.message ?? ev?.error?.message ?? ev?.error?.name
+      if (msg) return String(msg).slice(0, 240)
+    } catch {
+      return line.trim().slice(0, 240)
+    }
+  }
+  return null
+}
+
+function quotaReason(stdout, stderr) {
+  const text = `${stdout}\n${stderr}`
+  if (!QUOTA_RE.test(text)) return null
+  const line = text.split("\n").find((l) => QUOTA_RE.test(l)) ?? ""
+  return line.trim().slice(0, 240)
+}
+
 if (!target || (!task && !suitePath)) {
-  console.error('usage: node eval/ab.mjs --target <dir> (--task "<prompt>" | --suite <file>) [--reps 3] [--expect token]... [--timeout 600]')
+  console.error('usage: node eval/ab.mjs --target <dir> (--task "<prompt>" | --suite <file>) [--reps 3] [--expect token]... [--timeout 600] [--fail-fast 2]')
   process.exit(1)
 }
 if (!existsSync(target)) {
@@ -409,6 +494,13 @@ function liveModel() {
 // The comparison this file exists to make is "the same model, with the kit
 // and without it". The model is now pinned to make that literally true, and
 // the report prints it so a reader can check.
+//
+// The SMALL model is pinned too. `model` only covers the main loop; the title
+// agent runs on `small_model`, and an empty config leaves that at opencode's
+// default - which is a PAID model. A real A/B run made 1,073 title calls to
+// `gpt-5.4-nano` and every one came back "Insufficient account funds". It did
+// not consume the free window, but on a funded account it would have spent
+// money on titles for a benchmark that is supposed to be free.
 function makeNokitHome(model) {
   const home = mkdtempSync(join(tmpdir(), "opencode-ab-"))
   mkdirSync(join(home, ".config/opencode"), { recursive: true })
@@ -418,12 +510,26 @@ function makeNokitHome(model) {
   // not exist" off for exactly that case.
   /** @type {Record<string, string>} */
   const config = { $schema: "https://opencode.ai/config.json" }
-  if (model) config.model = model
+  if (model) {
+    config.model = model
+    config.small_model = model
+  }
   writeFileSync(join(home, ".config/opencode/opencode.json"), JSON.stringify(config, null, 2))
   const link = join(home, ".local/share/opencode")
   try {
     symlinkSync(join(REAL_HOME, ".local/share/opencode"), link, "dir")
   } catch {}
+  // And the cache, which is where opencode keeps `models.json` - the catalogue
+  // it resolves a `provider/model` string against. Without it a bare
+  // `model: "opencode/deepseek-v4.1-flash-free"` is a model the off arm has
+  // never heard of: every off run failed with "Unexpected server error" and
+  // the cause was only visible in `--print-logs` as ProviderModelNotFoundError.
+  // The cache is opencode's, not the kit's, so sharing it keeps the arm
+  // kit-free without making it a different installation.
+  mkdirSync(join(home, ".cache"), { recursive: true })
+  try {
+    symlinkSync(join(REAL_HOME, ".cache/opencode"), join(home, ".cache/opencode"), "dir")
+  } catch {}
   return home
 }
 
@@ -450,7 +556,7 @@ function fingerprint(dir) {
 
 function parseStream(file, root = TARGET_REAL) {
   const reads = [], searches = [], others = [], escaped = new Set(), readOutside = new Set()
-  let steps = 0, inputTokens = 0, reasoning = 0, finalCtx = 0, text = "", first = null, last = null
+  let steps = 0, inputTokens = 0, contextTokens = 0, reasoning = 0, finalCtx = 0, text = "", first = null, last = null
   for (const line of readFileSync(file, "utf8").split("\n")) {
     const s = line.trim()
     if (!s) continue
@@ -485,7 +591,14 @@ function parseStream(file, root = TARGET_REAL) {
     } else if (d.type === "step_finish") {
       const t = d.part?.tokens ?? {}
       steps++
+      // `inputTokens` is the uncached part only. It is kept because it is what
+      // every table before this one reported, and comparing against those
+      // numbers needs the same definition. `contextTokens` is the cost: the
+      // same step reports a few hundred uncached against tens of thousands
+      // cached, so a suite budget built on `inputTokens` was 5.6 times
+      // too generous.
       inputTokens += t.input ?? 0
+      contextTokens += requestTokens(t)
       reasoning += t.reasoning ?? 0
       finalCtx = (t.input ?? 0) + (t.cache?.read ?? 0)
     } else if (d.type === "text") {
@@ -502,6 +615,7 @@ function parseStream(file, root = TARGET_REAL) {
     searches: searches.length,
     other: others.reduce((a, t) => (a[t] = (a[t] ?? 0) + 1, a), {}),
     inputTokens,
+    contextTokens,
     reasoning,
     finalCtx,
     seconds: first !== null && last !== null ? elapsedSeconds(first, last) : null,
@@ -511,51 +625,140 @@ function parseStream(file, root = TARGET_REAL) {
   }
 }
 
-function runArmCapture(arm, home, id, idx, outDir, prompt, runDir) {
-  const out = join(outDir, `${id}-${arm}-${idx}.json`)
-  const env = { ...process.env }
-  if (arm === "off") {
-    env.HOME = home
-    // The kit-free home is only kit-free if the config is not found
-    // somewhere else. `OPENCODE_CONFIG_DIR` is what opencode looks at
-    // FIRST, so with it set the "off" arm loaded the kit's own config, used
-    // its rules, and used its model - and the arm was not an off arm at all.
-    delete env.OPENCODE_CONFIG_DIR
+// The child that is running right now, if any. Needed because `spawn` is
+// async: the event loop stays free, so a Ctrl-C reaches this file instead of
+// being eaten by a blocked spawnSync. The old code could not react to one -
+// the parent died, the agent was reparented to systemd and kept running
+// against the real auth store. One such run survived 2h54m.
+/** @type {import("node:child_process").ChildProcess | null} */
+let activeChild = null
+
+function killActiveChild() {
+  const child = activeChild
+  if (!child || child.killed) return
+  // The child is its own process-group leader (detached), so a negative pid
+  // reaches the helpers `opencode` starts. A plain child.kill() left those
+  // running after the parent was gone.
+  try {
+    process.kill(-child.pid, "SIGKILL")
+  } catch {
+    try {
+      child.kill("SIGKILL")
+    } catch {}
   }
-  const t0 = Date.now()
-  // --dir is not optional. Nested inside an opencode session, `opencode run`
-  // otherwise inherits the *parent* session's project directory and reads the
-  // wrong repository entirely - which it did on the first pilot.
-  // The LANE, when the suite wants one. The kit-off arm has no kit and
-  // therefore no lanes, so the flag is only for the arm that has them - which
-  // is the point: the lane is part of the kit, and measuring it means
-  // measuring the kit.
-  const argv = ["run", "--dir", runDir, "--format", "json"]
-  if (arm === "on" && agent) argv.push("--agent", agent)
-  argv.push(prompt)
-  const res = spawnSync("opencode", argv, {
-    cwd: runDir, env, encoding: "utf8", maxBuffer: 256 * 1024 * 1024,
-    timeout: runTimeoutMs, killSignal: "SIGKILL",
+}
+
+for (const [sig, code] of /** @type {const} */ ([["SIGINT", 130], ["SIGTERM", 143]])) {
+  process.on(sig, () => {
+    killActiveChild()
+    process.exit(code)
+  })
+}
+
+/**
+ * Run one arm and capture its stream.
+ *
+ * Async on purpose. The timeout is a timer that kills the process GROUP; the
+ * signal handlers above kill the same group on Ctrl-C. Neither is possible
+ * with spawnSync, which blocks the event loop for the whole run.
+ *
+ * @param {"on" | "off"} arm
+ * @param {string | null} home
+ * @param {string} id
+ * @param {number} idx
+ * @param {string} outDir
+ * @param {string} prompt
+ * @param {string} runDir
+ */
+function runArmCapture(arm, home, id, idx, outDir, prompt, runDir) {
+  return new Promise((resolve) => {
+    const out = join(outDir, `${id}-${arm}-${idx}.json`)
+    const env = { ...process.env }
+    // The kit's own guard must not decide the outcome of the suite. 120 runs
+    // spend more than the shared window's stop, so from the run that crosses it
+    // the guard would refuse tool calls - and every run after that point would
+    // be scored as a wrong answer for a reason the suite is not measuring. The
+    // suite has its own cap (`--budget`), and it is the one that counts here.
+    // The ledger is pointed at the run directory for the same reason: a
+    // benchmark should not charge the account's real window, or leave it
+    // looking spent to the session that started the benchmark.
+    env.OC_HARNESS_BUDGET_WARN = "0"
+    env.OC_HARNESS_BUDGET_STOP = "0"
+    env.OC_HARNESS_BUDGET_WINDOW_WARN = "0"
+    env.OC_HARNESS_BUDGET_WINDOW_STOP = "0"
+    env.OC_HARNESS_BUDGET_FILE = join(outDir, "budget.json")
+    if (arm === "off" && home) {
+      env.HOME = home
+      // The kit-free home is only kit-free if the config is not found
+      // somewhere else. `OPENCODE_CONFIG_DIR` is what opencode looks at
+      // FIRST, so with it set the "off" arm loaded the kit's own config, used
+      // its rules, and used its model - and the arm was not an off arm at all.
+      delete env.OPENCODE_CONFIG_DIR
+    }
+    const t0 = Date.now()
+    // --dir is not optional. Nested inside an opencode session, `opencode run`
+    // otherwise inherits the *parent* session's project directory and reads the
+    // wrong repository entirely - which it did on the first pilot.
+    // The LANE, when the suite wants one. The kit-off arm has no kit and
+    // therefore no lanes, so the flag is only for the arm that has them - which
+    // is the point: the lane is part of the kit, and measuring it means
+    // measuring the kit.
+    const argv = ["run", "--dir", runDir, "--format", "json"]
+    if (arm === "on" && agent) argv.push("--agent", agent)
+    argv.push(prompt)
+    const child = spawn("opencode", argv, {
+      cwd: runDir,
+      env,
+      // A process-group leader, so the timeout and the signal handlers can
+      // take the agent's helpers with it.
+      detached: true,
+      stdio: ["ignore", "pipe", "pipe"],
+    })
+    activeChild = child
+    let stdout = ""
+    let stderr = ""
+    let timedOut = false
+    let settled = false
+    /** @type {NodeJS.Timeout | undefined} */
+    let timer
+
+    const finish = (code) => {
+      if (settled) return
+      settled = true
+      clearTimeout(timer)
+      if (activeChild === child) activeChild = null
+      // The stream is the evidence --rescore needs later, so it is written
+      // even when the run was killed or refused.
+      writeFileSync(out, stdout)
+      resolve({
+        out,
+        code: code === 0 ? 0 : 1,
+        timedOut,
+        wall: elapsedSeconds(t0, Date.now()),
+        stderr,
+        quota: quotaReason(stdout, stderr),
+      })
+    }
+
+    child.stdout?.setEncoding("utf8")
+    child.stdout?.on("data", (d) => {
+      stdout += d
+    })
+    child.stderr?.setEncoding("utf8")
+    child.stderr?.on("data", (d) => {
+      stderr += d
+    })
+    child.on("error", (err) => {
+      stderr += `\n${errorMessage(err)}`
+      finish(null)
+    })
+    child.on("close", (code) => finish(code))
+
+    timer = setTimeout(() => {
+      timedOut = true
+      killActiveChild()
+    }, runTimeoutMs)
   })
-  const stdout = res.stdout ?? ""
-  writeFileSync(out, stdout)
-  // Whether the run was killed by the ceiling.
-  //
-  // The old check was `stdout === null`, and `spawnSync` NEVER returns null
-  // for stdout - it returns the partial output and reports the kill through
-  // `error` and `signal`. So the timeout branch could never fire, every run
-  // scored as a success, and a run that hung until the ceiling was counted in
-  // the correctness ratio and in every median.
-  const timedOut =
-    /** @type {any} */ (res.error)?.code === "ETIMEDOUT" ||
-    res.signal === "SIGKILL" ||
-    (res.status === null && Date.now() - t0 >= runTimeoutMs)
-  return {
-    out,
-    code: res.status === 0 ? 0 : 1,
-    timedOut,
-    wall: elapsedSeconds(t0, Date.now()),
-  }
 }
 
 const armModel = liveModel()
@@ -564,19 +767,227 @@ if (!armModel) {
   console.warn("        which is not necessarily the model the kit-on arm uses.")
 }
 const nokit = rescoreDir ? null : makeNokitHome(armModel)
-const outDir = rescoreDir ?? mkdtempSync(join(tmpdir(), "opencode-ab-out-"))
+// `--out` is what makes a suite resumable: the streams and their verdicts go
+// to a directory the caller names, so a second invocation can pick up where
+// the first stopped instead of paying for the same runs again.
+const outDir = rescoreDir ?? outFlag ?? mkdtempSync(join(tmpdir(), "opencode-ab-out-"))
+if (outFlag && !rescoreDir) mkdirSync(outFlag, { recursive: true })
 const before = fingerprint(target)
 
+// A resume trusts the sidecars in `--out`, and every one of them was measured
+// against a target, a suite and a model. Two of those can change between
+// invocations without any file name changing, and the mixed run reads as one
+// arm's result. The manifest is written once, when the directory is new; a
+// directory from before the manifest existed is adopted with a warning rather
+// than refused, because the runs in it were still measured against something
+// and the warning says which.
+const suiteHash = createHash("sha256")
+  .update(suitePath ? readFileSync(suitePath) : (task ?? ""))
+  .digest("hex")
+const manifestFile = join(outDir, "run-manifest.json")
+const manifest = { v: 1, suiteHash, task: task ?? null, target: TARGET_REAL, fingerprint: before, model: armModel, reps, timeoutMs: runTimeoutMs }
+if (outFlag && !rescoreDir) {
+  if (existsSync(manifestFile)) {
+    const old = JSON.parse(readFileSync(manifestFile, "utf8"))
+    const changed = ["fingerprint", "suiteHash", "model"].filter((k) => old[k] !== manifest[k])
+    if (changed.length) {
+      console.error(`FAIL: --out ${outDir} was measured against a different ${changed.join(", ")}, so resuming it would mix two experiments.`)
+      console.error(`  manifest: ${manifestFile}`)
+      console.error(`  recorded ${changed.map((k) => `${k}=${JSON.stringify(old[k])}`).join(", ")}`)
+      console.error(`  now      ${changed.map((k) => `${k}=${JSON.stringify(manifest[k])}`).join(", ")}`)
+      console.error(`  use a new --out, or restore the target/suite/model the manifest names.`)
+      process.exit(2)
+    }
+  } else {
+    const sidecars = readdirSync(outDir).filter((f) => f.endsWith(".result.json"))
+    if (sidecars.length) {
+      console.warn(`  warn: ${outDir} has ${sidecars.length} result(s) but no run-manifest.json;`)
+      console.warn(`        adopting them as measured against this target, suite and model.`)
+      manifest.adopted = true
+    }
+    writeFileSync(manifestFile, JSON.stringify(manifest, null, 2) + "\n")
+  }
+}
+
+/**
+ * The recorded verdict for one run, if `--out` was used and it is there.
+ *
+ * A sidecar rather than the stream alone, because grading a mutation needs the
+ * copy of the repository the run produced, and that copy is deleted. The
+ * verdict is written when it is known, and read back unchanged on a resume -
+ * re-grading a saved answer would be a second, possibly different, opinion.
+ *
+ * @param {string} id
+ * @param {"on" | "off"} arm
+ * @param {number} i
+ * @returns {any | null}
+ */
+function resumeRun(id, arm, i) {
+  if (!outFlag || rescoreDir) return null
+  const file = join(outDir, `${id}-${arm}-${i}.result.json`)
+  if (!existsSync(file)) return null
+  try {
+    const saved = JSON.parse(readFileSync(file, "utf8"))
+    // A sidecar from a different suite or a half-written file is not a
+    // result: the id and arm have to match, and there has to be a verdict.
+    if (saved?.id !== id || saved?.arm !== arm || saved?.i !== i) return null
+    if (!("correct" in saved)) return null
+    // A sidecar with no verdict and no step is not a result: the route was
+    // already dead when it was written. Skipping it would make the hole
+    // permanent on every later resume - a dead window that leaves one empty
+    // sidecar per attempt never fills in. A timeout that did work is a
+    // censored result, and stays skipped.
+    if (saved.correct === null && (saved.steps ?? 0) === 0) return null
+    // A sidecar written by an older version is missing fields the report
+    // reads - `escaped` crashed the whole report on a resumed run, after every
+    // run had already been paid for. The report treats a missing list as an
+    // empty one; a resume must not turn a finished suite into a stack trace.
+    if (!Array.isArray(saved.escaped)) saved.escaped = []
+    if (!Array.isArray(saved.tools)) saved.tools = []
+    if (typeof saved.text !== "string") saved.text = ""
+    if (typeof saved.steps !== "number") saved.steps = 0
+    // The sidecar holds the verdict, the stream holds the cost. A sidecar
+    // written before the counter counted cached context would otherwise resume
+    // with a number 5.6 times too small, and the suite's own cap would
+    // read as unspent. Only the token fields are taken: `escaped` is relative
+    // to a run copy that is gone, and re-deriving it here would report every
+    // path as an escape.
+    const stream = join(outDir, `${id}-${arm}-${i}.json`)
+    if (existsSync(stream)) {
+      try {
+        const m = parseStream(stream, TARGET_REAL)
+        if (m.steps > 0) {
+          saved.inputTokens = m.inputTokens
+          saved.contextTokens = m.contextTokens
+          // The report prints medians of these too, and a sidecar from before
+          // the counter counted the cached prefix has both at zero. Leaving
+          // them stale would make "final ctx" disagree with "context tokens"
+          // for the same run.
+          saved.finalCtx = m.finalCtx
+          saved.reasoning = m.reasoning
+          // Write it back: the correction is the whole reason the stream is
+          // read, and a second resume should not depend on the stream still
+          // being there.
+          saveRun(saved)
+        }
+      } catch {
+        // An unreadable stream is not a reason to throw away a scored run.
+      }
+    }
+    return saved
+  } catch {
+    return null
+  }
+}
+
+/**
+ * Record one run's verdict so a later invocation can reuse it.
+ *
+ * @param {any} run
+ */
+function saveRun(run) {
+  if (!outFlag || rescoreDir) return
+  const file = join(outDir, `${run.id}-${run.arm}-${run.i}.result.json`)
+  const tmp = `${file}.${process.pid}.tmp`
+  writeFileSync(tmp, JSON.stringify(run))
+  renameSync(tmp, file)
+}
+
 console.log(`A/B: ${jobs.length} task(s) x ${reps} rep(s) per arm${rescoreDir ? ", rescored" : ", alternating"}`)
 console.log(`  target  ${target}`)
 for (const j of jobs) console.log(`  task    ${j.id}`)
-console.log(`  timeout ${runTimeoutMs / 1000}s per run`)
+console.log(`  timeout ${runTimeoutMs / 1000}s per run, giving up after ${failFast} empty run(s)`)
 console.log(`  model   ${armModel ?? "(opencode's default - NOT pinned)"}`)
 if (nokit) console.log(`  kit-off home ${nokit}`)
 console.log("")
 
+// One cheap run before the suite, because the expensive failure is not a
+// wrong answer, it is spending the first twenty runs discovering the route is
+// closed. A preflight costs a few hundred tokens and turns that into one.
+//
+// `--no-preflight` skips it: a paid route has no window to spend, and the
+// check is a round trip.
+if (!rescoreDir && !args.includes("--no-preflight")) {
+  const probeDir = mkdtempSync(join(tmpdir(), "opencode-ab-preflight-"))
+  process.stdout.write("preflight ... ")
+  // Into the probe directory, not `--out`: a preflight stream has no task and
+  // would only be skipped with a warning by a later --rescore.
+  const probe = await runArmCapture("off", nokit, "preflight", 0, probeDir, "Reply with the single word: ok", probeDir)
+  if (probe.quota) {
+    console.log("")
+    console.error("FAIL: the model route is out of quota before the suite started, so nothing was measured.")
+    console.error(`  ${probe.quota}`)
+    console.error("  The free tier is one shared window: this suite and every chat session draw on it.")
+    console.error(`  preflight stream kept: ${probe.out}`)
+    process.exit(2)
+  }
+  if (probe.timedOut) {
+    console.log("")
+    console.error(`FAIL: the preflight did not answer within ${runTimeoutMs / 1000}s.`)
+    console.error("  A spent or unreachable route looks like this. Nothing was measured.")
+    console.error("  Check it by hand: opencode run --print-logs -m <model> hi")
+    console.error(`  preflight stream kept: ${probe.out}`)
+    process.exit(2)
+  }
+  // Answering is the test, not exiting. A run that reports an error and exits
+  // in three seconds used to print "ok" here - the first suite with this
+  // preflight passed it and then spent eighteen runs discovering that every
+  // off arm had failed.
+  const probeErr = streamError(readFileSync(probe.out, "utf8"))
+  if (probeErr) {
+    console.log("")
+    console.error("FAIL: the preflight run failed before the suite started, so nothing was measured.")
+    console.error(`  ${probeErr}`)
+    console.error("  A model the installation cannot resolve, or a provider that refuses the request,")
+    console.error("  fails every run the same way. Run one by hand to see the whole log:")
+    console.error("    opencode run --print-logs -m <model> hi")
+    console.error(`  preflight stream kept: ${probe.out}`)
+    process.exit(2)
+  }
+  // Exiting is not answering. A spent route can exit 0 with an empty stream -
+  // the shape the fail-fast test uses - and a preflight that reads only the
+  // exit code calls that "ok", after which the suite spends one timeout per
+  // run learning otherwise. The suite needs a step and a word.
+  const probeMetrics = parseStream(probe.out, realpathSync(probeDir))
+  if (probe.code !== 0 || probeMetrics.steps === 0 || probeMetrics.text.trim() === "") {
+    console.log("")
+    console.error("FAIL: the preflight produced no answer, so nothing was measured.")
+    console.error(`  exit code ${probe.code}, ${probeMetrics.steps} step(s), ${probeMetrics.text.trim().length} character(s) of text.`)
+    console.error("  A spent route or a broken install looks like this. Nothing was measured.")
+    console.error("  Check it by hand: opencode run --print-logs -m <model> hi")
+    console.error(`  preflight stream kept: ${probe.out}`)
+    process.exit(2)
+  }
+  console.log(`ok (${probe.wall}s)`)
+}
+
 const runs = []
 const keptCopies = []
+// Consecutive runs that produced neither a tool call nor a word. See
+// --fail-fast above: this is the shape of a spent route, not of a slow task.
+let deadStreak = 0
+
+/**
+ * Stop the suite when the route, not the task, is what failed.
+ *
+ * @param {number} n
+ * @param {string} detail
+ * @param {{ id: string }} job
+ * @param {"on" | "off"} arm
+ * @param {number} i
+ */
+function stopDeadRoute(n, detail, job, arm, i) {
+  console.log("")
+  console.error(`FAIL: the model route is not answering.`)
+  console.error(`  ${n} runs in a row ${detail},`)
+  console.error(`  most recently ${job.id} ${arm}-${i} on ${armModel ?? "opencode's default model"}.`)
+  console.error(`  A spent free route looks exactly like this; the provider's reason is only`)
+  console.error(`  printed by \`opencode run --print-logs\`. Nothing was measured, so this`)
+  console.error(`  stops instead of timing out one run at a time.`)
+  console.error(`  raw streams kept: ${outDir}`)
+  console.error(`  rescore them later: node eval/ab.mjs --rescore ${outDir}, with the same --suite or --task`)
+  process.exit(2)
+}
 if (rescoreDir) {
   const byId = new Map(jobs.map((j) => [j.id, j]))
   for (const f of readdirSync(rescoreDir).filter((x) => x.endsWith(".json")).sort()) {
@@ -589,11 +1000,29 @@ if (rescoreDir) {
     }
     const file = join(rescoreDir, f)
     const s = parseStream(file, TARGET_REAL)
-    const invalid = s.steps === 0
+    // The sidecar is the only place that knows a run hit the ceiling: its
+    // stream stops mid-sentence and looks exactly like a run that answered
+    // nothing. Grading it as wrong is the mistake this file warns about, and
+    // the re-score path made it - four runs that timed out at 12 steps came
+    // back as four wrong answers.
+    let timedOut = false
+    let sideSaved = null
+    const side = join(rescoreDir, f.replace(/\.json$/, ".result.json"))
+    if (existsSync(side)) {
+      try {
+        sideSaved = JSON.parse(readFileSync(side, "utf8"))
+        timedOut = sideSaved.timedOut === true
+      } catch {
+        // A sidecar that will not parse leaves the run scored on its stream.
+      }
+    }
+    const invalid = s.steps === 0 || timedOut
     let ok = null
     let why
     if (job.mutate) {
       why = "mutation not rescored"
+    } else if (timedOut) {
+      why = "TIMED OUT, not scored"
     } else if (invalid) {
       why = "no output, not scored"
     } else {
@@ -601,17 +1030,55 @@ if (rescoreDir) {
       // the answers differently from the run it is re-scoring would be a
       // measurement of the grader, and the number it produced would not be
       // comparable with the one it is meant to explain.
-      const g = gradeAnswer(target, s.text, job.want, { exclude: SUITE_FILES })
+      // `want` is the suite's field and `expect` is the one a bare `--task`
+      // sets; reading only the first graded every single-task re-score against
+      // the string "undefined" and reported "none held" for a right answer.
+      const g = gradeAnswer(target, s.text, job.want ?? job.expect, { exclude: SUITE_FILES })
       ok = g.ok
       why = g.ok ? `cited ${g.cited}` : g.cited === 0 ? "no citation" : `cited ${g.cited}, none held`
     }
-    runs.push({ id: job.id, arm: m[2], i: Number(m[3]), ...s, wall: s.seconds, code: 0, timedOut: false, invalid: invalid || job.mutate, correct: ok, why })
+    // The wall time is the ceiling the run was measured against, not the
+    // span of its last event. A run killed at 180s has a stream that stops
+    // at 58s, and reading the stream alone reported it as a 58s run.
+    runs.push({ id: job.id, arm: m[2], i: Number(m[3]), ...s, wall: sideSaved?.wall ?? s.seconds, code: 0, timedOut, invalid: invalid || job.mutate, correct: ok, why })
     console.log(`  ${job.id} ${m[2]}-${m[3]} ... ${why}`)
   }
   console.log("")
 } else for (const job of jobs) {
   for (let i = 1; i <= reps; i++) {
-    for (const arm of ["on", "off"]) {
+    for (const arm of /** @type {const} */ (["on", "off"])) {
+      // A run already in `--out` is not run again. This is the difference
+      // between an interrupted suite costing its remaining runs and costing
+      // all of them: a 36-run suite stopped at 13 paid for 13 twice.
+      const resumed = resumeRun(job.id, arm, i)
+      if (resumed) {
+        runs.push(resumed)
+        console.log(`  ${job.id} ${arm}-${i} ... resumed: ${resumed.why ?? (resumed.correct ? "correct" : "wrong")}`)
+        continue
+      }
+      // Say it out loud when a sidecar was there but is not a result. Silence
+      // would make the re-run look like a missing run, and the extra cost
+      // like a bug in the resume.
+      if (outFlag && !rescoreDir && existsSync(join(outDir, `${job.id}-${arm}-${i}.result.json`))) {
+        console.log(`  ${job.id} ${arm}-${i} ... retrying (previous run was not scored)`)
+      }
+      // A ceiling on the suite's own spend, checked before each run rather
+      // than after: the point is not to buy the last run, it is to stop with
+      // room left in the window.
+      if (suiteBudget > 0) {
+        // An older sidecar has no `contextTokens`; fall back to the uncached
+        // count rather than to zero, and let resumeRun() fill the real number
+        // in from the stream when the stream is still there.
+        const spent = runs.reduce((n, r) => n + (r.contextTokens ?? r.inputTokens ?? 0), 0)
+        if (spent >= suiteBudget) {
+          console.log("")
+          console.error(`STOP: the suite has spent ${spent.toLocaleString("en-US")} context tokens, its --budget is ${suiteBudget.toLocaleString("en-US")}.`)
+          console.error(`  ${runs.length} run(s) measured, the rest were not started. A run's cost is only`)
+          console.error(`  known after it finishes, so the cap stops the run after the one that crossed it.`)
+          console.error(`  resume later with the same --out: node eval/ab.mjs ... --out ${outDir}`)
+          process.exit(2)
+        }
+      }
       // Every run gets its own throwaway copy of the subject. A read-only task
       // does not need one, but sharing a tree between an editing task and
       // anything else is how a run inherits the previous run's work.
@@ -619,9 +1086,57 @@ if (rescoreDir) {
       cpSync(target, runDir, { recursive: true })
       if (sanitize && leaks.length) sanitizeCopy(runDir)
       process.stdout.write(`  ${job.id} ${arm}-${i} ... `)
-      const r = runArmCapture(arm, nokit, job.id, i, outDir, job.task, runDir)
+      const r = await runArmCapture(arm, nokit, job.id, i, outDir, job.task, runDir)
+      // The provider refusing the run is not a result. It used to look like
+      // one: the error event was on stdout, the A/B read it as an empty
+      // stream, and --fail-fast needed two of them before stopping. One is
+      // enough to know the window is closed.
+      if (r.quota) {
+        console.log("")
+        console.error("FAIL: the model route is out of quota, so this measured the provider, not the kit.")
+        console.error(`  ${r.quota}`)
+        console.error("  The free tier is shared: a long chat session and this suite draw on the same window.")
+        console.error("  Nothing after this run was measured. Wait for the window to roll, or use a paid route.")
+        console.error(`  raw stream kept: ${r.out}`)
+        process.exit(2)
+      }
       const runReal = realpathSync(runDir)
       const m = parseStream(r.out, runReal)
+      if (m.steps === 0 && m.text.trim() === "") {
+        // An error the installation can state is not a dead route to wait out:
+        // it will fail every run the same way, and --fail-fast would need two
+        // of them per task. Stop now and show the message.
+        const why = streamError(readFileSync(r.out, "utf8"))
+        if (why) {
+          console.log("")
+          console.error("FAIL: the run failed before it started, so this measured the installation, not the kit.")
+          console.error(`  ${why}`)
+          console.error(`  ${job.id} ${arm}-${i} produced no tool calls and no text. Nothing after it was measured.`)
+          console.error(`  A model the arm cannot resolve is the usual cause: the off arm has its own HOME.`)
+          console.error(`  raw stream kept: ${r.out}`)
+          process.exit(2)
+        }
+        deadStreak++
+        if (deadStreak >= failFast) {
+          stopDeadRoute(deadStreak, "produced no tool calls and no text", job, arm, i)
+        }
+      } else {
+        // A timeout that kept emitting events is a slow task. A timeout that
+        // went quiet well before the ceiling is a route that stopped
+        // answering after one step - the 10-rep run had four of those at
+        // 58-164s against a 180s ceiling, and resetting the streak on each
+        // one measured the rest of the suite at 180s per run.
+        const quietS = m.seconds === null ? r.wall : r.wall - m.seconds
+        const stalled = r.timedOut && quietS >= Math.min(60, runTimeoutMs / 2000)
+        if (stalled) {
+          deadStreak++
+          if (deadStreak >= failFast) {
+            stopDeadRoute(deadStreak, "stalled before the ceiling", job, arm, i)
+          }
+        } else if (!r.timedOut) {
+          deadStreak = 0
+        }
+      }
       // Grade against the repository, not against a string I typed. The
       // substring grader produced three wrong numbers in RESULTS.md; this
       // resolves every citation the agent made and reads the real line.
@@ -682,7 +1197,11 @@ if (rescoreDir) {
         ok = g.ok
         why = g.ok ? `cited ${g.cited}` : g.cited === 0 ? "no citation" : `cited ${g.cited}, none held`
       }
-      runs.push({ id: job.id, arm, i, ...m, wall: r.wall, code: r.code, timedOut: r.timedOut, invalid, correct: ok, why })
+      const run = { id: job.id, arm, i, ...m, wall: r.wall, code: r.code, timedOut: r.timedOut, invalid, correct: ok, why }
+      runs.push(run)
+      // Written before the escape check, because a run that escaped is still
+      // a result and re-running it would repeat the escape.
+      saveRun(run)
       // Stop the moment an arm reaches outside the run directory. The first
       // pilot wrote to the real repository while the task said not to modify
       // anything, so a second arm is not worth the risk once this has happened.
@@ -712,11 +1231,14 @@ if (rescoreDir) {
 
 const after = fingerprint(target)
 // Median, never mean. See median() in ./metrics.js for the run that proves why.
-// Invalid runs are excluded everywhere: a process that emitted nothing has no
-// tools, no tokens and no answer, and including its zeros would drag every
-// median toward looking as though the agent did nothing at all.
+// Invalid runs are excluded from the work metrics: a process that emitted
+// nothing has no tools, no tokens and no answer, and including its zeros would
+// drag every median toward looking as though the agent did nothing at all.
+// Wall time is the exception. A run that timed out still spent its ceiling,
+// and hiding it would report a suite that cost less time than it did.
 const scored = runs.filter((r) => !r.invalid)
 const stat = (arm, key) => median(scored.filter((r) => r.arm === arm).map((r) => r[key]))
+const wallStat = (arm) => median(runs.filter((r) => r.arm === arm).map((r) => r.wall))
 const scoreOf = (arm) => {
   const rs = scored.filter((r) => r.arm === arm)
   return `${rs.filter((r) => r.correct).length}/${rs.length}`
@@ -734,9 +1256,10 @@ const rows = [
   ["narrow %", stat("on", "narrow_pct"), stat("off", "narrow_pct")],
   ["grep/glob", stat("on", "searches"), stat("off", "searches")],
   ["steps", stat("on", "steps"), stat("off", "steps")],
-  ["input tokens", stat("on", "inputTokens"), stat("off", "inputTokens")],
+  ["context tokens", stat("on", "contextTokens"), stat("off", "contextTokens")],
+  ["uncached input", stat("on", "inputTokens"), stat("off", "inputTokens")],
   ["final ctx", stat("on", "finalCtx"), stat("off", "finalCtx")],
-  ["wall seconds", stat("on", "wall"), stat("off", "wall")],
+  ["wall seconds", wallStat("on"), wallStat("off")],
   ["not scored", runs.filter((r) => r.arm === "on" && r.invalid).length, runs.filter((r) => r.arm === "off" && r.invalid).length],
   // The escape check compares against a per-run copy that no longer exists when
   // rescoring, so it would report every path as an escape. Report nothing rather