opencode-harness-kit 0.5.7 → 0.6.0

package/bin/oc-harness.mjs

@@ -47,6 +47,7 @@ import { generateConfig, globalInstructionsPath, utilityModel, withMemoryPlugin,
 import { ruleNames, BASH_RULES, WRITE_RULES } from "../src/rules.js"
 import { planEffort } from "../src/effort.js"
 import { findTelemetryFiles, readLaneUsage, averageMs } from "../src/lane-usage.js"
+import { researchModels, isWithdrawn, pruneWithdrawnChains } from "../src/research.js"
 import { main as trustMain } from "../src/trust.js"
 
 const CONFIG_DIR =
@@ -64,10 +65,22 @@ const has = (f) => args.includes(f)
  * the second one is the conventional one. Reading only the first form made
  * `--profile=fleet-safe` look like a flag with a missing value, which is a
  * confusing way to be told you got the syntax wrong.
+ *
+ * A value that is itself a flag is NOT a value. `oc-harness run --role
+ * --max-rounds 5` used to set the role to the literal string "--max-rounds",
+ * and the loop then ran `opencode run --agent --max-rounds ...`, which failed
+ * on every round and marked every task blocked. The typo was silent because
+ * the value was plausible. There is no way to spell a value that starts with
+ * `--`, and there never needs to be: no flag of this CLI takes one.
  */
 const opt = (f) => {
   const i = args.indexOf(f)
-  if (i >= 0) return args[i + 1]
+  if (i >= 0) {
+    const next = args[i + 1]
+    // A missing value, or the next flag. Both mean the flag was not given
+    // what it needs - which is a typo, not an empty value.
+    return next === undefined || next.startsWith("--") ? undefined : next
+  }
   const inline = args.find((a) => a.startsWith(`${f}=`))
   return inline === undefined ? undefined : inline.slice(f.length + 1)
 }
@@ -109,7 +122,11 @@ const red = (s) => `\x1b[31m${s}\x1b[0m`
 // The colour out. The human report wants it; a JSON field does not, and a
 // script that greps the message gets escape bytes it has to know to strip.
 const plain = (s) => String(s).replace(/\x1b\[[0-9;]*m/g, "")
-const interactive = () => process.stdin.isTTY && !flag("--yes")
+// A command that must not ask. `auto` sets it: it reconciles, so a question
+// has no good answer - the one thing it can do about the user's own file is
+// leave it alone and say so.
+let nonInteractive = false
+const interactive = () => process.stdin.isTTY && !flag("--yes") && !nonInteractive
 
 /**
  * A yes question with a yes default. Only asked on a terminal; scripts and
@@ -499,13 +516,7 @@ function removeLegacyAgentFiles() {
     for (const name of readdirSync(dir)) {
       if (!name.endsWith(".md")) continue
       const full = join(dir, name)
-      let ours = false
-      try {
-        ours = readlinkSync(full).includes(join("space-harness", "agents"))
-      } catch {
-        ours = false
-      }
-      if (!ours) continue
+      if (!isLegacyAgentLink(full)) continue
       try {
         rmSync(full, { force: true })
         removed.push(name)
@@ -515,6 +526,39 @@ function removeLegacyAgentFiles() {
   return removed
 }
 
+/**
+ * The agent directory an earlier version of this kit installed from.
+ *
+ * The kit was named `space-harness` then, and it put its agent files in
+ * `<kit>/agents/`.
+ */
+const LEGACY_AGENT_DIR = ["space-harness", "agents"]
+
+/**
+ * Whether a path is a symlink into an earlier version's agent directory.
+ *
+ * The match is on the parent DIRECTORY, compared segment by segment. A plain
+ * substring test - `target.includes("space-harness/agents")` - also matched
+ * `space-harness/agents-backup/`, and this function DELETES what it matches.
+ * A user who symlinked their own agent file out of a directory with a similar
+ * name lost it, because the kit thought it had installed it.
+ *
+ * @param {string} full
+ * @returns {boolean}
+ */
+function isLegacyAgentLink(full) {
+  let target
+  try {
+    target = readlinkSync(full)
+  } catch {
+    return false
+  }
+  const parts = dirname(target).split(/[\\/]/)
+  const n = LEGACY_AGENT_DIR.length
+  if (parts.length < n) return false
+  return LEGACY_AGENT_DIR.every((seg, i) => parts[parts.length - n + i] === seg)
+}
+
 /**
  * Replace `link` with `link` plus the agents, skills and GLOBAL.md.
  *
@@ -726,6 +770,9 @@ async function cmdInit() {
   let id = opt("--profile")
   if (!id) {
     if (!process.stdin.isTTY) {
+      // No question can be asked without a terminal, and picking for the user
+      // is not this command's job: `init` is the front door for a person, and
+      // `auto` is the one that runs without asking. Say what to type.
       fail("usage: oc-harness init --profile <id> [--yes] (no terminal to ask)")
     }
     const def = Math.max(free.findIndex((e) => e.id === index().default), 0)
@@ -742,7 +789,7 @@ async function cmdInit() {
   console.log(`\n  ${stamp(`profile: ${id}`)}`)
   await cmdInstall()
   await cmdUse(id)
-  cmdVerify()
+  if (cmdVerify().length) process.exitCode = 1
   if (memorySpec()) {
     console.log(
       `\n  ${stamp(
@@ -758,6 +805,143 @@ async function cmdInit() {
   console.log(`\n  ${green("✓")} done. ${stamp("restart opencode, then ask it something")}\n`)
 }
 
+/**
+ * Bring the install back to the profile, without asking anything.
+ *
+ * The four commands that keep an install alive - refresh, install, use,
+ * verify - are each a step somebody has to remember, in the right order, with
+ * the right flags. An agent that has to keep an install working has to know
+ * all of that, and it has to know which of them are allowed to destroy
+ * something. This is one command that does the whole chain.
+ *
+ *   oc-harness auto [--profile <id>] [--offline] [--json]
+ *
+ * The profile is the one the installed config names, or the default. Nothing
+ * is asked: a file of the user's that the installer would have overwritten is
+ * left alone and reported, rather than being a question with no good answer.
+ *
+ * `--yes` still means consent. A profile that may train on prompts is
+ * refused without it, exactly as `use` refuses it - the automation is about
+ * the mechanics, not about the consent.
+ *
+ * `--offline` skips the refresh, for a machine that cannot reach models.dev.
+ * `--json` prints a report a script can read, and nothing else on stdout.
+ */
+async function cmdAuto() {
+  const wantJson = flag("--json")
+  const idx = index()
+  const steps = []
+  const notes = []
+
+  // The profile. What was asked for, else what is installed, else the default.
+  // A profile that was ASKED FOR and cannot be loaded is an error - falling
+  // back to another one would run the install against a profile the caller
+  // did not name.
+  const requested = opt("--profile")
+  let id = null
+  if (requested) {
+    try {
+      loadProfile(requested)
+      id = requested
+    } catch (err) {
+      fail(`profile "${requested}" cannot be loaded: ${err instanceof Error ? err.message : String(err)}`, wantJson)
+    }
+  } else {
+    const active = profileIdFromConfig()
+    if (active) {
+      try {
+        loadProfile(active)
+        id = active
+      } catch (err) {
+        notes.push(
+          `the installed profile ${active} cannot be loaded (${err instanceof Error ? err.message : String(err)}); using ${idx.default}`,
+        )
+      }
+    }
+    if (!id) {
+      id = idx.default
+      try {
+        loadProfile(id)
+      } catch (err) {
+        fail(`no usable profile: ${err instanceof Error ? err.message : String(err)}`, wantJson)
+      }
+    }
+  }
+
+  // The steps. Their human output is silenced in JSON mode, so stdout carries
+  // the report and nothing else.
+  const realLog = console.log
+  if (wantJson) console.log = () => {}
+  const problems = []
+  try {
+    // 1. The catalogue. A model that was withdrawn has to be caught before
+    //    it is used, and the profile files are the only place that lives.
+    if (flag("--offline")) {
+      steps.push({ step: "refresh", status: "skipped", detail: "--offline" })
+    } else {
+      // The refresh's findings are part of the report, not decoration. In JSON
+      // mode the refresh's own output is silenced, so a withdrawn model or a
+      // profile that could not be read used to vanish from the report - and
+      // the report said "ok": true while a profile was never checked against
+      // the live catalogue.
+      const refreshed = await cmdRefreshAll()
+      for (const g of refreshed.gone) {
+        problems.push(`profile ${g} is no longer in the live catalogue`)
+      }
+      for (const s of refreshed.skipped) {
+        problems.push(
+          `profile ${s} was skipped by the refresh, so it was not checked against the live catalogue`,
+        )
+      }
+      const findings = refreshed.gone.length + refreshed.skipped.length
+      steps.push({
+        step: "refresh",
+        status: findings ? "failed" : "ok",
+        ...(findings ? { detail: `${findings} finding(s)` } : {}),
+      })
+    }
+
+    // 2. The behavioural layer: the plugin link, the prompts, the skills, the
+    //    global rules. Without them the config means nothing.
+    await cmdInstall()
+    steps.push({ step: "install", status: "ok" })
+
+    // 3. The config, recompiled from the profile.
+    await cmdUse(id)
+    steps.push({ step: "use", status: "ok", detail: id })
+
+    // 4. The check that says whether all of that worked.
+    const verifyProblems = cmdVerify()
+    problems.push(...verifyProblems)
+    steps.push({
+      step: "verify",
+      status: verifyProblems.length ? "failed" : "ok",
+      detail: verifyProblems.length ? `${verifyProblems.length} problem(s)` : "the install is consistent",
+    })
+  } finally {
+    console.log = realLog
+  }
+
+  const report = {
+    ok: problems.length === 0,
+    profile: id,
+    steps,
+    problems,
+    ...(notes.length ? { notes } : {}),
+  }
+
+  if (wantJson) {
+    process.stdout.write(`${JSON.stringify(report, null, 2)}\n`)
+  } else {
+    console.log(`\n${bold("auto")}  ${stamp(report.ok ? "the install is in step with the profile" : `${problems.length} problem(s)`)}\n`)
+    for (const s of steps) console.log(`  ${green("✓")} ${s.step}${s.detail ? ` - ${s.detail}` : ""}`)
+    for (const n of notes) console.log(`  ${yellow("!")} ${n}`)
+    for (const pr of problems) console.log(`  ${red("✗")} ${pr}`)
+    console.log("")
+  }
+  if (problems.length) process.exitCode = 1
+}
+
 async function askChoice(question) {
   const rl = createInterface({ input: process.stdin, output: process.stdout })
   try {
@@ -852,13 +1036,16 @@ function linkOne(src, dest, opts = {}) {
  * the profile, an instruction path that resolves to nothing, and a set of role
  * agents that were never installed while the config still declared them.
  *
- * Exits non-zero when anything fails, so it can gate a commit or a CI job.
+ * Returns the problems it found rather than exiting on them, so a caller that
+ * has more to do - `oc-harness auto` - can carry on and report the whole
+ * picture. The exit code is the caller's to set.
+ *
+ * @returns {string[]} the problems, empty when the install is consistent
  */
 function cmdVerify() {
   const problems = []
   const notes = []
   const ok = []
-
   const activeId = profileIdFromConfig() ?? index().default
   let p = null
   try {
@@ -1001,7 +1188,11 @@ function cmdVerify() {
     for (const name of readdirSync(dir).filter((f) => f.endsWith(".md"))) {
       const where = `${scope}:${dir.replace(`${homedir()}/`, "~/")}`
       const lane = name.replace(/\.md$/, "")
-      if (lane in (/** @type {any} */ (actual?.agent) ?? {})) {
+      // `Object.hasOwn`, not `in`. The object is the user's parsed config, and
+      // `in` walks the prototype chain - so an agent file named `toString.md`
+      // or `constructor.md` matched a property every object has, and `verify`
+      // reported a clean install as shadowing a lane that does not exist.
+      if (Object.hasOwn(/** @type {any} */ (actual?.agent) ?? {}, lane)) {
         problems.push(
           `agent file ${where}/${name} shadows the ${lane} lane: an agent file replaces the config entry, so the lane would not run on its own model. Run: oc-harness install`,
         )
@@ -1152,9 +1343,10 @@ function cmdVerify() {
 
   if (problems.length) {
     console.log(`\n  ${red(`${problems.length} problem(s)`)}\n`)
-    process.exit(1)
+    return problems
   }
   console.log(`\n  ${green("install is consistent with the profile")}\n`)
+  return problems
 }
 
 /**
@@ -1511,6 +1703,11 @@ function diffProfile(p, live) {
  * Only machine-derived fields are rewritten. `retention`, `notes`, `roles` and
  * the fallback queue are human judgements and are never touched by a script:
  * losing a privacy verdict to an automated rewrite would be worse than drift.
+ *
+ * The ONE exception is a chain rung the provider has withdrawn. A chain names
+ * the models that could serve a lane, and a withdrawn model is not one of them
+ * any more - it cannot run. Removing it is a derivation from live metadata
+ * and it only ever removes; nothing is ever put INTO a lane.
  */
 /**
  * Refresh every shipped profile against the live catalogue.
@@ -1525,22 +1722,44 @@ async function cmdRefreshAll() {
   const changed = []
   const gone = []
   const same = []
+  // The profiles that could not be read, and the profiles whose backup could
+  // not be written. Both used to disappear without a word: the first was a
+  // `continue` with no output, the second was a swallowed error - and the
+  // summary line counted every id it was given, so a broken profile was
+  // reported as a refreshed one.
+  const skipped = []
+  let checked = 0
   for (const id of ids) {
     const path = profilePath(id)
-    if (!path) continue
+    if (!path) {
+      skipped.push(`${id} (no file)`)
+      continue
+    }
     let p
     try {
       p = JSON.parse(readFileSync(path, "utf8"))
-    } catch {
+    } catch (err) {
+      skipped.push(`${id} (${err instanceof Error ? err.message : String(err)})`)
       continue
     }
+    checked++
     const live = catalog?.[p.provider]?.models?.[p.model]
     if (!live) {
       gone.push(`${id} (${p.provider}/${p.model})`)
       continue
     }
+    // A model the provider has withdrawn is still IN the catalogue, so the
+    // check above passes it. A profile whose own model was withdrawn has no
+    // chain behind it to fall back to, so there is nothing a refresh can
+    // repair - it has to be retired, and that is a person's call.
+    const prune = pruneWithdrawnChains(p, catalog, p.provider)
+    if (prune.dead) {
+      gone.push(`${id} (${p.provider}/${p.model} was withdrawn from the provider)`)
+      continue
+    }
     const before = diffProfile(p, live)
-    if (!before.length) {
+    const pruned = prune.pruned.length > 0
+    if (!before.length && !pruned) {
       same.push(id)
       continue
     }
@@ -1548,21 +1767,47 @@ async function cmdRefreshAll() {
     // `verified` is the field that says when a profile was last checked
     // against live metadata. Leaving it stale after a refresh makes it a lie.
     p.verified = new Date().toISOString().slice(0, 10)
+    const notes = [...before]
+    if (pruned) {
+      notes.push(`withdrawn rungs removed: ${prune.pruned.join(", ")}`)
+      for (const lane of prune.lanes) {
+        notes.push(`lane ${lane.lane}: ${lane.from.join(", ")} -> ${lane.to.join(", ")}`)
+      }
+      if (prune.utilityMoved) {
+        notes.push(`utility route: ${prune.utilityMoved} -> ${p.utility.model}`)
+      }
+    }
     if (flag("--dryrun")) {
-      changed.push(`${id}: ${before.join(", ")} (dry run, nothing written)`)
+      changed.push(`${id}: ${notes.join("; ")} (dry run, nothing written)`)
       continue
     }
     // A refresh overwrites a file the user may have edited by hand, and for
     // an npm install that file is inside node_modules, where the next
-    // `npm install` destroys it anyway. The backup is the only copy.
-    keepProfileBackup(path, readFileSync(path, "utf8"))
+    // `npm install` destroys it anyway. The backup is the only copy - so a
+    // backup that could not be written means the refresh must not happen.
+    const kept = keepProfileBackup(path, readFileSync(path, "utf8"))
+    if (!kept) {
+      skipped.push(`${id} (the backup could not be written, so the profile was left alone)`)
+      continue
+    }
     writeFileSync(path, `${JSON.stringify(p, null, 2)}\n`)
-    changed.push(`${id}: ${before.join(", ")}`)
+    changed.push(`${id}: ${notes.join("; ")}`)
   }
-  console.log(`\n${bold("refreshed")}  ${stamp(`${ids.length} profile(s) against the live catalogue`)}\n`)
+  console.log(
+    `\n${bold("refreshed")}  ${stamp(`${checked} of ${ids.length} profile(s) against the live catalogue`)}\n`,
+  )
   for (const c of changed) console.log(`  ${green("~")} ${c}`)
   for (const g of gone) console.log(`  ${red("!")} ${g} is no longer in the catalogue`)
   for (const id of same) console.log(`  ${green("✓")} ${id} already matches`)
+  for (const s of skipped) console.log(`  ${red("!")} ${s} - skipped`)
+  if (skipped.length) {
+    console.log("")
+    console.log(
+      stamp(
+        "A profile that was skipped was not checked against the live catalogue. Its model may have been withdrawn without anyone noticing.",
+      ),
+    )
+  }
   if (gone.length) {
     console.log("")
     console.log(
@@ -1572,6 +1817,15 @@ async function cmdRefreshAll() {
     )
   }
   console.log("")
+  // A skipped profile is a failure, not a footnote. `refresh --all` is run by
+  // a job that assumes every profile was checked, and a non-zero exit is the
+  // only way that job finds out.
+  if (skipped.length) process.exitCode = 1
+  // The findings come back with the command. They used to be printed and
+  // dropped, so a caller that silences the human output - `auto --json` -
+  // never saw them: a withdrawn model or a profile that was never checked
+  // against the catalogue went into a report that said everything was fine.
+  return { changed, gone, same, skipped }
 }
 
 /**
@@ -1688,6 +1942,10 @@ const BACKUP_DIR = "backups"
  *
  * @param {string} path the profile file
  * @param {string} text its current contents
+ * @returns {string | null} the backup's name, or null when it could not be
+ *   written. The caller has to treat null as a refusal to go on: a profile
+ *   the user edited by hand has no other copy, and for an npm install that
+ *   file is inside node_modules, where the next `npm install` destroys it.
  */
 function keepProfileBackup(path, text) {
   try {
@@ -1698,9 +1956,20 @@ function keepProfileBackup(path, text) {
     const dir = join(CONFIG_DIR, "profile-backups")
     if (!existsSync(dir)) mkdirSync(dir, { recursive: true })
     const stamp = new Date().toISOString().replace(/[:.]/g, "-")
-    writeFileSync(join(dir, `${basename(path, ".json")}-${stamp}.json`), text)
+    const name = `${basename(path, ".json")}-${stamp}.json`
+    writeFileSync(join(dir, name), text)
+    return name
   } catch {
-    // A backup that cannot be written is not worth failing over.
+    // A backup that cannot be written is not worth failing over - unless the
+    // caller was about to destroy the only copy of the file it holds.
+    //
+    // This used to swallow the error and return nothing, and both callers
+    // then overwrote the profile anyway. A user who had edited
+    // `profiles/<id>.json` by hand - the retention, the roles, the fallback,
+    // the parts the code itself calls "human judgement" - lost those edits
+    // with no backup and no message. `keepBackup` below already had the right
+    // shape: return null, and let the caller refuse to go on.
+    return null
   }
 }
 
@@ -2060,8 +2329,30 @@ async function cmdRefresh(id) {
     )
   }
 
+  // A withdrawn model is still in the catalogue, so the check above passes
+  // it. A profile whose own model was withdrawn has no chain behind it, so
+  // there is nothing to fall back to and nothing a refresh can repair.
+  const prune = pruneWithdrawnChains(p, catalog, p.provider)
+  if (prune.dead) {
+    fail(
+      `${p.provider}/${p.model} was withdrawn from the provider.\n` +
+        `  This profile has no chain to fall back to, so it cannot be repaired.\n` +
+        `  Remove it, or point it at a model that is still live.`,
+    )
+  }
+
   const before = diffProfile(p, live)
-  if (before.length === 0) {
+  const notes = [...before]
+  if (prune.pruned.length > 0) {
+    notes.push(`withdrawn rungs removed: ${prune.pruned.join(", ")}`)
+    for (const lane of prune.lanes) {
+      notes.push(`lane ${lane.lane}: ${lane.from.join(", ")} -> ${lane.to.join(", ")}`)
+    }
+    if (prune.utilityMoved) {
+      notes.push(`utility route: ${prune.utilityMoved} -> ${p.utility.model}`)
+    }
+  }
+  if (notes.length === 0) {
     console.log(`\n  ${green("✓")} ${id} already matches live metadata\n`)
     return
   }
@@ -2094,18 +2385,24 @@ async function cmdRefresh(id) {
 
   if (flag("--dryrun")) {
     console.log(`\n  ${stamp("dry run, nothing written")}`)
-    for (const line of before) console.log(`      ${line}`)
+    for (const line of notes) console.log(`      ${line}`)
     return
   }
   // A refresh overwrites a file the user may have edited by hand, and for
   // an npm install that file is inside node_modules, where the next
-  // `npm install` destroys it anyway. The backup is the only copy.
-  keepProfileBackup(path, readFileSync(path, "utf8"))
+  // `npm install` destroys it anyway. The backup is the only copy - so a
+  // backup that could not be written means the refresh must not happen.
+  const kept = keepProfileBackup(path, readFileSync(path, "utf8"))
+  if (!kept) {
+    fail(
+      `could not write a backup of ${path}, so the profile was left alone. Fix the permissions on ${join(CONFIG_DIR, "profile-backups")} and run it again`,
+    )
+  }
   writeFileSync(path, `${JSON.stringify(p, null, 2)}\n`)
   console.log(`\n  ${green("✓")} refreshed ${path}`)
-  for (const line of before) console.log(`      ${line}`)
+  for (const line of notes) console.log(`      ${line}`)
   console.log(
-    `\n  ${stamp("kept as-is: retention, roles, notes, fallback (human judgement)")}\n`,
+    `\n  ${stamp("kept as-is: retention, roles, notes (human judgement)")}\n`,
   )
 }
 
@@ -2163,17 +2460,26 @@ function recordWithdrawal(model, profiles) {
  * chain has one fewer rung, and nobody is told.
  *
  * @param {any} catalog
- * @returns {Map<string, string[]>} the model ids that are gone, and the
- *   profiles that used each one
+ * @returns {{gone: Map<string, string[]>, skipped: {id: string, why: string}[]}}
+ *   the model ids that are gone and the profiles that used each one, and the
+ *   profiles that could not be checked at all
  */
 function withdrawnModels(catalog) {
   /** @type {Map<string, string[]>} the model, and the profiles that use it */
   const gone = new Map()
+  // The profiles that could not be read. They used to disappear without a
+  // word - the check was a `continue` with no output - and the summary then
+  // said every shipped model was still in the catalogue. So a profile that
+  // was NEVER CHECKED was reported as a profile that was checked and found
+  // fine. A skipped profile is a failure, not a footnote.
+  /** @type {{id: string, why: string}[]} */
+  const skipped = []
   for (const id of listProfileIds()) {
     let p
     try {
       p = loadProfile(id)
-    } catch {
+    } catch (err) {
+      skipped.push({ id, why: err instanceof Error ? err.message : String(err) })
       continue
     }
     for (const model of new Set([
@@ -2190,7 +2496,7 @@ function withdrawnModels(catalog) {
       gone.set(model, users)
     }
   }
-  return gone
+  return { gone, skipped }
 }
 
 /**
@@ -2211,13 +2517,20 @@ async function cmdDoctorWatch() {
   // times a second.
   const interval = Math.max(30, positiveNumber(opt("--interval"), 300))
   const seen = new Set()
+  // The profiles that could not be checked, so the watch says them once and
+  // not every poll.
+  const seenSkipped = new Set()
   let first = true
   for (;;) {
     /** @type {Map<string, string[]>} */
     let gone = new Map()
+    /** @type {{id: string, why: string}[]} */
+    let skipped = []
     try {
       const catalog = await fetchCatalog()
-      gone = withdrawnModels(catalog)
+      const found = withdrawnModels(catalog)
+      gone = found.gone
+      skipped = found.skipped
     } catch (err) {
       // A failed fetch is NOT an empty catalogue. Treating it as one
       // announced every missing model as "is back", emptied the seen-set,
@@ -2236,6 +2549,14 @@ async function cmdDoctorWatch() {
       console.log(stamp(`    fix: oc-harness doctor && oc-harness refresh <profile>`))
       recordWithdrawal(m, users.join(","))
     }
+    for (const s of skipped) {
+      if (seenSkipped.has(s.id)) continue
+      seenSkipped.add(s.id)
+      console.error(red(`  ${s.id} could not be checked against the catalogue: ${s.why}`))
+      console.error(
+        stamp("    a profile that was skipped was not checked. Its model may have been withdrawn without anyone noticing."),
+      )
+    }
     if (!first) {
       for (const m of seen) {
         if (!current.has(m)) {
@@ -2253,6 +2574,15 @@ async function cmdDoctorWatch() {
         console.error(stamp(`  recorded in ${join(CONFIG_DIR, WITHDRAWN_LOG)}`))
         process.exit(1)
       }
+      // A profile that could not be read is a profile that was not checked.
+      // The run above says "every shipped model is still in the catalogue",
+      // and that sentence is only true of the profiles it actually looked at.
+      if (skipped.length) {
+        console.error(
+          red(`  ${skipped.length} profile(s) could not be checked: ${skipped.map((s) => s.id).join(", ")}`),
+        )
+        process.exit(1)
+      }
       console.log(green("  every shipped model is still in the catalogue"))
       return
     }
@@ -2312,6 +2642,111 @@ async function cmdDoctor() {
 // second doctor in the same process does not inherit the first one's findings.
 const doctorReport = { ok: true, problems: 0, drift: 0, strict: false, findings: [], json: false }
 
+/**
+ * Research the live tier for models the kit could use.
+ *
+ * `doctor` says when a model has gone. This says what has come to replace
+ * it, and what is still a person's decision.
+ *
+ * It writes nothing. The shortlist it prints is a list to argue with, not a
+ * list to accept: `retention` is a judgement about a provider's terms, and
+ * that is the one field a script must not fill in.
+ */
+async function cmdResearch() {
+  const wantJson = flag("--json")
+  let catalog
+  try {
+    catalog = await fetchCatalog()
+  } catch (err) {
+    fail(err instanceof Error ? err.message : String(err), wantJson)
+  }
+
+  // A profile that cannot be read is `doctor`'s finding, not this command's.
+  // Research is about the tier, so a broken profile is left out of the
+  // comparison rather than failing the whole report.
+  const profiles = []
+  for (const id of listProfileIds()) {
+    try {
+      profiles.push(loadProfile(id))
+    } catch {
+      // doctor reports this, with the profile's own name on it.
+    }
+  }
+
+  const report = researchModels({
+    catalog,
+    profiles,
+    retentionOf: (model) => modelRetention(model),
+    defaultProfileId: index().default ?? null,
+  })
+
+  if (wantJson) {
+    console.log(JSON.stringify({ ok: true, ...report }, null, 2))
+    return
+  }
+
+  console.log(
+    `\n${bold("research")}  ${stamp(`tier ${report.tier.provider}, ${report.tier.live} live of ${report.tier.total} free`)}\n`,
+  )
+
+  // The models the kit uses that the provider has pulled. This is the part
+  // that has to be acted on, so it comes first.
+  if (report.shipped.gone.length) {
+    console.log(`  ${red("gone")} ${report.shipped.gone.length} model(s) in use have been withdrawn\n`)
+    for (const g of report.shipped.gone) {
+      console.log(`      ${red("✗")} ${g.id}`)
+      console.log(`        ${g.reason}`)
+      if (g.asBase.length) {
+        console.log(`        ${stamp("base model of")} ${g.asBase.join(", ")}`)
+      }
+      for (const [profile, laneList] of Object.entries(g.inChains ?? {})) {
+        console.log(`        ${stamp("fallback rung in")} ${profile}: ${laneList.join(", ")}`)
+      }
+    }
+    console.log("")
+  } else {
+    console.log(`  ${green("✓")} every model in use is still live\n`)
+  }
+
+  // Per lane: which rungs of its chain are alive. A lane whose whole chain
+  // has gone is a lane with no fallback at all.
+  if (report.lanes.length) {
+    console.log(`  ${bold("lanes")}`)
+    for (const l of report.lanes) {
+      const dead = l.dead.length
+        ? yellow(` ${l.dead.length} dead (${l.dead.join(", ")})`)
+        : green(" 0 dead")
+      console.log(`      ${l.lane.padEnd(12)} ${String(l.live.length).padStart(2)} live${dead}`)
+    }
+    console.log("")
+  }
+
+  // The shortlist. What each candidate can do, which lanes it could serve,
+  // and what is still undecided about it.
+  if (!report.candidates.length) {
+    console.log(`  ${stamp("no live free model outside the ones the kit already uses")}\n`)
+    return
+  }
+  console.log(`  ${bold("candidates")} ${stamp("live, free, and not in use yet")}\n`)
+  for (const c of report.candidates) {
+    const effort = c.effort.length ? c.effort.join(",") : "none published"
+    console.log(
+      `      ${bold(c.id)}  ${stamp(
+        `ctx ${Number(c.context).toLocaleString("en-US")}  out ${Number(c.output).toLocaleString("en-US")}  effort ${effort}`,
+      )}`,
+    )
+    console.log(`        fits: ${c.fits.length ? c.fits.join(", ") : "no lane on its own terms"}`)
+    console.log(
+      `        ${c.retention === "unknown" ? yellow("retention: unknown") : stamp(`retention: ${c.retention}`)}`,
+    )
+    for (const b of c.blockers) console.log(`        ${yellow(b)}`)
+    console.log("")
+  }
+  console.log(
+    `  ${stamp("nothing here is written anywhere. A retention verdict is a person reading a provider's terms.")}\n`,
+  )
+}
+
 async function runDoctor(catalog) {
   const retention = loadRetention().models
   const ids = listProfileIds()
@@ -2359,7 +2794,16 @@ async function runDoctor(catalog) {
     // with no model at all, which no amount of `refresh` on the base profile
     // would reveal.
     for (const [lane, chain] of Object.entries(chainOf(p))) {
-      const alive = chain.filter((rung) => catalog?.[p.provider]?.models?.[rung])
+      // A withdrawn model is still IN the catalog: models.dev keeps the
+      // record and marks it deprecated rather than deleting it. So "is it
+      // still there" is not the same question as "can it still run", and
+      // asking the first one reported a dead model as a healthy fallback.
+      // The message below always said "withdrawn" - only the filter was
+      // wrong, which is why the fleet's dead rung stayed invisible.
+      const alive = chain.filter((rung) => {
+        const record = catalog?.[p.provider]?.models?.[rung]
+        return record && !isWithdrawn(record)
+      })
       if (!alive.length) {
         issues.push(red(`lane ${lane}: every model in its chain has left the catalog`))
       } else if (alive.length < chain.length) {
@@ -2701,7 +3145,12 @@ function cmdAgents() {
     }
     const name = file.split("/").pop().replace(/\.md$/, "")
     const front = stripFrontmatter(text)
-    if (name in /** @type {any} */ (config.agent)) {
+    // `Object.hasOwn`, not `in`. `in` walks the prototype chain, so an agent
+    // file named `toString.md` or `constructor.md` matched a property every
+    // object has - and the file was reported as shadowing a lane that does not
+    // exist, and its missing model was never reported either. `verify` had the
+    // same check and the same fix; this one was left behind.
+    if (Object.hasOwn(/** @type {any} */ (config.agent) ?? {}, name)) {
       console.log(
         `  ${yellow("!")} ${name.padEnd(14)} ${yellow("shadows a lane in the config")}`,
       )
@@ -2733,7 +3182,10 @@ function cmdAgents() {
  */
 function fail(msg, wantJson = false) {
   if (wantJson) {
-    console.log(JSON.stringify({ ok: false, error: plain(msg) }, null, 2))
+    // `process.stdout.write`, not `console.log`: a JSON command silences
+    // `console.log` so the report is the only thing on stdout, and the error
+    // has to survive that.
+    process.stdout.write(`${JSON.stringify({ ok: false, error: plain(msg) }, null, 2)}\n`)
     console.error(`\n  ${red("error")} ${msg}\n`)
     process.exit(1)
   }
@@ -2758,9 +3210,29 @@ async function cmdRun() {
 
   const { runLoop } = await import("../src/driver/run.mjs")
 
+  // A role that is not a lane is a typo, and the loop would fail every round
+  // on it. It is refused here, while the answer is still "that is not a lane".
+  //
+  // A `--role` with no value is refused too, rather than falling back to the
+  // default. `oc-harness run --role --max-rounds 5` used to set the role to
+  // the literal string "--max-rounds" - and the loop then ran
+  // `opencode run --agent --max-rounds ...`, which failed on every round and
+  // marked every task blocked. A typo that quietly becomes the default is
+  // still a typo that quietly becomes the wrong agent.
+  const lanes = laneNames()
+  const roleGiven = args.includes("--role") || args.some((a) => a.startsWith("--role="))
+  const roleValue = opt("--role")
+  if (roleGiven && roleValue === undefined) {
+    fail(`--role needs a lane: ${lanes.join(", ")}`)
+  }
+  const role = roleValue ?? "build"
+  if (!lanes.includes(role)) {
+    fail(`--role ${role} is not one of ${lanes.join(", ")}`)
+  }
+
   const result = runLoop({
     cwd: process.cwd(),
-    role: opt("--role") ?? "build",
+    role,
     maxRounds,
     budgetMinutes,
   })
@@ -2811,8 +3283,13 @@ try {
     case "init":
       await cmdInit()
       break
+    case "auto":
+      await cmdAuto()
+      break
     case "verify":
-      cmdVerify()
+      // The exit code is the gate. `verify` runs as the last step of the
+      // install job in CI, and a verify that cannot fail is not a gate.
+      if (cmdVerify().length) process.exitCode = 1
       break
     case "agents":
       cmdAgents()
@@ -2827,6 +3304,9 @@ try {
       if (flag("--watch") || flag("--check-models")) await cmdDoctorWatch()
       else await cmdDoctor()
       break
+    case "research":
+      await cmdResearch()
+      break
     case "baseline":
       await import("./baseline.mjs")
       break
@@ -2847,6 +3327,8 @@ try {
 
     list      show profiles with retention and caveats
     init      pick a free profile, then install + use + verify in one go
+    auto      refresh + install + use + verify, asking nothing - the
+              unattended reconcile, for an agent or a job
     use <id>  compile a profile into ${CONFIG_FILE}
     show <id> print the resolved lane plan and derived numbers
     refresh <id>  update capability fields from models.dev
@@ -2857,6 +3339,8 @@ try {
     lanes         report which lanes the fleet actually used
     trust         set up npm trusted publishing for this package
     doctor    re-check profiles against models.dev and the installed config
+    research  find the live free models the kit could use, and the ones in
+              use that have been withdrawn
     upgrade   check for a newer kit, refresh the profiles, rewrite the config
     backups   the configs that were replaced, newest first
     uninstall remove the plugin, prompts, skills and GLOBAL.md again
@@ -2874,8 +3358,10 @@ try {
     --strict       doctor: fail on perishable catalogue drift too
     --watch        doctor: keep watching the catalogue, report what disappeared
     --check-models doctor: one run, non-zero when a shipped model is gone
-    --json        doctor, show, lanes: print the machine-readable form, and
-                  nothing else
+    --json        doctor, show, lanes, auto, research: print the
+                  machine-readable form, and nothing else
+    --offline     auto: skip the refresh, for a machine that cannot reach
+                  models.dev
     --hook        install: also install the pre-commit hook into this
                   project. It is the project's call, so it has to be asked
                   for.