opencode-harness-kit 0.1.0 → 0.2.1

package/src/driver/run.mjs

@@ -0,0 +1,237 @@
+#!/usr/bin/env node
+// `oc-harness run` -- drive the space-plan -> build -> review -> finish loop
+// without a human at the keyboard.
+//
+// The four skills already know how to do their own job. What was missing was
+// the thing that decides *which* one runs next and *when to stop*. That is
+// here, and it is deliberately small: it reads `.space/tasks.md`, hands one
+// task at a time to `opencode run --agent <role>`, records the real verify
+// output, and stops.
+//
+// What it will not do, ever:
+//   - push, open a PR, or cut a release. Publishing is a human decision, and an
+//     unattended loop that can do it is an unattended loop that will.
+//   - retry a task it already blocked, or start a task with no verify command.
+//     A task it cannot check is a task it cannot claim is done.
+//   - loop forever. Every stop condition is explicit and every one of them is
+//     reported, so a run that ends is never a mystery.
+
+import { spawnSync } from "node:child_process"
+import { existsSync, readFileSync, writeFileSync } from "node:fs"
+import { join } from "node:path"
+
+import {
+  parseTasks,
+  nextPending,
+  markTask,
+  taskPrompt,
+  summarize,
+  isBlocked,
+} from "../tasks.js"
+
+const TASKS_PATH = join(".space", "tasks.md")
+
+const bold = (s) => `\x1b[1m${s}\x1b[0m`
+const stamp = (s) => `\x1b[2m${s}\x1b[0m`
+const green = (s) => `\x1b[32m${s}\x1b[0m`
+const yellow = (s) => `\x1b[33m${s}\x1b[0m`
+const red = (s) => `\x1b[31m${s}\x1b[0m`
+
+/**
+ * Run one command to completion and report what happened.
+ *
+ * Never throws: the caller has to be able to write the failure into the task
+ * file, which is the only record anyone gets of why a task did not land.
+ */
+export function sh(command, { cwd, timeoutMs = 900_000 } = {}) {
+  const started = Date.now()
+  const res = spawnSync(command, {
+    shell: true,
+    cwd,
+    timeout: timeoutMs,
+    encoding: "utf8",
+    maxBuffer: 32 * 1024 * 1024,
+    env: { ...process.env, NO_COLOR: process.env.NO_COLOR ?? "" },
+  })
+  const stdout = `${res.stdout ?? ""}${res.stderr ?? ""}`.trimEnd()
+  return {
+    ok: res.status === 0,
+    status: res.status,
+    // A spawn that never started has no stderr; say so instead of implying the
+    // command ran and said nothing.
+    output: stdout || (res.error ? String(res.error.message) : ""),
+    seconds: Math.round((Date.now() - started) / 1000),
+  }
+}
+
+/** The tail of a verify run -- the last few lines are the verdict. */
+export function tail(text, n = 12) {
+  const lines = String(text).split("\n").filter((l) => l.trim())
+  return lines.slice(-n).join("\n")
+}
+
+/**
+ * Ask `opencode` to do the task. Returns the transcript so a human can read what
+ * the agent actually did rather than being told about it.
+ */
+export function invokeAgent(role, prompt, { cwd, timeoutMs }) {
+  const res = spawnSync(
+    "opencode",
+    ["run", "--agent", role, "--format", "json", prompt],
+    {
+      cwd,
+      encoding: "utf8",
+      timeout: timeoutMs,
+      maxBuffer: 64 * 1024 * 1024,
+    },
+  )
+  return {
+    ok: res.status === 0,
+    status: res.status,
+    output: `${res.stdout ?? ""}${res.stderr ?? ""}`.trimEnd(),
+    error: res.error ? String(res.error.message) : "",
+  }
+}
+
+/** The agent's own instructions for one task, plus what the loop expects back. */
+export function buildPrompt(task) {
+  return [
+    `You are the space-build agent. Do exactly this one task and stop.`,
+    ``,
+    taskPrompt(task),
+    ``,
+    task.verify
+      ? `When the change is done, run this command yourself: ${task.verify}`
+      : `There is no verify command for this task. Do the work, say what you changed, and stop.`,
+    ``,
+    `Rules for this run:`,
+    `  - Change only the files named above. If the task is wrong or blocked, say so and stop.`,
+    `  - Never run git push, never create a PR, never publish. A human does that.`,
+    `  - Report the real output of the verify command. Never invent a passing result.`,
+  ].join("\n")
+}
+
+/**
+ * Run the loop.
+ *
+ * Stops at the first of: nothing pending, nothing verifiable, a blocked task, a
+ * failing verify, the round cap, or the time budget. Whichever it was gets
+ * printed, so a run that stops early always says why.
+ */
+export function runLoop({
+  cwd = process.cwd(),
+  role = "build",
+  maxRounds = 20,
+  budgetMinutes = 120,
+  runAgent = invokeAgent,
+  runVerify = sh,
+} = {}) {
+  const started = Date.now()
+  const budgetMs = budgetMinutes * 60_000
+  const log = []
+
+  if (!existsSync(join(cwd, TASKS_PATH))) {
+    return stop(log, {
+      reason: `no task file at ${TASKS_PATH}. Run space-plan first -- this loop needs a plan to walk.`,
+      code: 1,
+    })
+  }
+
+  let text = readFileSync(join(cwd, TASKS_PATH), "utf8")
+  let rounds = 0
+
+  for (;;) {
+    const parsed = parseTasks(text)
+    const task = nextPending(parsed)
+
+    if (!task) {
+      const s = summarize(parsed)
+      return stop(log, {
+        reason:
+          s.blocked > 0
+            ? `nothing left to run: ${s.done} done, ${s.blocked} blocked.`
+            : `every task is done (${s.done}/${s.total}).`,
+        code: s.blocked > 0 ? 2 : 0,
+        summary: s,
+      })
+    }
+
+    if (isBlocked(task)) {
+      return stop(log, { reason: `task is blocked and needs a person: ${task.title}`, code: 2 })
+    }
+
+    if (!task.verify) {
+      text = markTask(text, task, {
+        status: "blocked",
+        verifyOutput: "blocked by the loop: this task has no verify command, so nothing could prove it finished.",
+      })
+      writeFileSync(join(cwd, TASKS_PATH), text)
+      return stop(log, {
+        reason: `"${task.title}" has no verify command. It cannot be checked, so it cannot be run unattended. Add one, or do it yourself.`,
+        code: 2,
+      })
+    }
+
+    rounds++
+    if (rounds > maxRounds) {
+      return stop(log, { reason: `stopped at the round cap (${maxRounds}). Raise it with --max-rounds, or run fewer tasks.`, code: 2 })
+    }
+
+    const elapsed = Date.now() - started
+    if (elapsed >= budgetMs) {
+      return stop(log, { reason: `stopped at the time budget (${budgetMinutes} min, ${Math.round(elapsed / 60000)} min used).`, code: 2 })
+    }
+
+    console.log(`\n  ${bold(`[${rounds}]`)} ${task.title}`)
+    if (task.files) console.log(`      ${stamp(task.files)}`)
+
+    const agent = runAgent(role, buildPrompt(task), {
+      cwd,
+      timeoutMs: Math.max(60_000, budgetMs - (Date.now() - started)),
+    })
+    if (!agent.ok) {
+      const why = agent.error || agent.output || `exit ${agent.status}`
+      text = markTask(text, task, {
+        status: "blocked",
+        verifyOutput: `blocked by the loop: the ${role} agent failed.\n${tail(why)}`,
+      })
+      writeFileSync(join(cwd, TASKS_PATH), text)
+      return stop(log, {
+        reason: `the ${role} agent failed on "${task.title}": ${tail(why, 3).split("\n")[0]}`,
+        code: 3,
+      })
+    }
+
+    // The agent's claim is not the evidence. Re-run the verify here, because a
+    // loop that trusts the agent's own word about whether it passed is a loop
+    // that reports success it never checked.
+    console.log(`      ${stamp(`verify: ${task.verify}`)}`)
+    const check = runVerify(task.verify, { cwd, timeoutMs: 1800_000 })
+    log.push({ round: rounds, task: task.title, ok: check.ok, seconds: check.seconds })
+
+    if (!check.ok) {
+      text = markTask(text, task, {
+        status: "blocked",
+        verifyOutput: `verify failed (exit ${check.status}, ${check.seconds}s)\n${tail(check.output)}`,
+      })
+      writeFileSync(join(cwd, TASKS_PATH), text)
+      return stop(log, {
+        reason: `verify failed on "${task.title}". Marked blocked -- the real output is in ${TASKS_PATH}.`,
+        code: 4,
+      })
+    }
+
+    text = markTask(text, task, {
+      status: "done",
+      verifyOutput: `exit ${check.status} in ${check.seconds}s\n${tail(check.output)}`,
+    })
+    writeFileSync(join(cwd, TASKS_PATH), text)
+    console.log(`      ${green("done")} ${stamp(`${check.seconds}s`)}`)
+  }
+}
+
+function stop(log, { reason, code, summary }) {
+  return { reason, code: code ?? 0, rounds: log.length, log, summary }
+}
+
+export { TASKS_PATH }

package/src/profile.js

@@ -17,6 +17,7 @@ export const ROOT = resolve(HERE, "..")
 /**
  * @typedef {"zero-retention" | "may-train" | "unknown"} Retention
  * @typedef {string | null} EffortRequest
+ * @typedef {{ model: string, levels?: string[] }} RoleModel
  * @typedef {object} Profile
  * @property {string} id
  * @property {string} [title]
@@ -31,6 +32,7 @@ export const ROOT = resolve(HERE, "..")
  *             structuredOutput?: boolean, attachment?: boolean }} [supports]
  * @property {{ levels?: string[], default?: string | null, reserveShare?: number }} [effort]
  * @property {Record<string, EffortRequest>} [roles]
+ * @property {Record<string, RoleModel>} [roleModels]
  * @property {{ enabled?: boolean, keepRecentToolCalls?: number,
  *             keepRecentTurns?: number, keepUserMessages?: boolean }} [prune]
  * @property {string[]} [fallback]
@@ -135,6 +137,35 @@ export function validate(p, path = "<memory>") {
       `${where}: effort.reserveShare must be between 0 and 0.5`,
     )
   }
+  for (const [role, rm] of Object.entries(p.roleModels ?? {})) {
+    if (!SPLITTABLE_ROLES.includes(role)) {
+      throw new ProfileError(
+        `${where}: roleModels.${role} is not one of ${SPLITTABLE_ROLES.join(", ")}`,
+      )
+    }
+    // resolveRoles walks `roles`, so a split for a role nobody asks for never
+    // reaches an agent. That reads as "split installed" while doing nothing.
+    if (!(role in (p.roles ?? {}))) {
+      throw new ProfileError(
+        `${where}: roleModels.${role} has no matching entry in roles, so it would never apply`,
+      )
+    }
+    if (!rm?.model || typeof rm.model !== "string") {
+      throw new ProfileError(`${where}: roleModels.${role} needs a model`)
+    }
+    if (rm.levels !== undefined && !Array.isArray(rm.levels)) {
+      throw new ProfileError(
+        `${where}: roleModels.${role}.levels must be an array`,
+      )
+    }
+    // retention.yml is the only place retention is decided, so a role model
+    // missing from it is a profile we cannot honestly write a header for.
+    if (!retentionFor(rm.model)) {
+      throw new ProfileError(
+        `${where}: roleModels.${role} names ${rm.model}, which has no retention.yml entry`,
+      )
+    }
+  }
 }
 
 // Canonical ordering for reasoning effort, used to snap a requested level onto
@@ -178,15 +209,45 @@ export function resolveEffort(levels, requested) {
   return best ?? levels[levels.length - 1] ?? null
 }
 
+/**
+ * Model and effort levels for one role, or null when the role stays on the
+ * profile's base model.
+ *
+ * @param {Profile} p
+ * @param {string} role
+ * @returns {{ model: string, levels: string[] } | null}
+ */
+export function resolveRoleModel(p, role) {
+  const rm = p.roleModels?.[role]
+  if (!rm) return null
+  return { model: rm.model, levels: rm.levels ?? [] }
+}
+
+/**
+ * Effort levels a role's own model offers. Falls back to the profile's when the
+ * role is not split out, so an unsplit profile behaves exactly as before.
+ *
+ * @param {Profile} p
+ * @param {string} role
+ * @returns {string[]}
+ */
+function levelsForRole(p, role) {
+  const rm = resolveRoleModel(p, role)
+  if (rm) return rm.levels
+  return p.effort?.levels ?? []
+}
+
 /**
  * @param {Profile} p
  * @returns {Record<string, string | null>}
  */
 export function resolveRoles(p) {
-  const levels = p.effort?.levels ?? []
   const out = /** @type {Record<string, string | null>} */ ({})
   for (const [role, want] of Object.entries(p.roles ?? {})) {
-    out[role] = resolveEffort(levels, want)
+    // Resolve against the levels of the model this role actually runs on. A
+    // role asking for `max` on a model topping out at `xhigh` gets `xhigh`,
+    // which is only correct if the profile knows the role's real model.
+    out[role] = resolveEffort(levelsForRole(p, role), want)
   }
   return out
 }
@@ -201,10 +262,39 @@ export function resolveDefaultVariant(p) {
  * tool output. Scaled from the model, never hardcoded.
  * @param {Profile} p @returns {number}
  */
+// The roles a profile may split onto another model. Kept in step with
+// `roleModels` in profiles/schema.json, which nothing validates at runtime.
+const SPLITTABLE_ROLES = ["plan", "build", "reviewer", "locate", "verify"]
+
+/**
+ * The model whose limits govern context management: the one with the widest
+ * input cap, not necessarily the base model.
+ *
+ * A split profile derives `reserved` and `prune` from a single `limits`, but
+ * the roles do not share it. Sizing for the narrowest input cap silently
+ * throttles every wider role to under half its real window, which is the whole
+ * point of running them on a bigger model. Sizing for the widest means no role
+ * can be under-reserved, which is the direction that has to be safe.
+ *
+ * @param {Profile} p
+ * @returns {number}
+ */
+export function inputCap(p) {
+  let cap = p.limits.input ?? p.limits.context
+  for (const s of roleSplitModels(p)) {
+    const other = s.limits?.input ?? s.limits?.context
+    if (typeof other === "number" && other > cap) cap = other
+  }
+  return cap
+}
+
+/**
+ * @param {Profile} p
+ * @returns {number}
+ */
 export function reservedTokens(p) {
-  const input = p.limits.input ?? p.limits.context
   const share = p.effort?.reserveShare ?? 0.12
-  return Math.max(8000, Math.round(input * share))
+  return Math.max(8000, Math.round(inputCap(p) * share))
 }
 
 /**
@@ -212,7 +302,7 @@ export function reservedTokens(p) {
  * @param {Profile} p @returns {number}
  */
 export function pruneThreshold(p) {
-  return (p.limits.input ?? p.limits.context) - reservedTokens(p)
+  return inputCap(p) - reservedTokens(p)
 }
 
 /**
@@ -296,6 +386,67 @@ export function retentionFor(id) {
   return table[id] ?? table[id.includes("/") ? id : `opencode/${id}`]
 }
 
+/**
+ * Every model a profile can put behind the user, base plus any split roles.
+ *
+ * @param {Profile} p
+ * @returns {string[]}
+ */
+export function modelsInPlay(p) {
+  const out = [p.model]
+  for (const rm of Object.values(p.roleModels ?? {})) {
+    if (!out.includes(rm.model)) out.push(rm.model)
+  }
+  return out
+}
+
+/**
+ * The split roles, each with the limits of the model that role actually runs
+ * on. A role on a different model has a different output cap, and the header
+ * has to say so: reading "output cap 524,288" and assuming `build` can spend
+ * all of it is wrong whenever build is split onto a smaller model.
+ *
+ * @param {Profile} p
+ * @returns {{ role: string, model: string, levels: string[], limits: Profile["limits"] | null }[]}
+ */
+export function roleSplitModels(p) {
+  return Object.entries(p.roleModels ?? {}).map(([role, rm]) => {
+    let limits = null
+    try {
+      limits = loadProfile(rm.model)?.limits ?? null
+    } catch {
+      limits = null
+    }
+    return { role, model: rm.model, levels: rm.levels ?? [], limits }
+  })
+}
+
+// Ordered safest-first: a profile is only as private as its least private model.
+const RETENTION_SEVERITY = ["zero-retention", "unknown", "may-train"]
+
+/**
+ * The retention that actually applies once every role is considered, which is
+ * what a config header has to print. `retention.yml` is the source of truth.
+ *
+ * @param {Profile} p
+ * @returns {{ retention: string, worst: string | null }}
+ */
+export function effectiveRetention(p) {
+  let worst = null
+  let retention = "zero-retention"
+  for (const id of modelsInPlay(p)) {
+    const declared = retentionFor(id)?.retention ?? "unknown"
+    if (declared === "may-train") {
+      return { retention: "may-train", worst: id }
+    }
+    if (!worst || RETENTION_SEVERITY.indexOf(declared) > RETENTION_SEVERITY.indexOf(worst)) {
+      worst = declared
+    }
+    retention = worst
+  }
+  return { retention, worst }
+}
+
 /** Where opencode keeps its config, honouring the same env var it does. */
 export function configFilePath() {
   const dir =
@@ -304,14 +455,118 @@ export function configFilePath() {
   return join(dir, "opencode.jsonc")
 }
 
+/**
+ * Parse JSONC the way OpenCode does: comments and trailing commas are legal.
+ *
+ * A regex that strips `//` cannot be used here. The generated config carries
+ * `"$schema": "https://opencode.ai/config.json"`, so a naive strip corrupts the
+ * URL; and a trailing comma -- ordinary in a hand-edited `.jsonc` -- makes
+ * `JSON.parse` throw, which used to surface as an empty object and silently
+ * dropped every hand-tuned key in `oc-harness use`. This walks the text once,
+ * tracking string state, so both are handled correctly.
+ *
+ * @param {string} text
+ * @returns {Record<string, unknown> | null} null when the text is not valid JSONC
+ */
+export function parseJsonc(text) {
+  let out = ""
+  let i = 0
+  let inString = false
+  let escaped = false
+  /** True once a `,` has been emitted that a `}` or `]` may turn out to cancel. */
+  let pendingComma = false
+
+  const n = text.length
+  while (i < n) {
+    const ch = text[i]
+
+    if (inString) {
+      out += ch
+      if (escaped) escaped = false
+      else if (ch === "\\") escaped = true
+      else if (ch === '"') inString = false
+      i++
+      continue
+    }
+
+    if (ch === '"') {
+      if (pendingComma) {
+        out += ","
+        pendingComma = false
+      }
+      out += ch
+      inString = true
+      i++
+      continue
+    }
+
+    if (ch === "/" && text[i + 1] === "/") {
+      while (i < n && text[i] !== "\n") i++
+      continue
+    }
+
+    if (ch === "/" && text[i + 1] === "*") {
+      i += 2
+      while (i < n && !(text[i] === "*" && text[i + 1] === "/")) i++
+      i += 2
+      continue
+    }
+
+    if (ch === ",") {
+      // A second comma before any value is malformed. Repairing `,,,` into one
+      // separator would mean guessing at a file we are about to rewrite, which
+      // is the exact failure this parser exists to prevent.
+      if (pendingComma) return null
+      // Hold the comma back: if the next significant character closes an
+      // object or array, this comma was illegal trailing punctuation.
+      pendingComma = true
+      i++
+      continue
+    }
+
+    if (ch === " " || ch === "\t" || ch === "\n" || ch === "\r") {
+      out += ch
+      i++
+      continue
+    }
+
+    if (ch === "}" || ch === "]") {
+      // A closer cancels any comma held back before it.
+      pendingComma = false
+      out += ch
+      i++
+      continue
+    }
+
+    // Every other significant character settles a comma held back across
+    // whitespace or a comment. Miss this on the string branch below and a
+    // comment after `1, // note` swallows the separator entirely.
+    if (pendingComma) {
+      out += ","
+      pendingComma = false
+    }
+    out += ch
+    i++
+  }
+
+  try {
+    const parsed = JSON.parse(out)
+    return parsed && typeof parsed === "object" ? parsed : null
+  } catch {
+    // An empty or comments-only file is not malformed JSONC, it is an empty
+    // config, and `oc-harness use` has to be able to fill one in. Only report
+    // a parse failure when there was actually something to parse.
+    if (!out.trim()) return {}
+    return null
+  }
+}
+
 /**
  * Read the active profile id from the config file on disk.
  *
- * This exists because the `config` hook cannot be the source. OpenCode's
- * Config schema sets `additionalProperties: false`, so a custom `harness` key
- * written into the file is stripped from the object handed to the hook. The
- * file survives; the in-memory config does not. Verified against the 1.18.35
- * schema.
+ * Returns null when the file is absent, unreadable, or not valid JSONC. Callers
+ * fall back to the profile named in the `config` hook, so a null here degrades
+ * to the default profile rather than failing.
  *
  * @returns {string | null}
  */
@@ -322,10 +577,10 @@ export function profileIdFromConfig() {
   const file = configFilePath()
   if (!existsSync(file)) return null
   try {
-    const text = readFileSync(file, "utf8")
-    const json = text.slice(text.indexOf("{"))
-    const parsed = JSON.parse(json)
-    const id = parsed?.harness?.profile
+    const parsed = parseJsonc(readFileSync(file, "utf8"))
+    const harness = parsed?.harness
+    const id =
+      harness && typeof harness === "object" ? /** @type {{profile?: unknown}} */ (harness).profile : null
     return typeof id === "string" && id ? id : null
   } catch {
     return null

package/src/rules.js

@@ -349,24 +349,9 @@ export const BASH_RULES = [
     test: /\b(curl|wget)\b[^|]*\|\s*(sudo\s+)?(sh|bash|zsh|python[0-9.]*)\b/i,
     why: "downloads and executes a script in one step",
   },
-  {
-    name: "git push with force",
-    scope: "segment",
-    // A dry run publishes nothing, so it is not the thing being gated.
-    check: (seg) =>
-      gitSubcommand(seg) === "push" &&
-      tokenize(seg).some((w) => w === "--force" || w === "-f" || w.startsWith("--force")),
-    why: "overwrites published history",
-  },
-  {
-    name: "git push",
-    scope: "segment",
-    // Flags are matched as whole words: a `-f` in `my-feature-f` is a branch
-    // name, not a force flag.
-    check: (seg) =>
-      gitSubcommand(seg) === "push" && !tokenize(seg).some((w) => w === "--dry-run" || w === "-n"),
-    why: "publishing is a human decision",
-  },
+  // `git push` is ask-gated via permissions (`*` ask in the generated
+  // config), not guard-blocked. Publishing needs human approval, but the
+  // guard lets it through so opencode can prompt first.
   {
     name: "recursive delete",
     scope: "segment",