opencode-harness-kit 0.2.0 → 0.3.3

package/src/router.ts

@@ -1,12 +1,14 @@
-// Model fallback across a profile's queue.
+// Model fallback across a profile's lanes.
 //
 // The honest version of this feature: opencode has no hook that can change a
 // live session's model. `chat.params` only sees temperature and sampling, the
 // model on `chat.message` is read-only, and `config` runs once at startup. So
 // failover here means two things, and only two:
 //
-//   1. At startup, if a health file marks the preferred model unhealthy, pick
-//      the next healthy profile before the session begins.
+//   1. At startup, if a health file marks a route unhealthy, the lane that
+//      used it moves to the next rung of its own chain. A fleet has ten lanes
+//      and ten different models, so one spent route must not take the other
+//      nine down with it.
 //   2. When a request fails, tell the agent plainly that the route is spent and
 //      a new session is needed, instead of letting it retry into a wall.
 //
@@ -25,6 +27,9 @@ import {
   profileIdFromConfig,
   reservedTokens,
   resolveEffort,
+  resolveLaneChain,
+  resolveLaneModel,
+  resolveRoles,
 } from "./profile.js"
 
 interface Health {
@@ -53,6 +58,29 @@ function writeHealth(worktree: string, h: Health) {
   }
 }
 
+/**
+ * The models a previous session found spent.
+ *
+ * Exported so the effort hook resolves a lane's variant against the model the
+ * lane actually ended up on. A lane that failed over from a model with a
+ * `none` effort level to one without levels has to lose its variant, or the
+ * config asks for an effort the new model does not publish.
+ *
+ * @param {string} worktree
+ * @returns {Set<string>}
+ */
+export function unhealthyModels(worktree: string): Set<string> {
+  const health = readHealth(worktree)
+  // The health file is keyed by the qualified route, `opencode/<model>`. The
+  // lane chains list bare model ids, so the prefix is dropped here. Comparing
+  // the two directly would never match, and a spent route would look healthy.
+  const out = new Set<string>()
+  for (const key of Object.keys(health.unhealthy ?? {})) {
+    out.add(key.includes("/") ? key.slice(key.indexOf("/") + 1) : key)
+  }
+  return out
+}
+
 /** Patterns that mean "this route is spent", not "this request was bad". */
 const SPENT = [
   /FreeUsageLimitError/i,
@@ -72,8 +100,8 @@ function classify(text: string): string | null {
 const WARN_TEXT =
   "[opencode-harness-kit] This session's model route is rate limited or withdrawn. " +
   "No plugin can swap a live session's model, so stop retrying and ask the human to start a " +
-  "new session. Run `oc-harness list` to see the fallback queue, or `oc-harness doctor` to " +
-  "check whether the profile still matches live metadata."
+  "new session. Run `oc-harness list` to see the lanes and their chains, or `oc-harness doctor` " +
+  "to check whether the profile still matches live metadata."
 
 export const RouterPlugin: Plugin = async ({ worktree }) => {
   let failedSessions = 0
@@ -85,32 +113,75 @@ export const RouterPlugin: Plugin = async ({ worktree }) => {
       try {
         const cfg = config as Record<string, any>
         const health = readHealth(worktree)
+        const unhealthy = unhealthyModels(worktree)
 
         // config.harness is stripped by schema validation, so the active
         // profile comes from the config file on disk.
         const activeId = cfg.harness?.profile ?? profileIdFromConfig()
-        if (activeId) {
-          const id: string = activeId
-          const mark = health.unhealthy[`opencode/${id}`]
-          if (mark) {
-            // Walk the queue for the first profile that is not marked down.
-            let chosen: string | null = id
+        if (!activeId) return
+        const id: string = activeId
+        let profile
+        try {
+          profile = loadProfile(id)
+        } catch {
+          return
+        }
+
+        // 1. The session's own model. When it is spent, the profile's
+        //    fallback queue names the profiles that can take over.
+        //
+        //    A fleet keeps its lanes. Every lane has its own chain, and step 2
+        //    moves a lane down that chain when its route is spent, so
+        //    overwriting the lanes here would collapse ten jobs onto one
+        //    model and throw away the chain that was supposed to survive the
+        //    outage. Only the session model moves.
+        const baseKey = `opencode/${profile.model}`
+        if (health.unhealthy?.[baseKey]) {
+          const queue = [id, ...(profile.fallback ?? [])]
+          for (const candidate of queue) {
+            if (health.unhealthy?.[`opencode/${candidate}`]) continue
+            let fallback
             try {
-              const queue = [id, ...(loadProfile(id).fallback ?? [])]
-              for (const candidate of queue) {
-                if (!health.unhealthy[`opencode/${candidate}`]) {
-                  chosen = candidate
-                  break
-                }
-              }
+              fallback = loadProfile(candidate)
             } catch {
-              // A broken profile must not stop opencode from starting.
-            }
-            if (chosen !== id) {
-              applyProfileToConfig(cfg, chosen)
-              health.active = chosen
-              writeHealth(worktree, health)
+              continue
             }
+            const key = `opencode/${fallback.model}`
+            if (health.unhealthy?.[key]) continue
+            const fleet = isFleet(profile)
+            applySessionModel(cfg, fallback, {
+              keepLanes: fleet,
+              // The lanes did not change, so the numbers derived from them did
+              // not change either.
+              profileId: fleet ? profile.id : fallback.id,
+              reserved: fleet ? reservedTokens(profile) : undefined,
+              threshold: fleet ? pruneThreshold(profile) : undefined,
+            })
+            health.active = candidate
+            writeHealth(worktree, health)
+            break
+          }
+        }
+
+        // 2. Every lane. A lane whose model is spent moves to the next rung of
+        //    its own chain, so one dead route costs one lane, not the fleet.
+        const roles = resolveRoles(profile, { unhealthy })
+        for (const [lane, level] of Object.entries(roles)) {
+          if (level === undefined) continue
+          const agent = cfg.agent?.[lane]
+          if (!agent) continue
+          const resolved = resolveLaneModel(profile, lane, { unhealthy })
+          if (resolved.model === profile.model && !profile.roleModels?.[lane]) continue
+          const wantModel = `opencode/${resolved.model}`
+          if (agent.model !== wantModel) {
+            agent.model = wantModel
+            // The variant has to be resolved against the model the lane ended
+            // up on. A lane that failed over to a model with fewer effort
+            // levels has to lose the variant the old model had.
+            const requested = (profile.roles?.[lane] ?? null) as any
+            const next = resolveEffort(resolved.levels, requested)
+            if (next === null) delete agent.variant
+            else agent.variant = next
           }
         }
       } catch {
@@ -160,30 +231,68 @@ export const RouterPlugin: Plugin = async ({ worktree }) => {
   }
 }
 
-/** Point the config at another profile, including its derived numbers. */
-function applyProfileToConfig(cfg: Record<string, any>, profileId: string) {
+/**
+ * Whether a profile runs more than one model behind its lanes.
+ * @param {any} profile
+ * @returns {boolean}
+ */
+function isFleet(profile: any): boolean {
+  return Object.keys(profile?.roleModels ?? {}).length > 0
+}
+
+/**
+ * Move the SESSION onto another profile's model.
+ *
+ * A fleet keeps its lanes. The lanes have their own chains, and the config
+ * hook has already moved each one down its chain past a spent route, so
+ * overwriting them here would collapse ten jobs onto one model and discard
+ * the chain that was supposed to survive the outage.
+ *
+ * A profile with no lanes of its own is a single-model profile: there is
+ * nothing to keep, so the fallback takes the lanes too.
+ *
+ * The session model is published on `harness.sessionModel` so the effort hook
+ * - which runs after this one - pins the session to the model that was chosen
+ * here rather than back to the base model that is spent.
+ */
+function applySessionModel(
+  cfg: Record<string, any>,
+  profile: any,
+  opts: {
+    keepLanes?: boolean
+    profileId?: string
+    reserved?: number
+    threshold?: number
+  } = {},
+) {
   try {
-    const p = loadProfile(profileId)
-    const model = `${p.provider}/${p.model}`
+    const model = `${profile.provider}/${profile.model}`
     cfg.model = model
     cfg.small_model = model
-    const variant = p.effort?.default
-    if (p.effort?.levels?.length && variant) cfg.variant = variant
+    const variant = profile.effort?.default
+    if (profile.effort?.levels?.length && variant) cfg.variant = variant
     else delete cfg.variant
 
-    for (const [role, want] of Object.entries(p.roles ?? {})) {
-      const agent = cfg.agent ?? (cfg.agent = {})
-      const existing = agent[role] ?? (agent[role] = {})
-      existing.model = model
-      const level = resolveEffort(p.effort?.levels ?? [], want)
-      if (level) existing.variant = level
-      else delete existing.variant
+    if (!opts.keepLanes) {
+      for (const [lane, want] of Object.entries(profile.roles ?? {})) {
+        const agent = cfg.agent ?? (cfg.agent = {})
+        const existing = agent[lane] ?? (agent[lane] = {})
+        existing.model = model
+        const level = resolveEffort(profile.effort?.levels ?? [], want as any)
+        if (level) existing.variant = level
+        else delete existing.variant
+      }
     }
     if (cfg.compaction) {
       cfg.compaction.prune = true
-      cfg.compaction.reserved = reservedTokens(p)
+      cfg.compaction.reserved = opts.reserved ?? reservedTokens(profile)
+    }
+    cfg.harness = {
+      ...(cfg.harness ?? {}),
+      profile: opts.profileId ?? profile.id,
+      threshold: opts.threshold ?? pruneThreshold(profile),
+      sessionModel: model,
     }
-    cfg.harness = { ...(cfg.harness ?? {}), profile: profileId, threshold: pruneThreshold(p) }
   } catch {
     // Leave the config alone if the fallback profile cannot be read.
   }

package/src/tasks.js

@@ -0,0 +1,248 @@
+// Parse and rewrite `.space/tasks.md`, the one artifact the whole loop reads.
+//
+// The driver reads it to decide what to do next, so a mistake here is a mistake
+// in the loop itself rather than in a report about the loop. It is kept pure and
+// line-based on purpose: every byte of the file that the loop does not own --
+// the goal, the constraints, the decisions, any prose a human added -- survives
+// a rewrite untouched.
+
+export const TASK_FIELDS = ["files", "change", "verify", "status"]
+
+const SECTIONS = ["Constraints", "Decisions", "Risks", "Tasks"]
+
+const headingRe = /^##\s+(.+?)\s*$/
+const taskRe = /^\s*-\s*\[([ xX])\]\s*(.*)$/
+const fieldRe = new RegExp(`^\\s+-\\s+(${TASK_FIELDS.join("|")})\\s*:\\s*(.*)$`)
+
+/**
+ * @typedef {object} Task
+ * @property {number} index      position in the file, zero based
+ * @property {boolean} checked   the checkbox was ticked
+ * @property {string} title      the task on one line
+ * @property {string} files      which files it is allowed to touch
+ * @property {string} change     what it is supposed to change
+ * @property {string} verify     the command that proves it
+ * @property {string} status     pending | done | blocked
+ * @property {number} start      first line of the task block
+ * @property {number} end        last line of the task block
+ * @property {string} [key]      indexed by field name when a block is written
+ */
+
+/**
+ * @typedef {object} TaskFile
+ * @property {string} goal
+ * @property {string} constraints
+ * @property {string} decisions
+ * @property {string} risks
+ * @property {Task[]} tasks
+ */
+
+/**
+ * Read a task file into its sections and tasks.
+ *
+ * Only the Tasks section is parsed structurally. Everything else is kept as raw
+ * lines so that rendering a document back never reformats a decision someone
+ * wrote in prose.
+ *
+ * @param {string} text
+ * @returns {TaskFile}
+ */
+export function parseTasks(text) {
+  const lines = String(text ?? "").split("\n")
+
+  const goal = (() => {
+    for (const line of lines) {
+      if (!line.trim()) continue
+      if (line.startsWith("# ")) return line.slice(2).trim()
+      return line.trim() // a file with no heading still has a goal on line 1
+    }
+    return ""
+  })()
+
+  /** @type {Record<string, string[]>} */
+  const sections = {}
+  let current = ""
+  for (const line of lines) {
+    const h = line.match(headingRe)
+    if (h) {
+      current = h[1]
+      if (!sections[current]) sections[current] = []
+      continue
+    }
+    if (current) sections[current].push(line)
+  }
+
+  /** @type {Task[]} */
+  const tasks = []
+  for (let i = 0; i < lines.length; i++) {
+    const m = lines[i].match(taskRe)
+    if (!m) continue
+    // A checkbox outside the Tasks section is prose, not a task.
+    if (!inSection(lines, i, "Tasks")) continue
+
+    /** @type {Record<string, any> & Task} */
+    const t = {
+      index: tasks.length,
+      checked: m[1].toLowerCase() === "x",
+      title: m[2].trim(),
+      files: "",
+      change: "",
+      verify: "",
+      status: "",
+      start: i,
+      end: i,
+    }
+
+    let last = null
+    let j = i + 1
+    for (; j < lines.length; j++) {
+      if (taskRe.test(lines[j]) || headingRe.test(lines[j])) break
+      const f = lines[j].match(fieldRe)
+      if (f) {
+        t[f[1]] = f[2].trim()
+        last = f[1]
+        continue
+      }
+      // A wrapped value continues the field above it, indented deeper.
+      if (last && lines[j].trim() && /^\s{4,}\S/.test(lines[j])) {
+        t[last] = `${t[last]} ${lines[j].trim()}`.trim()
+        continue
+      }
+      if (!lines[j].trim()) continue
+      break
+    }
+    t.end = j - 1
+    if (!t.status) t.status = t.checked ? "done" : "pending"
+    tasks.push(t)
+  }
+
+  return {
+    goal,
+    constraints: raw(sections, "Constraints"),
+    decisions: raw(sections, "Decisions"),
+    risks: raw(sections, "Risks"),
+    tasks,
+  }
+}
+
+/**
+ * @param {string[]} lines
+ * @param {number} idx
+ * @param {string} name
+ * @returns {boolean}
+ */
+function inSection(lines, idx, name) {
+  for (let i = idx; i >= 0; i--) {
+    const h = lines[i].match(headingRe)
+    if (h) return h[1] === name
+  }
+  return false
+}
+
+/**
+ * @param {Record<string, string[]>} sections
+ * @param {string} name
+ * @returns {string}
+ */
+function raw(sections, name) {
+  return (sections[name] ?? []).join("\n").trim()
+}
+
+/**
+ * The next task the loop is allowed to touch. Blocked work is never retried.
+ *
+ * @param {TaskFile} parsed
+ * @returns {Task | null}
+ */
+export function nextPending(parsed) {
+  return parsed.tasks.find((t) => !isDone(t) && !isBlocked(t)) ?? null
+}
+
+/** @param {Task} t */
+export const isDone = (t) => t.checked || t.status === "done"
+
+/** @param {Task} t */
+export const isBlocked = (t) => t.status === "blocked"
+
+/**
+ * Record the outcome of a task: flip the checkbox, set the status, and paste the
+ * verify output that was actually observed.
+ *
+ * The verify text is written under the `verify:` line rather than replacing it,
+ * because the contract asks for the command and the real output together.
+ *
+ * @param {string} text
+ * @param {Task} task
+ * @param {{ status: string, verifyOutput?: string }} outcome
+ * @returns {string}
+ */
+export function markTask(text, task, { status, verifyOutput }) {
+  const lines = String(text).split("\n")
+  const boxLine = lines[task.start] ?? ""
+  lines[task.start] = boxLine.replace(
+    /^\s*-\s*\[[ xX]\]/,
+    status === "done" ? "- [x]" : "- [ ]",
+  )
+
+  let statusLine = -1
+  for (let i = task.start; i <= task.end + 1 && i < lines.length; i++) {
+    const f = lines[i]?.match(fieldRe)
+    if (f && f[1] === "status") {
+      statusLine = i
+      break
+    }
+  }
+  if (statusLine >= 0) lines[statusLine] = lines[statusLine].replace(
+    /(\s-\s+status\s*:).*$/,
+    `$1 ${status}`,
+  )
+
+  if (verifyOutput) {
+    let verifyLine = -1
+    for (let i = task.start; i <= task.end + 1 && i < lines.length; i++) {
+      const f = lines[i]?.match(fieldRe)
+      if (f && f[1] === "verify") {
+        verifyLine = i
+        break
+      }
+    }
+    const verifySource = verifyLine >= 0 ? lines[verifyLine] : undefined
+    if (verifySource) {
+      const indent = (verifySource.match(/^\s*/) ?? [""])[0] + "  "
+      const quoted = String(verifyOutput)
+        .trimEnd()
+        .split("\n")
+        .map((l) => `${indent}${l}`)
+      lines.splice(verifyLine + 1, 0, ...quoted)
+    }
+  }
+
+  return lines.join("\n")
+}
+
+/**
+ * One task rendered as the prompt handed to the agent that will do it.
+ *
+ * @param {Task} task
+ * @returns {string}
+ */
+export function taskPrompt(task) {
+  const out = [task.title]
+  if (task.files) out.push("", `Files: ${task.files}`)
+  if (task.change) out.push("", `Change: ${task.change}`)
+  if (task.verify) out.push("", `Verify: ${task.verify}`)
+  return out.join("\n")
+}
+
+/**
+ * The summary the driver prints when it stops, whatever stopped it.
+ *
+ * @param {TaskFile} parsed
+ * @returns {{ done: number, blocked: number, pending: number, total: number }}
+ */
+export function summarize(parsed) {
+  const done = parsed.tasks.filter(isDone).length
+  const blocked = parsed.tasks.filter(isBlocked).length
+  const pending = parsed.tasks.filter((t) => !isDone(t) && !isBlocked(t)).length
+  return { done, blocked, pending, total: parsed.tasks.length }
+}

package/src/telemetry.ts

@@ -8,6 +8,29 @@
 // What is worth recording: context size per request (to see pruning working),
 // latency by role and effort (to see tiering paying off), tool denials (to see
 // the guard being useful rather than annoying), and provider rate limits.
+//
+// A fleet adds one more question, and it is the one that decides whether the
+// fleet stays at ten lanes or is cut down: which lane is actually used. Ten
+// lanes that are never called are ten descriptions in the model\'s context and
+// ten models on the free tier doing nothing. `oc-harness lanes` reads these
+// records and reports the ones that were never called.
+//
+// WHAT A CALL IS
+//
+// One assistant message is one call. opencode emits `message.updated` several
+// times for a single message - created, streamed, completed - so writing a
+// record per event counted every call three times. A lane record therefore
+// carries the message id, and a lane is counted once per id.
+//
+// Two records are written per message, at most: an open one when the message
+// appears, and a closed one when it completes. The closed record is the only
+// one with a duration, so "how long does this lane take" and "was this lane
+// called at all" are answered from the same file without guessing.
+//
+// The `chat.message` hook is deliberately not used for this. Measured on
+// OpenCode 1.19, its `agent` field is undefined, so it cannot attribute a
+// call to a lane; and a second counter that cannot be correlated with the
+// first would double-count every call.
 
 import type { Plugin } from "@opencode-ai/plugin"
 import { appendFileSync, mkdirSync, existsSync, statSync, renameSync } from "node:fs"
@@ -45,6 +68,17 @@ export const TelemetryPlugin: Plugin = async ({ worktree }) => {
   const dir = join(worktree, ".space")
   const started = Date.now()
 
+  // The message id of every lane record already written, and of every one
+  // already written with a duration.
+  //
+  // opencode emits `message.updated` several times for a single assistant
+  // message: once when the message is created, once per streaming update, and
+  // once when it completes. Writing a record per event counted one call three
+  // times, which made the "cut the lanes nobody calls" counter wrong by a
+  // factor of three. A lane is counted once per message id instead.
+  const opened = new Set<string>()
+  const closed = new Set<string>()
+
   return {
     event: async ({ event }) => {
       // Events are provider-specific and opencode may add shapes we have never
@@ -56,6 +90,30 @@ export const TelemetryPlugin: Plugin = async ({ worktree }) => {
       // they get logged even though the event shape is provider-specific.
       if (/error|rate|limit|quota/i.test(type)) {
         write(dir, { kind: "provider-event", type, detail: truncate(safeStringify(event)) })
+        return
+      }
+
+      // An assistant message is the only place the kit can see which lane ran,
+      // on which model, and how long it took. That record is what makes "cut
+      // the lanes nobody uses" a measurement instead of a guess.
+      if (type === "message.updated") {
+        const lane = laneOf(event)
+        if (!lane || !lane.id) return
+        if (lane.ms !== undefined) {
+          // The completed event. Written once, and only if the open record
+          // did not already carry a duration.
+          if (closed.has(lane.id)) return
+          closed.add(lane.id)
+          write(dir, { kind: "lane", ...lane })
+          return
+        }
+        // The open event: the lane was called. Written even if the call never
+        // finishes, because "this lane is useless" and "this lane is slow"
+        // are different questions and only the first one is settled by a
+        // record with no duration.
+        if (opened.has(lane.id)) return
+        opened.add(lane.id)
+        write(dir, { kind: "lane", ...lane })
       }
     },
 
@@ -77,6 +135,46 @@ export const TelemetryPlugin: Plugin = async ({ worktree }) => {
   }
 }
 
+/**
+ * The lane, model and duration of one assistant message, read defensively.
+ *
+ * The event shape is opencode's, not ours, and it has changed between
+ * releases. Every field is optional and a shape we do not recognise yields
+ * null rather than a wrong number.
+ *
+ * The message id is what makes the record countable: one message is one call.
+ */
+function laneOf(event: unknown): {
+  id?: string
+  agent?: string
+  model?: string
+  provider?: string
+  ms?: number
+} | null {
+  const e = event as { properties?: any } | undefined
+  const info = e?.properties?.info ?? e?.properties
+  if (!info || typeof info !== "object") return null
+  if (info.role && info.role !== "assistant") return null
+
+  const id = typeof info.id === "string" ? info.id : undefined
+  const agent = typeof info.agent === "string" ? info.agent : undefined
+  const modelId = info.model?.modelID ?? info.modelID
+  const providerId = info.model?.providerID ?? info.providerID
+  const time = info.time ?? {}
+  const created = typeof time.created === "number" ? time.created : undefined
+  const completed = typeof time.completed === "number" ? time.completed : undefined
+  const ms = created !== undefined && completed !== undefined ? completed - created : undefined
+
+  if (!id || !agent) return null
+  return {
+    id,
+    agent,
+    model: typeof modelId === "string" ? modelId : undefined,
+    provider: typeof providerId === "string" ? providerId : undefined,
+    ms: typeof ms === "number" && ms >= 0 ? ms : undefined,
+  }
+}
+
 function extractRule(output: string): string {
   const m = output.match(/blocked:\s*([^\n]+)/)
   return m ? m[1].trim() : "unknown"