opencode-harness-kit 0.2.1 → 0.3.3

package/src/router.ts

@@ -1,12 +1,14 @@
-// Model fallback across a profile's queue.
+// Model fallback across a profile's lanes.
 //
 // The honest version of this feature: opencode has no hook that can change a
 // live session's model. `chat.params` only sees temperature and sampling, the
 // model on `chat.message` is read-only, and `config` runs once at startup. So
 // failover here means two things, and only two:
 //
-//   1. At startup, if a health file marks the preferred model unhealthy, pick
-//      the next healthy profile before the session begins.
+//   1. At startup, if a health file marks a route unhealthy, the lane that
+//      used it moves to the next rung of its own chain. A fleet has ten lanes
+//      and ten different models, so one spent route must not take the other
+//      nine down with it.
 //   2. When a request fails, tell the agent plainly that the route is spent and
 //      a new session is needed, instead of letting it retry into a wall.
 //
@@ -25,6 +27,9 @@ import {
   profileIdFromConfig,
   reservedTokens,
   resolveEffort,
+  resolveLaneChain,
+  resolveLaneModel,
+  resolveRoles,
 } from "./profile.js"
 
 interface Health {
@@ -53,6 +58,29 @@ function writeHealth(worktree: string, h: Health) {
   }
 }
 
+/**
+ * The models a previous session found spent.
+ *
+ * Exported so the effort hook resolves a lane's variant against the model the
+ * lane actually ended up on. A lane that failed over from a model with a
+ * `none` effort level to one without levels has to lose its variant, or the
+ * config asks for an effort the new model does not publish.
+ *
+ * @param {string} worktree
+ * @returns {Set<string>}
+ */
+export function unhealthyModels(worktree: string): Set<string> {
+  const health = readHealth(worktree)
+  // The health file is keyed by the qualified route, `opencode/<model>`. The
+  // lane chains list bare model ids, so the prefix is dropped here. Comparing
+  // the two directly would never match, and a spent route would look healthy.
+  const out = new Set<string>()
+  for (const key of Object.keys(health.unhealthy ?? {})) {
+    out.add(key.includes("/") ? key.slice(key.indexOf("/") + 1) : key)
+  }
+  return out
+}
+
 /** Patterns that mean "this route is spent", not "this request was bad". */
 const SPENT = [
   /FreeUsageLimitError/i,
@@ -72,8 +100,8 @@ function classify(text: string): string | null {
 const WARN_TEXT =
   "[opencode-harness-kit] This session's model route is rate limited or withdrawn. " +
   "No plugin can swap a live session's model, so stop retrying and ask the human to start a " +
-  "new session. Run `oc-harness list` to see the fallback queue, or `oc-harness doctor` to " +
-  "check whether the profile still matches live metadata."
+  "new session. Run `oc-harness list` to see the lanes and their chains, or `oc-harness doctor` " +
+  "to check whether the profile still matches live metadata."
 
 export const RouterPlugin: Plugin = async ({ worktree }) => {
   let failedSessions = 0
@@ -85,32 +113,75 @@ export const RouterPlugin: Plugin = async ({ worktree }) => {
       try {
         const cfg = config as Record<string, any>
         const health = readHealth(worktree)
+        const unhealthy = unhealthyModels(worktree)
 
         // config.harness is stripped by schema validation, so the active
         // profile comes from the config file on disk.
         const activeId = cfg.harness?.profile ?? profileIdFromConfig()
-        if (activeId) {
-          const id: string = activeId
-          const mark = health.unhealthy[`opencode/${id}`]
-          if (mark) {
-            // Walk the queue for the first profile that is not marked down.
-            let chosen: string | null = id
+        if (!activeId) return
+        const id: string = activeId
+        let profile
+        try {
+          profile = loadProfile(id)
+        } catch {
+          return
+        }
+
+        // 1. The session's own model. When it is spent, the profile's
+        //    fallback queue names the profiles that can take over.
+        //
+        //    A fleet keeps its lanes. Every lane has its own chain, and step 2
+        //    moves a lane down that chain when its route is spent, so
+        //    overwriting the lanes here would collapse ten jobs onto one
+        //    model and throw away the chain that was supposed to survive the
+        //    outage. Only the session model moves.
+        const baseKey = `opencode/${profile.model}`
+        if (health.unhealthy?.[baseKey]) {
+          const queue = [id, ...(profile.fallback ?? [])]
+          for (const candidate of queue) {
+            if (health.unhealthy?.[`opencode/${candidate}`]) continue
+            let fallback
             try {
-              const queue = [id, ...(loadProfile(id).fallback ?? [])]
-              for (const candidate of queue) {
-                if (!health.unhealthy[`opencode/${candidate}`]) {
-                  chosen = candidate
-                  break
-                }
-              }
+              fallback = loadProfile(candidate)
             } catch {
-              // A broken profile must not stop opencode from starting.
-            }
-            if (chosen !== id) {
-              applyProfileToConfig(cfg, chosen)
-              health.active = chosen
-              writeHealth(worktree, health)
+              continue
             }
+            const key = `opencode/${fallback.model}`
+            if (health.unhealthy?.[key]) continue
+            const fleet = isFleet(profile)
+            applySessionModel(cfg, fallback, {
+              keepLanes: fleet,
+              // The lanes did not change, so the numbers derived from them did
+              // not change either.
+              profileId: fleet ? profile.id : fallback.id,
+              reserved: fleet ? reservedTokens(profile) : undefined,
+              threshold: fleet ? pruneThreshold(profile) : undefined,
+            })
+            health.active = candidate
+            writeHealth(worktree, health)
+            break
+          }
+        }
+
+        // 2. Every lane. A lane whose model is spent moves to the next rung of
+        //    its own chain, so one dead route costs one lane, not the fleet.
+        const roles = resolveRoles(profile, { unhealthy })
+        for (const [lane, level] of Object.entries(roles)) {
+          if (level === undefined) continue
+          const agent = cfg.agent?.[lane]
+          if (!agent) continue
+          const resolved = resolveLaneModel(profile, lane, { unhealthy })
+          if (resolved.model === profile.model && !profile.roleModels?.[lane]) continue
+          const wantModel = `opencode/${resolved.model}`
+          if (agent.model !== wantModel) {
+            agent.model = wantModel
+            // The variant has to be resolved against the model the lane ended
+            // up on. A lane that failed over to a model with fewer effort
+            // levels has to lose the variant the old model had.
+            const requested = (profile.roles?.[lane] ?? null) as any
+            const next = resolveEffort(resolved.levels, requested)
+            if (next === null) delete agent.variant
+            else agent.variant = next
           }
         }
       } catch {
@@ -160,30 +231,68 @@ export const RouterPlugin: Plugin = async ({ worktree }) => {
   }
 }
 
-/** Point the config at another profile, including its derived numbers. */
-function applyProfileToConfig(cfg: Record<string, any>, profileId: string) {
+/**
+ * Whether a profile runs more than one model behind its lanes.
+ * @param {any} profile
+ * @returns {boolean}
+ */
+function isFleet(profile: any): boolean {
+  return Object.keys(profile?.roleModels ?? {}).length > 0
+}
+
+/**
+ * Move the SESSION onto another profile's model.
+ *
+ * A fleet keeps its lanes. The lanes have their own chains, and the config
+ * hook has already moved each one down its chain past a spent route, so
+ * overwriting them here would collapse ten jobs onto one model and discard
+ * the chain that was supposed to survive the outage.
+ *
+ * A profile with no lanes of its own is a single-model profile: there is
+ * nothing to keep, so the fallback takes the lanes too.
+ *
+ * The session model is published on `harness.sessionModel` so the effort hook
+ * - which runs after this one - pins the session to the model that was chosen
+ * here rather than back to the base model that is spent.
+ */
+function applySessionModel(
+  cfg: Record<string, any>,
+  profile: any,
+  opts: {
+    keepLanes?: boolean
+    profileId?: string
+    reserved?: number
+    threshold?: number
+  } = {},
+) {
   try {
-    const p = loadProfile(profileId)
-    const model = `${p.provider}/${p.model}`
+    const model = `${profile.provider}/${profile.model}`
     cfg.model = model
     cfg.small_model = model
-    const variant = p.effort?.default
-    if (p.effort?.levels?.length && variant) cfg.variant = variant
+    const variant = profile.effort?.default
+    if (profile.effort?.levels?.length && variant) cfg.variant = variant
     else delete cfg.variant
 
-    for (const [role, want] of Object.entries(p.roles ?? {})) {
-      const agent = cfg.agent ?? (cfg.agent = {})
-      const existing = agent[role] ?? (agent[role] = {})
-      existing.model = model
-      const level = resolveEffort(p.effort?.levels ?? [], want)
-      if (level) existing.variant = level
-      else delete existing.variant
+    if (!opts.keepLanes) {
+      for (const [lane, want] of Object.entries(profile.roles ?? {})) {
+        const agent = cfg.agent ?? (cfg.agent = {})
+        const existing = agent[lane] ?? (agent[lane] = {})
+        existing.model = model
+        const level = resolveEffort(profile.effort?.levels ?? [], want as any)
+        if (level) existing.variant = level
+        else delete existing.variant
+      }
     }
     if (cfg.compaction) {
       cfg.compaction.prune = true
-      cfg.compaction.reserved = reservedTokens(p)
+      cfg.compaction.reserved = opts.reserved ?? reservedTokens(profile)
+    }
+    cfg.harness = {
+      ...(cfg.harness ?? {}),
+      profile: opts.profileId ?? profile.id,
+      threshold: opts.threshold ?? pruneThreshold(profile),
+      sessionModel: model,
     }
-    cfg.harness = { ...(cfg.harness ?? {}), profile: profileId, threshold: pruneThreshold(p) }
   } catch {
     // Leave the config alone if the fallback profile cannot be read.
   }

package/src/telemetry.ts

@@ -8,6 +8,29 @@
 // What is worth recording: context size per request (to see pruning working),
 // latency by role and effort (to see tiering paying off), tool denials (to see
 // the guard being useful rather than annoying), and provider rate limits.
+//
+// A fleet adds one more question, and it is the one that decides whether the
+// fleet stays at ten lanes or is cut down: which lane is actually used. Ten
+// lanes that are never called are ten descriptions in the model\'s context and
+// ten models on the free tier doing nothing. `oc-harness lanes` reads these
+// records and reports the ones that were never called.
+//
+// WHAT A CALL IS
+//
+// One assistant message is one call. opencode emits `message.updated` several
+// times for a single message - created, streamed, completed - so writing a
+// record per event counted every call three times. A lane record therefore
+// carries the message id, and a lane is counted once per id.
+//
+// Two records are written per message, at most: an open one when the message
+// appears, and a closed one when it completes. The closed record is the only
+// one with a duration, so "how long does this lane take" and "was this lane
+// called at all" are answered from the same file without guessing.
+//
+// The `chat.message` hook is deliberately not used for this. Measured on
+// OpenCode 1.19, its `agent` field is undefined, so it cannot attribute a
+// call to a lane; and a second counter that cannot be correlated with the
+// first would double-count every call.
 
 import type { Plugin } from "@opencode-ai/plugin"
 import { appendFileSync, mkdirSync, existsSync, statSync, renameSync } from "node:fs"
@@ -45,6 +68,17 @@ export const TelemetryPlugin: Plugin = async ({ worktree }) => {
   const dir = join(worktree, ".space")
   const started = Date.now()
 
+  // The message id of every lane record already written, and of every one
+  // already written with a duration.
+  //
+  // opencode emits `message.updated` several times for a single assistant
+  // message: once when the message is created, once per streaming update, and
+  // once when it completes. Writing a record per event counted one call three
+  // times, which made the "cut the lanes nobody calls" counter wrong by a
+  // factor of three. A lane is counted once per message id instead.
+  const opened = new Set<string>()
+  const closed = new Set<string>()
+
   return {
     event: async ({ event }) => {
       // Events are provider-specific and opencode may add shapes we have never
@@ -56,6 +90,30 @@ export const TelemetryPlugin: Plugin = async ({ worktree }) => {
       // they get logged even though the event shape is provider-specific.
       if (/error|rate|limit|quota/i.test(type)) {
         write(dir, { kind: "provider-event", type, detail: truncate(safeStringify(event)) })
+        return
+      }
+
+      // An assistant message is the only place the kit can see which lane ran,
+      // on which model, and how long it took. That record is what makes "cut
+      // the lanes nobody uses" a measurement instead of a guess.
+      if (type === "message.updated") {
+        const lane = laneOf(event)
+        if (!lane || !lane.id) return
+        if (lane.ms !== undefined) {
+          // The completed event. Written once, and only if the open record
+          // did not already carry a duration.
+          if (closed.has(lane.id)) return
+          closed.add(lane.id)
+          write(dir, { kind: "lane", ...lane })
+          return
+        }
+        // The open event: the lane was called. Written even if the call never
+        // finishes, because "this lane is useless" and "this lane is slow"
+        // are different questions and only the first one is settled by a
+        // record with no duration.
+        if (opened.has(lane.id)) return
+        opened.add(lane.id)
+        write(dir, { kind: "lane", ...lane })
       }
     },
 
@@ -77,6 +135,46 @@ export const TelemetryPlugin: Plugin = async ({ worktree }) => {
   }
 }
 
+/**
+ * The lane, model and duration of one assistant message, read defensively.
+ *
+ * The event shape is opencode's, not ours, and it has changed between
+ * releases. Every field is optional and a shape we do not recognise yields
+ * null rather than a wrong number.
+ *
+ * The message id is what makes the record countable: one message is one call.
+ */
+function laneOf(event: unknown): {
+  id?: string
+  agent?: string
+  model?: string
+  provider?: string
+  ms?: number
+} | null {
+  const e = event as { properties?: any } | undefined
+  const info = e?.properties?.info ?? e?.properties
+  if (!info || typeof info !== "object") return null
+  if (info.role && info.role !== "assistant") return null
+
+  const id = typeof info.id === "string" ? info.id : undefined
+  const agent = typeof info.agent === "string" ? info.agent : undefined
+  const modelId = info.model?.modelID ?? info.modelID
+  const providerId = info.model?.providerID ?? info.providerID
+  const time = info.time ?? {}
+  const created = typeof time.created === "number" ? time.created : undefined
+  const completed = typeof time.completed === "number" ? time.completed : undefined
+  const ms = created !== undefined && completed !== undefined ? completed - created : undefined
+
+  if (!id || !agent) return null
+  return {
+    id,
+    agent,
+    model: typeof modelId === "string" ? modelId : undefined,
+    provider: typeof providerId === "string" ? providerId : undefined,
+    ms: typeof ms === "number" && ms >= 0 ? ms : undefined,
+  }
+}
+
 function extractRule(output: string): string {
   const m = output.match(/blocked:\s*([^\n]+)/)
   return m ? m[1].trim() : "unknown"

package/examples/muse-spark-1.3-contributor-free.json

@@ -1,41 +0,0 @@
-{
-  "$comment": "Contrarian example. Kept out of profiles/index.json on purpose: free, 1M context, and widely used, but its provider is permitted to train on your prompts and it has no `max` effort. `oc-harness list` shows it struck through with the reason. Install it with `oc-harness use muse-spark-1.3-contributor-free` and the warning is repeated on install.",
-  "id": "muse-spark-1.3-contributor-free",
-  "title": "Muse Spark 1.3 Contributor Free (OpenCode Zen)",
-  "provider": "opencode",
-  "model": "muse-spark-1.3-contributor-free",
-  "family": "muse-free",
-  "verified": "2026-10-07",
-  "retention": "may-train",
-  "notes": "Capability/privacy tradeoff, not a recommendation. Served over /v1/responses, a better cache dialect than chat/completions, and it supports temperature. But: 'Heavily discounted token pricing in exchange for permission to use your prompts and completions to train future Meta models', and its top effort is xhigh rather than max. Do not make this the default for private code.",
-  "limits": {
-    "context": 1048576,
-    "output": 131072
-  },
-  "supports": {
-    "reasoning": true,
-    "temperature": true,
-    "toolCall": true,
-    "structuredOutput": true,
-    "attachment": true
-  },
-  "effort": {
-    "levels": ["minimal", "low", "medium", "high", "xhigh"],
-    "default": "xhigh",
-    "reserveShare": 0.12
-  },
-  "roles": {
-    "plan": "max",
-    "build": "high",
-    "reviewer": "xhigh",
-    "locate": "low",
-    "verify": "low"
-  },
-  "prune": {
-    "enabled": true,
-    "keepRecentToolCalls": 6,
-    "keepRecentTurns": 3,
-    "keepUserMessages": true
-  },
-  "fallback": ["x-preview-f-free", "longcat-2.5-preview-free"]
-}

package/profiles/deepseek-v4-flash-free.json

@@ -1,41 +0,0 @@
-{
-  "$comment": "Fallback tier 3. The smallest context in the queue at 200k, which is why prune.keepRecentToolCalls drops: the same settings tuned for a 1M model overflow this one.",
-  "id": "deepseek-v4-flash-free",
-  "title": "DeepSeek V4 Flash Free (OpenCode Zen)",
-  "provider": "opencode",
-  "model": "deepseek-v4-flash-free",
-  "family": "deepseek-flash",
-  "verified": "2026-10-07",
-  "retention": "zero-retention",
-  "notes": "Text only, no attachment, 200k context. Included to show the compiler scaling pruning to the model: reserveShare and retention fall together, so a 1M-tuned profile cannot silently overflow a 200k model.",
-  "limits": {
-    "context": 200000,
-    "output": 128000
-  },
-  "supports": {
-    "reasoning": true,
-    "temperature": true,
-    "toolCall": true,
-    "structuredOutput": true,
-    "attachment": false
-  },
-  "effort": {
-    "levels": ["low", "high", "max"],
-    "default": "max",
-    "reserveShare": 0.12
-  },
-  "roles": {
-    "plan": "max",
-    "build": "high",
-    "reviewer": "max",
-    "locate": "low",
-    "verify": "low"
-  },
-  "prune": {
-    "enabled": true,
-    "keepRecentToolCalls": 4,
-    "keepRecentTurns": 2,
-    "keepUserMessages": true
-  },
-  "fallback": []
-}

package/profiles/muse-split-free.json

@@ -1,64 +0,0 @@
-{
-  "$comment": "Split profile. The heavy reasoning roles run on Muse Spark 1.3 because it is the strongest free model on Zen by a wide margin (Terminal-Bench 2.1 88.8% vs Nemotron 3 Ultra's 56.4%, DeepSWE v1.1 75.4%). The bulk-output roles stay on Space Bunny because its 524,288-token output cap is 4x Muse Spark's 131,072, and scaffolding or migrating many files is where that ceiling is actually felt. `retention` below describes the BASE model only; the config header prints the effective retention, which is may-train because three roles run on Muse Spark.",
-  "id": "muse-split-free",
-  "title": "Muse Spark 1.3 (heavy) + Space Bunny Free (bulk output)",
-  "provider": "opencode",
-  "model": "space-bunny-free",
-  "family": null,
-  "verified": "2026-10-07",
-  "retention": "zero-retention",
-  "notes": "Opt-in split, not a default. Muse Spark is a Meta contributor-tier free endpoint: its provider may train on your prompts and completions. Three of the five roles are therefore may-train even though the base model is zero-retention. Install it only for public or throwaway repositories. Effort requests below are resolved against each role's own model, so plan and reviewer ask for `max` and land on `xhigh`, which is the top Muse Spark offers.",
-  "limits": {
-    "context": 1048576,
-    "input": 524288,
-    "output": 524288
-  },
-  "supports": {
-    "reasoning": true,
-    "temperature": false,
-    "toolCall": true,
-    "structuredOutput": false,
-    "attachment": true
-  },
-  "effort": {
-    "levels": ["low", "medium", "high", "xhigh", "max"],
-    "default": "max",
-    "reserveShare": 0.12
-  },
-  "roles": {
-    "plan": "max",
-    "build": "high",
-    "reviewer": "max",
-    "locate": "low",
-    "verify": "low"
-  },
-  "roleModels": {
-    "plan": {
-      "model": "muse-spark-1.3-contributor-free",
-      "levels": ["minimal", "low", "medium", "high", "xhigh"]
-    },
-    "build": {
-      "model": "muse-spark-1.3-contributor-free",
-      "levels": ["minimal", "low", "medium", "high", "xhigh"]
-    },
-    "reviewer": {
-      "model": "muse-spark-1.3-contributor-free",
-      "levels": ["minimal", "low", "medium", "high", "xhigh"]
-    },
-    "locate": {
-      "model": "space-bunny-free",
-      "levels": ["low", "medium", "high", "xhigh", "max"]
-    },
-    "verify": {
-      "model": "space-bunny-free",
-      "levels": ["low", "medium", "high", "xhigh", "max"]
-    }
-  },
-  "prune": {
-    "enabled": true,
-    "keepRecentToolCalls": 6,
-    "keepRecentTurns": 3,
-    "keepUserMessages": true
-  },
-  "fallback": ["x-preview-f-free", "longcat-2.5-preview-free", "deepseek-v4-flash-free"]
-}

package/profiles/space-bunny-free.json

@@ -1,42 +0,0 @@
-{
-  "$comment": "Validated against models.dev on 2026-10-07. Regenerate with `oc-harness use --refresh`.",
-  "id": "space-bunny-free",
-  "title": "Space Bunny Free (OpenCode Zen)",
-  "provider": "opencode",
-  "model": "space-bunny-free",
-  "family": null,
-  "verified": "2026-10-07",
-  "retention": "zero-retention",
-  "notes": "Stealth preview reasoning model. Served over /v1/chat/completions (openai-compatible), which is the weaker cache dialect compared with models on /v1/responses. At the time of writing it is the only free model on Zen whose top reasoning effort is `max` and whose provider is zero-retention. Free for a limited time; expect the slot to rotate.",
-  "limits": {
-    "context": 1048576,
-    "input": 524288,
-    "output": 524288
-  },
-  "supports": {
-    "reasoning": true,
-    "temperature": false,
-    "toolCall": true,
-    "structuredOutput": false,
-    "attachment": true
-  },
-  "effort": {
-    "levels": ["low", "medium", "high", "xhigh", "max"],
-    "default": "max",
-    "reserveShare": 0.12
-  },
-  "roles": {
-    "plan": "max",
-    "build": "max",
-    "reviewer": "max",
-    "locate": "low",
-    "verify": "low"
-  },
-  "prune": {
-    "enabled": true,
-    "keepRecentToolCalls": 6,
-    "keepRecentTurns": 3,
-    "keepUserMessages": true
-  },
-  "fallback": ["x-preview-f-free", "longcat-2.5-preview-free", "deepseek-v4-flash-free"]
-}

package/profiles/x-preview-f-free.json

@@ -1,41 +0,0 @@
-{
-  "$comment": "Fallback tier 1 for space-bunny-free. Zero-retention, 1M context, and a `max` effort, but only three levels: there is no medium or xhigh, so roles asking for those resolve down to high.",
-  "id": "x-preview-f-free",
-  "title": "x-preview-f Free (OpenCode Zen)",
-  "provider": "opencode",
-  "model": "x-preview-f-free",
-  "family": null,
-  "verified": "2026-10-07",
-  "retention": "zero-retention",
-  "notes": "Only three effort levels (low, high, max) and a 131k completion ceiling, so long single answers are not its strength. Kept as first fallback: same 1M context class and the same zero-retention terms as the default.",
-  "limits": {
-    "context": 1000000,
-    "output": 131072
-  },
-  "supports": {
-    "reasoning": true,
-    "temperature": true,
-    "toolCall": true,
-    "structuredOutput": true,
-    "attachment": true
-  },
-  "effort": {
-    "levels": ["low", "high", "max"],
-    "default": "max",
-    "reserveShare": 0.12
-  },
-  "roles": {
-    "plan": "max",
-    "build": "high",
-    "reviewer": "max",
-    "locate": "low",
-    "verify": "low"
-  },
-  "prune": {
-    "enabled": true,
-    "keepRecentToolCalls": 6,
-    "keepRecentTurns": 3,
-    "keepUserMessages": true
-  },
-  "fallback": ["longcat-2.5-preview-free", "deepseek-v4-flash-free"]
-}