opencode-harness-kit 0.0.0-stage → 0.1.0

package/src/profile.js

@@ -0,0 +1,333 @@
+// Profile loading, effort resolution, and retention lookup.
+//
+// The rule this file exists to enforce: no plugin in src/ names a model. Every
+// model-specific decision comes from a profile JSON, so switching models means
+// editing data, not code.
+//
+// Plain JS with JSDoc types on purpose: the opencode plugin loads .ts, but the
+// CLI has to run under plain node, so the shared core stays dependency-free.
+
+import { readFileSync, existsSync, readdirSync } from "node:fs"
+import { dirname, join, resolve } from "node:path"
+import { fileURLToPath } from "node:url"
+
+const HERE = dirname(fileURLToPath(import.meta.url))
+export const ROOT = resolve(HERE, "..")
+
+/**
+ * @typedef {"zero-retention" | "may-train" | "unknown"} Retention
+ * @typedef {string | null} EffortRequest
+ * @typedef {object} Profile
+ * @property {string} id
+ * @property {string} [title]
+ * @property {string} provider
+ * @property {string} model
+ * @property {string | null} [family]
+ * @property {string} [verified]
+ * @property {Retention} retention
+ * @property {string} [notes]
+ * @property {{ context: number, input?: number, output: number }} limits
+ * @property {{ reasoning?: boolean, temperature?: boolean, toolCall?: boolean,
+ *             structuredOutput?: boolean, attachment?: boolean }} [supports]
+ * @property {{ levels?: string[], default?: string | null, reserveShare?: number }} [effort]
+ * @property {Record<string, EffortRequest>} [roles]
+ * @property {{ enabled?: boolean, keepRecentToolCalls?: number,
+ *             keepRecentTurns?: number, keepUserMessages?: boolean }} [prune]
+ * @property {string[]} [fallback]
+ * @typedef {{ retention: Retention, note?: string, verified?: string, source?: string }} RetentionRecord
+ */
+
+export class ProfileError extends Error {}
+
+/**
+ * Profiles live in profiles/ (verified) and examples/ (contrarian, still
+ * installable). schema.json and index.json are not profiles.
+ */
+export function profileDirs() {
+  return [join(ROOT, "profiles"), join(ROOT, "examples")]
+}
+
+/** @param {string} id @returns {string | null} */
+export function profilePath(id) {
+  for (const dir of profileDirs()) {
+    const full = join(dir, `${id}.json`)
+    if (existsSync(full)) return full
+  }
+  return null
+}
+
+/** @returns {string[]} */
+export function listProfileIds() {
+  const out = []
+  for (const dir of profileDirs()) {
+    if (!existsSync(dir)) continue
+    for (const f of readdirSync(dir)) {
+      if (!f.endsWith(".json")) continue
+      if (f === "schema.json" || f === "index.json") continue
+      out.push(f.replace(/\.json$/, ""))
+    }
+  }
+  return out.sort()
+}
+
+/** @param {string} id @returns {Profile} */
+export function loadProfile(id) {
+  const path = profilePath(id)
+  if (!path) {
+    throw new ProfileError(`no profile named "${id}". Run: oc-harness list`)
+  }
+  let raw
+  try {
+    raw = JSON.parse(readFileSync(path, "utf8"))
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err)
+    throw new ProfileError(`cannot read ${path}: ${msg}`)
+  }
+  validate(raw, path)
+  return normalize(raw)
+}
+
+/**
+ * Fill in derived fields so nothing downstream has to guess.
+ * @param {Profile} p @returns {Profile}
+ */
+export function normalize(p) {
+  return {
+    ...p,
+    limits: { ...p.limits, input: p.limits.input ?? p.limits.context },
+    effort: {
+      levels: p.effort?.levels ?? [],
+      default: p.effort?.default ?? null,
+      reserveShare: p.effort?.reserveShare ?? 0.12,
+    },
+    prune: {
+      enabled: p.prune?.enabled ?? false,
+      keepRecentToolCalls: p.prune?.keepRecentToolCalls ?? 6,
+      keepRecentTurns: p.prune?.keepRecentTurns ?? 3,
+      keepUserMessages: p.prune?.keepUserMessages ?? true,
+    },
+    fallback: p.fallback ?? [],
+  }
+}
+
+const VALID_RETENTION = ["zero-retention", "may-train", "unknown"]
+
+/** @param {Profile} p @param {string} [path] */
+export function validate(p, path = "<memory>") {
+  const where = p?.id ?? path
+  if (!p?.id) throw new ProfileError(`${path}: missing "id"`)
+  if (!p.provider) throw new ProfileError(`${where}: missing "provider"`)
+  if (!p.model) throw new ProfileError(`${where}: missing "model"`)
+  if (!VALID_RETENTION.includes(p.retention)) {
+    throw new ProfileError(
+      `${where}: retention must be one of ${VALID_RETENTION.join(", ")}`,
+    )
+  }
+  if (!p.limits?.context || p.limits.context <= 0) {
+    throw new ProfileError(`${where}: limits.context must be a positive integer`)
+  }
+  if (!p.limits?.output || p.limits.output <= 0) {
+    throw new ProfileError(`${where}: limits.output must be a positive integer`)
+  }
+  const share = p.effort?.reserveShare
+  if (share !== undefined && (share < 0 || share > 0.5)) {
+    throw new ProfileError(
+      `${where}: effort.reserveShare must be between 0 and 0.5`,
+    )
+  }
+}
+
+// Canonical ordering for reasoning effort, used to snap a requested level onto
+// whatever the target model actually offers.
+const CANON = ["none", "minimal", "low", "medium", "high", "xhigh", "max"]
+
+/**
+ * Pick the available level closest to what a role asked for.
+ *
+ * A role asking for `max` on a model topping out at `xhigh` gets `xhigh`. A
+ * model with no tunable effort resolves to null, meaning "emit no variant"
+ * rather than "emit a level the model does not have".
+ *
+ * @param {string[]} levels
+ * @param {EffortRequest} requested
+ * @returns {string | null}
+ */
+export function resolveEffort(levels, requested) {
+  /** @type {string | null} */
+  if (!requested) return null
+  if (!levels || levels.length === 0) return null
+  if (levels.includes(requested)) return requested
+
+  const want = CANON.indexOf(requested)
+  if (want < 0) {
+    // Unknown label. Snap to the top level, which is the safe reading of a
+    // role that asked for something strong.
+    return levels[levels.length - 1] ?? null
+  }
+  let best = null
+  let bestDist = Infinity
+  for (const level of levels) {
+    const have = CANON.indexOf(level)
+    if (have < 0) continue
+    const d = Math.abs(have - want)
+    if (d < bestDist) {
+      bestDist = d
+      best = level
+    }
+  }
+  return best ?? levels[levels.length - 1] ?? null
+}
+
+/**
+ * @param {Profile} p
+ * @returns {Record<string, string | null>}
+ */
+export function resolveRoles(p) {
+  const levels = p.effort?.levels ?? []
+  const out = /** @type {Record<string, string | null>} */ ({})
+  for (const [role, want] of Object.entries(p.roles ?? {})) {
+    out[role] = resolveEffort(levels, want)
+  }
+  return out
+}
+
+/** @param {Profile} p @returns {string | null} */
+export function resolveDefaultVariant(p) {
+  return resolveEffort(p.effort?.levels ?? [], p.effort?.default ?? null)
+}
+
+/**
+ * Tokens held back from the input ceiling for reasoning growth and in-flight
+ * tool output. Scaled from the model, never hardcoded.
+ * @param {Profile} p @returns {number}
+ */
+export function reservedTokens(p) {
+  const input = p.limits.input ?? p.limits.context
+  const share = p.effort?.reserveShare ?? 0.12
+  return Math.max(8000, Math.round(input * share))
+}
+
+/**
+ * Context budget above which pruning should already be working.
+ * @param {Profile} p @returns {number}
+ */
+export function pruneThreshold(p) {
+  return (p.limits.input ?? p.limits.context) - reservedTokens(p)
+}
+
+/**
+ * Minimal YAML reader for retention.yml: a two-level map of small records,
+ * including folded block scalars (`note: >-`). Handling indentation, folded
+ * blocks and plain scalars costs about forty lines, which is less than a
+ * dependency for a file this shape.
+ *
+ * @returns {Record<string, RetentionRecord>}
+ */
+export function loadRetention() {
+  const file = join(ROOT, "retention.yml")
+  if (!existsSync(file)) return {}
+  const out = /** @type {Record<string, any>} */ ({})
+  const lines = readFileSync(file, "utf8").split("\n")
+
+  let current = null
+  let i = 0
+
+  while (i < lines.length) {
+    const raw = lines[i]
+    const line = raw.trim()
+    const indent = raw.length - raw.trimStart().length
+
+    if (!line || line.startsWith("#")) {
+      i++
+      continue
+    }
+    // A model key sits at two-space indent under `models:`. Keys are written
+    // provider-qualified already, so do not re-prefix them.
+    if (indent === 2 && line.endsWith(":")) {
+      const key = line.slice(0, -1).trim()
+      current = key.includes("/") ? key : `opencode/${key}`
+      out[current] = { retention: "unknown" }
+      i++
+      continue
+    }
+    if (!current) {
+      i++
+      continue
+    }
+
+    const field = line.match(/^(retention|verified|source|note):\s*(.*)$/)
+    if (field) {
+      const name = field[1]
+      const rest = field[2].trim()
+      if (/^[>|][-+]?$/.test(rest)) {
+        // Folded or literal block: gather every following line indented deeper
+        // than its key, then collapse whitespace into a one-line note.
+        const parts = []
+        let j = i + 1
+        for (; j < lines.length; j++) {
+          const l = lines[j]
+          if (!l.trim()) {
+            parts.push("")
+            continue
+          }
+          if (l.length - l.trimStart().length <= indent) break
+          parts.push(l.trim())
+        }
+        out[current][name] = parts.join(" ").replace(/\s+/g, " ").trim()
+        i = j
+        continue
+      }
+      out[current][name] = rest.replace(/^["']|["']$/g, "")
+      i++
+      continue
+    }
+    i++
+  }
+  return out
+}
+
+/**
+ * Retention as declared in retention.yml, which wins over the profile copy.
+ * Accepts either a bare profile id or an already-qualified provider/model.
+ * @param {string} id @returns {RetentionRecord | undefined}
+ */
+export function retentionFor(id) {
+  const table = loadRetention()
+  return table[id] ?? table[id.includes("/") ? id : `opencode/${id}`]
+}
+
+/** Where opencode keeps its config, honouring the same env var it does. */
+export function configFilePath() {
+  const dir =
+    process.env.OPENCODE_CONFIG_DIR ??
+    join(process.env.HOME ?? "", ".config", "opencode")
+  return join(dir, "opencode.jsonc")
+}
+
+/**
+ * Read the active profile id from the config file on disk.
+ *
+ * This exists because the `config` hook cannot be the source. OpenCode's
+ * Config schema sets `additionalProperties: false`, so a custom `harness` key
+ * written into the file is stripped from the object handed to the hook. The
+ * file survives; the in-memory config does not. Verified against the 1.18.35
+ * schema.
+ *
+ * @returns {string | null}
+ */
+export function profileIdFromConfig() {
+  const fromEnv = process.env.OPENCODE_HARNESS_PROFILE
+  if (fromEnv) return fromEnv.trim() || null
+
+  const file = configFilePath()
+  if (!existsSync(file)) return null
+  try {
+    const text = readFileSync(file, "utf8")
+    const json = text.slice(text.indexOf("{"))
+    const parsed = JSON.parse(json)
+    const id = parsed?.harness?.profile
+    return typeof id === "string" && id ? id : null
+  } catch {
+    return null
+  }
+}

package/src/prune.ts

@@ -0,0 +1,258 @@
+// Deterministic context pruning.
+//
+// Measured on one install: 77.9% of stored message bytes were tool output and
+// 15.6% reasoning, against 2.1% of actual text. Cache reads ran 20.6x fresh
+// input, and p50 context per request was ~99k tokens with only ~534 fresh
+// tokens. The session was re-sending old command output on every turn.
+//
+// Why not let the model decide when to compact: the model-driven approach needs
+// the model to remember to call a tool, and this kit targets models with weak
+// instruction following. This hook runs unconditionally and keeps pointers
+// instead of bodies, so nothing is lost, only re-read on demand.
+//
+// Invariants, because this rewrites history on every turn:
+//   - never touch the last message
+//   - never touch a tool call from the turn in flight
+//   - never touch patch or file parts, which carry diffs the model needs
+//   - never stub a user message unless asked to
+
+import type { Plugin } from "@opencode-ai/plugin"
+import { loadProfile, pruneThreshold, ROOT } from "./profile.js"
+import { readFileSync, existsSync } from "node:fs"
+import { join } from "node:path"
+
+/**
+ * Which profile is active. In order: an explicit environment variable, the
+ * marker the plugin writes into the project, then a marker inside the package.
+ * Returns null when nothing is set, and pruning then stays off rather than
+ * guessing at thresholds.
+ */
+function activeProfileId(worktree: string): string | null {
+  const candidates = [
+    process.env.OPENCODE_HARNESS_PROFILE,
+    join(worktree, ".space", "active"),
+    join(ROOT, "profiles", ".active"),
+  ]
+  for (const f of candidates) {
+    if (!f) continue
+    try {
+      if (existsSync(f)) {
+        const v = readFileSync(f, "utf8").trim()
+        if (v) return v
+      }
+    } catch {
+      // Try the next location.
+    }
+  }
+  return null
+}
+
+interface StubStats {
+  stubbed: number
+  kept: number
+  bytesBefore: number
+  bytesAfter: number
+}
+
+/** Rough size of a part, used only for the stats line. */
+function sizeOf(part: any): number {
+  if (typeof part?.text === "string") return part.text.length
+  if (typeof part?.state?.output === "string") return part.state.output.length
+  return 0
+}
+
+/**
+ * One line standing in for a tool result. The agent keeps the shape of what
+ * happened and a way to get the rest, which is what it actually needs to decide
+ * what to do next.
+ */
+function stub(part: any): string {
+  const tool = part?.tool ?? "tool"
+  const state = part?.state ?? {}
+  const input = state.input ?? {}
+
+  if (tool === "bash") {
+    const cmd = String(input.command ?? "").replace(/\s+/g, " ").slice(0, 80)
+    const status = state.status ?? "done"
+    let tail = ""
+    const out = String(state.output ?? "")
+    const m = out.match(/(passed|failed|error|Error|ok|OK|\d+ (?:passed|failed))/g)
+    if (m && m.length) tail = `, ${m[m.length - 1]}`
+    return `[${tool}] ${cmd} -> ${status}${tail} | full output was pruned from context; re-run the command if you need it`
+  }
+  if (tool === "read" || tool === "grep" || tool === "glob" || tool === "list") {
+    const target = String(input.filePath ?? input.path ?? input.pattern ?? "")
+    return `[${tool}] ${target.slice(0, 100)} | result pruned from context; re-read it if you need it`
+  }
+  return `[${tool}] result pruned from context (${state.status ?? "done"}); re-run if you need it`
+}
+
+export const PrunePlugin: Plugin = async ({ worktree, $ }) => {
+  let profile = null as ReturnType<typeof loadProfile> | null
+  try {
+    const id = activeProfileId(worktree)
+    if (id) profile = loadProfile(id)
+  } catch {
+    profile = null
+  }
+  if (!profile?.prune?.enabled) {
+    // Pruning is opt-in per profile. With no active profile we do nothing
+    // rather than guess at thresholds.
+    return {}
+  }
+
+  const keepCalls = profile.prune.keepRecentToolCalls ?? 6
+  const keepTurns = profile.prune.keepRecentTurns ?? 3
+  const keepUsers = profile.prune.keepUserMessages ?? true
+  const threshold = pruneThreshold(profile)
+
+  return {
+    "experimental.chat.messages.transform": async (_input, output) => {
+      try {
+        const messages = output.messages
+        if (!Array.isArray(messages) || messages.length === 0) return
+
+        // The trailing keepTurns are off limits, and the last message always is.
+        const floor = Math.max(0, messages.length - 1 - keepTurns)
+
+        // Walk backwards collecting tool parts to preserve verbatim.
+        // Walk backwards from the end until keepCalls tool parts are collected.
+        // This is deliberately not bounded by `floor`: how many recent calls
+        // stay verbatim and which turns are prunable are independent limits,
+        // and a call can be both recent and inside the prunable span.
+        const keepToolMessageIds = new Set<string>()
+        let seen = 0
+        for (let i = messages.length - 1; i >= 0 && seen < keepCalls; i--) {
+          const msg = messages[i]
+          if (!msg?.parts) continue
+          for (const part of msg.parts) {
+            if (part?.type !== "tool") continue
+            if (seen >= keepCalls) break
+            keepToolMessageIds.add(msg.info?.id ?? String(i))
+            seen++
+          }
+        }
+
+        const stats: StubStats = {
+          stubbed: 0,
+          kept: 0,
+          bytesBefore: 0,
+          bytesAfter: 0,
+        }
+
+        for (let i = 0; i < floor; i++) {
+          const msg = messages[i]
+          if (!msg?.parts) continue
+          const isUser = msg.info?.role === "user"
+          for (const part of msg.parts) {
+            const size = sizeOf(part)
+            stats.bytesBefore += size
+
+            if (isUser && keepUsers) {
+              stats.bytesAfter += size
+              stats.kept++
+              continue
+            }
+            if (keepToolMessageIds.has(msg.info?.id ?? String(i))) {
+              stats.bytesAfter += size
+              stats.kept++
+              continue
+            }
+            switch (part?.type) {
+              case "patch":
+              case "file":
+                // Diffs and file bodies are load-bearing; never stub them.
+                stats.bytesAfter += size
+                stats.kept++
+                break
+              case "reasoning":
+                // Old reasoning is stale by definition and safe to drop.
+                stats.bytesAfter += Math.min(size, 200)
+                stats.stubbed++
+                break
+              case "tool": {
+                // Only completed calls carry an output worth pruning. Pending
+                // and running states are small, and error states are usually
+                // the most useful thing in the transcript.
+                const state = part.state
+                if (state?.status !== "completed") {
+                  stats.bytesAfter += size
+                  stats.kept++
+                  break
+                }
+                part.state = {
+                  status: "completed",
+                  input: state.input,
+                  output: stub(part),
+                  title: state.title,
+                  time: state.time,
+                  metadata: { ...(state.metadata ?? {}), pruned: true, prunedAt: new Date().toISOString() },
+                }
+                stats.bytesAfter += sizeOf(part)
+                stats.stubbed++
+                break
+              }
+              case "text":
+                // Assistant prose carries the reasoning chain a human reads and
+                // the model later relies on. Keep it.
+                stats.bytesAfter += size
+                stats.kept++
+                break
+              default:
+                stats.bytesAfter += size
+                stats.kept++
+            }
+          }
+        }
+
+        if (stats.stubbed > 0) {
+          const saved = Math.max(0, stats.bytesBefore - stats.bytesAfter)
+          await logPrune(profile.id, {
+            ...stats,
+            saved,
+            threshold,
+          })
+        }
+      } catch (err) {
+        // A failed prune must never break the turn. Skipping is recoverable;
+        // corrupting history is not.
+        console.error(
+          `[opencode-harness-kit] prune skipped: ${
+            err instanceof Error ? err.message : String(err)
+          }`,
+        )
+      }
+    },
+  }
+}
+
+async function logPrune(
+  profileId: string,
+  stats: StubStats & {
+    saved: number
+    threshold: number
+  },
+) {
+  const dir = join(process.cwd(), ".space")
+  try {
+    const { mkdir, appendFile } = await import("node:fs/promises")
+    await mkdir(dir, { recursive: true })
+    await appendFile(
+      join(dir, "prune.jsonl"),
+      JSON.stringify({
+        t: new Date().toISOString(),
+        profile: profileId,
+        stubbed: stats.stubbed,
+        kept: stats.kept,
+        savedBytes: stats.saved,
+        threshold: stats.threshold,
+      }) + "\n",
+    )
+  } catch {
+    // Telemetry must never be load-bearing.
+  }
+}
+
+export const pruneMeta = { note: "opt-in via profile prune.enabled" }
+
+export default PrunePlugin