opencode-harness-kit 0.6.3 → 0.7.0

package/eval/compare.mjs

@@ -0,0 +1,267 @@
+#!/usr/bin/env node
+// compare.mjs - grade any agent CLI against the same suite, with the same
+// grader, in a fresh copy of the same subject.
+//
+// eval/ab.mjs measures this kit against opencode, because it has to read
+// opencode's event stream to get token and context numbers. That makes it
+// useless for comparing against Aider, OpenHands or SWE-agent, whose output
+// is a terminal transcript rather than the same JSON.
+//
+// This script takes the other route: it does not care how the agent prints.
+// It runs the command, treats its stdout as the answer, and grades the answer
+// or the resulting tree with eval/verify.mjs - the same citation resolver and
+// the same mutation comparison the kit is measured with. A harness that
+// writes a citation to the right line passes; one that writes prose does not.
+//
+//   node eval/compare.mjs --target <dir> --suite eval/tasks-self.json \
+//     --reps 3 --timeout 300 \
+//     --arm 'aider=aider --yes --no-auto-commits --message "$PROMPT"' \
+//     --arm 'swe=python -m sweagent run --problem "$PROMPT"'
+//
+// The command runs through `sh -c` in the run's copy, with:
+//   $PROMPT    the task text
+//   $TASK_ID   the task id
+//   $DIR       the run's copy of the subject
+//   $REP       1-based repetition
+//   $ARM       the arm's name
+//
+// It does NOT install anything. An arm whose binary is not on PATH is
+// reported as unusable and the run exits 2, because a missing harness
+// reported as "0% pass" is a lie about the harness.
+
+import { spawnSync } from "node:child_process"
+import { cpSync, existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"
+import { tmpdir } from "node:os"
+import { join } from "node:path"
+import { checkPremises } from "./premises.mjs"
+import { gradeAnswer, gradeMutation } from "./verify.mjs"
+
+const args = process.argv.slice(2)
+
+/** @param {string} name */
+function flag(name) {
+  const i = args.indexOf(name)
+  if (i < 0) return undefined
+  const value = args[i + 1]
+  if (value === undefined || value.startsWith("--")) {
+    console.error(`${name} needs a value`)
+    process.exit(2)
+  }
+  return value
+}
+
+/** @param {string} name */
+function flags(name) {
+  const out = []
+  for (let i = 0; i < args.length; i++) {
+    if (args[i] !== name) continue
+    const value = args[i + 1]
+    if (value === undefined) {
+      console.error(`${name} needs a value`)
+      process.exit(2)
+    }
+    out.push(value)
+  }
+  return out
+}
+
+function usage(code) {
+  const text = [
+    "usage: node eval/compare.mjs --target <dir> --suite <file> --arm name=command [...]",
+    "",
+    "  --target <dir>      prepared subject (eval/prepare.mjs)",
+    "  --suite <file>      task suite (eval/tasks-self.json)",
+    "  --arm name=command  repeatable; command runs through sh -c",
+    "  --reps <n>          repetitions per task per arm (default 3)",
+    "  --timeout <seconds> per run (default 300)",
+    "  --only <id,id>      run only these task ids",
+    "  --json              print one JSON object, and nothing else on stdout",
+  ].join("\n")
+  ;(code === 0 ? console.log : console.error)(text)
+  process.exit(code)
+}
+
+if (args.includes("--help") || args.includes("-h")) usage(0)
+
+const target = flag("--target")
+const suitePath = flag("--suite")
+const repsRaw = flag("--reps")
+const timeoutRaw = flag("--timeout")
+const only = flag("--only")?.split(",").map((s) => s.trim()).filter(Boolean)
+const asJson = args.includes("--json")
+const armSpecs = flags("--arm")
+
+if (!target || !suitePath || !armSpecs.length) usage(2)
+if (!existsSync(target)) {
+  console.error(`target not found: ${target}`)
+  process.exit(2)
+}
+if (!existsSync(suitePath)) {
+  console.error(`suite not found: ${suitePath}`)
+  process.exit(2)
+}
+
+const reps = repsRaw === undefined ? 3 : Number(repsRaw)
+if (!Number.isInteger(reps) || reps < 1) {
+  console.error(`--reps must be a positive integer, got ${JSON.stringify(repsRaw)}`)
+  process.exit(2)
+}
+const timeout = timeoutRaw === undefined ? 300 : Number(timeoutRaw)
+if (!Number.isInteger(timeout) || timeout < 1) {
+  console.error(`--timeout must be a positive integer of seconds, got ${JSON.stringify(timeoutRaw)}`)
+  process.exit(2)
+}
+
+const arms = armSpecs.map((spec) => {
+  const at = spec.indexOf("=")
+  if (at < 1) {
+    console.error(`--arm needs name=command, got ${JSON.stringify(spec)}`)
+    process.exit(2)
+  }
+  return { name: spec.slice(0, at), command: spec.slice(at + 1) }
+})
+
+const suite = JSON.parse(readFileSync(suitePath, "utf8"))
+const premises = checkPremises(target, suite)
+for (const w of premises.warnings) console.error(`warn: ${w.id}: ${w.problem}`)
+if (!premises.ok) {
+  console.error(`FAIL: ${premises.fatal.length} premise(s) in ${suitePath} do not hold against ${target}.`)
+  for (const f of premises.fatal) console.error(`  ${f.id}: ${f.problem}`)
+  process.exit(2)
+}
+
+const tasks = suite.tasks.filter((t) => !only || only.includes(t.id))
+if (!tasks.length) {
+  console.error("no tasks selected")
+  process.exit(2)
+}
+
+// A citation to the suite itself, to a build output or to a test is not
+// evidence about the source. Kept in step with eval/ab.mjs by the comment
+// there; this list is the grading rule, not a per-arm one.
+const SUITE_FILES = [
+  "tasks-self.json",
+  "tasks.json",
+  "tasks-ts.json",
+  "dist/",
+  // The tests quote the answer, so citing one proves it was grepped, not read.
+  "test/",
+  "eval/",
+]
+
+/** @type {{arm: string, id: string, rep: number, ok: boolean, invalid: boolean, seconds: number, why: string}[]} */
+const rows = []
+
+for (let rep = 1; rep <= reps; rep++) {
+  for (const task of tasks) {
+    for (const arm of arms) {
+      // cpSync copies INTO an existing directory. The run copy must be a
+      // new path, or the subject lands at <dir>/<basename(target)> and every
+      // command that names src/... looks at the wrong tree.
+      const base = mkdtempSync(join(tmpdir(), `compare-${arm.name.replace(/[^a-z0-9-]+/gi, "-")}-`))
+      const dir = join(base, "subject")
+      cpSync(target, dir, { recursive: true })
+      const env = {
+        ...process.env,
+        PROMPT: task.task,
+        TASK_ID: task.id,
+        DIR: dir,
+        REP: String(rep),
+        ARM: arm.name,
+        // opencode (and some wrappers) resolve the project from PWD, which a
+        // child inherits from this script rather than from `cwd`.
+        PWD: dir,
+      }
+      const started = Date.now()
+      const res = spawnSync(arm.command, {
+        shell: true,
+        cwd: dir,
+        env,
+        encoding: "utf8",
+        timeout: timeout * 1000,
+        maxBuffer: 32 * 1024 * 1024,
+      })
+      const seconds = (Date.now() - started) / 1000
+
+      // `sh -c` exits 127 when the command it was given is not on PATH, so
+      // the ENOENT is on the shell's stderr rather than on spawnSync's error.
+      if (
+        (res.error && /** @type {NodeJS.ErrnoException} */ (res.error).code === "ENOENT") ||
+        res.status === 127
+      ) {
+        console.error(`arm ${arm.name}: command not found: ${arm.command}`)
+        rmSync(base, { recursive: true, force: true })
+        process.exit(2)
+      }
+
+      const timedOut = res.error?.name === "Error" && /ETIMEDOUT|timed out/i.test(res.error.message)
+      const invalid = Boolean(res.error) || res.status === null
+      let ok = false
+      let why = ""
+
+      if (task.mutate) {
+        const g = gradeMutation(dir, task.mutate, { pristine: target })
+        ok = g.ok
+        why = g.ok ? "tree matches" : g.reason ?? "tree does not match"
+      } else {
+        const g = gradeAnswer(dir, res.stdout ?? "", task.want, { exclude: SUITE_FILES })
+        ok = g.ok
+        why = g.ok
+          ? `cited ${g.cited}`
+          : g.cited === 0
+            ? "no citation"
+            : `cited ${g.cited}, none held`
+      }
+
+      if (timedOut && !ok) why = `timed out after ${timeout}s; ${why}`
+      if (invalid && !ok) why = `${res.error?.message ?? "no exit status"}; ${why}`
+
+      rows.push({ arm: arm.name, id: task.id, rep, ok, invalid, seconds: +seconds.toFixed(1), why })
+      if (!asJson) {
+        console.log(`  ${task.id} ${arm.name}-${rep} ... ${ok ? "PASS" : "fail"}, ${seconds.toFixed(1)}s, ${why}`)
+      }
+      rmSync(base, { recursive: true, force: true })
+    }
+  }
+}
+
+function median(values) {
+  if (!values.length) return null
+  const s = [...values].sort((a, b) => a - b)
+  const m = Math.floor(s.length / 2)
+  return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2
+}
+
+const summary = arms.map((arm) => {
+  const rs = rows.filter((r) => r.arm === arm.name)
+  const valid = rs.filter((r) => !r.invalid)
+  return {
+    arm: arm.name,
+    attempts: rs.length,
+    pass: rs.filter((r) => r.ok).length,
+    invalid: rs.length - valid.length,
+    medianSeconds: median(valid.map((r) => r.seconds)),
+  }
+})
+
+if (asJson) {
+  console.log(JSON.stringify({ target, suite: suitePath, reps, timeout, arms: summary, runs: rows }, null, 2))
+} else {
+  console.log("")
+  console.log(`model-free comparison: ${tasks.length} task(s) x ${reps} rep(s) x ${arms.length} arm(s)`)
+  for (const s of summary) {
+    console.log(
+      `  ${s.arm.padEnd(12)} ${s.pass}/${s.attempts} pass` +
+        (s.invalid ? `, ${s.invalid} invalid` : "") +
+        (s.medianSeconds === null ? "" : `, median ${s.medianSeconds}s`),
+    )
+  }
+}
+
+// A run where every arm crashed measured nothing. Say so rather than
+// printing a 0% that looks like a result.
+const usable = summary.some((s) => s.attempts > s.invalid)
+if (!usable) {
+  console.error("no arm produced a usable run")
+  process.exit(2)
+}

package/eval/verify.mjs

@@ -161,6 +161,15 @@ export function resolveInTarget(target, ref) {
     const rel = target.replace(/^\/private/, "")
     if (ref.startsWith(rel)) candidates.push(`/private${ref}`)
     if (ref.startsWith(`/private${rel}`)) candidates.push(ref.replace(/^\/private/, ""))
+    // A run copy is a byte copy of the snapshot, so a path inside one names
+    // the same file relative to the snapshot. This is what makes `--rescore`
+    // grade an answer that cited `/tmp/opencode-ab-run-<id>/src/a.ts` after
+    // the copy is gone: the live run resolved it against the copy, the
+    // re-score has only the snapshot, and without this the same answer reads
+    // as "cited 0" - which is how a re-score reported 10/12 where the run it
+    // was re-scoring had scored 15/17.
+    const copy = /\/opencode-ab-run-[^/]+\/(.+)$/.exec(ref)
+    if (copy) candidates.push(join(target, copy[1]))
   } else {
     candidates.push(join(target, ref))
   }

package/lanes.json

@@ -4,8 +4,8 @@
   "lanes": {
     "build": {
       "mode": "primary",
-      "description": "Writes and verifies code. Use when the plan is settled and files need to change.",
-      "steps": 40,
+    "description": "Writes and verifies code. Use when the plan is settled and files need to change.",
+    "steps": 120,
       "permission": { "edit": "allow" },
       "prompt": "prompts/build.md"
     },
@@ -69,18 +69,22 @@
       "permission": {
         "edit": "deny",
         "bash": {
-          "*": "allow",
+          "*": "ask",
           "git status *": "allow",
           "git diff *": "allow",
-          "mkdir*": "allow",
           "ps*": "allow",
+          "node --test*": "allow",
+          "npm test*": "allow",
+          "npm run *": "allow",
+          "npx *": "allow",
+          "bun test*": "allow",
+          "deno test*": "allow",
+          "make*": "allow",
           "python3 -m pytest*": "allow",
           "pytest*": "allow",
           "ruff*": "allow",
           "mypy*": "allow",
           "black*": "allow",
-          "npm test*": "allow",
-          "npm run *": "allow",
           "npx tsc*": "allow",
           "cargo test*": "allow",
           "go test*": "allow"
@@ -102,11 +106,16 @@
       "permission": {
         "edit": "deny",
         "bash": {
-          "*": "allow",
+          "*": "deny",
           "pdftotext*": "allow",
-          "python3 -c*": "allow",
           "cat*": "allow",
-          "ls*": "allow"
+          "ls*": "allow",
+          "head*": "allow",
+          "tail*": "allow",
+          "wc*": "allow",
+          "file*": "allow",
+          "grep*": "allow",
+          "rg*": "allow"
         }
       },
       "prompt": "prompts/doc.md"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "opencode-harness-kit",
-  "version": "0.6.3",
+  "version": "0.7.0",
   "description": "Harness kit for OpenCode: capability profiles, a fleet of job lanes with one free model each, role-based reasoning effort, context pruning, and measured baselines.",
   "license": "MIT",
   "homepage": "https://github.com/pham-canh/opencode-harness-kit#readme",
@@ -67,7 +67,7 @@
     "typecheck": "tsc --noEmit && npm run typecheck:scripts",
     "typecheck:scripts": "tsc -p tsconfig.scripts.json",
     "pretest": "node -e \"const fs=require('fs');if(!fs.existsSync('test')){console.error('no test directory here - this is an installed package, not a checkout');process.exit(1)}\"",
-    "test": "node --test \"test/*.test.js\"",
+    "test": "OPENCODE_HARNESS_CATALOG_URL=file://test/fixtures/catalogue.json node --test \"test/*.test.js\"",
     "test:all": "npm run build && npm run typecheck && npm test",
     "test:dist": "git diff --exit-code -- dist/ || (echo \"dist/ is out of sync with src/. Run: npm run build && git add dist/\" && exit 1)",
     "check": "npm run build && npm run test:dist && npm run typecheck && npm test",
@@ -105,6 +105,10 @@
     "esbuild": "^0.28.2",
     "typescript": "^7.0.2"
   },
+  "overrides": {
+    "zod": "4.6.7",
+    "toml": "5.0.1"
+  },
   "publishConfig": {
     "access": "public"
   },

package/profiles/index.json

@@ -110,7 +110,7 @@
       "topEffort": "max",
       "context": 262144,
       "default": false,
-      "note": "Open-weight MoE with about 2.5B active parameters for local coding agents. Knowledge cutoff 2026-06, no image input. May train on prompts during its free period."
+      "note": "Open-weight MoE, 40B total with about 2.5B active parameters, built for coding agents and small enough to serve locally. Full effort ladder, 262k context, 64k output, no image input. Knowledge cutoff 2026-06. May train on prompts during its free period."
     },
     {
       "id": "big-pickle",
@@ -122,17 +122,6 @@
       "context": 160000,
       "default": false,
       "note": "Stealth model, no public identity, no effort levels, no image input. The provider may train on it during its free period."
-    },
-    {
-      "id": "flint-40b-free",
-      "file": "profiles/flint-40b-free.json",
-      "title": "Flint 40B Free",
-      "retention": "may-train",
-      "free": true,
-      "topEffort": "max",
-      "context": 262144,
-      "default": false,
-      "note": "Open-weight MoE, 40B total with about 2.5B active, built for coding agents and small enough to serve locally. Full effort ladder, 262k context, 64k output, no image input."
     }
   ],
   "excluded": [],

package/prompts/reviewer.md

@@ -38,5 +38,29 @@ Only if pass 1 is clean.
 Two sections, pass 1 then pass 2. If pass 1 blocked, pass 2 does not run and
 you say why.
 
+## Machine verdict
+
+When the request asks for a schema'd verdict (it names
+`oc-harness/review@1`), the two sections are replaced: your final message is
+exactly one JSON object and nothing else, with no code fence and no sentence
+around it.
+
+```json
+{
+  "schema": "oc-harness/review@1",
+  "verdict": "pass",
+  "blockers": [],
+  "notes": "short optional note"
+}
+```
+
+- `pass` means `blockers` is empty. `fail` means at least one blocker.
+- Each blocker is `{ "file": "path/from/repo", "line": 1, "issue": "...",
+  "consequence": "..." }`. A blocker without a line and a consequence is not
+  actionable and fails validation.
+- Anything that is not this object - prose, a fenced block, a missing field -
+  is treated as a failed review by the caller. When in doubt, `fail` with a
+  blocker that says what could not be checked.
+
 See also the `space-review` skill for the full plan -> build -> review -> finish
 loop and the shared task-file contract.

package/retention.yml

@@ -110,14 +110,6 @@ models:
     verified: 2026-10-19
     source: https://opencode.ai/docs/zen
 
-  opencode/flint-40b-free:
-    retention: may-train
-    note: >-
-      Free trial endpoint on opencode zen; treated as may-train until terms
-      state otherwise.
-    verified: 2026-10-22
-    source: https://opencode.ai/docs/zen
-
   # --- Paid routes, kept because the catalog still serves them and a profile
   # may be pointed at one. ---