opencode-harness-kit 0.6.4 → 0.7.0

package/dist/profile.d.ts

@@ -12,6 +12,7 @@ export type Lane = {
     steps?: number;
     permission?: Record<string, any>;
     prompt: string;
+    skills?: boolean;
 };
 export type Profile = {
     id: string;
@@ -69,7 +70,7 @@ export type RetentionRecord = {
  * @typedef {"zero-retention" | "retained" | "may-train" | "unknown"} Retention
  * @typedef {string | null} EffortRequest
  * @typedef {{ model: string, levels?: string[], chain?: string[] }} RoleModel
- * @typedef {{ description: string, mode?: string, steps?: number, permission?: Record<string, any>, prompt: string }} Lane
+ * @typedef {{ description: string, mode?: string, steps?: number, permission?: Record<string, any>, prompt: string, skills?: boolean }} Lane
  * @typedef {object} Profile
  * @property {string} id
  * @property {string} [title]

package/dist/prune.d.ts

@@ -1,5 +1,27 @@
 import type { Plugin } from "./plugin.js";
+interface StubStats {
+    stubbed: number;
+    kept: number;
+    bytesBefore: number;
+    bytesAfter: number;
+}
 export declare const PrunePlugin: Plugin;
+/**
+ * Append one line to the project's prune log, and rotate it when the file
+ * outgrew `maxBytes`.
+ *
+ * Exported so the rotation can be driven directly in the tests: reaching it
+ * through the hook needs a 4MB fixture, which is not a fixture.
+ *
+ * @param {string} worktree the project the log belongs to
+ * @param {string} profileId the profile the prune ran under
+ * @param {StubStats & { saved: number, threshold: number }} stats
+ * @param {number} [maxBytes] rotate above this; the telemetry default
+ */
+export declare function logPrune(worktree: string, profileId: string, stats: StubStats & {
+    saved: number;
+    threshold: number;
+}, maxBytes?: number): Promise<void>;
 export declare const pruneMeta: {
     note: string;
 };

package/dist/rules.d.ts

@@ -75,8 +75,12 @@ export declare function shellPayload(segment: string): string | null;
 export declare const ASK_GATED: RegExp[];
 /** @type {Rule[]} */
 export declare const BASH_RULES: Rule[];
-/** @type {Rule[]} */
-export declare const WRITE_RULES: Rule[];
+export declare const WRITE_RULES: {
+    name: string;
+    test: RegExp;
+    unless: RegExp;
+    why: string;
+}[];
 /**
  * The shared denylist, checked the way the rules are checked.
  *

package/docs/EXIT-CODES.md

@@ -9,18 +9,32 @@ stdout gets the answer and nothing else.
 ## `oc-harness run`
 
 The unattended loop. It walks `.space/tasks.md`, asks the agent for one task
-at a time, and re-runs the task's own verify command before believing it.
+at a time, and re-runs the task's own verify command before believing it. It
+builds and verifies; it does not plan, commit or push. With `--review` it also
+asks the reviewer lane for a schema'd verdict and stops unless that verdict
+passes.
 
 | Code | Meaning |
 |---|---|
 | 0 | Every task is done. |
 | 1 | There is no task file. The loop needs a plan to walk. |
-| 2 | Stopped early and a person is needed: a task is blocked, a task has no verify command, the round cap was reached, or the time budget ran out. |
-| 3 | The agent itself failed. The reason is the agent's own last words, and the task is marked blocked with them. |
+| 2 | Stopped early and a person is needed: a task is blocked, a task has no verify command or one that cannot fail (`true`, `echo ok`), a task file with no tasks in it, the agent changed files the task did not name, another run holds the lock, the round cap was reached, or the time budget ran out. |
+| 3 | The agent itself failed. The reason is the agent's own last words, and the task is marked blocked with them. This includes the agent exiting 0 with an empty transcript: that is a spent or unreachable model route, not a change worth verifying, so no verify runs. |
 | 4 | The verify command failed. The task is marked blocked and the real output is in `.space/tasks.md` - not in the exit code, which is no place for a stack trace. |
+| 5 | The review did not pass: the reviewer failed, returned prose instead of a schema'd verdict, or returned `fail`. The task is marked blocked with the blockers and the verdict file is in `.space/review/`. |
 
 Code 2 is the "asked for something impossible" code: nothing about it is a
-bug, and the reason is printed. Codes 3 and 4 mean the work did not land.
+bug, and the reason is printed. Codes 3, 4 and 5 mean the work did not land.
+
+A task's `files:` line is a scope, not a hint. When the agent changes a file
+the line does not name, the loop marks the task blocked and stops rather than
+letting the next task build on an unplanned edit. `--allow-extra-files` is
+the deliberate way to say the task may go wider; it is also the only way to
+run outside a git work tree, where the check has nothing to read.
+
+A live lock is not cleared automatically. `--force-unlock` breaks one
+explicitly, for the case where the recorded pid now belongs to an unrelated
+process. Do not use it to start a second run alongside a real one.
 
 ## `oc-harness doctor`
 

package/docs/REVIEW-VERDICT.md

@@ -0,0 +1,73 @@
+# The review verdict
+
+`oc-harness run --review` asks the `reviewer` lane for a machine-readable
+verdict. A verdict is accepted only when it matches this schema; anything else
+is a failed review, never a pass.
+
+The contract is deliberately narrow. A reviewer that writes prose, wraps the
+object in a sentence, or omits a field is rejected, because a loop that parses
+whatever it is given will eventually parse a compliment as a pass.
+
+## Schema
+
+```json
+{
+  "schema": "oc-harness/review@1",
+  "verdict": "pass",
+  "blockers": [],
+  "notes": ""
+}
+```
+
+| Field | Required | Rule |
+|---|---|---|
+| `schema` | yes | Exactly `oc-harness/review@1`. Bump the suffix to change the shape. |
+| `verdict` | yes | `pass` or `fail`. |
+| `blockers` | yes | An array. `pass` requires it empty; `fail` requires at least one. |
+| `blockers[].file` | yes | A path from the repository root. |
+| `blockers[].line` | yes | A non-negative integer. `0` is allowed for a file-level finding. |
+| `blockers[].issue` | yes | What is wrong. |
+| `blockers[].consequence` | yes | What breaks because of it. A finding with no consequence is a preference. |
+| `notes` | no | A short string. |
+
+The reviewer may output the object bare or inside one ` ```json ` fence. It
+may not put a sentence before or after it.
+
+## Where it is kept
+
+The orchestrator validates the verdict and writes it to
+`.space/review/<round>-<task-slug>.json`, atomically. A review that leaves no
+file is a review nobody can audit, and the next run cannot tell it happened.
+
+```json
+{
+  "schema": "oc-harness/review@1",
+  "at": "2026-10-24T12:00:00.000Z",
+  "task": "Add the retry budget",
+  "round": 3,
+  "reviewer": "reviewer",
+  "verify": { "command": "npm test", "exit": 0, "seconds": 12 },
+  "verdict": "fail",
+  "blockers": [
+    {
+      "file": "src/retry.js",
+      "line": 41,
+      "issue": "the budget is decremented after the sleep",
+      "consequence": "a timed-out attempt is refunded, so the loop can run forever"
+    }
+  ],
+  "notes": ""
+}
+```
+
+## What the loop does with it
+
+| Verdict | Outcome |
+|---|---|
+| `pass` | The task is marked `done`, and the task's `output:` records the verdict path. |
+| `fail` | The task is marked `blocked` with the blockers, and the run stops with exit 5. |
+| malformed | The task is marked `blocked`, and the run stops with exit 5. |
+| reviewer crashed | The task is marked `blocked`, and the run stops with exit 5. |
+
+Review runs after the task's own `verify:` has passed. A failed verify never
+reaches the reviewer, because there is nothing yet worth reviewing.

package/eval/RESULTS.md

@@ -18,6 +18,16 @@ bottom runs the same task with and without the kit, ten times each, on both
 arms at once - 120 agent runs, which is enough to be worth reading and not
 enough to be worth trusting.
 
+**One definition changed, and the tables are marked.** Until commit `8614d75`
+the A/B added up `tokens.input` - the part of each request that was not in the
+provider's prompt cache - and called it "input tokens". That is not what a
+request costs: a cached step reports a few hundred tokens there against tens of
+thousands in `cache.read`. The column is now called **uncached input**, and a
+**context tokens** row was added where the streams still exist to re-read it.
+Both are medians over the same runs; the context row is the one to compare
+against the guard's budget, and the uncached row is the one to compare against
+any table written before this note.
+
 ## Two measurements, and the difference between them
 
 **`npm run measure` is an OBSERVATION about one install.** It compares the work
@@ -175,7 +185,7 @@ so it entered the medians as a fast, empty, wrong answer.
 | narrow reads | 66.7% | 66.7% |
 | grep/glob calls | 3 | 2 |
 | steps | 9 | 6 |
-| input tokens | 56,205 | 10,540 |
+| uncached input | 56,205 | 10,540 |
 | final context | 39,928 | 12,718 |
 | wall seconds | 43 | 21 |
 | paths written outside the target | 0 | 0 |
@@ -199,8 +209,10 @@ traffic - gave **55/60 against 54/60**, the other way. Two measurements
 put the tie on either side of zero, which is what a tie looks like from
 both ends.
 
-**On everything else, the kit is a cost.** It spends **5.3x the input
-tokens** and takes **twice as long**, and it makes **more** tool calls.
+**On everything else, the kit is a cost.** It spends **5.3x the uncached input
+tokens** (the streams for this run are gone, so the context total cannot be
+re-derived; on the free route, where they survive, the same ratio is 6.4x) and
+takes **twice as long**, and it makes **more** tool calls.
 Those are the honest numbers.
 
 **The previous conclusion was the grader's fault, and it is worth writing
@@ -255,6 +267,117 @@ one-file question following it is waste either way. A fleet that routes
 each job to the model built for it is the other half of what the kit
 offers, and this suite uses one model on both sides, so it cannot see that.
 
+### The same suite on the free route, 3 reps per arm
+
+    node eval/ab.mjs --target <scratch>/self --suite eval/tasks-self.json \
+      --reps 3 --timeout 180 --out <scratch>/ab-self-02 --budget 2500000
+
+Run after the budget work, on the route this install actually uses,
+`opencode/deepseek-v4.1-flash-free`, with `--out` so a quota stop would
+resume rather than restart. 36 runs, 33 scored: three hit the 180-second
+ceiling (one on, two off) and are excluded rather than counted as wrong.
+
+**The token rows were re-read after the counter was fixed.** Until then the
+suite summed `tokens.input` - the part of the prompt that was not in the
+provider's cache - and the table called it "input tokens". The cost of a
+request is the whole context it reads. Re-running the same streams through the
+corrected reader puts the medians at 252,054 and 39,180 - **6.4x**, not 4.7x:
+
+    node eval/ab.mjs --target <scratch>/self --suite eval/tasks-self.json \
+      --rescore <scratch>/ab-self-02
+
+The context row is over the 12 runs per arm that can still be re-scored: the
+six edit runs per arm cannot, because the per-run copy of the subject they
+edited is gone and re-grading them would grade nothing. The uncached row is
+over the 17 and 16 runs the live suite scored, because that is the number it
+printed; it is kept because every table above it was written with that
+definition. Neither row is comparable to the other's count, and both are
+comparable between the arms.
+
+| metric (median) | kit on | kit off |
+| --- | --- | --- |
+| **correct** | **15/17** | **16/16** |
+| tool calls | 9 | 4 |
+| steps | 7 | 4 |
+| narrow reads | 60% | 83.4% |
+| context tokens (12 runs/arm) | 252,054 | 39,180 |
+| uncached input (17/16 runs) | 44,444 | 9,527 |
+| final context | 43,823 | 11,236 |
+| wall seconds | 53.8 | 18.6 |
+
+Same shape as the 10-rep run, and a little worse for the kit: 6.4x the context
+tokens, 2.9x the wall time, twice the tool calls, and the one task the kit
+lost outright was `read-control` (1/3 against 3/3) - the smallest task in the
+suite, where the kit's fixed overhead is the whole cost. Three reps per arm
+is not enough to call a correctness difference; it is enough to say the cost
+reproduces.
+
+### What the skill list cost
+
+The first request of a run is the clean place to measure fixed overhead: the
+prompt, the tool schemas, and the task, before any conversation. It is also
+nearly constant run to run, which makes a median over 20 runs meaningful to the
+token.
+
+| first request | kit on | kit off |
+| --- | --- | --- |
+| skills on (3-rep run) | 36,016 | 7,856 |
+| skills off (10-rep run) | 10,865 | 7,867 |
+
+`oc-harness use fleet-safe --no-skills` removed **25,151 tokens** from every
+kit-on request - 70% of its fixed cost, and more than the whole off arm. The
+spread across runs is ±20 tokens, so this is not sampling noise; the off arm
+did not move because the flag only changes the lanes' prompt.
+
+    node -e 'for (const f of <dir>/*-on-*.json) { first step_finish }'
+
+The residual 10,865 is the lane prompt, `GLOBAL.md` and the tool schemas. The
+skill list was the single largest line item in the kit's overhead, and it was
+paying for 219 skills that a lane with `tools.skill: false` could not open.
+
+### The 10-rep run: 47 of 120, stopped by the window
+
+    node eval/ab.mjs --target <scratch>/self --suite eval/tasks-self.json \
+      --reps 10 --timeout 180 --out <scratch>/ab-self-10 --budget 12000000
+
+It started 47 runs and then the account's free window ran out, which looks
+like `opencode run` hanging rather than like an error - see the preflight note
+in the README. 46 of the 47 produced a result: 40 were scored and six are
+listed as not scored. The 47th was cut off mid-run and is in no number below.
+The other 73 were never started. Three of the six tasks have runs at all, so
+this is a partial set and **not** the headline number. It is here because the
+direction is worth having and because the files it came from will not survive
+forever:
+
+    node eval/ab.mjs --target <scratch>/self --suite eval/tasks-self.json \
+      --rescore <scratch>/ab-self-10
+
+| metric | kit on | kit off |
+| --- | --- | --- |
+| correct, all three tasks | 18/19 | 22/22 |
+| `read-control` | 9/10 | 10/10 |
+| `read-multi-denylist` | 6/6 | 10/10 |
+| `read-multi-gate` | 3/3 | 2/2 |
+| context tokens (median) | 62,990 | 49,374 |
+| wall seconds (median) | 30.6 | 29.3 |
+| not scored | 5 | 1 |
+
+`wall seconds` is the harness's own timer, not the last event in the stream:
+four of the on runs were killed at 180s and their streams stop earlier, so a
+stream-derived column made the killed arm look faster.
+
+Five runs hit the 180-second ceiling: four of them are `read-multi-denylist`
+on the kit, at 12, 12, 15 and 20 steps, so on that task the kit did not merely
+answer worse, it did not finish four times in ten. The fifth is `read-multi-gate`
+off. A sixth run, `read-multi-gate` on, produced no output at all and is not
+scored either. The context ratio here (1.3x) is much smaller than the 3-rep
+run's 6.4x and should not be read as a contradiction: the task mix is
+different, and the unfinished runs are excluded from the median.
+
+Resuming is `--out` plus the same command; a finished run is read from its
+sidecar and not paid for again, and the sidecar's token count is re-read from
+the stream so the cap counts the corrected number.
+
 ## The A/B on TypeScript, against a PUBLIC subject
 
     node eval/prepare.mjs --clone https://github.com/typesense/typesense-js \
@@ -284,7 +407,7 @@ is the check that stops an ambiguous task.
 | tool calls | 12 | 12 |
 | read calls | 2 | 2 |
 | narrow reads | 50% | 58.4% |
-| input tokens | 46,025 | 17,324 |
+| uncached input | 46,025 | 17,324 |
 | final context | 45,455 | 18,962 |
 | wall seconds | **31.4** | 34.4 |
 | paths outside the target | 0 | 0 |
@@ -332,7 +455,7 @@ pristine subject file by file, and the answer is graded by citation.
 | **correct** | **2/2** | **2/2** |
 | tool calls | 47 | 46 |
 | read calls | 12 | 18 |
-| input tokens | 78,280 | 37,171 |
+| uncached input | 78,280 | 37,171 |
 | final context | 68,794 | 42,732 |
 | wall seconds | 121.4 | 104.7 |
 
@@ -360,7 +483,7 @@ off and on:
 | --- | --- | --- | --- |
 | tool calls | 67.5 | 51 | **-24%** |
 | read calls | 22.5 | 18.5 | -18% |
-| input tokens | 268,834 | 84,325 | **-69%** |
+| uncached input | 268,834 | 84,325 | **-69%** |
 | final context | 48,637 | 75,705 | +56% |
 | wall seconds | 307.5 | 123.5 | **-60%** |
 
@@ -401,7 +524,7 @@ task actually says, and the difference between the arms is the lane.
 | **correct** | **2/2** | **2/2** |
 | tool calls | 36 | 20 |
 | read calls | 5.5 | 7 |
-| input tokens | 85,499 | 38,934 |
+| uncached input | 85,499 | 38,934 |
 | wall seconds | 161.5 | 80.8 |
 
 ### What this says, and what it does not