opencode-harness-kit 0.5.4 → 0.5.6

package/dist/profile.d.ts

@@ -307,8 +307,21 @@ export declare function pruneThreshold(p: Profile, opts?: {
     unhealthy?: Set<string>;
 }): number;
 /**
- * Every model a profile can put behind the user: the base model plus the model
- * each lane actually resolves to.
+ * Every model a profile can put behind the user: the base model, the model
+ * each lane actually resolves to, and the utility route.
+ *
+ * The utility route belongs in this list because it is WRITTEN into the
+ * config as `small_model` and opencode sends real requests down it - a
+ * title, a one-line summary, a rewritten question. It used to be a chain
+ * rung only, so it never appeared here, and a profile could name a
+ * may-train utility route under a zero-retention claim: the file said
+ * "zero-retention" while a training provider served the calls.
+ *
+ * Every caller of this list wants the same thing - the models that can
+ * serve a request - so the list is widened rather than a second one
+ * invented beside it. The one caller that needs the lanes alone is
+ * `validate()`, which cross-checks the profile's DECLARED retention against
+ * the base model and the lanes; that is `laneModelsInPlay`.
  *
  * @param {Profile} p
  * @param {{ unhealthy?: Set<string> }} [opts]
@@ -317,6 +330,23 @@ export declare function pruneThreshold(p: Profile, opts?: {
 export declare function modelsInPlay(p: Profile, opts?: {
     unhealthy?: Set<string>;
 }): string[];
+/**
+ * The models the profile's own base and lanes put behind the user, without
+ * the utility route.
+ *
+ * This is the set the profile's own `retention` field is about: the base
+ * model and the lanes it resolves to. The utility route is an extra route
+ * the profile adds on top of them, and it is reported where it belongs - the
+ * config header names it, and `oc-harness use` warns about it - so it does
+ * not decide what the profile may declare about itself.
+ *
+ * @param {Profile} p
+ * @param {{ unhealthy?: Set<string> }} [opts]
+ * @returns {string[]}
+ */
+export declare function laneModelsInPlay(p: Profile, opts?: {
+    unhealthy?: Set<string>;
+}): string[];
 /**
  * The split lanes, each with the limits of the model that lane actually runs
  * on. A lane on a different model has a different output cap, and the header

package/dist/telemetry-store.d.ts

@@ -3,6 +3,28 @@ export declare const MAX_BYTES: number;
 export declare function logPath(dir: string): string;
 /** The telemetry file and its rotated predecessors, oldest first. */
 export declare function logFiles(dir: string): string[];
+/**
+ * Rotate by size so a long-running project cannot fill its disk.
+ *
+ * THREE generations are kept: the current file, `.1` and `.2`. The reader
+ * enumerates all three, so the writer has to produce all three.
+ *
+ * The rotation used to rename the current file straight to `.1`, and a
+ * rename REPLACES the destination on POSIX - so the previous generation was
+ * destroyed on every rotation and `.2` never came into existence. Two
+ * rotations after the fact, the oldest history was gone and the third
+ * generation the reader looks for was missing: the history a project lost
+ * was lost on a schedule nobody was watching.
+ *
+ * So the oldest is shifted out of the way first, and the oldest of the
+ * three is dropped - the disk use stays bounded at three files, and the
+ * generation that is dropped is the one that was already two rotations
+ * old.
+ *
+ * @param file the file to rotate
+ * @param maxBytes the size above which it is rotated
+ */
+export declare function rotate(file: string, maxBytes?: number): void;
 /**
  * Append one record.
  *

package/docs/EXIT-CODES.md

@@ -75,6 +75,13 @@ attributable measurement carries the gate.
 | 0 | The install is in place, or the change was confirmed. Something may have been left alone - a file that was already there, a prompt that was not answered - and the reason is printed. |
 | 1 | The install was refused, or a step failed. |
 
+A target that was left alone is NOT counted as installed: the install says
+so in place of its tick, and `verify` reports the file that is not the kit's
+- because a file of the user's own at `GLOBAL.md` means the kit's rules are
+not being loaded, and a tick next to that is a claim the installer cannot
+make. A link that could not be created is a step that failed, and the
+install exits 1.
+
 `doctor` also exits 1 when there is no config, or no profile recorded in
 it: a kit that is installed but points at nothing is a broken install, and
 a check that passes it is not a check.

package/docs/KNOWN-LIMITS.md

@@ -323,10 +323,28 @@ comparable, and the gate uses the number that IS comparable.
 `eval/measure-prune.mjs` is that number. opencode stores the message parts
 unpruned - the pruner rewrites the outgoing request, not the history - so the
 stored parts of a real session are exactly what the pruner was given, and
-replaying the real pruner over them measures exactly what it removed. On one
-install: **80.5% of the context
-across 60 real sessions**, 6,608 parts stubbed - the number
-`npm run measure:prune` prints.
+replaying the real pruner over them measures exactly what it removed.
+
+It reports **two** measures, because the script and the pruner's own log used
+to answer two different questions and only one of them is this kit's claim:
+
+- **REQUEST** - the size of the request that goes back to the model, counted
+  on the parts as they are serialised. This is the same definition
+  `savedChars` uses in `prune.jsonl`, so the script and the log ask the same
+  question of the same parts and their numbers reconcile rather than
+  disagree: the script's request delta IS the sum of the log's `savedChars`
+  for the same passes. It is the primary number and the one the `--min-saved`
+  gate is computed from, because "a smaller request" is what this kit claims.
+- **TEXT** - the text inside the parts: how much text was replaced. It is the
+  larger of the two, because the request measure also carries the parts the
+  pruner left alone, and because the stub's own metadata is part of the
+  request and counts against the saving.
+
+On one install: **80.5% of the text across 60 real sessions**, 6,608 parts
+stubbed. That is the TEXT measure - the number `npm run measure:prune`
+printed before the two were separated, when the script measured text and
+labelled it "context". The REQUEST measure on the same install is lower, and
+it is the one the gate checks.
 
 The controlled comparison of the two arms is `eval/ab.mjs`, and its result is
 in `eval/RESULTS.md`.
@@ -361,13 +379,37 @@ re-reads stop: **-69% input tokens and -60% wall time** on that task, at the
 cost of a bigger context. It is a setting - `prune.keepDistinctOutputs` - and
 turning it off brings the old behaviour back.
 
-A `savedChars` of 0 in `prune.jsonl` is not a second, redundant pass. It means
-`bytesAfter` did not come out below `bytesBefore` for that turn, which happens
-when a stub is no shorter than the output it replaced (the stub's own label
-costs more than a short body), or when a `reasoning` part was under the 200
-byte floor and so was not shortened at all. These lines are still logged
-because a part was stubbed, which is the event worth recording. Savings
-aggregate correctly either way, since each line is measured independently.
+`savedChars` in `prune.jsonl` is the size of the request before the pass minus
+the size after it, measured on the parts as they are serialised - not on the
+text inside them. It used to measure the text only, and to ignore the state
+the stub put in a part's place, which carries about fifty characters of its
+own metadata per stubbed part. So a pass whose request GREW was reported as
+a saving of the difference between two texts.
+
+A negative `savedChars` therefore means the pass made the request BIGGER: the
+stub was longer than the output it replaced, because the stub carries its own
+label and its own metadata. Such a pass is still logged, because a part was
+stubbed, which is the event worth recording, and the negative number is the
+honest report of it.
+
+The measurement script measures the same thing on purpose.
+`eval/measure-prune.mjs` counts a part with the pruner's own expression -
+`JSON.stringify(part).length` - before and after the pass, and reports it as
+the REQUEST measure, so the two reconcile by construction instead of by
+coincidence: replaying twelve real sessions through the real pruner, the
+script measured a request delta of **6,049** characters and the log the
+pruner wrote during that replay said `savedChars: 6049`. The script also
+reports the TEXT measure, because the two answer different questions and the
+difference is worth stating rather than hiding: the request measure counts
+the stub's own metadata against the saving and carries the parts the pruner
+left alone, so it is the smaller of the two, and it can be negative where the
+text measure is positive.
+
+There is no minimum size under which a part is left alone. This file used to
+claim a 200 byte floor for `reasoning` parts; there was never such a floor in
+the code, and a reasoning part shorter than the stub - about seventy-eight
+characters - is still replaced, so the request grows. The measurement above
+is what makes that visible instead of silent.
 
 `baseline.mjs` measures permission pressure with its own glob approximation of
 OpenCode's matcher, not OpenCode's matcher. It handles the cases that matter

package/eval/measure-prune.mjs

@@ -20,6 +20,31 @@
 //
 // The gate is on the CONTROLLED number. The install-wide one stays an
 // observation.
+//
+// TWO MEASURES, both reported, neither hidden, because they answer two
+// different questions:
+//
+//   REQUEST - how much did the REQUEST that goes back to the model shrink.
+//   Counted the way the pruner's own log counts it: the size of a PART,
+//   before the pass and after it. `partSize` in src/prune.ts is
+//   `JSON.stringify(part).length`, and `savedChars` in `.space/prune.jsonl`
+//   is the delta of that number. The measure below is the SAME expression,
+//   so the script and the log ask the same question of the same parts and
+//   cannot disagree about what they mean. The delta is the same number
+//   too: the parts outside the pruner's window are untouched, so they
+//   contribute nothing to it either way.
+//
+//   TEXT - how much TEXT was replaced. The text inside the parts, which is
+//   what this script used to measure and what it still reports.
+//
+// The two are not the same number twice. The request measure also carries
+// the parts the pruner left alone, and the stub's own metadata is part of
+// the request and is not text. So the request measure is the smaller of
+// the two, and it can be negative where the text measure is positive - a
+// stub that costs more than the output it replaced.
+//
+// The gate is on the REQUEST measure, because that is the claim the kit
+// makes: a smaller request.
 
 import { existsSync, mkdtempSync, rmSync } from "node:fs"
 import { homedir, tmpdir } from "node:os"
@@ -186,20 +211,79 @@ const hook = await HarnessPlugin(
 const transform = hook["experimental.chat.messages.transform"]
 
 /**
- * The size the PRUNER measures.
+ * The size of a PART, as it sits in the request.
+ *
+ * THIS IS THE PRUNER'S OWN DEFINITION, EXPRESSION FOR EXPRESSION.
+ * `partSize` in src/prune.ts is `JSON.stringify(part).length`, and
+ * `savedChars` in `.space/prune.jsonl` is that number's delta. This is the
+ * same expression on the same objects, so the script and the log measure
+ * the same thing and the two numbers are reconcilable by construction
+ * rather than by coincidence.
+ *
+ * CHARACTERS, not bytes: a UTF-8 character is more than one byte, and an
+ * agent's output is not ASCII. The pruner's measure counts characters
+ * too, and the byte figures are reported beside them, not instead of
+ * them.
+ *
+ * A part that cannot be serialised counts as zero, exactly as the
+ * pruner's own measure does: a measurement that throws is worse than a
+ * measurement that counts the part as nothing.
+ *
+ * @param {any} part
+ * @returns {number}
+ */
+function requestPartSize(part) {
+  try {
+    const json = JSON.stringify(part)
+    return typeof json === "string" ? json.length : 0
+  } catch {
+    return 0
+  }
+}
+
+/**
+ * The same part, in bytes.
  *
- * The pruner counts CHARACTERS, not bytes: `part.text.length`. The report
- * labelled that "bytes", which is wrong for anything that is not ASCII - and
- * an agent's output is not ASCII. Both numbers are reported.
+ * @param {any} part
+ * @returns {number}
  */
-function sizeOf(part) {
+function requestPartBytes(part) {
+  try {
+    const json = JSON.stringify(part)
+    return typeof json === "string" ? Buffer.byteLength(json, "utf8") : 0
+  } catch {
+    return 0
+  }
+}
+
+/**
+ * The TEXT inside a part - the SECONDARY measure.
+ *
+ * `part.text` for a text or reasoning part, `part.state.output` for a tool
+ * part. This is what the script measured before, and it is still reported,
+ * because it answers a different question: how much text was replaced.
+ *
+ * It is not the claim the kit makes. The text measure cannot see the
+ * metadata a stub puts in a part's place, and it cannot see the parts the
+ * pruner left alone, so it is the larger of the two numbers and it can be
+ * positive where the request measure is negative.
+ *
+ * @param {any} part
+ * @returns {number}
+ */
+function textSize(part) {
   if (typeof part?.text === "string") return part.text.length
   if (typeof part?.state?.output === "string") return part.state.output.length
   return 0
 }
 
-/** The same text, in bytes. */
-function byteSizeOf(part) {
+/**
+ * The same text, in bytes.
+ *
+ * @param {any} part
+ * @returns {number}
+ */
+function textBytes(part) {
   if (typeof part?.text === "string") return Buffer.byteLength(part.text, "utf8")
   if (typeof part?.state?.output === "string") {
     return Buffer.byteLength(part.state.output, "utf8")
@@ -207,10 +291,32 @@ function byteSizeOf(part) {
   return 0
 }
 
-let totalBefore = 0
-let totalAfter = 0
-let bytesBefore = 0
-let bytesAfter = 0
+/**
+ * The size of a whole message list, by one measure.
+ *
+ * @param {{parts: any[]}[]} messages
+ * @param {(part: any) => number} size
+ * @returns {number}
+ */
+function measureAll(messages, size) {
+  let total = 0
+  for (const m of messages) {
+    for (const p of m.parts) total += size(p)
+  }
+  return total
+}
+
+// The REQUEST measure: the request that goes back to the model, before the
+// pass and after it, counted the way the pruner's log counts it.
+let reqCharsBefore = 0
+let reqCharsAfter = 0
+let reqBytesBefore = 0
+let reqBytesAfter = 0
+// The TEXT measure: the text inside the parts, before and after.
+let textCharsBefore = 0
+let textCharsAfter = 0
+let textBytesBefore = 0
+let textBytesAfter = 0
 let measured = 0
 let stubbed = 0
 const perSession = []
@@ -218,35 +324,38 @@ const perSession = []
 for (const s of sessions) {
   const messages = sessionMessages(s.id)
   if (!messages.length) continue
-  const before = messages.reduce(
-    (n, m) => n + m.parts.reduce((k, p) => k + sizeOf(p), 0),
-    0,
-  )
-  const beforeBytes = messages.reduce(
-    (n, m) => n + m.parts.reduce((k, p) => k + byteSizeOf(p), 0),
-    0,
-  )
   // The pruner mutates in place, so the copy has to be the real thing.
-  const reasoningBefore = messages.flatMap((m) =>
-    m.parts.filter((p) => p.type === "reasoning").map((p) => p.text),
-  )
   const copy = messages.map((m) => ({
     info: m.info,
     parts: m.parts.map((p) => (p.state ? { ...p, state: { ...p.state } } : { ...p })),
   }))
-  await transform({}, { messages: copy })
-  const after = copy.reduce(
-    (n, m) => n + m.parts.reduce((k, p) => k + sizeOf(p), 0),
-    0,
-  )
-  if (!before) continue
-  totalBefore += before
-  totalAfter += after
-  bytesBefore += beforeBytes
-  bytesAfter += copy.reduce(
-    (n, m) => n + m.parts.reduce((k, p) => k + byteSizeOf(p), 0),
-    0,
+  // BEFORE is measured on the COPY - on the very objects the pruner is about
+  // to see. The copy serialises identically to the part it was made from
+  // (same keys, same order, same values), and measuring one set of objects
+  // before and after is the point: a "before" taken from a different object
+  // than the "after" is two measurements of two things.
+  const reqCharsBeforeSession = measureAll(copy, requestPartSize)
+  const reqBytesBeforeSession = measureAll(copy, requestPartBytes)
+  const textCharsBeforeSession = measureAll(copy, textSize)
+  const textBytesBeforeSession = measureAll(copy, textBytes)
+  const reasoningBefore = copy.flatMap((m) =>
+    m.parts.filter((p) => p.type === "reasoning").map((p) => p.text),
   )
+  await transform({}, { messages: copy })
+  const reqCharsAfterSession = measureAll(copy, requestPartSize)
+  const reqBytesAfterSession = measureAll(copy, requestPartBytes)
+  const textCharsAfterSession = measureAll(copy, textSize)
+  const textBytesAfterSession = measureAll(copy, textBytes)
+  // A session with nothing in it has no percentage to report.
+  if (reqCharsBeforeSession <= 0) continue
+  reqCharsBefore += reqCharsBeforeSession
+  reqCharsAfter += reqCharsAfterSession
+  reqBytesBefore += reqBytesBeforeSession
+  reqBytesAfter += reqBytesAfterSession
+  textCharsBefore += textCharsBeforeSession
+  textCharsAfter += textCharsAfterSession
+  textBytesBefore += textBytesBeforeSession
+  textBytesAfter += textBytesAfterSession
   measured++
   // The pruner marks TOOL parts with `metadata.pruned` and REPLACES reasoning
   // parts, so counting only the marked ones undercounts what was stubbed.
@@ -263,24 +372,60 @@ for (const s of sessions) {
   stubbed += cut
   perSession.push({
     session: s.id,
-    bytes_before: before,
-    bytes_after: after,
-    saved_pct: Math.round((1 - after / before) * 1000) / 10,
+    // REQUEST - the size of the request, before and after the pass.
+    request_chars_before: reqCharsBeforeSession,
+    request_chars_after: reqCharsAfterSession,
+    request_bytes_before: reqBytesBeforeSession,
+    request_bytes_after: reqBytesAfterSession,
+    request_saved_pct: pctOf(reqCharsBeforeSession, reqCharsAfterSession),
+    // TEXT - the text inside the parts, before and after the pass.
+    text_chars_before: textCharsBeforeSession,
+    text_chars_after: textCharsAfterSession,
+    text_saved_pct: pctOf(textCharsBeforeSession, textCharsAfterSession),
     parts_stubbed: cut,
     tool_parts_stubbed: toolCut,
     reasoning_parts_stubbed: reasoningCut,
   })
 }
 
-const savedPct =
-  totalBefore > 0 ? Math.round((1 - totalAfter / totalBefore) * 1000) / 10 : null
+/**
+ * A percentage, to two decimals, or null when there is no percentage to
+ * report.
+ *
+ * Two decimals, not one: an effect of 0.05% is a real effect, and one
+ * decimal renders it as "0%" - a number that says "nothing was removed"
+ * about a pass that removed seven thousand characters. That is the exact
+ * failure this script exists to catch, so it is not repeated in the
+ * reporting of it.
+ *
+ * @param {number} before
+ * @param {number} after
+ * @returns {number | null}
+ */
+function pctOf(before, after) {
+  if (!(before > 0)) return null
+  return Math.round((1 - after / before) * 10000) / 100
+}
+
+/** A percentage for printing. */
+function fmtPct(n) {
+  return n === null ? "n/a" : `${n}%`
+}
+
+// The REQUEST measure's percentage. This is the one the gate is computed
+// from, because it is the claim the kit makes: a smaller request.
+const requestSavedPct = pctOf(reqCharsBefore, reqCharsAfter)
+// The TEXT measure's percentage: how much text was replaced.
+const textSavedPct = pctOf(textCharsBefore, textCharsAfter)
 
 const failures = []
 if (minSaved !== null) {
   if (measured === 0) {
     failures.push("no session had assistant messages to measure")
-  } else if (savedPct === null || savedPct < minSaved) {
-    failures.push(`the pruner removed ${savedPct ?? "n/a"}% of the context, expected at least ${minSaved}%`)
+  } else if (requestSavedPct === null || requestSavedPct < minSaved) {
+    failures.push(
+      `the pruner removed ${requestSavedPct ?? "n/a"}% of the request, expected at least ${minSaved}%`,
+    )
   }
 }
 
@@ -294,11 +439,21 @@ process.on("exit", () => {
 const report = {
   db: DB,
   sessions_measured: measured,
-  characters_before: totalBefore,
-  characters_after: totalAfter,
-  bytes_before: bytesBefore,
-  bytes_after: bytesAfter,
-  saved_pct: savedPct,
+  // REQUEST measure - the size of the request, counted the way the pruner's
+  // own log counts it (the size of a PART). This is the number the gate is
+  // computed from.
+  request_chars_before: reqCharsBefore,
+  request_chars_after: reqCharsAfter,
+  request_bytes_before: reqBytesBefore,
+  request_bytes_after: reqBytesAfter,
+  request_saved_chars: reqCharsBefore - reqCharsAfter,
+  request_saved_pct: requestSavedPct,
+  // TEXT measure - the text inside the parts. How much text was replaced.
+  text_chars_before: textCharsBefore,
+  text_chars_after: textCharsAfter,
+  text_bytes_before: textBytesBefore,
+  text_bytes_after: textBytesAfter,
+  text_saved_pct: textSavedPct,
   parts_stubbed: stubbed,
   per_session: perSession,
   verdict: failures.length ? "FAIL" : "PASS",
@@ -316,17 +471,36 @@ console.log(
   `  sessions ${measured} (last ${DAYS}d, at least ${MIN_MESSAGES} assistant messages each)`,
 )
 console.log("")
-console.log(`  context before   ${totalBefore.toLocaleString("en-US")} characters`)
-console.log(`  context after    ${totalAfter.toLocaleString("en-US")} characters`)
-console.log(`  in bytes         ${bytesBefore.toLocaleString("en-US")} -> ${bytesAfter.toLocaleString("en-US")}`)
-console.log(`  removed          ${savedPct === null ? "n/a" : savedPct + "%"}`)
+console.log("  REQUEST measure - how much the request that goes back to the model shrank.")
+console.log("  The size of a PART, before and after the pass: the same definition the")
+console.log("  pruner's own log uses (src/prune.ts `partSize`).")
+console.log(`    request context before   ${reqCharsBefore.toLocaleString("en-US")} characters`)
+console.log(`    request context after    ${reqCharsAfter.toLocaleString("en-US")} characters`)
+console.log(`    request saved            ${(reqCharsBefore - reqCharsAfter).toLocaleString("en-US")} characters`)
+console.log(`    request in bytes         ${reqBytesBefore.toLocaleString("en-US")} -> ${reqBytesAfter.toLocaleString("en-US")}`)
+console.log(`    removed                  ${fmtPct(requestSavedPct)}`)
+console.log("    <- the --min-saved gate is computed from this number, and the")
+console.log("       saved figure is the same number the pruner's log writes as")
+console.log("       savedChars for the same pass.")
+console.log("")
+console.log("  TEXT measure - how much text was replaced. The text inside the parts.")
+console.log(`    text before              ${textCharsBefore.toLocaleString("en-US")} characters`)
+console.log(`    text after               ${textCharsAfter.toLocaleString("en-US")} characters`)
+console.log(`    text saved               ${(textCharsBefore - textCharsAfter).toLocaleString("en-US")} characters`)
+console.log(`    text in bytes            ${textBytesBefore.toLocaleString("en-US")} -> ${textBytesAfter.toLocaleString("en-US")}`)
+console.log(`    removed                  ${fmtPct(textSavedPct)}`)
+console.log("")
 console.log(`  parts stubbed    ${stubbed}`)
 if (perSession.length) {
   console.log("")
-  const worst = [...perSession].sort((a, b) => a.saved_pct - b.saved_pct).slice(0, 5)
-  console.log("  the five sessions the pruner helped least:")
+  const worst = [...perSession]
+    .sort((a, b) => a.request_saved_pct - b.request_saved_pct)
+    .slice(0, 5)
+  console.log("  the five sessions the pruner helped least, by the request measure:")
   for (const p of worst) {
-    console.log(`    ${p.saved_pct}%  ${p.bytes_before.toLocaleString("en-US")} bytes  ${p.session.slice(0, 12)}`)
+    console.log(
+      `    ${fmtPct(p.request_saved_pct)}  ${p.request_bytes_before.toLocaleString("en-US")} bytes  ${p.session.slice(0, 12)}`,
+    )
   }
 }
 if (failures.length) {
@@ -334,6 +508,6 @@ if (failures.length) {
   console.log(`  FAIL  ${failures.join("; ")}`)
 } else if (minSaved !== null) {
   console.log("")
-  console.log(`  PASS  the pruner removed at least ${minSaved}% of the context.`)
+  console.log(`  PASS  the pruner removed at least ${minSaved}% of the request.`)
 }
 process.exit(failures.length ? 1 : 0)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "opencode-harness-kit",
-  "version": "0.5.4",
+  "version": "0.5.6",
   "description": "Harness kit for OpenCode: capability profiles, a fleet of job lanes with one free model each, role-based reasoning effort, context pruning, and measured baselines.",
   "license": "MIT",
   "homepage": "https://github.com/pham-canh/opencode-harness-kit#readme",