opencode-harness-kit 0.4.3 → 0.4.4

package/eval/attribute.mjs

@@ -0,0 +1,170 @@
+#!/usr/bin/env node
+// attribute.mjs - why did context per request drop?
+//
+// measure-install.mjs shows context falling from 123,776 to 68,836 tokens per
+// assistant message. That is a correlation between two windows of different
+// work, and it does not say which mechanism did it. This splits the number into
+// its two parts, which are physically different:
+//
+//   fixed overhead - the system prompt and tool definitions, sent on every
+//                    request. Paid on turn one, before any work happens.
+//   history        - what the conversation has accumulated since: tool output,
+//                    files read, the agent's own earlier messages.
+//
+// Those move independently. A kit that only made the system prompt shorter
+// would show up in the first; a kit that made tool output smaller shows up in
+// the second. Knowing which one moved is the difference between "the numbers
+// look good" and "the numbers look good for a reason we can point at".
+//
+// Turn count is reported alongside, and history is also given per turn,
+// because shorter sessions accumulate less history on their own. That
+// confound is the reason the per-turn column exists.
+//
+//   node eval/attribute.mjs          human-readable
+//   node eval/attribute.mjs --json
+
+import { existsSync, statSync } from "node:fs"
+import { homedir } from "node:os"
+import { join } from "node:path"
+import { canQuerySqlite, sqliteMissingMessage, query as queryDb } from "../src/sqlite.js"
+
+import { historyPerTurn, payback } from "./metrics.js"
+
+const args = process.argv.slice(2)
+const flag = (...n) => {
+  const i = args.findIndex((a) => n.includes(a))
+  return i >= 0 ? args[i + 1] : undefined
+}
+const wantJson = args.includes("--json")
+const DB = flag("--db") ?? join(homedir(), ".local/share/opencode/opencode.db")
+
+const installDir = join(homedir(), ".config/opencode/agents")
+const split = flag("--split")
+  ? Date.parse(flag("--split"))
+  : existsSync(installDir)
+    ? statSync(installDir).mtimeMs
+    : Date.now() - 7 * 864e5
+
+if (!canQuerySqlite()) {
+  console.error(sqliteMissingMessage())
+  process.exit(1)
+}
+
+if (!existsSync(DB)) {
+  console.error(`opencode database not found: ${DB}`)
+  process.exit(1)
+}
+
+const SESSION_FLOOR = flag("--since") ? Date.parse(flag("--since")) : 0
+
+const sql = `
+with s as (
+  select id, time_created from session where time_created >= ${SESSION_FLOOR}
+),
+r as (
+  select m.session_id as sid,
+    s.time_created as st,
+    json_extract(m.data,'$.tokens.input')
+      + json_extract(m.data,'$.tokens.cache.read') as ctx,
+    row_number() over (partition by m.session_id order by m.time_created) as rn,
+    count(*)     over (partition by m.session_id) as n
+  from message m join s on s.id = m.session_id
+  where json_extract(m.data,'$.role') = 'assistant'
+    and json_extract(m.data,'$.tokens') is not null
+),
+per as (
+  select sid, min(st) as st, max(n) as turns,
+    max(case when rn = 1  then ctx end) as first_ctx,
+    max(case when rn = n  then ctx end) as last_ctx
+  from r group by sid
+)
+select case when st < ${split} then 'before' else 'after' end as win,
+  count(*) as sessions,
+  avg(turns) as turns,
+  avg(first_ctx) as fixed,
+  avg(last_ctx) as last_ctx,
+  avg(last_ctx - first_ctx) as history
+from per group by win`
+
+const rows = queryDb(DB, sql)
+
+const pick = (win) => {
+  const r = rows.find((x) => x.win === win)
+  if (!r) return null
+  const sessions = Number(r.sessions)
+  const turns = Number(r.turns)
+  const fixed = Math.round(Number(r.fixed))
+  const last = Math.round(Number(r.last_ctx))
+  const history = Math.round(Number(r.history))
+  return {
+    sessions,
+    turns: round(turns),
+    fixed,
+    last,
+    history,
+    history_per_turn: historyPerTurn(history, turns),
+  }
+}
+
+function round(n) {
+  return Math.round(n * 10) / 10
+}
+
+const before = pick("before")
+const after = pick("after")
+
+const report = {
+  db: DB,
+  split: new Date(split).toISOString(),
+  before,
+  after,
+  delta: {
+    // The kit's own price, paid on every request.
+    fixed: before && after ? after.fixed - before.fixed : null,
+    // What it was supposed to buy.
+    history: before && after ? after.history - before.history : null,
+    history_per_turn:
+      before && after && before.history_per_turn !== null
+        ? round(after.history_per_turn - before.history_per_turn)
+        : null,
+  },
+}
+
+if (wantJson) {
+  console.log(JSON.stringify(report, null, 2))
+  process.exit(0)
+}
+
+if (!before || !after) {
+  console.error("not enough data on one side of the split to compare")
+  process.exit(1)
+}
+
+const row = (label, a, b) =>
+  `  ${label.padEnd(24)} ${String(a).padStart(12)} ${String(b).padStart(12)}`
+
+console.log("CONTEXT ATTRIBUTION")
+console.log(`  db     ${DB}`)
+console.log(`  split  ${report.split}   <- kit installed`)
+console.log("")
+console.log(row("", "before", "after"))
+console.log(row("sessions", before.sessions, after.sessions))
+console.log(row("turns per session", before.turns, after.turns))
+console.log("")
+console.log(row("fixed overhead", before.fixed, after.fixed))
+console.log(row("context at end", before.last, after.last))
+console.log(row("history", before.history, after.history))
+console.log(row("history per turn", before.history_per_turn, after.history_per_turn))
+console.log("")
+console.log(`  fixed overhead   ${report.delta.fixed > 0 ? "+" : ""}${report.delta.fixed} tokens`)
+console.log(`  history          ${report.delta.history} tokens per session`)
+console.log(`  history per turn ${report.delta.history_per_turn > 0 ? "+" : ""}${report.delta.history_per_turn} tokens`)
+console.log("")
+
+const pays = payback(report.delta.fixed, report.delta.history)
+console.log(
+  pays
+    ? "  The kit costs more per request and buys back more per session. That is the whole claim, decomposed."
+    : "  The kit does not currently pay for itself on this measure. Report it as it is.",
+)
+process.exit(0)

package/eval/flags.mjs

@@ -0,0 +1,88 @@
+// flags.mjs - reading a number out of a command line, without the two ways it
+// used to go wrong.
+//
+// A flag that cannot be parsed must be an ERROR, not a disabled check.
+//
+// `--min-narrow` with no value became `undefined`, which became `null`, which
+// meant "no gate" - and the report said PASS. `--min-narrow abc` became NaN,
+// and `!Number.isNaN(NaN)` is false, so the comparison never ran and the gate
+// was skipped. A typo in a CI gate turned the gate off without saying so.
+//
+// The same thing happened to `--reps abc` in the A/B runner: NaN, the loop
+// body never executed, and the suite reported success having run nothing.
+
+/**
+ * A number from a flag, or null when the flag was not given.
+ *
+ * @param {string} name the flag, including the dashes
+ * @param {string[] | undefined} argv
+ * @returns {number | null}
+ */
+export function numberFlag(name, argv = process.argv.slice(2)) {
+  const raw = rawValue(name, argv)
+  if (raw === undefined) return null
+  const n = Number(raw)
+  if (!Number.isFinite(n)) {
+    throw new Error(`${name} must be a number, got: ${raw}`)
+  }
+  return n
+}
+
+/**
+ * A whole number from a flag, at least one, or null when the flag was not
+ * given.
+ *
+ * @param {string} name the flag, including the dashes
+ * @param {string[] | undefined} argv
+ * @returns {number | null}
+ */
+export function countFlag(name, argv = process.argv.slice(2)) {
+  const raw = rawValue(name, argv)
+  if (raw === undefined) return null
+  const n = Number(raw)
+  if (!Number.isInteger(n) || n < 1) {
+    throw new Error(`${name} must be a whole number of at least 1, got: ${raw}`)
+  }
+  return n
+}
+
+/**
+ * The raw text of a flag, or undefined when the flag was not given.
+ *
+ * The LAST occurrence wins, so an argument appended to an npm script
+ * overrides the one in the script: `npm run measure:prune -- --sessions 1`
+ * has to mean one session, not the sixty the script says.
+ *
+ * @param {string} name the flag, including the dashes
+ * @param {string[]} argv
+ * @returns {string | undefined}
+ */
+function rawValue(name, argv) {
+  let value
+  let seen = false
+  for (let i = 0; i < argv.length; i++) {
+    if (argv[i] === name) {
+      if (argv[i + 1] === undefined) throw new Error(`${name} needs a value`)
+      value = argv[i + 1]
+      seen = true
+      i++
+      continue
+    }
+    if (argv[i].startsWith(`${name}=`)) {
+      value = argv[i].slice(name.length + 1)
+      seen = true
+    }
+  }
+  return seen ? value : undefined
+}
+
+/**
+ * The value of a flag that takes a string.
+ *
+ * @param {string} name the flag, including the dashes
+ * @param {string[] | undefined} argv
+ * @returns {string | undefined}
+ */
+export function stringFlag(name, argv = process.argv.slice(2)) {
+  return rawValue(name, argv)
+}

package/eval/measure-install.mjs

@@ -0,0 +1,323 @@
+#!/usr/bin/env node
+// measure-install.mjs - compare a real OpenCode install against itself, before
+// and after the kit was installed, so "does this kit help" is a number instead
+// of an impression.
+//
+//   node eval/measure-install.mjs                      human-readable table
+//   node eval/measure-install.mjs --json               machine-readable
+//   node eval/measure-install.mjs --before <iso> --after <iso>
+//   node eval/measure-install.mjs --min-narrow 50 --min-ctx-drop 20
+//                                                     exit 1 when either regresses
+//
+// Node 22 has no node:sqlite, so this shells out to the sqlite3 CLI in
+// read-only mode, the same way bin/baseline.mjs does. Both windows are computed
+// in a single pass: the 988 MB install database is slow enough that a second
+// scan is worth avoiding.
+//
+// Why these three metrics. eval/run.mjs proves the pruner stubs bytes and the
+// allowlist rewrites permission globs. It cannot prove that the agent changed
+// how it reads, because that is a property of the transcripts, not of a
+// synthetic one. Narrow reads and context per request are that property.
+
+import { existsSync, statSync } from "node:fs"
+import { homedir } from "node:os"
+import { join } from "node:path"
+
+const args = process.argv.slice(2)
+
+function flag(...names) {
+  const i = args.findIndex((a) => names.includes(a))
+  return i >= 0 ? args[i + 1] : undefined
+}
+const has = (...names) => args.some((a) => names.includes(a))
+
+const wantJson = has("--json")
+const DB = flag("--db") ?? join(homedir(), ".local/share/opencode/opencode.db")
+
+// The install boundary is the installed agent symlinks: `oc-harness install`
+// creates them, and nothing else does. Falling back to the plugin symlink keeps
+// this working on an install made by hand.
+const installDir = join(homedir(), ".config/opencode/agents")
+const installStamp = existsSync(installDir)
+  ? statSync(installDir).mtimeMs
+  : Date.now() - 7 * 864e5
+
+// Three edges, not two, and the two outer windows are the SAME LENGTH.
+//
+// The after-window used to be "everything since the install", which grows
+// without bound. That makes the comparison a different one every time the
+// script runs: a week after the install it compares 7 days against 7 days,
+// and a month after it compares 7 days against 30, and the answer moves
+// because the WORK moved, not because the kit did. A gate whose number
+// changes because you ran it on a different day is not a gate.
+//
+// The windows are now fixed and equal around the split: `--days` of work on
+// each side. `--since` puts the old unbounded behaviour back, for the case
+// where the question really is "everything since".
+let days
+try {
+  days = numberFlag("--days") ?? 7
+} catch (err) {
+  console.error(String(errorMessage(err)))
+  process.exit(1)
+}
+if (days <= 0) {
+  console.error(`--days must be more than 0, got: ${days}`)
+  process.exit(1)
+}
+// `--since` is a BOOLEAN. It used to be read with the value-after-the-flag
+// helper, so `--since` as the last argument was indistinguishable from
+// `--since` followed by nothing, and the unbounded window was never taken.
+const since = args.includes("--since")
+const split = stringFlag("--split") ? Date.parse(stringFlag("--split")) : installStamp
+const before = stringFlag("--before")
+  ? Date.parse(stringFlag("--before"))
+  : split - days * 864e5
+const after = stringFlag("--after")
+  ? Date.parse(stringFlag("--after"))
+  : since
+    ? Date.now()
+    : Math.min(Date.now(), split + days * 864e5)
+
+if (Number.isNaN(before) || Number.isNaN(after) || Number.isNaN(split)) {
+  console.error("--before/--after/--split take an ISO timestamp, e.g. 2026-10-08T05:41:00Z")
+  process.exit(1)
+}
+if (!(before < split && split < after)) {
+  console.error(`need before < split < after, got ${new Date(before).toISOString()} / ${new Date(split).toISOString()} / ${new Date(after).toISOString()}`)
+  process.exit(1)
+}
+
+// A gate that cannot be parsed is an error, not a disabled gate. A typo used
+// to turn the check off silently and the report said PASS.
+let minNarrow
+let minCtxDrop
+try {
+  minNarrow = numberFlag("--min-narrow")
+  minCtxDrop = numberFlag("--min-ctx-drop")
+} catch (err) {
+  console.error(String(errorMessage(err)))
+  process.exit(1)
+}
+
+if (!canQuerySqlite()) {
+  console.error(sqliteMissingMessage())
+  process.exit(1)
+}
+
+if (!existsSync(DB)) {
+  console.error(`opencode database not found: ${DB}`)
+  console.error("pass --db <path> or run opencode at least once first")
+  process.exit(1)
+}
+
+// Both helpers live in ./metrics.js so they can be tested without a database.
+import { pct, readsPerSearch, ctxDropPct as computeCtxDropPct } from "./metrics.js"
+import { canQuerySqlite, sqliteMissingMessage, query as queryDb } from "../src/sqlite.js"
+import { numberFlag, stringFlag } from "./flags.mjs"
+
+/**
+ * The message of a caught value, whatever it turned out to be.
+ *
+ * `catch` gives `unknown` under strict mode, and the file is not strict -
+ * but a message is what these scripts exist to print.
+ *
+ * @param {unknown} err
+ * @returns {string}
+ */
+function errorMessage(err) {
+  return err instanceof Error ? err.message : String(err)
+}
+
+function query(sql) {
+  return queryDb(DB, sql)
+}
+
+const TOOLS = `json_extract(data,'$.type')='tool'`
+const TOOL_NAME = `json_extract(data,'$.tool')`
+const isRead = `(${TOOL_NAME}='read')`
+const isSearch = `(${TOOL_NAME} in ('grep','glob'))`
+// A read is narrow when the caller bounded it. `offset` is the strong signal;
+// `limit` alone still means the agent chose not to take the whole file.
+const isNarrow = `(json_extract(data,'$.state.input.offset') is not null or json_extract(data,'$.state.input.limit') is not null)`
+
+const window = (ms) =>
+  `case when time_created < ${ms} then 'before' when time_created < ${after} then 'after' else 'other' end`
+
+const toolRows = query(`
+  select ${window(split)} as win,
+    sum(${isRead}) as reads,
+    sum(case when ${isRead} and ${isNarrow} then 1 else 0 end) as narrow,
+    sum(${isSearch}) as searches,
+    sum(${TOOL_NAME}='bash') as bash
+  from part
+  where ${TOOLS} and time_created >= ${before} and time_created < ${after}
+    and ${TOOL_NAME} in ('read','grep','glob','bash')
+  group by win`)
+
+const ctxRows = query(`
+  select ${window(split)} as win,
+    count(*) as messages,
+    avg(json_extract(data,'$.tokens.input') + json_extract(data,'$.tokens.cache.read')) as avg_ctx,
+    avg(json_extract(data,'$.tokens.reasoning')) as avg_reasoning
+  from message
+  where json_extract(data,'$.role')='assistant'
+    and time_created >= ${before} and time_created < ${after}
+    and json_extract(data,'$.tokens') is not null
+  group by win`)
+
+function pick(rows, win) {
+  const r = rows.find((x) => x.win === win)
+  const reads = r ? Number(r.reads ?? 0) : 0
+  const narrow = r ? Number(r.narrow ?? 0) : 0
+  const searches = r ? Number(r.searches ?? 0) : 0
+  const c = ctxRows.find((x) => x.win === win)
+  return {
+    reads,
+    narrow,
+    narrow_pct: pct(narrow, reads),
+    searches,
+    reads_per_search: readsPerSearch(reads, searches),
+    bash: r ? Number(r.bash ?? 0) : 0,
+    messages: c ? Number(c.messages) : 0,
+    // NULL, not zero, when the window has no rows. Zero would say "the
+    // average context per request was zero", and the drop computed from it
+    // would be 100%.
+    // A NULL average is not zero. `avg()` over rows with no tokens is NULL,
+    // and turning that into 0 reported a 100% context improvement.
+    avg_ctx: c && c.avg_ctx !== null ? Math.round(Number(c.avg_ctx)) : null,
+    avg_reasoning: c ? Math.round(Number(c.avg_reasoning ?? 0)) : null,
+  }
+}
+
+const before_ = pick(toolRows, "before")
+const after_ = pick(toolRows, "after")
+
+const ctxDropPct = computeCtxDropPct(before_.avg_ctx, after_.avg_ctx)
+
+// A window of a few hundred reads moves with the day's task mix. Say so instead
+// of letting a small sample pass as a result. A window with NO reads is the
+// thinnest there is.
+//
+// BOTH windows are checked. A comparison against a thin before-window is a
+// comparison against noise, and the report said nothing about it.
+const thin = after_.reads < 500 || before_.reads < 500
+
+// The two windows are the same LENGTH now, and that is the most that can be
+// said for them. They are still not the same WORK, and a number computed
+// across two different jobs is a number about the jobs.
+const confounders = []
+if (before_.reads > 0 && after_.reads > 0) {
+  const ratio = after_.reads / before_.reads
+  if (ratio >= 2 || ratio <= 0.5) {
+    confounders.push(
+      `the two windows did ${ratio >= 2 ? "more" : "less"} than twice the work (${before_.reads} vs ${after_.reads} reads), so the comparison is between different jobs`,
+    )
+  }
+}
+if (before_.messages > 0 && after_.messages > 0) {
+  const ratio = after_.messages / before_.messages
+  if (ratio >= 2 || ratio <= 0.5) {
+    confounders.push(
+      `the two windows did ${ratio >= 2 ? "more" : "less"} than twice the messages (${before_.messages} vs ${after_.messages}), so the comparison is between different jobs`,
+    )
+  }
+}
+
+const failures = []
+/** @type {string[]} */
+const skipped = []
+if (minNarrow !== null) {
+  if (after_.reads === 0) {
+    failures.push("no reads in the after window, so the narrow-read share cannot be measured")
+  } else if (after_.narrow_pct < minNarrow) {
+    failures.push(`narrow reads ${after_.narrow_pct}% < ${minNarrow}%`)
+  }
+}
+if (minCtxDrop !== null) {
+  if (confounders.length) {
+    // The number is about the jobs, not about the kit. Gating on it anyway
+    // means the gate fails for reasons nobody can act on, and a gate that
+    // fails for reasons nobody can act on is a gate people switch off.
+    //
+    // The ATTRIBUTABLE measurement of the same thing - the pruner replayed
+    // over the stored history of real sessions - is eval/measure-prune.mjs,
+    // and that is the one the gate uses.
+    skipped.push("context per request")
+  } else if (ctxDropPct === null) {
+    failures.push("no context data in the after window, so the drop cannot be measured")
+  } else if (ctxDropPct < minCtxDrop) {
+    failures.push(`context per request fell ${ctxDropPct}% < ${minCtxDrop}%`)
+  }
+}
+
+const verdict = failures.length ? "FAIL" : "PASS"
+
+const report = {
+  db: DB,
+  before: new Date(before).toISOString(),
+  split: new Date(split).toISOString(),
+  after: new Date(after).toISOString(),
+  before_window: before_,
+  after_window: after_,
+  ctx_drop_pct: ctxDropPct,
+  thin_sample: thin,
+  confounders,
+  skipped,
+  verdict,
+  failures,
+}
+
+if (wantJson) {
+  console.log(JSON.stringify(report, null, 2))
+  process.exit(failures.length ? 1 : 0)
+}
+
+const row = (label, a, b) =>
+  `  ${label.padEnd(22)} ${String(a).padStart(14)} ${String(b).padStart(14)}`
+
+console.log("INSTALL EFFECTIVENESS")
+console.log(`  db      ${DB}`)
+console.log(`  before  ${report.before}`)
+console.log(`  split   ${report.split}   <- kit installed`)
+console.log(`  after   ${report.after}`)
+console.log(
+  `  window  ${Math.round((split - before) / 864e5)}d before, ${Math.round((after - split) / 864e5)}d after`,
+)
+console.log("")
+console.log(row("", "before", "after"))
+console.log(row("read calls", before_.reads, after_.reads))
+console.log(row("narrow reads", `${before_.narrow} (${before_.narrow_pct}%)`, `${after_.narrow} (${after_.narrow_pct}%)`))
+console.log(row("grep/glob", before_.searches, after_.searches))
+console.log(row("reads per search", before_.reads_per_search ?? "-", after_.reads_per_search ?? "-"))
+console.log(row("assistant msgs", before_.messages, after_.messages))
+console.log(row("avg ctx / request", before_.avg_ctx, after_.avg_ctx))
+console.log(row("avg reasoning", before_.avg_reasoning, after_.avg_reasoning))
+console.log("")
+console.log(`  context per request: ${ctxDropPct === null ? "n/a" : `${ctxDropPct}% ${ctxDropPct >= 0 ? "lower" : "HIGHER"}`}`)
+if (thin) {
+  console.log("")
+  console.log(
+    `  NOTE  ${before_.reads} reads before and ${after_.reads} after: a thin sample on at least one side.`,
+  )
+  console.log("        Treat the direction as a hint, not a result.")
+}
+for (const skip of skipped) {
+  console.log("")
+  console.log(`  NOTE  ${skip} is reported but NOT gated: the two windows are not comparable.`)
+  console.log("        eval/measure-prune.mjs is the attributable measurement of the same thing.")
+}
+for (const c of confounders) {
+  console.log("")
+  console.log(`  NOTE  ${c}.`)
+  console.log("        The A/B in eval/RESULTS.md is the controlled measurement;")
+  console.log("        this one is an observation about one install.")
+}
+if (failures.length) {
+  console.log("")
+  console.log(`  FAIL  ${failures.join("; ")}`)
+} else if (minNarrow !== null || minCtxDrop !== null) {
+  console.log("")
+  console.log(`  PASS  narrow reads and context per request both held.`)
+}
+process.exit(failures.length ? 1 : 0)