opencode-harness-kit 0.4.3 → 0.4.4

package/eval/measure-prune.mjs

@@ -0,0 +1,339 @@
+#!/usr/bin/env node
+// measure-prune.mjs - how much context the pruner actually removes, on real
+// work.
+//
+// The install-wide comparison in measure-install.mjs is an OBSERVATION: it
+// compares the work before the kit landed with the work after, and the work
+// is never the same. It reported the context per request going UP by 17% on
+// one install, with twice as many reads and twice as many messages in the
+// second window - a number about the jobs, not about the kit.
+//
+// This one is attributable. opencode stores the message parts UNPRUNED: the
+// pruner rewrites the outgoing request, not the history. So the stored parts
+// of a real session are exactly what the pruner was given, and replaying the
+// real pruner over them measures exactly what it removed - on the same
+// messages, with and without, on the work that actually happened.
+//
+//   node eval/measure-prune.mjs                    the last 20 sessions
+//   node eval/measure-prune.mjs --sessions 50
+//   node eval/measure-prune.mjs --min-saved 20    gate: at least 20% removed
+//
+// The gate is on the CONTROLLED number. The install-wide one stays an
+// observation.
+
+import { existsSync, mkdtempSync, rmSync } from "node:fs"
+import { homedir, tmpdir } from "node:os"
+import { dirname, join } from "node:path"
+import { fileURLToPath } from "node:url"
+
+import { canQuerySqlite, sqliteMissingMessage, query as queryDb } from "../src/sqlite.js"
+import { countFlag, numberFlag } from "./flags.mjs"
+
+/**
+ * The message of a caught value, whatever it turned out to be.
+ *
+ * `catch` gives `unknown` under strict mode, and the file is not strict -
+ * but a message is what these scripts exist to print.
+ *
+ * @param {unknown} err
+ * @returns {string}
+ */
+function errorMessage(err) {
+  return err instanceof Error ? err.message : String(err)
+}
+
+const HERE = dirname(fileURLToPath(import.meta.url))
+const ROOT = join(HERE, "..")
+const DIST = join(ROOT, "dist", "index.js")
+
+const args = process.argv.slice(2)
+const flag = (name) => {
+  const i = args.indexOf(name)
+  if (i >= 0) return args[i + 1]
+  const inline = args.find((a) => a.startsWith(`${name}=`))
+  return inline === undefined ? undefined : inline.slice(name.length + 1)
+}
+const wantJson = args.includes("--json")
+
+const DB = flag("--db") ?? join(homedir(), ".local/share/opencode/opencode.db")
+// A mistyped number is an error, not a disabled gate. `--sessions abc`
+// became NaN and was interpolated straight into the SQL, where it became a
+// column name.
+let SESSIONS
+let DAYS
+let MIN_MESSAGES
+let minSaved
+try {
+  SESSIONS = countFlag("--sessions") ?? 20
+  DAYS = countFlag("--days") ?? 30
+  MIN_MESSAGES = countFlag("--min-messages") ?? 6
+  // A gate of 0% is a valid gate, so this one is a plain number.
+  minSaved = numberFlag("--min-saved")
+} catch (err) {
+  console.error(String(errorMessage(err)))
+  process.exit(1)
+}
+// The population: sessions in a time window, with enough assistant messages
+// for the pruner to have anything to prune.
+//
+// The pruner leaves the last `keepTurns + 1` messages alone, so a session
+// with five messages has nothing older than that and the pruner correctly
+// does nothing. Measuring the last N sessions therefore measures whatever ran
+// last - a test suite, a batch of probes - rather than the work.
+//
+// A session needs more than `keepTurns + 1` assistant messages to be
+// prunable at all, so the default is six.
+
+
+
+if (!existsSync(DB)) {
+  console.error(`opencode database not found: ${DB}`)
+  console.error("pass --db <path> or run opencode at least once first")
+  process.exit(1)
+}
+if (!canQuerySqlite()) {
+  console.error(sqliteMissingMessage())
+  process.exit(1)
+}
+if (!existsSync(DIST)) {
+  console.error("dist/index.js is missing. Run: npm run build")
+  process.exit(1)
+}
+
+// The real pruner, out of the real bundle. A measurement of a re-implementation
+// is a measurement of the re-implementation.
+const { HarnessPlugin } = await import(DIST)
+
+/**
+ * The messages of a session, in the shape the pruner sees them.
+ * @param {string} sessionId
+ */
+function sessionMessages(sessionId) {
+  const msgs = queryDb(
+    DB,
+    `select id, time_created, data from message where session_id = '${sessionId.replace(/'/g, "''")}' order by time_created asc`,
+  )
+  const out = []
+  for (const m of msgs) {
+    let info
+    try {
+      info = JSON.parse(m.data)
+    } catch {
+      continue
+    }
+    // EVERY message, not only the assistant ones. The pruner computes its
+    // floor over the whole list - `messages.length - 1 - keepTurns` - so a
+    // replay that drops the user messages measures a different thing, and a
+    // lower bound rather than the real effect.
+    const parts = queryDb(
+      DB,
+      `select data from part where message_id = '${String(m.id).replace(/'/g, "''")}' order by time_created asc`,
+    )
+    const parsed = []
+    for (const p of parts) {
+      try {
+        parsed.push(JSON.parse(p.data))
+      } catch {
+        // A part that does not parse is a part the pruner cannot touch.
+      }
+    }
+    if (!parsed.length) continue
+    out.push({ info, parts: parsed })
+  }
+  return out
+}
+
+// The sessions to measure: in the window, with enough messages, newest first.
+const sessions = queryDb(
+  DB,
+  `select s.id as id, count(m.id) as msgs
+   from session s
+   join message m on m.session_id = s.id
+   where s.time_created >= ${Date.now() - DAYS * 864e5}
+     and json_extract(m.data,'$.role') = 'assistant'
+   group by s.id
+   having msgs >= ${MIN_MESSAGES}
+   order by s.time_created desc
+   limit ${Math.floor(SESSIONS)}`,
+)
+
+// The pruner has to be ACTIVE for the measurement, whatever the machine's
+// local state says. It finds its profile from `.space/active` in the project,
+// and that file is gitignored - so on a fresh clone there is none, the pruner
+// does nothing, and the gate reports 0% removed.
+//
+// The environment variable is the documented override, and it is set for the
+// measurement only.
+const PREV = process.env.OPENCODE_HARNESS_PROFILE
+process.env.OPENCODE_HARNESS_PROFILE = PREV ?? "fleet-safe"
+
+// The measurement must not write into the repository it is measuring: the
+// pruner logs every pass to `.space/prune.jsonl` in the project, and that is
+// the same file the kit's own runtime telemetry uses. A scratch directory
+// keeps the measurement a measurement.
+const scratch = mkdtempSync(join(tmpdir(), "ock-prune-measure-"))
+const hook = await HarnessPlugin(
+  {
+    client: { app: { get: async () => ({}), log: { on: async () => {} } } },
+    project: { id: "measure", worktree: scratch, vcs: "git" },
+    directory: scratch,
+    worktree: scratch,
+    serverUrl: new URL("http://127.0.0.1:0"),
+    $: Object.assign(() => ({ pipe: () => ({}) }), { env: () => process.env }),
+  },
+  { only: ["prune"] },
+)
+const transform = hook["experimental.chat.messages.transform"]
+
+/**
+ * The size the PRUNER measures.
+ *
+ * The pruner counts CHARACTERS, not bytes: `part.text.length`. The report
+ * labelled that "bytes", which is wrong for anything that is not ASCII - and
+ * an agent's output is not ASCII. Both numbers are reported.
+ */
+function sizeOf(part) {
+  if (typeof part?.text === "string") return part.text.length
+  if (typeof part?.state?.output === "string") return part.state.output.length
+  return 0
+}
+
+/** The same text, in bytes. */
+function byteSizeOf(part) {
+  if (typeof part?.text === "string") return Buffer.byteLength(part.text, "utf8")
+  if (typeof part?.state?.output === "string") {
+    return Buffer.byteLength(part.state.output, "utf8")
+  }
+  return 0
+}
+
+let totalBefore = 0
+let totalAfter = 0
+let bytesBefore = 0
+let bytesAfter = 0
+let measured = 0
+let stubbed = 0
+const perSession = []
+
+for (const s of sessions) {
+  const messages = sessionMessages(s.id)
+  if (!messages.length) continue
+  const before = messages.reduce(
+    (n, m) => n + m.parts.reduce((k, p) => k + sizeOf(p), 0),
+    0,
+  )
+  const beforeBytes = messages.reduce(
+    (n, m) => n + m.parts.reduce((k, p) => k + byteSizeOf(p), 0),
+    0,
+  )
+  // The pruner mutates in place, so the copy has to be the real thing.
+  const reasoningBefore = messages.flatMap((m) =>
+    m.parts.filter((p) => p.type === "reasoning").map((p) => p.text),
+  )
+  const copy = messages.map((m) => ({
+    info: m.info,
+    parts: m.parts.map((p) => (p.state ? { ...p, state: { ...p.state } } : { ...p })),
+  }))
+  await transform({}, { messages: copy })
+  const after = copy.reduce(
+    (n, m) => n + m.parts.reduce((k, p) => k + sizeOf(p), 0),
+    0,
+  )
+  if (!before) continue
+  totalBefore += before
+  totalAfter += after
+  bytesBefore += beforeBytes
+  bytesAfter += copy.reduce(
+    (n, m) => n + m.parts.reduce((k, p) => k + byteSizeOf(p), 0),
+    0,
+  )
+  measured++
+  // The pruner marks TOOL parts with `metadata.pruned` and REPLACES reasoning
+  // parts, so counting only the marked ones undercounts what was stubbed.
+  const toolCut = copy
+    .flatMap((m) => m.parts)
+    .filter((p) => p.state?.metadata?.pruned).length
+  const reasoningAfter = copy.flatMap((m) =>
+    m.parts.filter((p) => p.type === "reasoning").map((p) => p.text),
+  )
+  const reasoningCut = reasoningBefore.filter(
+    (t, i) => reasoningAfter[i] !== t,
+  ).length
+  const cut = toolCut + reasoningCut
+  stubbed += cut
+  perSession.push({
+    session: s.id,
+    bytes_before: before,
+    bytes_after: after,
+    saved_pct: Math.round((1 - after / before) * 1000) / 10,
+    parts_stubbed: cut,
+    tool_parts_stubbed: toolCut,
+    reasoning_parts_stubbed: reasoningCut,
+  })
+}
+
+const savedPct =
+  totalBefore > 0 ? Math.round((1 - totalAfter / totalBefore) * 1000) / 10 : null
+
+const failures = []
+if (minSaved !== null) {
+  if (measured === 0) {
+    failures.push("no session had assistant messages to measure")
+  } else if (savedPct === null || savedPct < minSaved) {
+    failures.push(`the pruner removed ${savedPct ?? "n/a"}% of the context, expected at least ${minSaved}%`)
+  }
+}
+
+// The measurement's own scratch directory, and the environment it borrowed.
+process.on("exit", () => {
+  rmSync(scratch, { recursive: true, force: true })
+  if (PREV === undefined) delete process.env.OPENCODE_HARNESS_PROFILE
+  else process.env.OPENCODE_HARNESS_PROFILE = PREV
+})
+
+const report = {
+  db: DB,
+  sessions_measured: measured,
+  characters_before: totalBefore,
+  characters_after: totalAfter,
+  bytes_before: bytesBefore,
+  bytes_after: bytesAfter,
+  saved_pct: savedPct,
+  parts_stubbed: stubbed,
+  per_session: perSession,
+  verdict: failures.length ? "FAIL" : "PASS",
+  failures,
+}
+
+if (wantJson) {
+  console.log(JSON.stringify(report, null, 2))
+  process.exit(failures.length ? 1 : 0)
+}
+
+console.log("PRUNER EFFECT ON REAL SESSIONS")
+console.log(`  db       ${DB}`)
+console.log(
+  `  sessions ${measured} (last ${DAYS}d, at least ${MIN_MESSAGES} assistant messages each)`,
+)
+console.log("")
+console.log(`  context before   ${totalBefore.toLocaleString("en-US")} characters`)
+console.log(`  context after    ${totalAfter.toLocaleString("en-US")} characters`)
+console.log(`  in bytes         ${bytesBefore.toLocaleString("en-US")} -> ${bytesAfter.toLocaleString("en-US")}`)
+console.log(`  removed          ${savedPct === null ? "n/a" : savedPct + "%"}`)
+console.log(`  parts stubbed    ${stubbed}`)
+if (perSession.length) {
+  console.log("")
+  const worst = [...perSession].sort((a, b) => a.saved_pct - b.saved_pct).slice(0, 5)
+  console.log("  the five sessions the pruner helped least:")
+  for (const p of worst) {
+    console.log(`    ${p.saved_pct}%  ${p.bytes_before.toLocaleString("en-US")} bytes  ${p.session.slice(0, 12)}`)
+  }
+}
+if (failures.length) {
+  console.log("")
+  console.log(`  FAIL  ${failures.join("; ")}`)
+} else if (minSaved !== null) {
+  console.log("")
+  console.log(`  PASS  the pruner removed at least ${minSaved}% of the context.`)
+}
+process.exit(failures.length ? 1 : 0)

package/eval/metrics.js

@@ -0,0 +1,79 @@
+// metrics.js - the arithmetic behind measure-install.mjs, with no I/O so
+// test/metrics.test.js can pin it without a database.
+//
+// Kept separate from measure-install.mjs on purpose: importing that file runs
+// the whole measurement and calls process.exit, which would make the numbers
+// untestable and the module unusable anywhere else.
+
+/** Share of reads that were bounded by offset or limit, as a percentage. */
+export function pct(narrow, total) {
+  if (!total) return 0
+  return Math.round((narrow / total) * 1000) / 10
+}
+
+/** Read calls per search call. Null when there were no searches to divide by. */
+export function readsPerSearch(reads, searches) {
+  if (!searches) return null
+  return Math.round((reads / searches) * 10) / 10
+}
+
+/**
+ * Percentage drop in average context per request. Null when there is no
+ * baseline, or no measurement on the other side.
+ *
+ * The other side being NULL means there was no data. Treating that as zero
+ * would report a 100% improvement - "context per request: 100% lower" from a
+ * window that contained nothing at all. The caller passes null for "no
+ * rows", and this returns null rather than a number.
+ */
+export function ctxDropPct(before, after) {
+  if (!before || after === null || after === undefined) return null
+  return Math.round((1 - after / before) * 1000) / 10
+}
+
+/** Tokens of accumulated history per assistant turn. Null with no turns to divide by. */
+export function historyPerTurn(history, turns) {
+  if (!turns) return null
+  return Math.round((history / turns) * 10) / 10
+}
+
+/**
+ * Wall time between two opencode event timestamps.
+ *
+ * Those timestamps are milliseconds, and dividing by 100 instead of 1000 made
+ * every duration in eval/RESULTS.md ten times too large for one commit - a
+ * 4.4 minute run was reported as 44 minutes. The ratios survived because the
+ * error is a constant factor, which is exactly why it went unnoticed until the
+ * absolute numbers were checked against a wall clock.
+ */
+export function elapsedSeconds(fromMs, toMs) {
+  if (!Number.isFinite(fromMs) || !Number.isFinite(toMs)) return null
+  return Math.round((toMs - fromMs) / 100) / 10
+}
+
+/**
+ * Median of a list, rounded to one decimal.
+ *
+ * Not an average. One kit-off run took 262.7s against 42.4s and 7.3s, and a mean
+ * over three turns that into an 8x slowdown that never happened. Outliers are
+ * the normal case for agent wall time, so every reported figure is a median.
+ * Null rather than NaN for an empty list.
+ */
+export function median(values) {
+  const v = values.filter((x) => Number.isFinite(x)).sort((a, b) => a - b)
+  if (!v.length) return null
+  const mid = Math.floor(v.length / 2)
+  const m = v.length % 2 ? v[mid] : (v[mid - 1] + v[mid]) / 2
+  return Math.round(m * 10) / 10
+}
+
+/**
+ * Whether the kit pays for itself: the fixed overhead it adds to every request
+ * versus the history it removes per session. Positive means it costs more.
+ *
+ * Pinned because the whole conclusion in eval/RESULTS.md rests on the sign, and
+ * turning `< 0` into `<= 0` would flip the verdict without any visible change.
+ */
+export function payback(fixedDelta, historyDelta) {
+  return fixedDelta + historyDelta < 0
+}

package/eval/premises.mjs

@@ -0,0 +1,200 @@
+#!/usr/bin/env node
+// premises.mjs - check that a suite's premises are true of the subject before
+// anyone spends twenty agent runs finding out.
+//
+// Seven numbers in eval/RESULTS.md were wrong, and every one of them was a
+// grader agreeing with the agent about something that was not there:
+//
+//   - a task asked for a total of 100, the profiles summed to 85/80/80/80/80/75,
+//     so four agents correctly preserved sums that were never 100 and the
+//     grader passed them
+//   - a `mustNotContain` check on a string the subject never contained is
+//     vacuous: an agent that changes nothing scores a perfect pass
+//   - a `mustContain` string already present in the subject means the work was
+//     done before the run started
+//
+// A premise that can be checked by machine should never be checked by reading
+// it. This checks every premise in the suite that is mechanically checkable:
+//
+//   read task    `want` occurs in the subject at least once
+//   mutate task  mutate.file exists
+//                mustNotContain is present in the ORIGINAL file (not vacuous)
+//                mustContain / mustContainMin is absent from the original
+//
+// What it cannot check is a premise written in prose -- "each profile has three
+// dimensions". For that the honest answer is the same one the task file already
+// gives: read the code. What this can do is make sure the parts that are
+// checkable are true, so a false premise that is still prose is the only kind
+// that gets through.
+//
+//   node eval/premises.mjs --target <dir> --suite eval/tasks.json
+//   node eval/premises.mjs --target <dir> --suite eval/tasks.json --json
+
+import { existsSync, readFileSync, readdirSync } from "node:fs"
+import { join, relative } from "node:path"
+
+const TEXT_FILE = /\.(md|py|ts|tsx|js|json|txt|ya?ml|toml|sh|ini|cfg)$/
+
+/**
+ * Every text file in the subject, as {rel, lines}.
+ * @param {string} root
+ * @param {number} [depth]
+ * @returns {{rel: string, lines: string[]}[]}
+ */
+export function walk(root, depth = 0) {
+  if (depth > 8) return []
+  /** @type {{rel: string, lines: string[]}[]} */
+  const out = []
+  let entries
+  try {
+    entries = readdirSync(root, { withFileTypes: true })
+  } catch {
+    return []
+  }
+  for (const e of entries) {
+    if (e.name === ".git" || e.name === "node_modules" || e.name === ".space") continue
+    const abs = join(root, e.name)
+    if (e.isDirectory()) {
+      out.push(...walk(abs, depth + 1))
+      continue
+    }
+    if (!TEXT_FILE.test(e.name)) continue
+    try {
+      out.push({ rel: relative(root, abs), lines: readFileSync(abs, "utf8").split("\n") })
+    } catch {
+      /* unreadable file is not a premise failure */
+    }
+  }
+  return out
+}
+
+/**
+ * Every place `needle` occurs, as file:line.
+ * @param {{rel: string, lines: string[]}[]} files
+ * @param {string} needle
+ */
+export function findAll(files, needle) {
+  const hits = []
+  for (const f of files) {
+    for (let i = 0; i < f.lines.length; i++) {
+      if (f.lines[i].includes(needle)) hits.push(`${f.rel}:${i + 1}`)
+    }
+  }
+  return hits
+}
+
+/**
+ * @param {string} target subject root
+ * @param {{tasks?: any[], snapshot?: string}} suite
+ * @returns {{ok: boolean, fatal: {id: string, problem: string}[], warnings: {id: string, problem: string}[]}}
+ */
+export function checkPremises(target, suite) {
+  /** @type {{id: string, problem: string}[]} */
+  const fatal = []
+  /** @type {{id: string, problem: string}[]} */
+  const warnings = []
+  const files = walk(target)
+  if (!files.length) {
+    return { ok: false, fatal: [{ id: "(subject)", problem: `no text files found under ${target}` }], warnings }
+  }
+
+  for (const t of suite.tasks ?? []) {
+    const id = t.id ?? "(no id)"
+
+    if (t.mutate) {
+      const abs = join(target, t.mutate.file)
+      if (!existsSync(abs)) {
+        fatal.push({ id, problem: `mutate.file ${t.mutate.file} is not in the subject` })
+        continue
+      }
+      const body = readFileSync(abs, "utf8")
+      const count = (s) => body.split(s).length - 1
+      for (const s of t.mutate.mustNotContain ?? []) {
+        if (count(s) === 0) {
+          fatal.push({
+            id,
+            problem: `mustNotContain ${JSON.stringify(s)} is not in ${t.mutate.file} to begin with, so the check passes no matter what the agent does`,
+          })
+        }
+      }
+      for (const s of t.mutate.mustContain ?? []) {
+        if (count(s) > 0) {
+          fatal.push({
+            id,
+            problem: `mustContain ${JSON.stringify(s)} is already in ${t.mutate.file}: the task is already done before the run`,
+          })
+        }
+      }
+      for (const [s, n] of Object.entries(t.mutate.mustContainMin ?? {})) {
+        if (count(s) > 0) {
+          fatal.push({
+            id,
+            problem: `mustContainMin ${JSON.stringify(s)} already appears ${count(s)} time(s) in ${t.mutate.file}`,
+          })
+        }
+        if (typeof n !== "number" || n < 1 || !Number.isInteger(n)) {
+          fatal.push({ id, problem: `mustContainMin ${JSON.stringify(s)} needs a whole number of at least 1` })
+        }
+      }
+      continue
+    }
+
+    const hits = findAll(files, String(t.want ?? ""))
+    if (hits.length === 0) {
+      fatal.push({
+        id,
+        problem: `want ${JSON.stringify(t.want)} occurs nowhere in the subject, so no answer can be graded correct`,
+      })
+      continue
+    }
+    const where = hits.map((h) => h.split(":")[0])
+    const spread = new Set(where)
+    if (spread.size > 1) {
+      // Not fatal: read-multi-out is exactly this shape, and being able to name
+      // which of several files answers the question is the work being measured.
+      warnings.push({ id, problem: `want occurs in ${spread.size} files: ${[...spread].join(", ")}` })
+    }
+    if (hits.length > 1 && spread.size === 1) {
+      warnings.push({ id, problem: `want occurs ${hits.length} times in ${[...spread][0]}` })
+    }
+  }
+  return { ok: fatal.length === 0, fatal, warnings }
+}
+
+// --------------------------------------------------------------------------
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  const args = process.argv.slice(2)
+  const flag = (...n) => {
+    const i = args.findIndex((a) => n.includes(a))
+    return i >= 0 ? args[i + 1] : undefined
+  }
+  const target = flag("--target")
+  const suitePath = flag("--suite")
+  if (!target || !suitePath) {
+    console.error("usage: node eval/premises.mjs --target <dir> --suite <file> [--json]")
+    process.exit(1)
+  }
+  const suite = JSON.parse(readFileSync(suitePath, "utf8"))
+  const report = checkPremises(target, suite)
+  if (args.includes("--json")) {
+    console.log(JSON.stringify(report, null, 2))
+    process.exit(report.ok ? 0 : 1)
+  }
+  const label = suite.snapshot ?? target
+  console.log(`premises for ${suitePath} against ${label}`)
+  console.log("")
+  if (report.warnings.length) {
+    console.log(`  ${report.warnings.length} warning(s):`)
+    for (const w of report.warnings) console.log(`    ${w.id}: ${w.problem}`)
+    console.log("")
+  }
+  if (report.fatal.length) {
+    console.log(`  ${report.fatal.length} FATAL. The suite would score runs against a subject that does not say what it claims:`)
+    for (const f of report.fatal) console.log(`    ${f.id}: ${f.problem}`)
+    console.log("")
+    console.log("  Fix the task, or fix the subject. Do not run the suite against this.")
+    process.exit(1)
+  }
+  console.log(`  all premises hold against ${label}`)
+}