opencode-harness-kit 0.4.2 → 0.4.4

package/eval/prepare.mjs

@@ -0,0 +1,233 @@
+#!/usr/bin/env node
+// prepare.mjs - build the subject the A/B suite runs against.
+//
+// Everything measured in eval/RESULTS.md was run against a directory somebody
+// made by hand in a temp folder. That is not a result anyone else can check, and
+// the directory is already gone on some of the older runs. This makes it a
+// command.
+//
+//   node eval/prepare.mjs --repo ~/work/lablab-harness --ref 8a674f5
+//   node eval/prepare.mjs --repo ~/work/lablab-harness --ref 8a674f5 --json
+//
+//   node eval/prepare.mjs --from ~/work/gibwork/vendor/gibwork-sentinel
+//
+// --repo uses `git archive`, so the subject is exactly a committed tree: no
+// uncommitted edits, no .git, no build output that was never committed. --from
+// copies a plain directory, for sources that are not in git.
+//
+// Either way the subject lands somewhere stable under the scratch root rather
+// than a fresh mktemp, so a run can be pointed at the same bytes twice.
+
+import { execFileSync, spawnSync } from "node:child_process"
+import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"
+import { homedir, tmpdir } from "node:os"
+import { basename, isAbsolute, join, relative, resolve } from "node:path"
+
+const args = process.argv.slice(2)
+const flag = (...n) => {
+  const i = args.findIndex((a) => n.includes(a))
+  return i >= 0 ? args[i + 1] : undefined
+}
+
+const repo = flag("--repo")
+const ref = flag("--ref") ?? "HEAD"
+const from = flag("--from")
+const wantJson = args.includes("--json")
+const name = flag("--name")
+
+if (!repo && !from) {
+  console.error("usage: node eval/prepare.mjs (--repo <path> [--ref <sha>] | --from <path>) [--name <n>] [--json]")
+  process.exit(1)
+}
+if (repo && from) {
+  console.error("pass --repo or --from, not both")
+  process.exit(1)
+}
+
+/**
+ * The same scan ab.mjs runs before it will start. Repeated here because the
+ * whole point is that a subject which leaks is a subject an agent can walk out
+ * of, and the person preparing it should find out now, not after twenty runs.
+ */
+function findLeakedPaths(dir, depth = 0) {
+  if (depth > 6) return []
+  const found = []
+  let entries = []
+  try {
+    entries = readdirSync(dir, { withFileTypes: true })
+  } catch {
+    return []
+  }
+  for (const e of entries) {
+    if (e.name === ".git" || e.name === ".space" || e.name === "node_modules") continue
+    const p = join(dir, e.name)
+    if (e.isDirectory()) {
+      found.push(...findLeakedPaths(p, depth + 1))
+      continue
+    }
+    if (!/\.(md|py|ts|tsx|js|json|txt|ya?ml|toml|sh)$/.test(e.name)) continue
+    let body
+    try {
+      body = readFileSync(p, "utf8")
+    } catch {
+      continue
+    }
+    for (const m of body.matchAll(/\/(?:Users|home)\/[A-Za-z0-9._-]+(?:\/[A-Za-z0-9._-]+)*/g)) {
+      const abs = m[0].replace(/[.,;:)]+$/, "")
+      if (!existsSync(abs)) continue
+      if (abs === dir || abs.startsWith(dir + "/")) continue
+      found.push({ file: p.slice(dir.length + 1), path: abs })
+    }
+  }
+  const seen = new Set()
+  return found.filter((f) => (seen.has(f.path) ? false : seen.add(f.path)))
+}
+
+const scratch = resolve(process.env.SPACE_HARNESS_SCRATCH ?? join(tmpdir(), "space-harness-subjects"))
+mkdirSync(scratch, { recursive: true })
+
+/**
+ * The directory NAME of a subject.
+ *
+ * A name, not a path. `--name ../..` used to resolve outside the scratch
+ * root, and the `rmSync` below then deleted whatever was there - a subject
+ * label is a command-line argument, and the command below it removes a
+ * directory tree recursively.
+ *
+ * The check is made twice on purpose: once here, on the text, and once on
+ * the resolved path, below. Either alone would be enough; both together mean
+ * a mistake in either cannot delete anything.
+ *
+ * @param {string | undefined} raw
+ * @returns {string | null}
+ */
+function safeLabel(raw) {
+  if (!raw) return null
+  if (/[\\/]/.test(raw)) return null
+  if (raw === "." || raw === "..") return null
+  if (raw.includes("..")) return null
+  if (raw.length === 0) return null
+  return raw
+}
+
+const requested = name ?? (from ? basename(from) : `${basename(repo ?? "repo")}-${ref}`)
+const label = safeLabel(requested)
+if (!label) {
+  console.error(`--name must be a single directory name, not a path: ${requested}`)
+  process.exit(1)
+}
+const out = join(scratch, label)
+
+// Belt and braces: the resolved path has to still be inside the scratch root.
+const rel = relative(scratch, resolve(out))
+if (!rel || rel.startsWith("..") || isAbsolute(rel)) {
+  console.error(`refusing to write outside the scratch root: ${out}`)
+  process.exit(1)
+}
+
+if (existsSync(out)) rmSync(out, { recursive: true, force: true })
+mkdirSync(out, { recursive: true })
+
+let source
+let pinned
+if (from) {
+  if (!existsSync(from)) {
+    console.error(`--from path not found: ${from}`)
+    process.exit(1)
+  }
+  cpSync(from, out, { recursive: true })
+  source = from
+  pinned = "not a git repo, copied as found"
+} else {
+  if (!existsSync(join(repo, ".git"))) {
+    console.error(`not a git repository: ${repo}`)
+    process.exit(1)
+  }
+  let sha
+  try {
+    sha = execFileSync("git", ["-C", repo, "rev-parse", ref], { encoding: "utf8" }).trim()
+  } catch {
+    console.error(`cannot resolve ref ${ref} in ${repo}`)
+    process.exit(1)
+  }
+  // git archive writes a tar to stdout. Piping it into tar keeps this in one
+  // process rather than leaving a tarball behind for someone to trip over.
+  //
+  // No shell. The previous version built one `sh -c` string and quoted the
+  // paths with `JSON.stringify`, which is not shell quoting: it escapes `"` and
+  // `\` and leaves `$`, backticks and `!` alone, so a path containing `$(...)`
+  // was executed by the shell. `git` and `tar` are called with argument arrays
+  // and the tar goes through a pipe, so nothing is ever parsed as a command.
+  //
+  // `eval/` and `test/` are EXCLUDED, and that is not a detail. The subject of
+  // the self suite is this repository: `eval/` holds the grader and the
+  // answer key, and `test/` quotes the answer in its own assertions. An agent
+  // that found `eval/verify.mjs` would use the grader to check its own work,
+  // and one that found the answer in a test would cite the test.
+  //
+  // Both were measured. An agent that read the grader made 48 tool calls to a
+  // task that needed 5, and the answer was somewhere else in the file it was
+  // asked about.
+  const archive = spawnSync(
+    "git",
+    [
+      "-C", repo, "archive", "--format=tar", sha, "--",
+      ".",
+      ":(exclude)eval",
+      ":(exclude)test",
+    ],
+    {
+      maxBuffer: 512 * 1024 * 1024,
+    },
+  )
+  if (archive.status !== 0 || archive.error) {
+    console.error(`cannot archive ${repo} at ${sha}`)
+    if (archive.stderr?.length) console.error(archive.stderr.toString().trimEnd())
+    process.exit(1)
+  }
+  const untar = spawnSync("tar", ["-x", "-C", out], {
+    input: archive.stdout,
+    maxBuffer: 512 * 1024 * 1024,
+  })
+  if (untar.status !== 0 || untar.error) {
+    console.error(`cannot unpack the archive into ${out}`)
+    if (untar.stderr?.length) console.error(untar.stderr.toString().trimEnd())
+    process.exit(1)
+  }
+  source = repo
+  pinned = sha.slice(0, 12)
+}
+
+// Record what this subject is, next to it, so a later run can be checked against
+// it without having to reconstruct the command.
+const manifest = {
+  source,
+  ref: from ? null : ref,
+  commit: from ? null : pinned,
+  built_at: new Date().toISOString(),
+  command: `node eval/prepare.mjs ${from ? "--from " + from : `--repo ${repo} --ref ${ref}`}`,
+}
+writeFileSync(join(out, ".space-harness-subject.json"), JSON.stringify(manifest, null, 2) + "\n")
+
+const leaks = findLeakedPaths(out)
+const report = { path: out, ...manifest, leaked_paths: leaks.map((l) => l.path) }
+
+if (wantJson) {
+  console.log(JSON.stringify(report, null, 2))
+} else {
+  console.log(`subject: ${out}`)
+  console.log(`  from   ${source}`)
+  console.log(`  pinned ${pinned ?? "-"}`)
+  if (leaks.length) {
+    console.log("")
+    console.log(`  ${leaks.length} absolute path(s) inside this subject resolve on this machine:`)
+    for (const l of leaks.slice(0, 8)) console.log(`    ${l.path}`)
+    console.log("")
+    console.log("  ab.mjs will refuse to run against this unless you pass --sanitize,")
+    console.log("  which rewrites them in each run copy. Do that, or scrub the source.")
+  } else {
+    console.log("  no absolute paths resolve outside it")
+  }
+  console.log("")
+  console.log(`  next: node eval/ab.mjs --target ${out} --suite eval/tasks.json --reps 2 --sanitize`)
+}

package/eval/run.mjs

@@ -0,0 +1,376 @@
+#!/usr/bin/env node
+// eval/run.mjs - did the harness actually change anything?
+//
+// A harness that claims to improve efficiency has to be measurable, or the
+// claim is decoration. This runs the shipped transformers over a synthetic
+// transcript shaped like a real one and reports what changed.
+//
+// It measures the mechanical effects that are deterministic and offline:
+//   - how much context the pruner removes
+//   - how many permission prompts the allowlist removes
+//   - whether effort tiering cuts the latency it predicts to cut
+//
+// It does NOT measure model quality. That needs real runs against real tasks,
+// and a synthetic transcript cannot tell you whether the code got better.
+//
+// Usage:
+//   node eval/run.mjs                 human-readable
+//   node eval/run.mjs --json         machine-readable
+//   node eval/run.mjs --compare      measure with the kit disabled, for a diff
+
+import { execFileSync } from "node:child_process"
+import { existsSync } from "node:fs"
+import { homedir } from "node:os"
+import { dirname, join } from "node:path"
+import { fileURLToPath } from "node:url"
+import { query as queryDb } from "../src/sqlite.js"
+
+const ROOT = join(dirname(fileURLToPath(import.meta.url)), "..")
+const DIST = join(ROOT, "dist", "index.js")
+const wantJson = process.argv.includes("--json")
+const compare = process.argv.includes("--compare")
+
+if (!existsSync(DIST)) {
+  console.error("dist/index.js missing. Run: npm run build")
+  process.exit(1)
+}
+
+/**
+ * A transcript shaped like the one that produced the real baseline: tool calls
+ * with kilobyte-scale output, a small amount of prose, and reasoning.
+ */
+function syntheticTranscript({ turns = 40, toolOutputBytes = 9000 } = {}) {
+  const messages = []
+  for (let i = 0; i < turns; i++) {
+    messages.push({
+      info: { id: `a${i}`, role: "assistant", time: { created: 1789000000000 + i } },
+      parts: [
+        { id: `s${i}`, sessionID: "ses", messageID: `a${i}`, type: "step-start" },
+        {
+          id: `r${i}`,
+          sessionID: "ses",
+          messageID: `a${i}`,
+          type: "reasoning",
+          text: "thinking ".repeat(400),
+        },
+        {
+          id: `t${i}`,
+          sessionID: "ses",
+          messageID: `a${i}`,
+          type: "tool",
+          callID: `c${i}`,
+          tool: i % 3 === 0 ? "bash" : "read",
+          state: {
+            status: "completed",
+            input: i % 3 === 0 ? { command: `pytest -q tests/test_${i}.py` } : { filePath: `src/mod_${i}.py` },
+            output: "x".repeat(toolOutputBytes),
+            title: "tool",
+            metadata: {},
+            time: { start: 1, end: 2 },
+          },
+        },
+        {
+          id: `x${i}`,
+          sessionID: "ses",
+          messageID: `a${i}`,
+          type: "text",
+          text: "Short prose conclusion for this step.",
+        },
+      ],
+    })
+    if (i % 8 === 0) {
+      messages.push({
+        info: { id: `u${i}`, role: "user", time: { created: 1789000000000 + i } },
+        parts: [{ id: `ut${i}`, sessionID: "ses", messageID: `u${i}`, type: "text", text: "continue" }],
+      })
+    }
+  }
+  return messages
+}
+
+function bytesOf(messages) {
+  return JSON.stringify(messages).length
+}
+
+async function measurePruning() {
+  const { HarnessPlugin } = await import(DIST)
+  const worktree = join(ROOT, ".space-eval")
+  const { mkdirSync, writeFileSync, rmSync } = await import("node:fs")
+  mkdirSync(join(worktree, ".space"), { recursive: true })
+  writeFileSync(join(worktree, ".space", "active"), "fleet-safe\n")
+
+  const hooks = await HarnessPlugin(
+    { worktree, directory: worktree, project: {}, client: {}, $: () => {} },
+    { only: ["prune"], profile: "fleet-safe" },
+  )
+  const transform = hooks["experimental.chat.messages.transform"]
+  if (!transform) {
+    rmSync(worktree, { recursive: true, force: true })
+    throw new Error("prune hook did not activate; check the active marker path")
+  }
+
+  const messages = syntheticTranscript()
+  const before = bytesOf(messages)
+  await transform({}, { messages })
+  const after = bytesOf(messages)
+
+  const toolParts = messages.flatMap((m) => m.parts).filter((p) => p.type === "tool")
+  const pruned = toolParts.filter((p) => p.state?.metadata?.pruned).length
+  const assistantTurns = messages.filter((m) => m.info.role === "assistant").length
+  const usersIntact = messages
+    .filter((m) => m.info.role === "user")
+    .every((m) => m.parts[0].text === "continue")
+
+  rmSync(worktree, { recursive: true, force: true })
+  return {
+    bytesBefore: before,
+    bytesAfter: after,
+    reductionPct: +(((before - after) / before) * 100).toFixed(1),
+    assistantTurns,
+    toolParts: toolParts.length,
+    pruned,
+    usersIntact,
+  }
+}
+
+/** Replay a corpus of bash commands through an allowlist. */
+async function measurePrompts() {
+  const { BASH_ALLOWLIST } = await import(join(ROOT, "src", "config.js"))
+  const corpus = [
+    // Shape from the measured 5,239 real commands, compressed.
+    ...Array(1742).fill("cd /Users/x/proj && ls -la"),
+    ...Array(326).fill("ls -la"),
+    ...Array(219).fill("# build the app"),
+    ...Array(217).fill('python3 -c "print(1)"'),
+    ...Array(151).fill("gh pr list"),
+    ...Array(148).fill("echo hello"),
+    ...Array(141).fill("sleep 2"),
+    ...Array(133).fill("ssh -o StrictHostKeyChecking=no deploy@host uptime"),
+    ...Array(93).fill("curl -s https://example.test/api"),
+    ...Array(80).fill("cat src/app.py"),
+    ...Array(67).fill("TOKEN=$(curl -s https://example.test/token)"),
+    ...Array(66).fill("rm -rf build/"),
+    ...Array(59).fill("git add -A"),
+    ...Array(47).fill("osascript -e 'tell app \"Finder\" to quit'"),
+    ...Array(43).fill("pkill -f node"),
+  ]
+  const glob = (cmd, pat) =>
+    new RegExp("^" + pat.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, "[\\s\\S]*") + "$").test(cmd)
+  const prompts = corpus.filter((c) => !BASH_ALLOWLIST.some((p) => glob(c, p))).length
+  return {
+    commands: corpus.length,
+    prompts,
+    promptSharePct: +((prompts / corpus.length) * 100).toFixed(1),
+  }
+}
+
+/**
+ * Effort tiering, measured from the install database where that is possible.
+ *
+ * Per-variant medians are read for the profile's own model, so this no longer
+ * mixes models the way it used to. When every variant the role map depends on is
+ * present, the output is labelled measured; otherwise it falls back to the
+ * fixed constants below and says so, rather than substituting one tier's median
+ * for another.
+ *
+ * Even when measured, the saving is a median mix projected over a session, not
+ * an end-to-end session measurement. `oc-harness baseline` reports the same
+ * per-variant medians with sample sizes.
+ */
+/**
+ * Per-variant median request latency for one model, straight from the install
+ * database. Returns null when the database is missing or has no data for that
+ * model, so the caller can fall back rather than invent a number.
+ *
+ * Median ranks match bin/baseline.mjs: rank (n+1)/2 for odd n, the mean of the
+ * two middle ranks for even n.
+ *
+ * The query is transcribed rather than imported because bin/baseline.mjs runs
+ * its whole report at module top level and calls process.exit, so importing it
+ * would print the baseline report and kill this run. Keep the two in step when
+ * either changes.
+ */
+function mediansForModel(modelID, providerID) {
+  const db = join(homedir(), ".local/share/opencode/opencode.db")
+  if (!existsSync(db)) return null
+
+  const sql = `
+    select variant, printf('%.1f', avg(ms)) as median_ms, n
+    from (
+      select variant, ms, rn, n
+      from (
+        select coalesce(json_extract(data,'$.variant'),'(default)') as variant,
+               (json_extract(data,'$.time.completed')
+                - json_extract(data,'$.time.created')) / 1000.0 as ms,
+               row_number() over (
+                 partition by coalesce(json_extract(data,'$.variant'),'(default)')
+                 order by (json_extract(data,'$.time.completed')
+                           - json_extract(data,'$.time.created')) / 1000.0) as rn,
+               count(*) over (
+                 partition by coalesce(json_extract(data,'$.variant'),'(default)')) as n
+        from message
+        where json_extract(data,'$.role') = 'assistant'
+          ${providerID ? `and json_extract(data,'$.providerID') = '${providerID.replace(/'/g, "''")}'` : ""}
+          and json_extract(data,'$.modelID') = '${modelID.replace(/'/g, "''")}'
+          and (json_extract(data,'$.time.completed')
+               - json_extract(data,'$.time.created')) / 1000.0 > 0
+      )
+    )
+    where rn = cast((n + 1) / 2 as integer)
+       or (n % 2 = 0 and rn = cast((n + 2) / 2 as integer))
+    group by variant, n`
+
+  let rows
+  try {
+    // The shared helper, so this script runs on a machine without the sqlite3
+    // CLI like every other measurement script.
+    rows = queryDb(db, sql)
+  } catch {
+    return null
+  }
+  if (!rows.length) return null
+
+  const MEDIAN = {}
+  const samples = {}
+  for (const r of rows) {
+    MEDIAN[r.variant] = Number(r.median_ms)
+    samples[r.variant] = r.n
+  }
+  return { MEDIAN, samples, source: "measured" }
+}
+
+async function measureEffort() {
+  const { loadProfile, resolveRoles } = await import(join(ROOT, "src", "profile.js"))
+
+  const FALLBACK = { max: 18.7, xhigh: 7.8, high: 7.8, medium: 7.8, low: 7.8 }
+  const p = loadProfile("fleet-safe")
+  const roles = resolveRoles(p)
+
+  // A realistic session: one plan, eight builds, two reviews, four verifications.
+  const mix = { plan: 1, build: 8, reviewer: 2, verify: 4, locate: 0 }
+
+  // Every variant this report depends on has to have been measured. A missing
+  // one must not be substituted with another tier's median: that would print a
+  // number under a "measured" header for a tier that was never measured.
+  const needed = new Set([...Object.values(roles).filter(Boolean), "max"])
+  const found = mediansForModel(p.model, p.provider)
+  const complete = found && [...needed].every((v) => typeof found.MEDIAN[v] === "number")
+
+  const { MEDIAN, samples, source } = complete
+    ? found
+    : { MEDIAN: FALLBACK, samples: {}, source: "projection" }
+
+  let allMax = 0
+  let tiered = 0
+  for (const [role, count] of Object.entries(mix)) {
+    allMax += count * MEDIAN.max
+    const level = roles[role]
+    tiered += count * MEDIAN[level ?? "max"]
+  }
+  // Only the variants this report actually depends on: the completeness gate
+  // covers these, so listing anything else would print unchecked numbers under
+  // a measured header.
+  const shown = Object.entries(samples).filter(([v]) => needed.has(v))
+  const thin = shown.filter(([, n]) => n < 30)
+
+  return {
+    mix,
+    source,
+    medians: MEDIAN,
+    samples: Object.fromEntries(shown),
+    allMaxSeconds: +allMax.toFixed(1),
+    tieredSeconds: +tiered.toFixed(1),
+    savedPct: +(((allMax - tiered) / allMax) * 100).toFixed(1),
+    measured: source === "measured",
+    caveat:
+      source === "measured"
+        ? thin.length
+          ? `Median per request on this model from the install database. ` +
+            `Treat the tiers with few samples as indicative only: ` +
+            thin.map(([v, n]) => `${v} (n=${n})`).join(", ") +
+            `. The saving below is that median mix projected over a session, ` +
+            `not a measured end-to-end session.`
+          : `Median per request on this model from the install database, ` +
+            `projected over a session mix. Not an end-to-end session measurement.`
+        : `PROJECTION, not a measurement: no complete per-variant data for ` +
+          `${p.model} in the install database, so fixed constants are used. ` +
+          `The cause may be a missing or unreadable database, or a variant ` +
+          `that has not been exercised yet. \`oc-harness baseline\` reports ` +
+          `per-variant medians with sample sizes and will say which is which.`,
+  }
+}
+
+const [pruning, prompts, effort] = await Promise.all([
+  measurePruning(),
+  measurePrompts(),
+  measureEffort(),
+])
+
+const report = { generated: new Date().toISOString(), pruning, prompts, effort }
+
+if (wantJson) {
+  console.log(JSON.stringify(report, null, 2))
+  process.exit(0)
+}
+
+const n = (x) => Number(x).toLocaleString("en-US")
+const pad = (k) => k.padEnd(34)
+const stamp = (s) => `\x1b[2m${s}\x1b[0m`
+const wrap = (text, width) => {
+  const words = text.split(/\s+/)
+  const lines = []
+  let line = ""
+  for (const w of words) {
+    if ((line + " " + w).trim().length > width) {
+      lines.push(line.trim())
+      line = w
+    } else {
+      line += " " + w
+    }
+  }
+  if (line.trim()) lines.push(line.trim())
+  return lines
+}
+
+console.log(`\nopencode-harness-kit eval  ${report.generated}\n`)
+
+console.log(`\nCONTEXT PRUNING  (synthetic transcript: ${pruning.assistantTurns} assistant turns, ${pruning.toolParts} tool calls)`)
+console.log(`  ${pad("bytes before")} ${n(pruning.bytesBefore)}`)
+console.log(`  ${pad("bytes after")} ${n(pruning.bytesAfter)}`)
+console.log(`  ${pad("reduction")} ${pruning.reductionPct}%`)
+console.log(`  ${pad("tool parts pruned")} ${pruning.pruned} / ${pruning.toolParts}`)
+console.log(`  ${pad("user messages intact")} ${pruning.usersIntact ? "yes" : "NO"}`)
+
+console.log("\nPERMISSION PROMPTS  (corpus shaped from 5,239 measured commands)")
+console.log(`  ${pad("commands")} ${n(prompts.commands)}`)
+console.log(`  ${pad("prompts")} ${n(prompts.prompts)}  (${prompts.promptSharePct}%)`)
+console.log(`  ${pad("previously")} 95.3% with a narrow allowlist`)
+
+const tierHeader = effort.measured
+  ? "EFFORT TIERING  (medians measured on this model, projected over a session mix)"
+  : "EFFORT TIERING  (PROJECTION - cross-model medians, not a measurement)"
+console.log(`\n${tierHeader}`)
+console.log(`  ${pad("all-max session")} ${effort.allMaxSeconds}s`)
+console.log(`  ${pad("tiered session")} ${effort.tieredSeconds}s`)
+console.log(`  ${pad("implied saving")} ${effort.savedPct}%`)
+console.log(`  ${pad("measured")} ${effort.measured ? "per-variant medians yes, session no" : "no"}`)
+if (effort.measured) {
+  // Print from samples, not medians: both are restricted to the variants the
+  // completeness gate checked, so they cannot drift apart here.
+  const med = Object.entries(effort.samples)
+    .map(([v, n]) => `${v} ${effort.medians[v]}s (n=${n})`)
+    .join(", ")
+  console.log(`  ${pad("median per request")} ${stamp(med)}`)
+}
+for (const line of wrap(effort.caveat, 60)) console.log(`  ${pad("")} ${stamp(line)}`)
+
+if (compare) {
+  const off = syntheticTranscript()
+  console.log(
+    `\n  ${pad("kit disabled")} ${n(bytesOf(off))} bytes ` +
+      `(${(((bytesOf(off) - pruning.bytesAfter) / bytesOf(off)) * 100).toFixed(1)}% larger than pruned)`,
+  )
+}
+
+console.log("\n  Not measured here: code quality, task completion, review defects.")
+console.log("  Those need real runs against real tasks. See docs/RESEARCH.md.\n")

package/eval/tasks-self.json

@@ -0,0 +1,81 @@
+{
+  "$comment": [
+    "Verification tasks for eval/ab.mjs --suite, written against THIS repository.",
+    "",
+    "The earlier suite was written against `lablab-harness@8a674f5`, which is",
+    "not public. A subject nobody else can fetch is a result nobody else can",
+    "check, and the A/B is the one number in this repository that decides",
+    "whether the kit is worth its cost. This suite runs against the kit's own",
+    "tree at a pinned commit, so `git clone && git checkout <sha>` reproduces",
+    "it exactly.",
+    "",
+    "Grading is by citation, not by prose: each read task names `want`, a",
+    "substring of one real source line, and eval/verify.mjs resolves the path:line",
+    "the agent cited and reads that line out of the snapshot. An answer can only",
+    "pass by pointing at the code.",
+    "",
+    "The control comes first and is kept, because it is the control: one file,",
+    "one constant. The rest need enumeration, discrimination between two similar",
+    "things, or two files - which is what the multi-step tasks are for.",
+    "",
+    "Every `want` below was read out of the tree with grep before it was written",
+    "here. Re-check with: git archive | tar -x && grep -n"
+  ],
+  "snapshot": "this repository, at the commit the suite is committed with",
+  "tasks": [
+    {
+      "id": "read-control",
+      "$comment": "Control. One file, one constant.",
+      "task": "In this repository, one exported constant names the five lanes that are part of the plan -> build -> review -> finish loop. Report the exact file path and line number where that constant is defined, and quote the line. Do not modify any files.",
+      "want": "export const WORKFLOW_LANES = [\"build\", \"plan\", \"reviewer\", \"locate\", \"verify\"]"
+    },
+    {
+      "id": "read-multi-denylist",
+      "$comment": "One file, eleven candidates, and two of them are the same command written with and without a space - so the agent cannot stop at the first `chmod 777`.",
+      "task": "In this repository, one source file lists the bash commands the kit denies outright. The list has eleven entries, and two of them are the same command written with and without a space. Report the file and the line number of the entry that denies `gh repo delete`, and quote it. Do not modify any files.",
+      "want": "\"gh repo delete*\""
+    },
+    {
+      "id": "read-multi-gate",
+      "$comment": "Three similar checks in one file, and the task names which one. The earlier suite had the same shape and the same reason: a check that reads as a match to the wrong line is a correct answer scored wrong.",
+      "task": "One source file in this repository has a rule that decides whether a git command rewrites history. It checks three git subcommands, and each one matches a different flag. Report the file and the line number of the check that rejects `git clean -fd`, and quote the comparison. Do not modify any files.",
+      "want": "return /(^|\\s)-[a-zA-Z]*f/.test(text)"
+    },
+    {
+      "id": "read-two-files",
+      "$comment": "Needs two files: the lane table is a JSON file and the list of names is a constant in the code, so the agent has to know the same ten names live in both.",
+      "task": "The ten job lanes in this repository are named in two files: a JSON table, and a source file that holds them as a constant. Report the file and the line number of the constant, and quote the line that declares it. Do not modify any files.",
+      "want": "export const LANE_NAMES = ["
+    },
+    {
+      "id": "edit-rename",
+      "$comment": "A rename, in a file with four uses of the name. mustContainMin counts the occurrences, because a plain substring check passes a partial rename. mustNotContain is as important as mustContain: leaving the old name in one place is not the change that was asked for.",
+      "task": "In src/guard.ts the constant that marks the kit's own denial message is used in four places. Rename it to `DENIAL_MARKER` everywhere in that file, without changing what it holds. Make no other changes and do not reformat anything.",
+      "mutate": {
+        "file": "src/guard.ts",
+        "mustContainMin": {
+          "DENIAL_MARKER": 4
+        },
+        "mustNotContain": [
+          "const MARKER"
+        ]
+      }
+    },
+    {
+      "id": "edit-multi",
+      "$comment": "Two separate edits in one file, both of which have to land. An agent that does one and stops is the obvious failure, and the grader catches it because both strings are required.",
+      "task": "In src/denylist.js the BASH_DENYLIST holds the commands the kit never allows. Change the entry `gh pr close*` to `gh pr close *`, and change the entry `gh release*` to `gh release *`. Make no other changes and do not reformat anything.",
+      "mutate": {
+        "file": "src/denylist.js",
+        "mustContain": [
+          "gh pr close *",
+          "gh release *"
+        ],
+        "mustNotContain": [
+          "gh pr close*",
+          "gh release*"
+        ]
+      }
+    }
+  ]
+}