opencode-harness-kit 0.4.3 → 0.4.4

package/eval/tasks-ts.json

@@ -0,0 +1,50 @@
+{
+  "$comment": [
+    "TypeScript subject. The kit is TypeScript and the Python suite in tasks.json",
+    "never tested that, which is the gap this file exists to close.",
+    "The subject is a PRIVATE repository, so this suite is a pilot that",
+    "nobody can repeat. The repeatable one is tasks-self.json.",
+    "",
+    "Subject is vendor/gibwork-sentinel from gibwork-agent-harness: 692 lines of",
+    "TypeScript across 7 modules, shipped alongside compiled JavaScript in dist/.",
+    "Every answer below occurs in src/ and again in dist/. Each task says so",
+    "explicitly, because picking the build output instead of the source is the",
+    "trap and the reason this subject is worth having.",
+    "",
+    "All three wants were checked to occur in exactly one file under src/.",
+    "That check is what stops an ambiguous task - two of the earlier ones were",
+    "wrong for exactly that reason."
+  ],
+  "snapshot": "gibwork-agent-harness/vendor/gibwork-sentinel (not a git repo, copied as-is)",
+  "tasks": [
+    {
+      "id": "ts-rpc-default",
+      "task": "This package is shipped as both TypeScript sources and compiled JavaScript. In the TypeScript source only, not the build output, which module declares the default Solana RPC endpoint, and what is that URL? Report the file and line. Do not modify any files.",
+      "want": "https://api.mainnet-beta.solana.com"
+    },
+    {
+      "id": "ts-rest-client",
+      "task": "This package is shipped as both TypeScript sources and compiled JavaScript. In the TypeScript source only, not the build output, which module exports the class that talks to the Gibwork REST API? Report the file and line. Do not modify any files.",
+      "want": "export class GibworkClient"
+    },
+    {
+      "id": "ts-pr-template",
+      "task": "This package is shipped as both TypeScript sources and compiled JavaScript. In the TypeScript source only, not the build output, one module writes a pull request template file into the generated workspace. Which module writes it, and what is the filename? Report the file and line. Do not modify any files.",
+      "want": "PULL_REQUEST_TEMPLATE.md"
+    },
+    {
+      "id": "ts-edit-rpc",
+      "$comment": "The edit task, which the TypeScript suite was missing. mustNotContain is as important as mustContain: the same string is in dist/solana.js too, so an agent that edits the build output instead of the source gets mustNotContain for free and is caught.",
+      "task": "In the TypeScript source of this package, the Solana verifier defaults to the mainnet RPC endpoint. Change that default to the devnet endpoint at https://api.devnet.solana.com. Make no other changes and do not reformat anything.",
+      "mutate": {
+        "file": "src/solana.ts",
+        "mustContain": [
+          "api.devnet.solana.com"
+        ],
+        "mustNotContain": [
+          "api.mainnet-beta.solana.com"
+        ]
+      }
+    }
+  ]
+}

package/eval/tasks.json

@@ -0,0 +1,83 @@
+{
+  "$comment": [
+    "Verification tasks for eval/ab.mjs --suite.",
+    "Written against a `git archive` snapshot of lablab-harness at 8a674f5.",
+    "",
+    "Grading is by citation, not by prose: each read task names `want`, a",
+    "substring of one real source line, and eval/verify.mjs resolves the path:line",
+    "the agent cited and reads that line out of the snapshot. An answer can only",
+    "pass by pointing at the code.",
+    "",
+    "The single-file lookups came first and are kept, because they are the",
+    "control. The multi-step ones are the point: this repository defines OUT in",
+    "6 modules and BASE in 19, so answering them means enumerating candidates and",
+    "then choosing, not hitting the right file first try.",
+    "",
+    "Every want below was read out of the snapshot with sed before it was written",
+    "here. Re-check with: git archive | tar -x && grep -n"
+  ],
+  "snapshot": "lablab-harness@8a674f5",
+  "tasks": [
+    {
+      "id": "read-control",
+      "$comment": "Control. One module, one constant.",
+      "task": "In this repository one module builds a result object containing a boolean field named pass. Report the exact file path and line number where that field is assigned, and quote the line. Do not modify any files.",
+      "want": "\"pass\": total >= 55"
+    },
+    {
+      "id": "read-multi-out",
+      "$comment": "Six modules define OUT at module level. The agent has to see all six to pick the one whose value is the weekly plan.",
+      "task": "Six modules in agent/ each define a module-level constant OUT pointing at a different output file. Which one of them points at work/plan-week.md? Report the file and the line. Do not modify any files.",
+      "want": "plan-week.md"
+    },
+    {
+      "id": "read-multi-gate",
+      "$comment": "judge.py has two real submit gates: line 345 rejects total < 5 with a hard failure, line 348 rejects total < 7 asking for a FIX. An earlier wording of this task said 'the comparison that rejects a too-low total' and a kit-off run cited line 345, which is a correct answer, and was scored wrong. Naming the threshold removes the ambiguity without naming the file.",
+      "task": "One module refuses to submit an entry and prints a message asking for a FIX before doing so. It has two such guards; report the one whose threshold is 7. Report the file, the line number and quote the comparison. Do not modify any files.",
+      "want": "total < 7"
+    },
+    {
+      "id": "read-multi-skills",
+      "$comment": "Needs two different files: research.py:20 for the output, research.py:21 for the skills path. want matches the second line.",
+      "task": "One module writes its findings to work/winners.md and reads its playbooks from the user's OpenCode skills directory rather than from the repository. Report both line numbers in that module, and quote the line that builds the skills path. Do not modify any files.",
+      "want": "\"opencode\" / \"skills\""
+    },
+    {
+      "id": "read-multi-skills-collision",
+      "$comment": "Two modules define SKILLS as a list of skill names and one defines it as a Path. A grep for SKILLS returns three hits, and only one of them is the path the task asks about. Premise checked with premises.mjs: the two list definitions are byte-identical, so the agent cannot tell them apart by content and has to read which variable each module goes on to use.",
+      "task": "Three modules define a module-level name SKILLS. In two of them it is a list of skill names; in one it is a Path built from the user's home directory. Report the file and line of the one that is a Path, and quote that line. Do not modify any files.",
+      "want": "Path.home() / \".config\" / \"opencode\" / \"skills\""
+    },
+    {
+      "id": "read-multi-weights-outliers",
+      "$comment": "Six profiles, and four of them share the same weights dict. An agent that reads the first profile and assumes the rest match is wrong twice over. The task cannot be answered by grepping one line: the answer requires comparing six dicts and finding which two differ. Premise verified by hand: 20/20/40/20 appears four times, and the two outliers are 20/15/40/25 and 25/25/30/20, so 'Business Value: 15' and 'Technology: 30' each occur exactly once in the file.",
+      "task": "One module defines six stack profiles. Four of the six have identical weights dictionaries. The other two differ from the majority. Report both outliers: the profile name and the exact weights line for each, quoting them. Do not modify any files.",
+      "want": "\"Presentation\": 25, \"Business Value\": 25, \"Technology\": 30"
+    },
+    {
+      "id": "edit-rename-output",
+      "$comment": "The first task that needs the kit to do work rather than to shortcut it. Graded on the file after the run, not on what the agent says it did. mustNotContain is as important as mustContain: appending the new path without removing the old one is not the change that was asked for.",
+      "task": "In agent/portfolio.py the weekly plan is written to work/plan-week.md. Change it so it is written to work/plan-week-v2.md instead. Make no other changes and do not reformat anything.",
+      "mutate": {
+        "file": "agent/portfolio.py",
+        "mustContain": [
+          "plan-week-v2.md"
+        ],
+        "mustNotContain": [
+          "plan-week.md\""
+        ]
+      }
+    },
+    {
+      "id": "edit-multi-profile",
+      "$comment": "The first task that is real work rather than a shortcut: six profiles, six edits, and an agent that stops after the first two is the obvious failure. mustContainMin counts occurrences, because a plain substring check passes that. An earlier version of this task said the weights sum to 100 and asked for a compensating cut. That premise was false, the sums were 85/80/75 and not 100, so all four agents correctly preserved sums that were never 100 and the grader passed them anyway. A grader that only counts 'Robustness' occurrences cannot tell that difference, which is the whole reason eval/premises.mjs and the prose premise in the task text both have to be right. Premise for the current wording re-verified against lablab-harness@8a674f5: six profiles in STACK_OVERRIDE, each with a four-dimension weights dict, so adding a fifth is correct, and the fallback dict at the bottom of the file has four dimensions too and must be left alone.",
+      "task": "In agent/rubric.py the STACK_OVERRIDE dict defines six stack profiles, and each profile's weights dictionary has four dimensions. Add a fifth dimension named Robustness with weight 10 to every one of those six profile dictionaries, so each profile has five dimensions. Leave the fallback dictionary at the bottom of the file alone. Make no other changes and do not reformat anything.",
+      "mutate": {
+        "file": "agent/rubric.py",
+        "mustContainMin": {
+          "Robustness": 6
+        }
+      }
+    }
+  ]
+}

package/eval/thresholds.mjs

@@ -0,0 +1,134 @@
+#!/usr/bin/env node
+// thresholds.mjs - propose the floor for measure:check instead of guessing it.
+//
+// The gate had 50% because that looked about right. The right floor comes from
+// what the install has actually done: the lowest day since the kit landed, less
+// one observed spread, so ordinary day-to-day movement cannot trip it.
+//
+//   node eval/thresholds.mjs              human-readable
+//   node eval/thresholds.mjs --json
+//
+// It reports a floor. It does not apply one. A floor that rewrites itself is a
+// floor that cannot catch a regression, because the day the kit breaks is also
+// the day the floor moves down. Changing measure:check stays a human decision.
+
+import { existsSync, statSync } from "node:fs"
+import { homedir } from "node:os"
+import { join } from "node:path"
+
+import { median } from "./metrics.js"
+import { countFlag } from "./flags.mjs"
+import { canQuerySqlite, sqliteMissingMessage, query as queryDb } from "../src/sqlite.js"
+
+/**
+ * The message of a caught value, whatever it turned out to be.
+ *
+ * `catch` gives `unknown` under strict mode, and the file is not strict -
+ * but a message is what these scripts exist to print.
+ *
+ * @param {unknown} err
+ * @returns {string}
+ */
+function errorMessage(err) {
+  return err instanceof Error ? err.message : String(err)
+}
+
+const args = process.argv.slice(2)
+const flag = (...n) => {
+  const i = args.findIndex((a) => n.includes(a))
+  return i >= 0 ? args[i + 1] : undefined
+}
+const wantJson = args.includes("--json")
+const DB = flag("--db") ?? join(homedir(), ".local/share/opencode/opencode.db")
+
+const installDir = join(homedir(), ".config/opencode/agents")
+const split = existsSync(installDir) ? statSync(installDir).mtimeMs : Date.now() - 7 * 864e5
+
+if (!existsSync(DB)) {
+  console.error(`opencode database not found: ${DB}`)
+  process.exit(1)
+}
+
+// Per day, since the install: how many reads, and how many were bounded. A day
+// with very few reads says nothing, so it is excluded rather than allowed to
+// drag the distribution.
+// A malformed number used to become NaN and land in the SQL as
+// `having reads >= NaN`, which sqlite rejects with a raw error.
+let minReads
+try {
+  minReads = countFlag("--min-reads") ?? 10
+} catch (err) {
+  console.error(String(errorMessage(err)))
+  process.exit(1)
+}
+const sql = `
+-- The day boundary is LOCAL, like every other day bucket in this repo. It
+-- was hard-coded to UTC+7, so the threshold this script proposes was
+-- bucketed on a different calendar than the window measure-install.mjs
+-- compares it against.
+select date(time_created/1000,'unixepoch','localtime') as day,
+  sum(json_extract(data,'$.tool')='read') as reads,
+  sum(case when json_extract(data,'$.tool')='read'
+            and (json_extract(data,'$.state.input.offset') is not null
+              or json_extract(data,'$.state.input.limit') is not null)
+       then 1 else 0 end) as narrow
+from part
+where json_extract(data,'$.type')='tool' and time_created >= ${split}
+group by day having reads >= ${minReads} order by day`
+
+const rows = queryDb(DB, sql)
+
+const days = rows.map((r) => ({
+  day: r.day,
+  reads: Number(r.reads),
+  pct: Math.round((Number(r.narrow) / Number(r.reads)) * 1000) / 10,
+}))
+
+if (days.length < 2) {
+  console.error(`only ${days.length} day(s) with at least ${minReads} reads since the install.`)
+  console.error("A floor needs a spread to sit below. Nothing to propose.")
+  process.exit(1)
+}
+
+const pcts = days.map((d) => d.pct)
+const low = Math.min(...pcts)
+// Rounded before the subtraction: 75.2 - 60.9 is 14.299999999999997 otherwise,
+// and a threshold report should not print that.
+const spread = Math.round((Math.max(...pcts) - low) * 10) / 10
+// One whole observed spread below the worst day.
+const floor = Math.max(0, Math.round((low - spread) / 5) * 5)
+
+const report = {
+  db: DB,
+  split: new Date(split).toISOString(),
+  min_reads_per_day: minReads,
+  days,
+  lowest_day: low,
+  median_day: median(pcts),
+  spread,
+  proposed_floor: floor,
+}
+
+if (wantJson) {
+  console.log(JSON.stringify(report, null, 2))
+  process.exit(0)
+}
+
+console.log("THRESHOLD PROPOSAL")
+console.log(`  db     ${DB}`)
+console.log(`  split  ${report.split}`)
+console.log("")
+console.log(`  ${"day".padEnd(14)} ${"reads".padStart(8)} ${"narrow".padStart(8)}`)
+for (const d of days) console.log(`  ${d.day.padEnd(14)} ${String(d.reads).padStart(8)} ${`${d.pct}%`.padStart(8)}`)
+console.log("")
+console.log(`  lowest day  ${low}%`)
+console.log(`  median day  ${report.median_day}%`)
+console.log(`  spread      ${spread} points`)
+console.log(`  floor       ${floor}%   (lowest day, one spread below)`)
+console.log("")
+console.log(`  to adopt: npm run measure -- --min-narrow ${floor} --min-ctx-drop <floor>`)
+console.log("")
+if (days.length < 7) {
+  console.log(`  NOTE  ${days.length} days is a small sample. One spread is not a confidence`)
+  console.log("        interval and this floor should be revisited as days accumulate.")
+}

package/eval/verify.mjs

@@ -0,0 +1,287 @@
+#!/usr/bin/env node
+// verify.mjs - grade an answer against the snapshot, not against my guess.
+//
+// Three numbers in eval/RESULTS.md were wrong because the grader matched
+// strings I had typed by hand:
+//
+//   `--expect "judge.py|7"`  passed answers containing the digit 7, which is
+//                             every line number, and reported 3/3
+//   `expect: ["score.py"]`  scored a correct answer wrong because the agent
+//                             wrote "the rubric" instead of the filename
+//   `--expect judge.py`      pointed at judge.py:348, which is `if total < 7`,
+//                             a submit gate, not the field the task asked for
+//
+// None of those are wrong answers. They are a grader that trusts the agent's
+// prose. This does not.
+//
+// The agent's answer is treated as a *claim about the repository*. Every
+// file:line reference in it is extracted, resolved against the real snapshot,
+// and the line at that position is read. The answer is correct when one of the
+// references actually points at a line containing what was asked for. An agent
+// that says "the rubric file" without a path names no reference and is scored
+// wrong - which is the right default for a question that demanded a path.
+//
+// Nothing here is hand-written per task except the thing being looked for, and
+// that is a substring of one line of real source that is verified to exist.
+
+import { readFileSync, existsSync } from "node:fs"
+import { join, isAbsolute, relative, resolve } from "node:path"
+
+/**
+ * The message of a caught value, whatever it turned out to be.
+ *
+ * @param {unknown} err
+ * @returns {string}
+ */
+function errorMessage(err) {
+  return err instanceof Error ? err.message : String(err)
+}
+
+/**
+ * Pull file:line references out of free text.
+ *
+ * Handles the shapes an agent actually writes:
+ *   agent/score.py:214        path:line
+ *   `agent/score.py` line 214 path ... line N
+ *   agent/score.py (214)      path (line)
+ *   /abs/path/to/file.py:12   absolute
+ *
+ * Returns {file, line, text} where text is the span the match came from, so a
+ * caller can report what the agent claimed rather than only whether it held.
+ */
+export function extractRefs(text) {
+  if (typeof text !== "string" || !text) return []
+  const refs = []
+  const push = (file, line) => {
+    const f = String(file).trim().replace(/^["'`]|["'`;,.)]+$/g, "")
+    if (!f) return
+    // Number(null) is 0, not NaN, so a bare path would be recorded as line 0
+    // and then window-checked against the top of the file.
+    const n = line === null || line === undefined ? NaN : Number(line)
+    refs.push({
+      file: f,
+      line: Number.isFinite(n) ? n : null,
+      text: f + (Number.isFinite(n) ? `:${n}` : ""),
+    })
+  }
+
+  // path:line
+  for (const m of text.matchAll(/([\w./@~-]+\.[A-Za-z0-9]+):(\d+)/g)) push(m[1], m[2])
+
+  // path ... line 214, on ONE line.
+  //
+  // The gap may not contain a full stop. A full stop is a sentence end, and
+  // an answer that says "the answer is in src/a.js. Unrelated note: see line
+  // 2 of the report" would otherwise pair the path with a line number that is
+  // about something else - inventing a claim the agent never made, and then
+  // scoring the answer against it. Under-claiming a line is better than
+  // claiming a wrong one.
+  //
+  // An ellipsis is not a sentence end. "see src/a.js ... line 5" is the
+  // shape the pattern is named after, and a full stop inside the gap lost
+  // it.
+  //
+  // The gap may not contain a full stop, EXCEPT the ones that are part of
+  // an ellipsis. "see src/a.js ... line 5" is the shape this pattern is
+  // named after, and a full stop inside the gap lost it.
+  for (const m of text.matchAll(
+    /([\w./@~-]+\.[A-Za-z0-9]+)(?:[^\n.]|\.(?=\.)|(?<=\.)\.){0,40}?\blines?\s*:?\s*[*:_`\s]*(\d+)/gi,
+  ))
+    push(m[1], m[2])
+
+  // path, then a line number on the NEXT line.
+  //
+  // The number has to be introduced by a label at the start of that line.
+  // An agent that answers in a markdown field writes the file on one line and
+  // the number on the next:
+  //
+  //     **File:** `src/profile.js`
+  //     **Line:** 105
+  //
+  // and a gap that stops at the newline never pairs them, so the citation is
+  // lost and a correct answer is scored wrong. Requiring the label is what
+  // keeps this from pairing a path with a line number that belongs to the
+  // next sentence.
+  //
+  // One blank line is allowed between the two fields. Markdown puts a blank
+  // line between a label and its value all the time, and a pattern that
+  // only allows adjacent lines loses the citation on a shape that is
+  // ordinary.
+  for (const m of text.matchAll(
+    /([\w./@~-]+\.[A-Za-z0-9]+)[^\n]{0,40}?\n(?:[ \t]*\n)?[ \t]*[*_`]{0,2}[Ll]ines?[*_`]{0,2}\s*:?\s*[*_`\s]*(\d+)/g,
+  ))
+    push(m[1], m[2])
+
+  // The markdown field form, paired explicitly: the file and the number are
+  // named as fields, so the pairing does not depend on how far apart they are.
+  for (const m of text.matchAll(
+    /[*_`]{0,2}[Ff]ile[*_`]{0,2}\s*:?\s*[*_`]*([\w@~-]+(?:\/[\w.@~-]+)*\.[A-Za-z0-9]+)[*_`]*[\s\S]{0,80}?[*_`]{0,2}[Ll]ine[*_`]{0,2}\s*:?\s*[*_`\s]*(\d+)/g,
+  ))
+    push(m[1], m[2])
+
+  // Markdown emphasis between the words, which is what a model writes when it
+  // answers: **File:** `agent/score.py` **Line:** 214
+  for (const m of text.matchAll(/\*{0,2}[Ll]ines?\*{0,2}\s*:?\s*[*`_\s]*(\d+)[\s\S]{0,40}?([\w@~-]+(?:\/[\w.@~-]+)*\.[A-Za-z0-9]+)/g))
+    push(m[2], m[1])
+
+  // path (214)
+  for (const m of text.matchAll(/([\w./@~-]+\.[A-Za-z0-9]+)\s*\((\d+)\)/g)) push(m[1], m[2])
+
+  // Bare paths, no line. Still worth resolving: a task may only ask which file.
+  for (const m of text.matchAll(/(?<![\w/])([\w@~-]+(?:\/[\w.@~-]+)+\.[A-Za-z0-9]+)/g)) push(m[1], null)
+
+// De-duplicate on the claimed position, keeping order. A bare path is dropped
+// when the same file was also cited with a line, since the line version is the
+// stronger claim and grading both double-counts one answer.
+//
+// The pairing that produces the line claim is what has to be careful, then -
+// and it is: a line that belongs to another sentence must not be paired with
+// the file at all. That is fixed where the pairs are made, not here.
+const seen = new Set()
+const unique = refs.filter((r) => {
+  const key = `${r.file}:${r.line}`
+  if (seen.has(key)) return false
+  seen.add(key)
+  return true
+})
+const withLines = new Set(unique.filter((r) => r.line !== null).map((r) => r.file))
+return unique.filter((r) => r.line !== null || !withLines.has(r.file))
+}
+
+/**
+ * Resolve a claimed path against the snapshot root, tolerating the absolute
+ * form an agent prints when it saw the real path. Returns null when the file is
+ * not in the snapshot, which is how a hallucinated citation fails.
+ */
+export function resolveInTarget(target, ref) {
+  const candidates = []
+  if (isAbsolute(ref)) {
+    candidates.push(ref)
+    // The agent may print the temp path with /private stripped or added.
+    const rel = target.replace(/^\/private/, "")
+    if (ref.startsWith(rel)) candidates.push(`/private${ref}`)
+    if (ref.startsWith(`/private${rel}`)) candidates.push(ref.replace(/^\/private/, ""))
+  } else {
+    candidates.push(join(target, ref))
+  }
+  for (const c of candidates) {
+    const norm = resolve(c)
+    if (norm.startsWith(target) && existsSync(norm)) return norm
+  }
+  return null
+}
+
+/**
+ * Whether a cited path is one of the excluded files or inside an excluded
+ * directory.
+ *
+ * @param {string} file
+ * @param {string[]} exclude
+ * @returns {boolean}
+ */
+export function isExcluded(file, exclude = []) {
+  const base = file.split("/").pop()
+  return exclude.some((x) => {
+    if (!x) return false
+    if (x.endsWith("/")) return file.startsWith(x)
+    // A file NAME excludes that file wherever it is: the suite is excluded as
+    // "tasks-self.json", and the citation is "eval/tasks-self.json".
+    if (file === x || base === x) return true
+    return file.startsWith(`${x}/`)
+  })
+}
+
+/**
+ * Grade one answer.
+ *
+ * `want` is a substring of the line that answers the task. A window of
+ * `window` lines is read around the claim, because an agent that names
+ * line 214 for a definition on 213 has still located it, and a task should not
+ * fail over an off-by-one in a natural-language citation.
+ *
+ * Returns every reference with its verdict rather than a single boolean, so a
+ * wrong answer can be read instead of guessed at.
+ */
+export function gradeAnswer(target, text, want, { window = 1, exclude = [] } = {}) {
+  const refs = extractRefs(text)
+  const checked = refs.map((ref) => {
+    const abs = resolveInTarget(target, ref.file)
+    if (!abs) return { ...ref, status: "not-in-snapshot" }
+    // A citation to the SUITE is not evidence. The subject of the self
+    // suite is this repository, and the suite's answer key is inside it, so
+    // an agent that greps for the symbol can cite the answer key and pass
+    // without reading the source.
+    //
+    // The test is on the RESOLVED path, relative to the target, not on the
+    // string the agent typed. "./dist/index.js", "src/../dist/index.js" and
+    // "/tmp/scratch/self/dist/index.js" are three spellings of the same
+    // file, and the build output is the build output however it is written.
+    const rel = relative(target, abs)
+    if (isExcluded(rel, exclude) || isExcluded(ref.file, exclude)) {
+      return { ...ref, status: "excluded", resolved: rel }
+    }
+    let lines
+    try {
+      lines = readFileSync(abs, "utf8").split("\n")
+    } catch {
+      return { ...ref, status: "unreadable" }
+    }
+    const needle = String(want)
+    const at = ref.line
+    if (at !== null && at >= 1) {
+      const lo = Math.max(0, at - 1 - window)
+      const hi = Math.min(lines.length, at + window)
+      const slice = lines.slice(lo, hi).join("\n")
+      if (slice.includes(needle)) return { ...ref, status: "match", matched: slice.trim().slice(0, 160) }
+      return { ...ref, status: "line-mismatch", matched: slice.trim().slice(0, 160) }
+    }
+    // No line claimed: accept if the whole file contains the needle.
+    const body = lines.join("\n")
+    if (body.includes(needle)) return { ...ref, status: "match-file", matched: "" }
+    return { ...ref, status: "file-mismatch" }
+  })
+
+  const ok = checked.some((c) => c.status === "match" || c.status === "match-file")
+  // A citation is a file the agent named. One file cited twice - once with a
+  // line and once without - is one citation, not two.
+  const cited = new Set(
+    checked.filter((c) => c.status !== "not-in-snapshot").map((c) => c.file),
+  ).size
+  return { ok, refs: checked, cited }
+}
+
+/**
+ * Grade an edit, by looking at the file rather than at what the agent said it
+ * did. This is the only shape that tests whether the kit helps with work, as
+ * opposed to work it can shortcut.
+ *
+ * `mustContain` and `mustNotContain` are substrings of the final file. The
+ * second list matters as much as the first: an agent that appends the new value
+ * without removing the old one has not made the change it was asked for.
+ *
+ * `mustContainMin` counts occurrences instead of testing presence. A task that
+ * says "in every one of the six profiles" is not done by editing the first two,
+ * and a plain substring check would pass it happily.
+ */
+export function gradeMutation(runDir, spec) {
+  const abs = resolveInTarget(runDir, spec.file)
+  if (!abs) return { ok: false, reason: `${spec.file} missing from the run copy` }
+  let body
+  try {
+    body = readFileSync(abs, "utf8")
+  } catch (e) {
+    return { ok: false, reason: `${spec.file} unreadable: ${errorMessage(e)}` }
+  }
+  const count = (s) => body.split(s).length - 1
+  for (const s of spec.mustContain ?? []) {
+    if (!body.includes(s)) return { ok: false, reason: `${spec.file} does not contain ${JSON.stringify(s)}` }
+  }
+  for (const [s, n] of Object.entries(spec.mustContainMin ?? {})) {
+    const got = count(s)
+    if (got < n) return { ok: false, reason: `${spec.file} contains ${JSON.stringify(s)} ${got} time(s), needs at least ${n}` }
+  }
+  for (const s of spec.mustNotContain ?? []) {
+    if (body.includes(s)) return { ok: false, reason: `${spec.file} still contains ${JSON.stringify(s)}` }
+  }
+  return { ok: true }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "opencode-harness-kit",
-  "version": "0.4.3",
+  "version": "0.4.4",
   "description": "Harness kit for OpenCode: capability profiles, a fleet of job lanes with one free model each, role-based reasoning effort, context pruning, and measured baselines.",
   "license": "MIT",
   "homepage": "https://github.com/pham-canh/space-harness#readme",
@@ -16,13 +16,28 @@
   "module": "./dist/index.js",
   "exports": {
     ".": {
-      "types": "./src/index.ts",
+      "types": "./dist/index.d.ts",
       "import": "./dist/index.js"
     },
-    "./profile": "./src/profile.js",
-    "./config": "./src/config.js",
-    "./rules": "./src/rules.js",
-    "./package.json": "./package.json"
+    "./config": {
+      "types": "./dist/config.d.ts",
+      "import": "./src/config.js"
+    },
+    "./denylist": {
+      "types": "./dist/denylist.d.ts",
+      "import": "./src/denylist.js"
+    },
+    "./lanes.schema.json": "./lanes.schema.json",
+    "./package.json": "./package.json",
+    "./profile": {
+      "types": "./dist/profile.d.ts",
+      "import": "./src/profile.js"
+    },
+    "./profiles/schema.json": "./profiles/schema.json",
+    "./rules": {
+      "types": "./dist/rules.d.ts",
+      "import": "./src/rules.js"
+    }
   },
   "bin": {
     "oc-harness": "bin/oc-harness.mjs"
@@ -42,25 +57,35 @@
     "CHANGELOG.md",
     "MIGRATION.md",
     "README.md",
-    "LICENSE"
+    "CONTRIBUTING.md",
+    "SECURITY.md",
+    "LICENSE",
+    "eval/"
   ],
   "scripts": {
-    "build": "esbuild src/index.ts --bundle --platform=node --format=esm --outfile=dist/index.js",
-    "typecheck": "tsc --noEmit",
+    "build": "npm run build:types && esbuild src/index.ts --bundle --platform=node --format=esm --outfile=dist/index.js",
+    "typecheck": "tsc --noEmit && npm run typecheck:scripts",
+    "typecheck:scripts": "tsc -p tsconfig.scripts.json",
+    "pretest": "node -e \"const fs=require('fs');if(!fs.existsSync('test')){console.error('no test directory here - this is an installed package, not a checkout');process.exit(1)}\"",
     "test": "node --test \"test/*.test.js\"",
     "test:all": "npm run build && npm run typecheck && npm test",
+    "test:dist": "git diff --exit-code -- dist/ || (echo \"dist/ is out of sync with src/. Run: npm run build && git add dist/\" && exit 1)",
+    "check": "npm run build && npm run test:dist && npm run typecheck && npm test",
     "test:install": "npm run measure:check && npm run attribute",
-    "check": "npm run test:all && npm run test:install && node bin/oc-harness.mjs verify",
-    "check:online": "npm run check && node bin/oc-harness.mjs doctor",
+    "check:install": "npm run test:all && npm run test:install && node bin/oc-harness.mjs verify",
+    "check:online": "npm run check:install && node bin/oc-harness.mjs doctor",
+    "doctor": "node bin/oc-harness.mjs doctor",
     "baseline": "node bin/baseline.mjs",
     "release:trust": "node bin/oc-harness.mjs trust",
     "eval": "node eval/run.mjs",
     "measure": "node eval/measure-install.mjs",
-    "measure:check": "node eval/measure-install.mjs --min-narrow 45 --min-ctx-drop 20",
+    "measure:prune": "node eval/measure-prune.mjs --sessions 60 --days 30 --min-saved 20",
+    "measure:check": "npm run measure:prune && node eval/measure-install.mjs --min-narrow 45 --min-ctx-drop 20",
     "attribute": "node eval/attribute.mjs",
     "attribute:json": "node eval/attribute.mjs --json",
     "thresholds": "node eval/thresholds.mjs",
-    "prepublishOnly": "npm run build"
+    "prepublishOnly": "npm run build",
+    "build:types": "tsc -p tsconfig.types.json"
   },
   "keywords": [
     "opencode",
@@ -73,13 +98,15 @@
   "engines": {
     "node": ">=22"
   },
-  "peerDependencies": {
-    "@opencode-ai/plugin": ">=1.18.0"
-  },
   "devDependencies": {
     "@opencode-ai/plugin": "^1.18.31",
     "@types/node": "^26.6.5",
+    "ajv": "^8.20.0",
     "esbuild": "^0.28.2",
     "typescript": "^7.0.2"
-  }
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "author": "pham-canh"
 }