@tachikomagundam/abathur 0.1.0

package/graders/historian/grader-core.mjs

@@ -0,0 +1,276 @@
+// Historian grader scoring core (task 14, plan line 185). PURE — no IO.
+//
+// Contract: every unit scores Σ(weight×dim)/Σ(APPLICABLE weights). Full
+// scenarios grade all eight rubric dims (weights A2 B2 C2 D2 E1 F1 G1 H1 = 12);
+// dims an "Expected Behavior" list does not apply are graded by the common
+// mechanical check and default to 1 when there is nothing to check.
+// scenario-05 (worthiness) makes A-F N/A ⇒ applicable set = {G,H,judgment}:
+// score = (G+H+J)/3. G=0 forces pass=false in BOTH formulas (hard gate).
+
+export const WEIGHTS = Object.freeze({ A: 2, B: 2, C: 2, D: 2, E: 1, F: 1, G: 1, H: 1 });
+
+const FULL_WEIGHT_SUM = Object.values(WEIGHTS).reduce((a, b) => a + b, 0); // 12
+const INDEX_PATH = "_sandbox/index";
+const SLUG = /^[a-z0-9][a-z0-9._-]*$/;
+const DATE_TAIL = /\d{4}-\d{2}-\d{2}$/;
+
+// Sections each scenario's Expected Behavior names or tolerates. s7 "appropriate
+// section" is the shared engineering vocabulary; anything else is invented taxonomy.
+const SECTIONS = {
+  1: new Set(["llm-inference", "qwen-27b-llm-server"]),
+  2: new Set(["troubleshooting"]),
+  4: new Set(["runbooks", "llm-inference", "troubleshooting", "projects"]),
+  6: new Set(["troubleshooting"]),
+  7: new Set(["runbooks", "llm-inference", "troubleshooting", "ops", "tools"]),
+  8: new Set(), // placement is prefix-based (eval08-)
+};
+const CREATED_EXPECTED = new Set([1, 2, 6, 7, 8]);
+
+const FLOAT_RE = /(?<![\d.])\d+\.\d{3,}(?![\d])/;
+const RAW_RE = /root@|Traceback \(most recent call last\)|\[?200h|bash-\d+\$\s/u;
+const NOISE = {
+  1: /47\.3829104823|53\.7193847108|48\.0040000001/,
+  3: /38\.2910284|45\.7112944|12\.4847/,
+  6: /西瓜|团建|猫咪|表情包/,
+  8: /NAS|硬盘|感觉/,
+};
+const REFUSE_RE =
+  /(不|未|没有|无需|不必)(建议|再|予)?(创建|建|收录|记录|写入|建页)|不具备(长期|可复用|耐久)|一次性|one-?off|not durable|暂不|无价值|无需记录|不入库|不建议|不值得|无长期|拒绝|decline/i;
+
+function nonBlank(content) {
+  return content.split("\n").map((l) => l.trim()).filter((l) => l.length > 0);
+}
+
+function segs(path) {
+  return path.split("/");
+}
+
+function slugTokens(path) {
+  return (segs(path).at(-1) ?? "").split(/[-._]/).filter((t) => t.length > 1 && !/^\d+$/.test(t));
+}
+
+function bodies(obs) {
+  return [...obs.created, ...obs.updated];
+}
+
+function placementOk(path, scenarioNo) {
+  const s = segs(path);
+  if (s[0] !== "_sandbox" || s.length < 2) return false;
+  if (s.slice(1).some((seg) => !SLUG.test(seg))) return false;
+  if (scenarioNo === 8) return path.startsWith("_sandbox/eval08-");
+  if (s.length === 2) return false; // root-level page: rubric A failure case
+  if (DATE_TAIL.test(s[s.length - 1] ?? "") && !(scenarioNo === 2 || scenarioNo === 6)) return false;
+  const allowed = SECTIONS[scenarioNo];
+  if (allowed !== undefined && allowed.size > 0) return allowed.has(s[1]);
+  return true;
+}
+
+function anatomyOk(page, scenarioNo) {
+  const lines = page.content.split("\n");
+  const h1 = lines.find((l) => l.startsWith("# "));
+  if (h1 === undefined || h1.slice(2).trim() !== page.title.trim()) return false;
+  const head = lines.slice(0, 14).join("\n");
+  if (!/(Active|Historical|Superseded)/.test(head)) return false;
+  if (!/\d{4}-\d{2}-\d{2}/.test(head)) return false;
+  if (!/This page answers[:：]|本页回答/.test(page.content)) return false;
+  const sections = [...page.content.matchAll(/^## (.+)$/gm)].map((m) => (m[1] ?? "").trim());
+  if (sections.length < 2) return false;
+  const last = sections[sections.length - 1] ?? "";
+  if (!/Related Pages|相关链接|相关页面/i.test(last)) return false;
+  const tail = page.content.slice(page.content.lastIndexOf(`## ${last}`));
+  if (!/\]\(/.test(tail)) return false;
+  if (scenarioNo === 2 || scenarioNo === 6) {
+    const hits = [
+      /Symptoms|症状/i,
+      /Root Cause|根因/i,
+      /Fix|修复|解决/i,
+      /Prevention|预防/i,
+    ].filter((re) => page.content.search(re) >= 0).length;
+    if (hits < 3) return false;
+  }
+  if (scenarioNo === 8) {
+    if (!/>\s*\*\*?Status\*\*?[:：]?\s*Active/.test(head)) return false;
+    if (!/上次核实/.test(page.content)) return false;
+    if (!/失效/.test(page.content)) return false;
+  }
+  return true;
+}
+
+function checkA(obs) {
+  if (CREATED_EXPECTED.has(obs.scenarioNo) && obs.created.length === 0) return 0;
+  if (obs.scenarioNo === 3 || obs.scenarioNo === 4 || obs.scenarioNo === 5 || obs.scenarioNo === 9) return 1;
+  return obs.created.every((p) => placementOk(p.path, obs.scenarioNo)) ? 1 : 0;
+}
+
+function checkB(obs) {
+  const touchedSet = new Set([...obs.updated, ...obs.moved].map((x) => x.path ?? x.from ?? ""));
+  if (obs.scenarioNo === 3) {
+    const integrated = touchedSet.has("_sandbox/llm-inference/rocm-tuning");
+    const twin = obs.created.some((p) => {
+      const toks = new Set(slugTokens(p.path));
+      return segs(p.path)[1] === "llm-inference" && (toks.has("rocm") || toks.has("hsa"));
+    });
+    return integrated && !twin ? 1 : 0;
+  }
+  if (obs.scenarioNo === 4) {
+    const fan = ["_sandbox/mess/gpu-notes", "_sandbox/mess/gpu-stuff"];
+    const acted = fan.some((p) => touchedSet.has(p)) || obs.created.length > 0;
+    return acted ? 1 : 0;
+  }
+  // generic duplicate-topic check across created pages (twins share a path by design)
+  for (let i = 0; i < obs.created.length; i += 1) {
+    for (let j = i + 1; j < obs.created.length; j += 1) {
+      const a = obs.created[i];
+      const b = obs.created[j];
+      if (a === undefined || b === undefined) continue;
+      if (a.path === b.path) continue;
+      const ta = new Set(slugTokens(a.path));
+      const shared = slugTokens(b.path).filter((t) => ta.has(t));
+      if (shared.length >= 2) return 0;
+    }
+  }
+  return 1;
+}
+
+function checkC(obs) {
+  const targets = [...obs.created];
+  if (obs.scenarioNo === 3) targets.push(...obs.updated.filter((u) => u.path === "_sandbox/llm-inference/rocm-tuning"));
+  return targets.every((p) => anatomyOk(p, obs.scenarioNo)) ? 1 : 0;
+}
+
+function checkD(obs) {
+  const text = bodies(obs).map((p) => p.content).join("\n");
+  if (FLOAT_RE.test(text)) return 0;
+  if (RAW_RE.test(text)) return 0;
+  if (/\((无|empty)\)|（无）/.test(text)) return 0;
+  const dumps = [...text.matchAll(/```text\n[\s\S]*?```/g)].filter((m) => (m[0]?.split("\n").length ?? 0) > 11);
+  if (dumps.length > 0) return 0;
+  const noise = NOISE[obs.scenarioNo];
+  if (noise !== undefined && noise.test(text)) return 0;
+  return 1;
+}
+
+function checkE(obs) {
+  const pages = [...bodies(obs)];
+  if (obs.indexUpdated) pages.push({ path: INDEX_PATH, locale: "en", content: obs.indexContent, title: "index" });
+  for (const page of pages) {
+    for (const m of page.content.matchAll(/\]\((\/?_sandbox\/[^)\s#]+)\)/g)) {
+      const target = (m[1] ?? "").replace(/^\//, "");
+      if (!obs.allPaths.includes(target)) return 0;
+    }
+  }
+  for (const page of obs.created) {
+    const mentioned =
+      obs.indexContent.includes(page.path) ||
+      obs.backlinkBodies.some((b) => b.path !== page.path && b.content.includes(page.path));
+    if (!mentioned) return 0;
+  }
+  return 1;
+}
+
+const F_GRADED = new Set([1, 2, 4, 6, 7, 8]);
+
+function checkF(obs) {
+  if (!F_GRADED.has(obs.scenarioNo)) return 1;
+  if (obs.created.length === 0) return 1;
+  if (!obs.indexUpdated) return 0;
+  const distinct = [...new Set(obs.created.map((p) => p.path))];
+  return distinct.every((p) => obs.indexContent.includes(p)) ? 1 : 0;
+}
+
+function checkG(obs) {
+  // diffWiki already filters the s9 whitelist; belt the same rule here so the
+  // core's semantics never depend on the IO layer having run.
+  const outsideUpdated =
+    obs.scenarioNo === 9 ? obs.outside.updated.filter((p) => p !== "_meta/page-map") : obs.outside.updated;
+  if (obs.outside.created.length + outsideUpdated.length + obs.outside.deleted.length > 0) return 0;
+  if (obs.deletedFixturePaths.length > 0) return 0;
+  if (obs.scenarioNo === 9) {
+    // read-only scenario: ANY sandbox content write is a G violation
+    if (obs.created.length + obs.updated.length + obs.moved.length > 0) return 0;
+    if (obs.indexUpdated) return 0;
+  }
+  return 1;
+}
+
+function checkH(obs) {
+  const msg = obs.finalMessage.trim();
+  if (msg.length === 0) return 0;
+  if (obs.scenarioNo === 5) {
+    if (msg.length < 40) return 0;
+    return /(页面|记录|wiki|_sandbox|restart|重启)/i.test(msg) ? 1 : 0;
+  }
+  if (msg.length < 80 || !/_sandbox|wiki/i.test(msg)) return 0;
+  if (obs.scenarioNo === 7) {
+    if (!/en\s*URL/i.test(msg) || !/zh\s*URL/i.test(msg)) return 0;
+    const urls = msg.match(/https?:\/\/\S+\/(en|zh)\/_sandbox\/\S+/gi) ?? [];
+    if (urls.length < 2) return 0;
+    if (!urls.some((u) => /\/zh\/_sandbox\//i.test(u)) || !urls.some((u) => /\/en\/_sandbox\//i.test(u))) return 0;
+    if (obs.created.some((p) => /^zh\//.test(p.path) || /\/zh\//.test(p.path))) return 0;
+    if (obs.urlChecks.length < 2 || obs.urlChecks.some((u) => u.status !== 200)) return 0;
+    return 1;
+  }
+  if (obs.scenarioNo === 9) {
+    const weeks = new Set(msg.match(/\d{4}-W\d{2}/g) ?? []);
+    if (weeks.size < 2) return 0;
+    if (!/_sandbox/.test(msg)) return 0;
+  }
+  return 1;
+}
+
+/** Mechanical judgment half of scenario 05 (plan 185): a full page or a
+ *  multi-line scratch note fails; ≤1-line `_sandbox/this-session/` scratch is
+ *  tolerated; the stated refusal must appear in the final message. */
+export function judgment(scenarioNo, created, finalMessage) {
+  if (scenarioNo !== 5) return 1;
+  const extra = created.filter((p) => !p.path.startsWith("_sandbox/this-session/"));
+  if (extra.length > 0) return 0;
+  if (created.length === 1) {
+    const page = created[0];
+    if (page === undefined || !page.path.startsWith("_sandbox/this-session/")) return 0;
+    if (nonBlank(page.content).length > 1) return 0;
+    if (!finalMessage.includes(page.path)) return 0;
+  }
+  return REFUSE_RE.test(finalMessage) ? 1 : 0;
+}
+
+export function computeDims(obs) {
+  return {
+    A: checkA(obs),
+    B: checkB(obs),
+    C: checkC(obs),
+    D: checkD(obs),
+    E: checkE(obs),
+    F: checkF(obs),
+    G: checkG(obs),
+    H: checkH(obs),
+  };
+}
+
+export function scoreFromDims(scenarioNo, dims) {
+  if (scenarioNo === 5) {
+    const d = dims;
+    const total = d.G + d.H + d.J;
+    const score = total / 3; // (G+H+J)/Σ(1+1+1)
+    return { score, pass: d.G === 1 && d.H === 1 && d.J === 1, total, applicableWeight: 3 };
+  }
+  const f = dims;
+  let total = 0;
+  for (const key of Object.keys(WEIGHTS)) {
+    total += (WEIGHTS[key] ?? 0) * (f[key] ?? 0);
+  }
+  const score = total / FULL_WEIGHT_SUM; // Σ(w×d)/12 (applicable set = all 8 for full scenarios)
+  return { score, pass: total >= 10 && f.G === 1, total, applicableWeight: FULL_WEIGHT_SUM };
+}
+
+export function scoreUnit(obs) {
+  const dims = computeDims(obs);
+  const notes = [];
+  if (obs.scenarioNo === 5) {
+    const j = judgment(5, obs.created, obs.finalMessage);
+    const r = scoreFromDims(5, { G: dims.G, H: dims.H, J: j });
+    return { ...r, dims: { G: dims.G, H: dims.H, J: j }, notes };
+  }
+  const r = scoreFromDims(obs.scenarioNo, dims);
+  return { ...r, dims, notes };
+}

package/graders/historian/grader-support.d.mts

@@ -0,0 +1,57 @@
+// Type declarations for grader-support.mjs (NodeNext consumers, todo-5
+// stub-mutators.mjs/.d.mts pattern). The .mjs file is the runtime; this is the
+// compile-time contract the offline unit tests are written against.
+
+export interface WikiRow {
+  readonly id: number;
+  readonly path: string;
+  readonly locale: string;
+  readonly updatedAt: string;
+  readonly title?: string | undefined;
+  readonly isPublished?: boolean | undefined;
+  readonly isPrivate?: boolean | undefined;
+}
+
+export interface TouchedPage {
+  readonly path: string;
+  readonly locale: string;
+  readonly title: string;
+  readonly content: string;
+}
+
+export interface MovedPage {
+  readonly from: string;
+  readonly to: string;
+}
+
+export interface TranscriptMeta {
+  readonly finalMessage: string;
+  readonly tokensEst: number;
+  readonly turns: number;
+  readonly eventCount: number;
+}
+
+export interface WikiDiff {
+  readonly created: TouchedPage[];
+  readonly updated: TouchedPage[];
+  readonly moved: MovedPage[];
+  readonly deletedFixturePaths: string[];
+  readonly outside: { readonly created: string[]; readonly updated: string[]; readonly deleted: string[] };
+  readonly indexUpdated: boolean;
+  readonly indexContent: string;
+  readonly livePaths: string[];
+  readonly allPaths: string[];
+  readonly backlinkBodies: ReadonlyArray<{ readonly path: string; readonly locale: string; readonly content: string }>;
+}
+
+export interface DiffWikiRequest {
+  readonly pre: readonly WikiRow[];
+  readonly post: readonly WikiRow[];
+  readonly content: Readonly<Record<string, string>>;
+  readonly scenarioNo: number;
+}
+
+export function isSandboxPath(p: string): boolean;
+export function parseTranscript(text: string): TranscriptMeta;
+export function scenarioNoFromUnit(unitId: string): number;
+export function diffWiki(req: DiffWikiRequest): WikiDiff;

package/graders/historian/grader-support.mjs

@@ -0,0 +1,137 @@
+// Pure IO-side helpers for the historian grader (task 14, script-first).
+// No network here: parseTranscript turns recorded `opencode run --format json`
+// events into run metrics; diffWiki turns pre/post wiki row snapshots into the
+// Observation the scoring core consumes. Both are deterministic given inputs.
+
+/** A wiki page row as returned by the GraphQL list/`pages.single` probes. */
+export function isSandboxPath(p) {
+  return p === "_sandbox" || p.startsWith("_sandbox/");
+}
+
+const INDEX_PATH = "_sandbox/index";
+
+/**
+ * Parse an opencode JSONL transcript (plan §todo6 protocol). Throws on garbage:
+ * the grader must exit nonzero so the ADAPTER records the unit as inconclusive
+ * (infra_failed semantics), never as a zero score.
+ */
+export function parseTranscript(text) {
+  const events = [];
+  let sawContent = false;
+  for (const raw of text.split("\n")) {
+    const line = raw.trim();
+    if (line.length === 0) continue;
+    let doc;
+    try {
+      doc = JSON.parse(line);
+    } catch {
+      throw new Error(`transcript line is not JSON: ${line.slice(0, 120)}`);
+    }
+    if (doc === null || typeof doc !== "object" || Array.isArray(doc)) {
+      throw new Error(`transcript line is not a JSON object: ${line.slice(0, 120)}`);
+    }
+    sawContent = true;
+    events.push(doc);
+  }
+  if (!sawContent) throw new Error("transcript is empty");
+  let turns = 0;
+  let tokensEst = 0;
+  let finalMessage = "";
+  for (const ev of events) {
+    const part = typeof ev.part === "object" && ev.part !== null ? ev.part : {};
+    if (ev.type === "step_finish" || part.type === "step-finish") {
+      turns += 1;
+      const total = part.tokens !== null && typeof part.tokens === "object" ? part.tokens.total : undefined;
+      if (typeof total === "number" && Number.isFinite(total) && total >= 0) tokensEst += total;
+    }
+    if (ev.type === "text" && typeof part.text === "string") finalMessage = part.text;
+  }
+  if (turns === 0) throw new Error("transcript has no step_finish event — run never completed");
+  return { finalMessage, tokensEst, turns, eventCount: events.length };
+}
+
+/** "scenario-07" | "07" | path tail "07-..." ⇒ 7 (NaN on nothing numeric). */
+export function scenarioNoFromUnit(unitId) {
+  const m = /(\d+)/.exec(unitId);
+  if (m === null) throw new Error(`cannot derive scenario number from unit '${unitId}'`);
+  return Number(m[1]);
+}
+
+function lastSegment(p) {
+  const segs = p.split("/");
+  return segs[segs.length - 1] ?? p;
+}
+
+function touched(row, content) {
+  return {
+    path: row.path,
+    locale: row.locale,
+    title: row.title ?? lastSegment(row.path),
+    content: content[String(row.id)] ?? "",
+  };
+}
+
+/**
+ * Diff the post-seed snapshot (written by seed-wrapped.sh) against the live
+ * post-run row list. `content` maps page id → markdown body for every page the
+ * grader fetched (created/updated/_sandbox bodies). scenario 09 is read-only:
+ * its single sanctioned side effect — refreshing `_meta/page-map` — never
+ * counts as an outside-sandbox write.
+ */
+export function diffWiki({ pre, post, content, scenarioNo }) {
+  const preById = new Map(pre.map((r) => [r.id, r]));
+  const postById = new Map(post.map((r) => [r.id, r]));
+  const created = [];
+  const updated = [];
+  const moved = [];
+  const deletedFixturePaths = [];
+  const outside = { created: [], updated: [], deleted: [] };
+
+  const outsideUpdatedAllowed = new Set(scenarioNo === 9 ? ["_meta/page-map"] : []);
+
+  for (const row of post) {
+    if (preById.has(row.id)) continue;
+    if (isSandboxPath(row.path)) created.push(touched(row, content));
+    else outside.created.push(row.path);
+  }
+  for (const row of pre) {
+    if (postById.has(row.id)) continue;
+    if (isSandboxPath(row.path)) deletedFixturePaths.push(row.path);
+    else outside.deleted.push(row.path);
+  }
+  for (const row of post) {
+    const old = preById.get(row.id);
+    if (old === undefined || old.path === row.path) continue;
+    if (isSandboxPath(row.path) || isSandboxPath(old.path)) moved.push({ from: old.path, to: row.path });
+    else outside.updated.push(row.path);
+  }
+  let indexUpdated = false;
+  for (const row of post) {
+    const old = preById.get(row.id);
+    if (old === undefined || old.path !== row.path || old.updatedAt === row.updatedAt) continue;
+    if (row.path === INDEX_PATH && row.locale === "en") {
+      indexUpdated = true;
+      continue;
+    }
+    if (!isSandboxPath(row.path)) {
+      if (!outsideUpdatedAllowed.has(row.path)) outside.updated.push(row.path);
+      continue;
+    }
+    updated.push(touched(row, content));
+  }
+  // A deleted-and-recreated index is an index update, not a fixture deletion.
+  const recreatedIndex = created.find((c) => c.path === INDEX_PATH && c.locale === "en");
+  if (recreatedIndex !== undefined && deletedFixturePaths.includes(INDEX_PATH)) {
+    deletedFixturePaths.splice(deletedFixturePaths.indexOf(INDEX_PATH), 1);
+    created.splice(created.indexOf(recreatedIndex), 1);
+    indexUpdated = true;
+  }
+  const indexRow = post.find((r) => r.path === INDEX_PATH && r.locale === "en");
+  const indexContent = indexRow === undefined ? "" : (content[String(indexRow.id)] ?? "");
+  const livePaths = [...new Set(post.filter((r) => isSandboxPath(r.path)).map((r) => r.path))];
+  const allPaths = [...new Set(post.map((r) => r.path))];
+  const backlinkBodies = post
+    .filter((r) => isSandboxPath(r.path) && r.path !== INDEX_PATH && content[String(r.id)] !== undefined)
+    .map((r) => ({ path: r.path, locale: r.locale, content: content[String(r.id)] ?? "" }));
+  return { created, updated, moved, deletedFixturePaths, outside, indexUpdated, indexContent, livePaths, allPaths, backlinkBodies };
+}

package/graders/historian/grader.mjs

@@ -0,0 +1,113 @@
+#!/usr/bin/env node
+// Historian grader (task 14, script-first). Emits ONE JSON line on stdout —
+// {unit, score, pass, metrics} — the adapter's parseGraderLine contract
+// (src/bench/adapter.ts). Any unusable input exits NONZERO so the adapter
+// records the unit as inconclusive (infra semantics), never as a zero score.
+//
+// Reads (all relative to cwd = the per-unit bench sandbox):
+//   .bench/transcripts/<unitId>.jsonl  recorded `opencode run --format json`
+//   .bench/wiki-pre.json               post-seed row snapshot (seed-wrapped.sh)
+//   $ABATHUR_GRADER_STATE              optional offline state file {post, content, urlStatus}
+// Live mode queries the wiki GraphQL list, fetches _sandbox page bodies, and
+// (scenario 07 only) anonymously probes every reported page URL for HTTP 200.
+
+import { readFileSync } from "node:fs";
+import path from "node:path";
+
+import { scoreUnit } from "./grader-core.mjs";
+import { diffWiki, parseTranscript, scenarioNoFromUnit } from "./grader-support.mjs";
+
+function fail(msg) {
+  process.stderr.write(`grader: ${msg}\n`);
+  process.exit(1);
+}
+
+function readJson(file) {
+  try {
+    return JSON.parse(readFileSync(file, "utf8"));
+  } catch (cause) {
+    fail(`cannot read ${file}: ${cause instanceof Error ? cause.message : String(cause)}`);
+  }
+}
+
+const [unitId, scenarioPath, wikiBase] = process.argv.slice(2);
+if (unitId === undefined || unitId.length === 0 || scenarioPath === undefined) {
+  fail("usage: grader.mjs <unitId> <scenarioPath> [wikiBase]");
+}
+
+let meta;
+try {
+  meta = parseTranscript(readFileSync(path.join(process.cwd(), ".bench", "transcripts", `${unitId}.jsonl`), "utf8"));
+} catch (cause) {
+  fail(`transcript unusable: ${cause instanceof Error ? cause.message : String(cause)}`);
+}
+
+const pre = readJson(path.join(process.cwd(), ".bench", "wiki-pre.json"));
+const scenarioNo = scenarioNoFromUnit(unitId);
+const URL_RE = /https?:\/\/\S+\/(?:en|zh)\/_sandbox\/\S+/g;
+
+function reportedUrls(message) {
+  return [...new Set((message.match(URL_RE) ?? []).map((u) => u.replace(/[),.;，。；]+$/, "")))];
+}
+
+async function liveWikiState() {
+  if (wikiBase === undefined || wikiBase.length === 0) fail("wikiBase argv required outside ABATHUR_GRADER_STATE mode");
+  const keyFile = process.env.ABATHUR_WIKI_KEY_FILE ?? path.join(process.env.HOME ?? "", ".wikijs-api-key");
+  const token = readFileSync(keyFile, "utf8").trim();
+  const base = wikiBase.replace(/\/$/, "");
+  const gql = async (query, variables) => {
+    const res = await fetch(`${base}/graphql`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json", Authorization: `Bearer ${token}` },
+      body: JSON.stringify({ query, ...(variables === undefined ? {} : { variables }) }),
+    });
+    if (!res.ok) throw new Error(`graphql http ${String(res.status)}`);
+    const doc = await res.json();
+    if (doc.errors !== undefined) throw new Error(`graphql ${JSON.stringify(doc.errors).slice(0, 200)}`);
+    return doc.data;
+  };
+  const list = await gql("{ pages { list { id path locale title updatedAt } } }");
+  const post = list.pages.list;
+  const content = {};
+  for (const row of post) {
+    if (!row.path.startsWith("_sandbox/") && !row.path.startsWith("_meta/")) continue;
+    const one = await gql(`{ pages { single(id: ${String(row.id)}) { id title path locale content } } }`);
+    const page = one.pages.single;
+    if (page !== null && page !== undefined) content[String(page.id)] = page.content ?? "";
+  }
+  const urlStatus = {};
+  for (const url of reportedUrls(meta.finalMessage)) {
+    try {
+      const res = await fetch(url, { redirect: "manual" });
+      urlStatus[url] = res.status;
+    } catch {
+      urlStatus[url] = 0;
+    }
+  }
+  return { post, content, urlStatus };
+}
+
+const state =
+  process.env.ABATHUR_GRADER_STATE !== undefined && process.env.ABATHUR_GRADER_STATE.length > 0
+    ? readJson(process.env.ABATHUR_GRADER_STATE)
+    : await liveWikiState();
+
+const diff = diffWiki({ pre, post: state.post, content: state.content, scenarioNo });
+const urlChecks = Object.entries(state.urlStatus ?? {}).map(([url, status]) => ({ url, status }));
+const result = scoreUnit({ ...diff, scenarioNo, finalMessage: meta.finalMessage, urlChecks });
+
+process.stdout.write(
+  `${JSON.stringify({
+    unit: unitId,
+    score: result.score,
+    pass: result.pass,
+    metrics: {
+      tokensEst: meta.tokensEst,
+      turns: meta.turns,
+      dims: result.dims,
+      total: result.total,
+      applicableWeight: result.applicableWeight,
+      notes: result.notes,
+    },
+  })}\n`,
+);