@supersuit/hyperspec 0.6.0 → 0.7.0

package/src/learn.mjs

@@ -0,0 +1,422 @@
+// `hyperspec learn prepare` and `hyperspec learn record`: the Learn stage. A factory wrote a first
+// draft; a person edited it into the draft they approved. Every edit is something the spec did not
+// say well enough, or at all. `prepare` diffs the two drafts sentence by sentence and writes a
+// PACKET: each edit as a hunk (E1..En), the spec's block names, fixed instructions and the verdict
+// shape. An outside judge, a person or a model, names for each hunk the one block that should have
+// prevented it. `record` validates that verdict, tallies it by block, names ONE next move for the
+// block with the most edits, and appends a ledger line. hyperspec never calls a model, and learn
+// never edits the spec: the move is a suggestion for whoever maintains it.
+//
+// The judge packets (src/judge.mjs) set the rules followed here: paths kept as given so the packet
+// is byte-identical on rerun, and record never trusts the packet file: it rebuilds the packet from
+// the files on disk and requires the file to be exactly those bytes.
+
+import { appendFileSync, existsSync, readFileSync, statSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { sha256 } from "./hash.mjs";
+import { writeFileAtomic } from "./fsutil.mjs";
+import { readDraft } from "./draft.mjs";
+import { loadWritingSpec, lintBlock } from "./check.mjs";
+import { openLedger, ledgerDraftKey } from "./ledger.mjs";
+import { BLOCKS, blockPresent } from "./writing.mjs";
+import { sentenceUnits, normalizeSentence } from "./sentences.mjs";
+import { truncate } from "./stations/util.mjs";
+import { hashMismatchMessage } from "./judge.mjs";
+
+export const LEARN_VERSION = "0.1";
+export const PACKET_NAME = "learn.packet.json";
+
+// The closed list a verdict names blocks from, in the order every tally breaks ties by: the nine
+// writing blocks in schema order, then "none" (no block of the spec could have prevented the edit).
+export const LEARN_BLOCKS = Object.freeze([...BLOCKS, "none"]);
+
+// The one next move per block, suggested for the block with the most edits. "none" has no move.
+export const LEARN_MOVES = Object.freeze({
+  materials: "mark or add the material the edit drew on",
+  dna: "add a golden or a style rule",
+  persona: "tighten the persona's stance, may_assert or will_not_say",
+  audience: "extend the audience's knows or terms",
+  goal: "tighten a goal condition's fails_when",
+  form: "adjust the form block's length or shape",
+  spine: "restate the spine's claim or its order",
+  sources: "add or cite a source in the claims ledger",
+  characters: "extend a character's speech or knowledge",
+});
+
+export const LEARN_INSTRUCTIONS = [
+  "Each hunk is one edit a person made between the first draft a factory produced (first) and the draft they approved (approved): deleted (only in first), inserted (only in approved) or replaced (first became approved).",
+  "For every hunk, name the one block of the spec (read it at the spec path) that, had it said more or said it better, would have had the factory write the approved text in the first place:",
+  "materials (the source material the text drew on), dna (the writer's voice: goldens and style rules), persona (who is speaking: stance, what they may assert, what they will not say), audience (the reader: what they know, the terms they use), goal (the change the draft must produce and its conditions), form (length and shape), spine (the claims and their order), sources (the claims ledger), characters (fiction: a character's speech or knowledge),",
+  "or none when no block of the spec could have prevented it.",
+  "A hunk never crosses a paragraph or a heading; its sentences field counts the sentences it touched.",
+  "Name only a block listed in blocks. Answer only in the verdict shape given in verdict_schema: one entry per hunk id, each exactly once, with a why saying what the block should have said, or why no block could have.",
+].join(" ");
+
+const present = (v) => typeof v === "string" && v.trim().length > 0;
+const isObject = (v) => Boolean(v) && typeof v === "object" && !Array.isArray(v);
+const plural = (n, word) => `${n} ${word}${n === 1 ? "" : "s"}`;
+
+// ---- the diff ----------------------------------------------------------------------------------
+
+// The most cells the LCS table may have: past it, prepare refuses with a usage error
+// rather than allocate gigabytes. The common start and end of the two drafts are trimmed first, so
+// only the span that actually differs counts.
+export const MAX_DIFF_CELLS = 10_000_000;
+
+// Thrown by diffSentences when the differing span is too large to diff; prepare and record turn it
+// into a usage error (exit 2) naming both sentence counts.
+export class DiffTooLarge extends Error {
+  constructor(firstCount, approvedCount) {
+    super(`the drafts differ over ${firstCount} sentences of the first draft against ${approvedCount} of the approved draft (after their common start and end), more than learn diffs at once (${MAX_DIFF_CELLS.toLocaleString("en-US")} comparisons); learn from a smaller pair, a chapter or a scene at a time`);
+    this.firstCount = firstCount;
+    this.approvedCount = approvedCount;
+  }
+}
+
+// A boundary between two units: a blank line between paragraphs, or the end of a heading that
+// opens its paragraph. Boundaries are tokens in the diff, equal to one another, so the LCS anchors
+// on them and no hunk crosses one. Only a paragraph's first line is ever a heading
+// (src/sentences.mjs), so a hard wrap that starts a line with "# " is text, not a
+// boundary, and a reflow cannot make one.
+const BOUNDARY = Symbol("boundary");
+
+function tokens(units) {
+  const out = [];
+  let leading = true;
+  units.forEach((u, k) => {
+    const prev = units[k - 1];
+    if (prev) {
+      if (u.para !== prev.para) { out.push({ key: BOUNDARY }); leading = true; }
+      else if (prev.heading && leading) out.push({ key: BOUNDARY });
+    }
+    if (!u.heading) leading = false;
+    out.push({ key: normalizeSentence(u.text), unit: k });
+  });
+  return out;
+}
+
+// diffSentences(firstText, approvedText): the edits between two drafts, as hunks in document order,
+// { id: "E1", kind: "deleted" | "inserted" | "replaced", first, approved, sentences }. Units
+// (src/sentences.mjs) are compared with their whitespace normalized and matched by a longest common
+// subsequence, boundaries included; every maximal run of unmatched units between two matches or
+// boundaries is one hunk. A hunk's texts are the drafts' own text from its first unit to its last,
+// as written (null on the side that has none), and `sentences` is the larger of its two unit
+// counts. A run whose two sides read the same once whitespace is normalized is a reflow, not an
+// edit: it gets no hunk and no id. Throws DiffTooLarge past MAX_DIFF_CELLS.
+export function diffSentences(firstText, approvedText) {
+  const a = sentenceUnits(firstText);
+  const b = sentenceUnits(approvedText);
+  const ta = tokens(a);
+  const tb = tokens(b);
+  // Trim the common start and end: they are matches whatever the LCS says.
+  let lo = 0;
+  while (lo < ta.length && lo < tb.length && ta[lo].key === tb[lo].key) lo++;
+  let hiA = ta.length;
+  let hiB = tb.length;
+  while (hiA > lo && hiB > lo && ta[hiA - 1].key === tb[hiB - 1].key) { hiA--; hiB--; }
+  const n = hiA - lo;
+  const m = hiB - lo;
+  if ((n + 1) * (m + 1) > MAX_DIFF_CELLS) {
+    const count = (t, from, to) => t.slice(from, to).filter((x) => x.key !== BOUNDARY).length;
+    throw new DiffTooLarge(count(ta, lo, hiA), count(tb, lo, hiB));
+  }
+  // lcs[i * (m + 1) + j]: the LCS length of ta[lo + i..hiA) and tb[lo + j..hiB).
+  const lcs = new Uint32Array((n + 1) * (m + 1));
+  const at = (i, j) => lcs[i * (m + 1) + j];
+  for (let i = n - 1; i >= 0; i--) {
+    for (let j = m - 1; j >= 0; j--) {
+      lcs[i * (m + 1) + j] = ta[lo + i].key === tb[lo + j].key ? at(i + 1, j + 1) + 1 : Math.max(at(i + 1, j), at(i, j + 1));
+    }
+  }
+
+  const hunks = [];
+  let run = { a: [], b: [] };
+  const slice = (units, text, ks) => (ks.length ? text.slice(units[ks[0]].start, units[ks.at(-1)].end) : null);
+  const close = () => {
+    const first = slice(a, firstText, run.a);
+    const approved = slice(b, approvedText, run.b);
+    const sentences = Math.max(run.a.length, run.b.length);
+    run = { a: [], b: [] };
+    if (first === null && approved === null) return;
+    if (first !== null && approved !== null && normalizeSentence(first) === normalizeSentence(approved)) return;
+    hunks.push({
+      id: `E${hunks.length + 1}`,
+      kind: first === null ? "inserted" : approved === null ? "deleted" : "replaced",
+      first,
+      approved,
+      sentences,
+    });
+  };
+  let i = 0;
+  let j = 0;
+  while (i < n || j < m) {
+    if (i < n && j < m && ta[lo + i].key === tb[lo + j].key) { close(); i++; j++; continue; }
+    if (j >= m || (i < n && at(i + 1, j) >= at(i, j + 1))) {
+      const t = ta[lo + i++];
+      if (t.key === BOUNDARY) close(); else run.a.push(t.unit);
+    } else {
+      const t = tb[lo + j++];
+      if (t.key === BOUNDARY) close(); else run.b.push(t.unit);
+    }
+  }
+  close();
+  return hunks;
+}
+
+// ---- the packet --------------------------------------------------------------------------------
+
+// The blocks a verdict may name for this spec: each writing block the spec has written, in schema
+// order, then "none". A deferred or absent block (characters outside fiction) is not listed.
+export function specBlocks(spec) {
+  const writing = isObject(spec.data?.writing) ? spec.data.writing : {};
+  return [...BLOCKS.filter((b) => blockPresent(b, writing[b])), "none"];
+}
+
+const packetJson = (packet) => `${JSON.stringify(packet, null, 2)}\n`;
+
+// The packet for this spec and these two drafts, built the one way both commands build it.
+function buildPacket(specPathArg, spec, specSha, first, approved) {
+  const hunks = diffSentences(first.text, approved.text);
+  const blocks = specBlocks(spec);
+  const packet = {
+    hyperspec_learn: LEARN_VERSION,
+    spec: specPathArg,
+    spec_sha256: specSha,
+    first: first.path,
+    first_sha256: first.sha256,
+    approved: approved.path,
+    approved_sha256: approved.sha256,
+    blocks,
+    instructions: LEARN_INSTRUCTIONS,
+    hunks,
+    verdict_schema: {
+      type: "object",
+      required: ["edits"],
+      properties: {
+        edits: {
+          type: "array",
+          description: "one entry per hunk id, each exactly once",
+          items: {
+            type: "object",
+            required: ["id", "block", "why"],
+            properties: {
+              id: { enum: hunks.map((h) => h.id) },
+              block: { enum: blocks },
+              why: { type: "string", description: "what the block should have said to prevent this edit, or why no block could have" },
+            },
+          },
+        },
+      },
+    },
+  };
+  return { packet, packetBytes: packetJson(packet) };
+}
+
+// "2 replaced, 1 deleted, 1 inserted": the kinds present, most first, ties in that fixed order.
+function kindCounts(hunks) {
+  const order = ["replaced", "deleted", "inserted"];
+  return order
+    .map((k) => [k, hunks.filter((h) => h.kind === k).length])
+    .filter(([, c]) => c > 0)
+    .sort((x, y) => y[1] - x[1] || order.indexOf(x[0]) - order.indexOf(y[0]))
+    .map(([k, c]) => `${c} ${k}`)
+    .join(", ");
+}
+
+const NOTHING = "the first draft and the approved draft match";
+
+// prepareLearn(specPathArg, { first, approved, out, force }): writes <out>/learn.packet.json.
+export function prepareLearn(specPathArg, { first: firstArg, approved: approvedArg, out: outDirArg, force = false } = {}) {
+  if (!present(specPathArg)) return { usage: true, error: "learn prepare needs a spec path" };
+  if (!present(firstArg)) return { usage: true, error: "learn prepare needs --first <draft>" };
+  if (!present(approvedArg)) return { usage: true, error: "learn prepare needs --approved <draft>" };
+  if (!present(outDirArg)) return { usage: true, error: "learn prepare needs --out <dir>" };
+
+  const loaded = loadWritingSpec(specPathArg, "learn prepare");
+  if (loaded.usage) return loaded;
+  const { spec } = loaded;
+
+  let outStat = null;
+  try { outStat = statSync(resolve(outDirArg)); } catch { /* reported below */ }
+  if (!outStat) return { usage: true, error: `--out folder does not exist: ${outDirArg}; create it first` };
+  if (!outStat.isDirectory()) return { usage: true, error: `--out is not a folder: ${outDirArg}` };
+
+  const blocked = lintBlock(spec, specPathArg);
+  if (blocked) return blocked;
+
+  const first = readDraft(firstArg);
+  if (!first) return { usage: true, error: `cannot read the first draft: ${firstArg}` };
+  const approved = readDraft(approvedArg);
+  if (!approved) return { usage: true, error: `cannot read the approved draft: ${approvedArg}` };
+  const specSha = sha256(readFileSync(resolve(specPathArg)));
+
+  let built;
+  try { built = buildPacket(specPathArg, spec, specSha, first, approved); }
+  catch (e) { if (e instanceof DiffTooLarge) return { usage: true, error: e.message }; throw e; }
+  const { packet, packetBytes } = built;
+  const path = join(outDirArg, PACKET_NAME);
+  if (existsSync(resolve(path)) && !force) return { usage: true, error: `refusing to overwrite ${path}; pass --force to replace it` };
+  writeFileAtomic(resolve(path), packetBytes);
+
+  const n = packet.hunks.length;
+  return {
+    ok: true,
+    specPath: specPathArg,
+    path,
+    edits: n,
+    kinds: Object.fromEntries(["replaced", "deleted", "inserted"].map((k) => [k, packet.hunks.filter((h) => h.kind === k).length])),
+    summary: n ? `${plural(n, "edit")} over ${plural(packet.hunks.reduce((sum, h) => sum + h.sentences, 0), "sentence")}: ${kindCounts(packet.hunks)}` : `no edits: ${NOTHING}; nothing to learn`,
+    code: 0,
+  };
+}
+
+// ---- record ------------------------------------------------------------------------------------
+
+const finding = (id, message, fix) => ({ station: "learn", id, severity: "fail", message, fix });
+const shape = (message) => finding("learn-verdict-shape", message, "Write one object: { edits: [{ id, block, why }] }, one entry per hunk id.");
+
+// Every problem with the verdict against the rebuilt packet; [] when it is valid.
+function validate(verdict, packet) {
+  if (!isObject(verdict)) return [shape("the verdict is not a JSON object")];
+  if (!Array.isArray(verdict.edits)) return [shape("edits is missing or not a list")];
+  const out = [];
+  const ids = packet.hunks.map((h) => h.id);
+  const seen = new Set();
+  const dup = new Set();
+  verdict.edits.forEach((e, i) => {
+    if (!isObject(e)) { out.push(shape(`edits[${i}] is not an object`)); return; }
+    if (!present(e.id)) { out.push(shape(`edits[${i}].id is missing or not a string`)); return; }
+    const id = e.id;
+    if (!ids.includes(id)) out.push(finding("learn-edit-unknown", `${id} is not a hunk in the packet (hunks: ${ids.join(", ") || "none"})`, "Classify only the packet's hunks, by their ids."));
+    else if (seen.has(id)) { if (!dup.has(id)) out.push(finding("learn-edit-duplicate", `${id} is classified more than once`, "Classify each hunk exactly once.")); dup.add(id); }
+    seen.add(id);
+    if (typeof e.block !== "string" || !LEARN_BLOCKS.includes(e.block)) {
+      out.push(finding("learn-block-unknown", `${id}: block ${JSON.stringify(e.block ?? null)} is not one of ${LEARN_BLOCKS.join(", ")}`, "Name one block from the closed list, or none."));
+    } else if (!packet.blocks.includes(e.block)) {
+      out.push(finding("learn-block-absent", `${id}: the spec has no ${e.block} block`, `Name a block the spec has (${packet.blocks.join(", ")}).`));
+    }
+    if (!present(e.why)) out.push(finding("learn-why-missing", `${id}: why is empty or not a string`, "Say what the block should have said to prevent the edit, or why no block could have."));
+  });
+  for (const id of ids) {
+    if (!seen.has(id)) out.push(finding("learn-edit-missing", `${id} is not classified`, "Give every hunk in the packet exactly one entry."));
+  }
+  return out;
+}
+
+// { block: { edits, sentences } } for the blocks named (sentences: the sum of each hunk's sentence
+// count), most sentences first, then most edits, then LEARN_BLOCKS order (a rewritten
+// paragraph is one edit of many sentences, and weighs as the sentences it rewrote).
+function tallyOf(edits, hunks) {
+  const size = new Map(hunks.map((h) => [h.id, h.sentences]));
+  const counts = new Map();
+  for (const e of edits) {
+    const c = counts.get(e.block) ?? { edits: 0, sentences: 0 };
+    c.edits += 1;
+    c.sentences += size.get(e.id);
+    counts.set(e.block, c);
+  }
+  return Object.fromEntries([...counts].sort((x, y) => y[1].sentences - x[1].sentences || y[1].edits - x[1].edits || LEARN_BLOCKS.indexOf(x[0]) - LEARN_BLOCKS.indexOf(y[0])));
+}
+
+// "1 edit, 4 sentences": one block's line of the tally.
+export const tallyLine = (c) => `${plural(c.edits, "edit")}, ${plural(c.sentences, "sentence")}`;
+
+// recordLearn(packetPathArg, verdictPathArg): usage (exit 2); { invalid } (exit 1, nothing
+// appended; also { stale } when a file changed since prepare); or the recorded result (exit 0).
+export function recordLearn(packetPathArg, verdictPathArg) {
+  if (!present(packetPathArg)) return { usage: true, error: "learn record needs a packet path" };
+  if (!present(verdictPathArg)) return { usage: true, error: "learn record needs --verdict <file>" };
+
+  let packetText;
+  try { packetText = readFileSync(resolve(packetPathArg), "utf8"); }
+  catch { return { usage: true, error: `cannot read packet: ${packetPathArg}` }; }
+  let packet;
+  try { packet = JSON.parse(packetText); } catch { packet = null; }
+  if (!isObject(packet) || packet.hyperspec_learn !== LEARN_VERSION || !present(packet.spec) || !present(packet.first) || !present(packet.approved)) {
+    return { usage: true, error: `not a hyperspec learn packet: ${packetPathArg}` };
+  }
+
+  let verdictBuf;
+  try { verdictBuf = readFileSync(resolve(verdictPathArg)); }
+  catch { return { usage: true, error: `cannot read verdict: ${verdictPathArg}` }; }
+
+  const loaded = loadWritingSpec(packet.spec, "learn record");
+  if (loaded.usage) return { usage: true, error: `the packet's spec: ${loaded.error}` };
+  const { spec } = loaded;
+  const first = readDraft(packet.first);
+  if (!first) return { usage: true, error: `cannot read the packet's first draft: ${packet.first}` };
+  const approved = readDraft(packet.approved);
+  if (!approved) return { usage: true, error: `cannot read the packet's approved draft: ${packet.approved}` };
+
+  const base = { packetPath: packetPathArg, verdictPath: verdictPathArg };
+  const refuse = (f, extra = {}) => ({ ...base, ok: false, invalid: true, ...extra, findings: [f], code: 1 });
+
+  // A verdict on bytes other than the ones the packet was built from classifies edits that no
+  // longer exist: say which file no longer matches, claiming neither cause.
+  const specSha = sha256(readFileSync(resolve(packet.spec)));
+  const changed = [
+    specSha !== packet.spec_sha256 && "the spec",
+    first.sha256 !== packet.first_sha256 && "the first draft",
+    approved.sha256 !== packet.approved_sha256 && "the approved draft",
+  ].filter(Boolean);
+  if (changed.length) {
+    return refuse(finding("learn-stale", hashMismatchMessage(changed), "Run learn prepare again (with --force) and classify the new packet."), { stale: true });
+  }
+
+  // Never trust the packet file: rebuild it and require these exact bytes, then validate against
+  // the rebuilt copy only.
+  let built;
+  try { built = buildPacket(packet.spec, spec, specSha, first, approved); }
+  catch (e) { if (e instanceof DiffTooLarge) return { usage: true, error: e.message }; throw e; }
+  const { packet: rebuilt, packetBytes } = built;
+  if (packetBytes !== packetText) {
+    return refuse(finding("learn-packet-altered", `${packetPathArg} is not the packet learn prepare builds from the spec and drafts on disk`, "Run learn prepare again (with --force) and classify the new packet; never edit a packet."));
+  }
+
+  let verdictJson;
+  try { verdictJson = JSON.parse(verdictBuf.toString("utf8").replace(/^/, "")); }
+  catch (e) {
+    return refuse(finding("learn-verdict-not-json", `the verdict is not valid JSON (${truncate(e instanceof Error ? e.message : String(e), 120)})`, "Write the verdict as one JSON object in the packet's verdict_schema shape."));
+  }
+  const problems = validate(verdictJson, rebuilt);
+  if (problems.length) return { ...base, ok: false, invalid: true, findings: problems, code: 1 };
+
+  const edits = verdictJson.edits.length;
+  const sentences = rebuilt.hunks.reduce((sum, h) => sum + h.sentences, 0);
+  const tally = tallyOf(verdictJson.edits, rebuilt.hunks);
+  const counts = Object.entries(tally).map(([b, c]) => `${b} ${plural(c.edits, "edit")} (${plural(c.sentences, "sentence")})`).join(", ");
+  const top = Object.entries(tally).find(([b]) => b !== "none");
+  const suggestion = top ? { block: top[0], edits: top[1].edits, sentences: top[1].sentences, move: LEARN_MOVES[top[0]] } : null;
+  // "none" means no block of the spec could have prevented the edit (the packet's instructions),
+  // so a verdict of nothing but none leaves the spec nothing to learn.
+  const NOTHING_TO_LEARN = "no block could have prevented any edit, so the spec has nothing to learn from this pair";
+  const next = suggestion
+    ? `${suggestion.block}, ${suggestion.sentences} of ${plural(sentences, "sentence")} (${suggestion.edits} of ${plural(edits, "edit")}): ${suggestion.move}`
+    : edits ? `none; ${NOTHING_TO_LEARN}` : `none; ${NOTHING}, so there is nothing to learn`;
+  // The spec is the one the edits were classified against (a changed spec is stale, above), and
+  // learn never edits it: every count naming a block is an edit the spec does not yet prevent.
+  const reason = !edits ? `no edits: ${NOTHING}` : `edits by block: ${counts}; ${suggestion ? "not yet applied to the spec" : NOTHING_TO_LEARN}`;
+
+  let ledgerPath = null;
+  let ledgerWarning = null;
+  const ledger = openLedger(spec);
+  if (ledger?.warning) ledgerWarning = ledger.warning;
+  else if (ledger) {
+    const line = {
+      at: new Date().toISOString(),
+      kind: "learn",
+      first: ledgerDraftKey(spec.dir, packet.first),
+      first_sha256: first.sha256,
+      approved: ledgerDraftKey(spec.dir, packet.approved),
+      approved_sha256: approved.sha256,
+      spec_sha256: specSha,
+      verdict: "not-improved",
+      reason,
+      tally,
+    };
+    appendFileSync(ledger.abs, `${JSON.stringify(line)}\n`);
+    ledgerPath = ledger.decl;
+  }
+
+  return { ...base, ok: true, edits, sentences, tally, suggestion, next, verdict: ledger && !ledger.warning ? "not-improved" : null, reason, ledgerPath, ledgerWarning, code: 0 };
+}

package/src/ledger.mjs

@@ -0,0 +1,108 @@
+// The runs ledger, shared by every command that appends a verdict to a spec's improvement.ledger
+// after grading a draft: `hyperspec check` (one line per run of the deterministic stations) and
+// `hyperspec judge record` (one line per recorded judgment). Both follow the same truth rules, so
+// the rules live here once: which file, how the draft is keyed, and which verdict a line earns.
+//
+// compare.mjs, the other ledger writer, grades recipes rather than drafts and keeps its own shape.
+
+import { readFileSync } from "node:fs";
+import { relative, resolve, sep } from "node:path";
+import { insideDir } from "./fsutil.mjs";
+
+const present = (v) => typeof v === "string" && v.trim().length > 0;
+
+// The spec's ledger, ready to append to: { decl, abs, priorText }, where decl is improvement.ledger
+// exactly as the spec wrote it (what a command reports) and priorText is the file's current text
+// ("" when not written yet; the first run creates it). { warning } when the declared path leads
+// outside the spec's folder, and null when the spec declares no ledger at all. Lint's test 9
+// already requires a ledger before any draft is graded; the checks here are defense in depth.
+export function openLedger(spec) {
+  const decl = spec.data?.improvement?.ledger;
+  if (!present(decl)) return null;
+  if (!insideDir(spec.dir, decl)) return { warning: "improvement.ledger escapes the spec's directory; not appended" };
+  const abs = resolve(spec.dir, decl);
+  let priorText = "";
+  try { priorText = readFileSync(abs, "utf8"); } catch { /* not written yet */ }
+  return { decl, abs, priorText };
+}
+
+// Every well-formed line of the given kind already in the ledger, oldest first. A line that is not
+// valid JSON, or not an object of that kind, is skipped: this is a read for verdict history, not a
+// lint pass, and a malformed line is lint test 9's finding to report.
+export function priorLines(text, kind) {
+  return (text ?? "")
+    .split("\n")
+    .filter((l) => l.trim())
+    .map((l) => { try { return JSON.parse(l); } catch { return null; } })
+    .filter((v) => v && typeof v === "object" && !Array.isArray(v) && v.kind === kind);
+}
+
+// The draft as the ledger records it: relative to the spec's folder, with forward slashes, so
+// "./draft.md", "draft.md" and an absolute path are one history, and no absolute path lands in a
+// ledger that is usually committed.
+export function ledgerDraftKey(specDir, draftPathArg) {
+  return relative(resolve(specDir), resolve(draftPathArg)).split(sep).join("/");
+}
+
+// The verdict a new line earns, compared with `last`, the most recent earlier line this one
+// continues (the caller picks it: check uses the last full check of the same draft, judge the last
+// judgment of the same station and draft). `last` is normalized to { stations, draft_sha256,
+// spec_sha256 }, stations mapping each station name to its status then; statusNow maps each
+// station run now to its status. "Changed" means the draft's bytes or the spec's bytes, unless the
+// caller says more (what, below).
+//
+//   none, and every station passes              -> one-shot
+//   none, and a station fails                   -> not-improved "failing stations: X"
+//   it failed, every station passes now         -> improved "stations now pass: X", exactly the
+//                                                  stations that failed then and pass now
+//   it passed, nothing changed, passing         -> not-improved "no change since the last passing <noun>"
+//   it passed, something changed, passing       -> not-improved "<what> changed; every station still passes"
+//   failing now                                 -> not-improved "[<what> changed; |no change since
+//                                                  the last <noun>; ]still failing: X" when every
+//                                                  failing station also failed then, else
+//                                                  "[<what> changed; ]failing stations: X"
+//
+// Returns { verdict, detail }, detail holding either change (improved) or reason (not-improved),
+// or nothing (one-shot). Every reason is literally true of the two lines compared.
+//
+// noun: what a line records, in the reasons that say nothing changed ("no change since the last
+// passing check"); a judge line passes "judgment". Default "check".
+//
+// what: the caller's own account of what changed since `last`, when the draft and spec hashes are
+// not the whole of it (a judge's packet also reads the DNA goldens or the claims ledger); a string
+// such as "draft" or "the claims ledger (claims.jsonl)", or null for nothing. Omitted, it is worked
+// out from the two hashes.
+//
+// unchangedImprovedReason: a caller whose station can flip from fail to pass with nothing changed
+// (a judge answering differently; check cannot, being deterministic over the same files) passes the
+// reason to record instead: "improved" then requires something changed since the failing line, an
+// unchanged flip is not-improved with this reason, and an improved line's change names what
+// changed ("draft changed; stations now pass: doctor").
+export function ledgerVerdict({ last, statusNow, draftSha, specSha, unchangedImprovedReason, noun = "check", what: whatGiven }) {
+  const list = (names) => names.join(", ");
+  const failing = Object.entries(statusNow).filter(([, st]) => st === "fail").map(([n]) => n);
+  const passedNow = failing.length === 0;
+  if (!last) {
+    return passedNow
+      ? { verdict: "one-shot", detail: {} }
+      : { verdict: "not-improved", detail: { reason: `failing stations: ${list(failing)}` } };
+  }
+  const failedThen = Object.entries(last.stations ?? {}).filter(([, st]) => st === "fail").map(([n]) => n);
+  const draftChanged = last.draft_sha256 !== draftSha;
+  const specChanged = last.spec_sha256 !== specSha;
+  const what = whatGiven !== undefined ? whatGiven
+    : draftChanged && specChanged ? "spec and draft" : specChanged ? "spec" : draftChanged ? "draft" : null;
+
+  if (passedNow && failedThen.length) {
+    const nowPass = failedThen.filter((n) => statusNow[n] === "pass");
+    if (nowPass.length && !what && unchangedImprovedReason) return { verdict: "not-improved", detail: { reason: unchangedImprovedReason } };
+    if (nowPass.length) return { verdict: "improved", detail: { change: `${unchangedImprovedReason ? `${what} changed; ` : ""}stations now pass: ${list(nowPass)}` } };
+    return { verdict: "not-improved", detail: { reason: `${what ? `${what} changed; ` : ""}stations that failed last time now skip: ${list(failedThen)}` } };
+  }
+  if (passedNow) {
+    return { verdict: "not-improved", detail: { reason: what ? `${what} changed; every station still passes` : `no change since the last passing ${noun}` } };
+  }
+  const still = failing.every((n) => failedThen.includes(n));
+  const prefix = what ? `${what} changed; ` : still ? `no change since the last ${noun}; ` : "";
+  return { verdict: "not-improved", detail: { reason: `${prefix}${still ? "still failing" : "failing stations"}: ${list(failing)}` } };
+}

package/src/sentences.mjs

@@ -0,0 +1,81 @@
+// Sentence units: the grain `hyperspec learn` diffs two drafts at. Deterministic, no judgment.
+//
+// A unit is one sentence, one markdown heading line, or one list item. Built from the splitters
+// src/segments.mjs already ships, so "what is a sentence" never disagrees between commands:
+// paragraphs first (a blank line always ends a unit, punctuation or not), then, inside each
+// paragraph, its first line on its own when it is a heading, and the other lines through the sentence
+// splitter (". ! ?" followed by whitespace, never inside a quotation on the same line, each list
+// item on its own). Three rules of its own on top:
+//   - curly double quotes count as quotes, the same as straight ones (the shared splitter only
+//     knows '"'; each curly mark is one UTF-16 unit, so mapping them keeps every offset);
+//   - a unit that ends in a common abbreviation ("Dr.", "e.g.", an initial like "J.") is joined
+//     to the next unit of the same run, since that period ends a word, not a sentence;
+//   - so is any unit followed by one that starts with a lowercase letter ("the U.S. economy",
+//     "9 a.m. in"), since a sentence does not start lowercase.
+
+import { splitSegments } from "./segments.mjs";
+
+const HEADING = /^ {0,3}#{1,6}(?:[ \t]|$)/;
+
+// Lowercased, leading punctuation stripped ("(e.g." reads as "e.g.").
+const ABBREVIATIONS = new Set(["mr.", "mrs.", "ms.", "dr.", "prof.", "sr.", "jr.", "st.", "vs.", "cf.", "e.g.", "i.e."]);
+
+function endsInAbbreviation(text) {
+  const last = text.trimEnd().split(/\s+/).at(-1) ?? "";
+  const word = last.replace(/^[^\p{L}]+/u, "");
+  return ABBREVIATIONS.has(word.toLowerCase()) || /^\p{Lu}\.$/u.test(word);
+}
+
+const startsLowercase = (text) => /^\p{Ll}/u.test(text);
+
+// The lines of text[start, end) as { start, end } spans, a CRLF line's "\r" outside its span.
+function lines(text, start, end) {
+  const out = [];
+  let at = start;
+  while (at <= end) {
+    let nl = text.indexOf("\n", at);
+    if (nl < 0 || nl > end) nl = end;
+    let e = nl;
+    if (e > at && text[e - 1] === "\r") e -= 1;
+    out.push({ start: at, end: e });
+    at = nl + 1;
+  }
+  return out;
+}
+
+// Sentences of text[start, end), split on `scan` (the text with curly double quotes straightened,
+// same offsets), with abbreviation and lowercase splits joined back.
+function sentencesOf(text, scan, start, end) {
+  const units = splitSegments(scan.slice(start, end), { by: "sentence" }).map((s) => ({ start: start + s.start, end: start + s.end }));
+  const out = [];
+  for (const u of units) {
+    const prev = out.at(-1);
+    if (prev && (endsInAbbreviation(text.slice(prev.start, prev.end)) || startsLowercase(text.slice(u.start, u.end)))) prev.end = u.end;
+    else out.push({ ...u });
+  }
+  return out;
+}
+
+// sentenceUnits(text): [{ text, start, end, para, heading }] in document order, text exactly
+// text.slice(start, end). para is the 0-based index of the paragraph the unit is in; heading is
+// true for a markdown heading line.
+export function sentenceUnits(text) {
+  const scan = text.replace(/[“”]/g, '"');
+  const out = [];
+  splitSegments(text).forEach((para, p) => {
+    let run = null;
+    const flush = () => { if (run) out.push(...sentencesOf(text, scan, run.start, run.end).map((u) => ({ ...u, para: p, heading: false }))); run = null; };
+    // Only a paragraph's first line can be a heading: a later line that starts with
+    // "#" is a hard wrap inside the text and is split into sentences like the rest of it.
+    lines(text, para.start, para.end).forEach((line, k) => {
+      if (k === 0 && HEADING.test(text.slice(line.start, line.end))) { out.push({ start: line.start, end: line.end, para: p, heading: true }); return; }
+      if (run) run.end = line.end; else run = { ...line };
+    });
+    flush();
+  });
+  return out.map((u) => ({ text: text.slice(u.start, u.end), start: u.start, end: u.end, para: u.para, heading: u.heading }));
+}
+
+// The form two units are compared in: every whitespace run one space, trimmed. For comparison
+// only; a hunk carries its units as written.
+export const normalizeSentence = (s) => s.replace(/\s+/g, " ").trim();