@supersuit/hyperspec 0.5.0 → 0.7.0

package/src/stations/terms.mjs

@@ -0,0 +1,131 @@
+// Station "terms" (hyperspec 0.6). Checks that every term in
+// writing.audience.terms (the new optional list: terms the piece uses that the reader may not
+// know) is defined the first time it appears in the draft. Pure and deterministic like every
+// station: no model call, no judgment about which words are jargon, just the closed rule below.
+//
+// Matching a term against the draft is case-insensitive and whole-word (Unicode-aware: a match
+// cannot start or end mid-word, using the same letter/number/apostrophe class dna.mjs's own word
+// definition uses, so "AI" does not match inside "said" and a multi-word term like "context
+// window" matches only as that exact phrase).
+//
+// "Defined at its first appearance" is read as: the sentence containing
+// the term's first appearance, OR the sentence right after it, contains an occurrence of the term
+// followed (within the next 6 words of that same sentence, or by a colon among them) by one of
+// "is", "means", "refers to"; OR the term is immediately followed by a parenthetical ("term
+// (short gloss)"). Sentence boundaries come from splitSegments(text, { by: "sentence" }) in
+// src/segments.mjs, the same splitter every other station that reads sentences uses, so "which
+// sentence is this in" never disagrees between stations.
+//
+// A term also listed in writing.audience.knows is never flagged, whatever the draft does with it:
+// the reader is assumed to already have it. A term that never appears in the draft at all is not
+// flagged either (nothing to define); writing.audience.terms with no real entries, or missing
+// entirely, skips the whole station.
+//
+// Fenced code blocks and inline code spans are masked out (src/stations/util.mjs's maskCode)
+// before any of the above runs: a term's only appearance inside example syntax like
+// `` `ledger: check` `` is not a prose use, and the mechanical colon/`is` rule below would
+// otherwise read that code span's own punctuation as a definition it never gave. Masking
+// preserves every character offset, so every 1-based line number reported below is still a line
+// of the ORIGINAL draft, never the masked copy.
+
+import { str } from "../placeholder.mjs";
+import { splitSegments } from "../segments.mjs";
+import { lineAt, maskCode } from "./util.mjs";
+
+export const name = "terms";
+
+// The same word/number class dna.mjs's WORD_RE tokenizes with (letters, digits, straight or
+// curly apostrophe for contractions), Unicode-aware so an accented or non-Latin term's edges are
+// read correctly too.
+const WORD_CHARS = "\\p{L}\\p{N}'’";
+
+function escapeRegExp(s) {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+// A case-insensitive, whole-phrase, Unicode-aware matcher for `term`: it cannot start or end
+// mid-word (a lookbehind/lookahead against the word-char class above), but the term itself may
+// contain internal spaces (a multi-word term matches only as that exact phrase, spaces and all).
+function termRegex(term, flags) {
+  const body = escapeRegExp(term.trim()).replace(/\s+/g, "\\s+");
+  return new RegExp(`(?<![${WORD_CHARS}])${body}(?![${WORD_CHARS}])`, flags);
+}
+
+// Every { word, start, end } token in `text`, in order, using the same word-char class as above.
+function tokenize(text) {
+  const re = new RegExp(`[${WORD_CHARS}]+`, "gu");
+  const out = [];
+  let m;
+  while ((m = re.exec(text))) out.push({ word: m[0].toLowerCase(), start: m.index, end: m.index + m[0].length });
+  return out;
+}
+
+// Whether `sentenceText` contains a definition of `term`: some occurrence of the term followed,
+// within the next 6 words of the SAME sentence (or by a colon somewhere among them), by "is",
+// "means" or "refers"+"to"; or immediately (skipping only whitespace) followed by "(".
+function definesTerm(sentenceText, term) {
+  const re = termRegex(term, "giu");
+  let m;
+  while ((m = re.exec(sentenceText))) {
+    const tailStart = m.index + m[0].length;
+    const tail = sentenceText.slice(tailStart);
+    if (/^\s*\(/.test(tail)) return true;
+
+    const tokens = tokenize(tail).slice(0, 6);
+    const windowEnd = tokens.length ? tokens[tokens.length - 1].end : tail.length;
+    const window = tail.slice(0, windowEnd);
+    if (window.includes(":")) return true;
+    const words = tokens.map((t) => t.word);
+    if (words.includes("is") || words.includes("means")) return true;
+    if (words.some((w, i) => w === "refers" && words[i + 1] === "to")) return true;
+    // re is not global-sticky across iterations by construction (lastIndex advances past this
+    // match automatically since the "g" flag is set), so a term repeated in one sentence is
+    // still checked occurrence by occurrence rather than looping forever.
+  }
+  return false;
+}
+
+export function run(spec, draft) {
+  const audience = spec?.data?.writing?.audience ?? {};
+  const terms = Array.isArray(audience.terms) ? audience.terms.map(str).filter(Boolean) : [];
+  if (!terms.length) {
+    return { station: name, status: "skip", findings: [], reason: "writing.audience.terms is empty or not set" };
+  }
+
+  const knows = new Set((Array.isArray(audience.knows) ? audience.knows : []).map((x) => str(x).toLowerCase()).filter(Boolean));
+
+  const scanText = maskCode(draft.text);
+  const sentences = splitSegments(scanText, { by: "sentence" });
+  const findings = [];
+  const usedIds = new Set();
+
+  for (const term of terms) {
+    if (knows.has(term.toLowerCase())) continue;
+
+    const first = termRegex(term, "iu").exec(scanText);
+    if (!first) continue; // the term never appears outside code; nothing to define
+
+    const pos = first.index;
+    const idx = sentences.findIndex((s) => pos >= s.start && pos < s.end);
+    const candidates = idx === -1 ? [] : [sentences[idx], sentences[idx + 1]].filter(Boolean);
+    const defined = candidates.some((s) => definesTerm(s.text, term));
+    if (defined) continue;
+
+    const slug = term.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "") || "term";
+    let id = `station-terms-undefined-${slug}`;
+    let n = 2;
+    while (usedIds.has(id)) { id = `station-terms-undefined-${slug}-${n}`; n += 1; }
+    usedIds.add(id);
+
+    findings.push({
+      station: name,
+      id,
+      severity: "fail",
+      line: lineAt(draft.text, pos),
+      message: `term "${term}" is not defined at its first appearance (need "is", "means", "refers to", a colon within a few words, or an immediate parenthetical, in that sentence or the next)`,
+      fix: `Define "${term}" where it first appears, e.g. "${term} is ..." or "${term} (a short gloss)".`,
+    });
+  }
+
+  return { station: name, status: findings.length ? "fail" : "pass", findings };
+}

package/src/stations/util.mjs

@@ -0,0 +1,99 @@
+import { resolve } from "node:path";
+import { str } from "../placeholder.mjs";
+import { readSegments } from "../segments.mjs";
+
+// Shared helpers for the deterministic stations (hyperspec 0.6). "One place this pattern is
+// defined" (src/placeholder.mjs's own header comment), so a helper more than one station needs
+// lives here rather than being copied station to station: lineAt, truncate, maskCode, maskRanges
+// and markedSegments.
+
+// The 1-based line containing character offset `pos` of `text`. Counts "\n" characters directly
+// rather than reading draft.lines: draft.lines' own CRLF handling ("\r" left attached to the
+// previous line's entry, or stripped, depending on how src/check.mjs built it) is none of any
+// station's business, and counting "\n" occurrences gives the right line number either way, since
+// every line, CRLF or not, still carries exactly one "\n".
+export function lineAt(text, pos) {
+  let line = 1;
+  for (let i = 0; i < pos && i < text.length; i++) if (text[i] === "\n") line++;
+  return line;
+}
+
+// Trims `text`, then truncates to at most `max` characters (an ellipsis replacing the last
+// character when it is longer), matching the general finding-message rule that a quoted span of
+// the draft is at most 80 characters. Trimming FIRST means a span with leading/trailing
+// whitespace never wastes truncation budget on it.
+export function truncate(text, max) {
+  const t = String(text).trim();
+  return t.length > max ? `${t.slice(0, max - 1)}…` : t;
+}
+
+// `text` with every [start, end) range in `ranges` blanked out: every character in the range
+// becomes a space, EXCEPT "\n", which is left alone. Preserving newlines (rather than blanking
+// them too) is what keeps every character offset, and therefore every 1-based line number
+// computed afterward, identical to `text` before masking, even when a masked range spans more
+// than one line (a fenced code block, in particular): blanking a "\n" would merge two lines into
+// one for every offset that comes after it, which is exactly the bug a single-line-only masker
+// would not have caught. Used by every "mask X out before scanning for Y" step across the
+// stations, so a masked-and-rescanned span can never be matched a second time.
+export function maskRanges(text, ranges) {
+  let out = text;
+  for (const { start, end } of ranges) {
+    const blanked = out.slice(start, end).replace(/[^\n]/g, " ");
+    out = out.slice(0, start) + blanked + out.slice(end);
+  }
+  return out;
+}
+
+// A fenced code block: a line starting (after up to 3 spaces of indent) with 3 or more backticks
+// or tildes, some content, then a line starting with at least as many of the SAME fence
+// character. Non-greedy so back-to-back fences in one draft are matched as separate blocks
+// rather than one block swallowing everything between the first open and the last close.
+const FENCE_RE = /^[ \t]{0,3}(`{3,}|~{3,})[^\n]*\n[\s\S]*?^[ \t]{0,3}\1[ \t]*$/gm;
+
+// An inline code span: a single backtick, a run of non-backtick, non-newline characters, a
+// closing backtick. Does not attempt CommonMark's full rule for spans opened with a run of two or
+// more backticks (rare in practice, and unneeded for the case this exists to fix: an inline
+// `like this` mention of syntax that must not be read as prose).
+const INLINE_CODE_RE = /`[^`\n]+`/g;
+
+// `text` with every fenced code block and inline code span blanked out (offsets and line numbers
+// preserved, see maskRanges above). Both `links` (a URL shown as example syntax inside a fence or
+// a span is not a real, followable link) and `terms` (a term's only appearance inside `` `code`
+// `` is not a prose use, and must not count toward "does this term appear/get defined") mask code
+// out before doing anything else, so later masking passes (Markdown links, reference definitions,
+// bare URLs, sentence splitting) never see inside a code span or fence.
+export function maskCode(text) {
+  const fenceRanges = [];
+  FENCE_RE.lastIndex = 0;
+  let m;
+  while ((m = FENCE_RE.exec(text))) fenceRanges.push({ start: m.index, end: m.index + m[0].length });
+  const withoutFences = maskRanges(text, fenceRanges);
+
+  const inlineRanges = [];
+  INLINE_CODE_RE.lastIndex = 0;
+  let im;
+  while ((im = INLINE_CODE_RE.exec(withoutFences))) inlineRanges.push({ start: im.index, end: im.index + im[0].length });
+  return maskRanges(withoutFences, inlineRanges);
+}
+
+// Every marked material of the spec, as [{ material, segments }], in writing.materials.items
+// order: an item with a segments: field whose file readSegments can read. The segments file
+// resolves relative to spec.dir, the way every writing path does. readSegments' own findings are
+// dropped here on purpose: lint reports them (test 1 and 4) and check only runs once lint passes,
+// so a station reading materials only needs the segments themselves. quotes and private both
+// read this, so it is cached on ctx (one read per check run, whichever station asks first); a
+// caller that passes no ctx simply reads the files again.
+export function markedSegments(spec, ctx) {
+  if (ctx && ctx.markedSegments) return ctx.markedSegments;
+  const items = spec?.data?.writing?.materials?.items;
+  const out = [];
+  for (const item of Array.isArray(items) ? items : []) {
+    const segPath = str(item?.segments);
+    if (!segPath) continue;
+    const material = str(item?.id) || segPath;
+    const { segments } = readSegments(resolve(spec?.dir || ".", segPath), { materialId: str(item?.id) || undefined });
+    out.push({ material, segments: segments.filter((s) => s && typeof s === "object") });
+  }
+  if (ctx) ctx.markedSegments = out;
+  return out;
+}

package/src/writing-fields.mjs

@@ -204,8 +204,10 @@ function scopeMismatch(out, idPrefix, scopeDirRaw, label, idSuffix, specVal, dis
 // changed by hash; scope fields that differ from scope.md; a dna format this linter does not
 // know; features that differ from a fresh measurement (only named when the goldens themselves
 // are unchanged, since changed goldens explain every number); and, when nothing more specific
-// differs, bytes dna measure would not have written.
-function featuresStaleness(featuresPath, diskScope, diskGoldens) {
+// differs, bytes dna measure would not have written. Exported so the check command's dna station
+// (src/stations/dna.mjs) skips on exactly the staleness lint fails a spec for, never a second
+// definition of it.
+export function featuresStaleness(featuresPath, diskScope, diskGoldens) {
   let text;
   let recorded;
   try {
@@ -365,6 +367,28 @@ function audienceFields(raw, d, here, idPrefix) {
   if (!list(raw.knows).some((x) => str(x))) out.push(f(1, `${idPrefix}-knows`, "fail", "writing.audience has no knows", "List at least one term the reader already has."));
   const reader = str(raw.reader);
   if (!READER_VALUES.includes(reader)) out.push(f(1, `${idPrefix}-reader`, "fail", `writing.audience.reader is "${reader || "(none)"}"`, "Set reader to person or agent."));
+
+  // terms (0.6, optional): the reader-may-not-know terms the check command's `terms` station
+  // checks are defined at first use. The KEY being absent is fine and does not change the
+  // audience block's completeness (the same "absence is 0.4/0.5 behavior" shape scope_dir and
+  // characters use elsewhere in this file): nothing below runs, and the block can still be
+  // complete with no terms: field at all. Present, though, it must be a list of real text: a
+  // scalar is named as not a list, an empty list fails as a whole (the "declared but empty" shape
+  // audience.knows already uses one line up), and each non-string, placeholder or empty entry is
+  // named on its own (mirroring the top-level `rejects` list's own item check in src/rules.mjs).
+  if (raw.terms !== undefined) {
+    if (!Array.isArray(raw.terms)) {
+      out.push(f(1, `${idPrefix}-terms`, "fail", "writing.audience.terms is not a list", "Write terms as a list, one term per line starting \"- \"."));
+    } else if (!raw.terms.length) {
+      out.push(f(1, `${idPrefix}-terms`, "fail", "writing.audience.terms is present but has no real entries", "List at least one term, or remove terms: entirely; it is optional."));
+    } else {
+      raw.terms.forEach((x, i) => {
+        if (x != null && typeof x !== "string") out.push(f(1, `${idPrefix}-terms`, "fail", `writing.audience.terms item ${i + 1} is not a plain string`, "Write each term as plain text, e.g. a word or short phrase."));
+        else if (!str(x)) out.push(f(1, `${idPrefix}-terms`, "fail", `writing.audience.terms item ${i + 1} is ${typeof x === "string" && x.trim() ? "a placeholder" : "empty"}`, "Replace it with the real term, or remove the item."));
+      });
+    }
+  }
+
   return out;
 }
 

package/src/writing.mjs

@@ -36,7 +36,7 @@ const required = (block, fiction) => block !== "characters" || fiction;
 // least one field. This is deliberately shallow: it is the bar for "something was written here",
 // not the bar for "this block is correct", which is what checkOwner and the field rules in
 // writing-fields.mjs are for.
-function blockPresent(block, raw) {
+export function blockPresent(block, raw) {
   if (block === "characters") return Array.isArray(raw) && raw.length > 0;
   if (block === "materials") return isObj(raw) && Array.isArray(raw.items) && raw.items.length > 0;
   return isObj(raw) && Object.keys(raw).length > 0;