explico 0.8.0 → 0.11.1

package/src/reference/gate-math-punctuation.ts

@@ -0,0 +1,381 @@
+/**
+ * The trailing-punctuation gate (rule N12): math never ends in the sentence's
+ * own `.`, `,`, `;` or `:`. It runs from the CLI alongside the completeness
+ * gate (`gate-math.ts`) over the same spans (`mathSpans`: every `$…$` /
+ * `$$…$$` in a lesson body, including table cells and component slots, and in
+ * an assessment item's `prompt` / `explanation`), and shares its
+ * `MathGateDiagnostic` shape and output array.
+ *
+ * "Ends" is read the way a reader sees it, not the raw source: trailing
+ * spacing (`\,`, `\quad`), `\label{…}` / `\caption{…}` / `\tag{…}`, a row break `\\`, an
+ * `\end{…}`, and the invisible delimiter `\right.` are looked through, and a
+ * closing `\text{…}` / `\mathrm{…}` is looked into — so `5\ \text{m,}` and
+ * `c &= d. \end{aligned}` are both caught. Every row of a multi-line block
+ * (the text before each `\\`) is checked the same way.
+ *
+ * Two shapes are called out by name because they look like notation: a
+ * trailing decimal point (`5.`) — a precision marker that cannot be told
+ * apart from the full stop, so it is never allowed — and a typed `...`
+ * (write `\ldots`). The one exception is a dotted abbreviation set upright
+ * that is itself notation (`+ \text{h.c.}`, `X_n \to X\ \text{a.s.}`): the
+ * shipped {@link MATH_END_ABBREVIATIONS}, edited per course with
+ * `rules.N12.allow` / `deny`.
+ */
+import { equationLinkFor, mathSpans, type MathGateDiagnostic } from './gate-math';
+import { lessonLine } from './gate-prose-notation';
+import type { NotationLessonInput } from './types';
+
+interface AssessmentItemLike {
+  readonly id?: string;
+  readonly prompt?: string;
+  readonly explanation?: string;
+}
+interface AssessmentLike {
+  readonly id: string;
+  readonly items?: readonly AssessmentItemLike[];
+}
+
+/** Dotted abbreviations that are notation, so they may end math (`\text{h.c.}`). */
+export const MATH_END_ABBREVIATIONS: readonly string[] = [
+  'a.e.',
+  'a.s.',
+  'c.c.',
+  'const.',
+  'h.c.',
+  'i.i.d.',
+];
+
+export interface MathPunctuationOptions {
+  /** Abbreviations allowed in addition to {@link MATH_END_ABBREVIATIONS}. */
+  readonly allow?: readonly string[];
+  /** Shipped abbreviations this course does not accept. */
+  readonly deny?: readonly string[];
+}
+
+export interface PunctuationFinding {
+  /** Index of the mark in the LaTeX. */
+  readonly index: number;
+  readonly mark: string;
+  readonly kind: 'sentence' | 'decimal' | 'ellipsis' | 'abbreviation';
+  /** True when the mark ends a row (text before a `\\`), not the whole span. */
+  readonly rowEnd: boolean;
+  /** The upright dotted word the mark closes (`etc.`), for `abbreviation`. */
+  readonly word?: string;
+}
+
+// --- tokens ------------------------------------------------------------------
+
+interface Token {
+  readonly kind: 'command' | 'open' | 'close' | 'space' | 'char';
+  /** A command's name without its backslash (`frac`, `,`, `\`); otherwise the character. */
+  readonly text: string;
+  readonly start: number;
+  /** For a brace, the index of its partner; -1 when unbalanced. */
+  match: number;
+  /** Index of the innermost enclosing `open` token; -1 at top level. */
+  readonly parent: number;
+}
+
+const LETTERS = /[A-Za-z]+/y;
+
+function tokenize(latex: string): Token[] {
+  const tokens: Token[] = [];
+  const stack: number[] = [];
+  let i = 0;
+  while (i < latex.length) {
+    const ch = latex[i]!;
+    const parent = stack.length === 0 ? -1 : stack[stack.length - 1]!;
+    if (ch === '%') {
+      // A comment runs to the end of the line; `\%` is consumed below as a command.
+      const newline = latex.indexOf('\n', i);
+      i = newline === -1 ? latex.length : newline;
+      continue;
+    }
+    if (ch === '\\') {
+      LETTERS.lastIndex = i + 1;
+      const letters = LETTERS.exec(latex);
+      const name = letters ? letters[0] : latex.charAt(i + 1);
+      tokens.push({ kind: 'command', text: name, start: i, match: -1, parent });
+      i += 1 + Math.max(name.length, 1);
+      continue;
+    }
+    if (ch === '{') {
+      tokens.push({ kind: 'open', text: ch, start: i, match: -1, parent });
+      stack.push(tokens.length - 1);
+    } else if (ch === '}') {
+      const open = stack.pop();
+      const index = tokens.length;
+      tokens.push({
+        kind: 'close',
+        text: ch,
+        start: i,
+        match: open ?? -1,
+        parent: open === undefined ? -1 : tokens[open]!.parent,
+      });
+      if (open !== undefined) tokens[open]!.match = index;
+    } else {
+      tokens.push({ kind: /\s/.test(ch) ? 'space' : 'char', text: ch, start: i, match: -1, parent });
+    }
+    i += 1;
+  }
+  return tokens;
+}
+
+// --- reading the end of a span -----------------------------------------------
+
+const MARKS = new Set(['.', ',', ';', ':']);
+
+/** Commands that render no glyph at the end of math: spacing, row breaks, numbering switches. */
+const INVISIBLE = new Set([
+  ',', ';', ':', '!', ' ', '\\',
+  'quad', 'qquad', 'enspace', 'thinspace', 'medspace', 'thickspace',
+  'negthinspace', 'negmedspace', 'negthickspace',
+  'nonumber', 'notag', 'displaystyle', 'textstyle',
+]);
+
+/** Commands whose `{…}` argument never shows as the last thing in the math. */
+const SKIPPED_GROUP = new Set(['label', 'caption', 'tag', 'begin', 'end', 'hspace', 'vspace', 'color']);
+
+/** Commands that set their argument upright as text — where a dotted abbreviation lives. */
+const UPRIGHT = new Set(['text', 'textrm', 'textup', 'textnormal', 'mathrm', 'mbox', 'operatorname']);
+
+/** `\right.` and its sized cousins: the `.` is the invisible null delimiter. */
+const SIZER = /^(?:left|right|middle|[bB]igg?[lrm]?)$/;
+
+/** The command a `{…}` group is the argument of, allowing `\tag*{…}`; undefined for a bare group. */
+function groupOwner(tokens: readonly Token[], open: number, from: number): number | undefined {
+  let k = open - 1;
+  while (k >= from && tokens[k]!.kind === 'space') k -= 1;
+  if (k >= from && tokens[k]!.kind === 'char' && tokens[k]!.text === '*') k -= 1;
+  return k >= from && tokens[k]!.kind === 'command' ? k : undefined;
+}
+
+type Tail = { readonly mark: number } | 'clean' | 'empty';
+
+/** The last visible token in `tokens[from, to)`: a punctuation mark, something else, or nothing. */
+function tail(tokens: readonly Token[], from: number, to: number): Tail {
+  let j = to - 1;
+  while (j >= from) {
+    const token = tokens[j]!;
+    if (token.kind === 'space') {
+      j -= 1;
+      continue;
+    }
+    if (token.kind === 'command' && INVISIBLE.has(token.text)) {
+      j -= 1;
+      continue;
+    }
+    if (token.kind === 'char' && token.text === '.') {
+      const before = tokens[j - 1];
+      if (j - 1 >= from && before?.kind === 'command' && SIZER.test(before.text)) {
+        j -= 2;
+        continue;
+      }
+    }
+    if (token.kind === 'char' && token.text === ']') {
+      // `\\[4pt]` — a row break's optional spacing.
+      let open = j - 1;
+      while (open >= from && !(tokens[open]!.kind === 'char' && tokens[open]!.text === '[')) open -= 1;
+      const breakToken = tokens[open - 1];
+      if (open - 1 >= from && breakToken?.kind === 'command' && breakToken.text === '\\') {
+        j = open - 2;
+        continue;
+      }
+      return 'clean';
+    }
+    if (token.kind === 'close') {
+      if (token.match < from) return 'clean';
+      const owner = groupOwner(tokens, token.match, from);
+      if (owner !== undefined && SKIPPED_GROUP.has(tokens[owner]!.text)) {
+        j = owner - 1;
+        continue;
+      }
+      const inner = tail(tokens, token.match + 1, j);
+      if (inner !== 'empty') return inner;
+      j = token.match - 1;
+      continue;
+    }
+    if (token.kind === 'char' && MARKS.has(token.text)) return { mark: j };
+    return 'clean';
+  }
+  return 'empty';
+}
+
+function classify(
+  tokens: readonly Token[],
+  j: number,
+  rowEnd: boolean,
+  allowed: ReadonlySet<string>,
+): PunctuationFinding | undefined {
+  const token = tokens[j]!;
+  const base = { index: token.start, mark: token.text, rowEnd };
+  if (token.text !== '.') return { ...base, kind: 'sentence' };
+
+  const isChar = (k: number, pattern: RegExp) =>
+    tokens[k]?.kind === 'char' && pattern.test(tokens[k]!.text);
+
+  if (isChar(j - 1, /^\.$/)) return { ...base, kind: 'ellipsis' };
+
+  if (isChar(j - 1, /^\d$/)) {
+    let k = j - 1;
+    while (isChar(k - 1, /^\d$/)) k -= 1;
+    // `x_2.` is a subscript followed by the full stop, not a number.
+    if (!isChar(k - 1, /^[_^]$/)) return { ...base, kind: 'decimal' };
+  }
+
+  const owner = token.parent >= 0 ? groupOwner(tokens, token.parent, 0) : undefined;
+  if (owner !== undefined && UPRIGHT.has(tokens[owner]!.text)) {
+    let k = j;
+    while (isChar(k - 1, /^[A-Za-z.]$/)) k -= 1;
+    const word = tokens
+      .slice(k, j + 1)
+      .map((t) => t.text)
+      .join('');
+    if (/[A-Za-z]/.test(word)) {
+      if (allowed.has(word)) return undefined;
+      return { ...base, kind: 'abbreviation', word };
+    }
+  }
+  return { ...base, kind: 'sentence' };
+}
+
+function allowedAbbreviations(options: MathPunctuationOptions): ReadonlySet<string> {
+  const deny = new Set(options.deny ?? []);
+  return new Set([
+    ...MATH_END_ABBREVIATIONS.filter((word) => !deny.has(word)),
+    ...(options.allow ?? []),
+  ]);
+}
+
+function findings(latex: string, allowed: ReadonlySet<string>): PunctuationFinding[] {
+  const tokens = tokenize(latex);
+  const byMark = new Map<number, PunctuationFinding>();
+  const record = (result: Tail, rowEnd: boolean) => {
+    if (typeof result !== 'object' || byMark.has(result.mark)) return;
+    const finding = classify(tokens, result.mark, rowEnd, allowed);
+    if (finding) byMark.set(result.mark, finding);
+  };
+
+  tokens.forEach((token, k) => {
+    if (token.kind === 'command' && token.text === '\\') record(tail(tokens, token.parent + 1, k), true);
+  });
+  record(tail(tokens, 0, tokens.length), false);
+
+  return [...byMark.values()].sort((left, right) => left.index - right.index);
+}
+
+/** Every N12 finding in one span of LaTeX (the text between the `$` delimiters). */
+export function trailingPunctuation(
+  latex: string,
+  options: MathPunctuationOptions = {},
+): PunctuationFinding[] {
+  return findings(latex, allowedAbbreviations(options));
+}
+
+// --- diagnostics ---------------------------------------------------------------
+
+const SEE = 'See docs/authoring/notation.md#n12.';
+
+function messageFor(finding: PunctuationFinding, snippet: string): string {
+  const where = finding.rowEnd ? 'ends a row of the math (before `\\\\`)' : 'ends the math';
+  switch (finding.kind) {
+    case 'decimal':
+      return (
+        `"${snippet}" ${where} with a trailing decimal point, which cannot be told apart from ` +
+        `the sentence's full stop. Drop the point, or write the digits that state the precision ` +
+        `(\`5.0\`); keep a full stop outside the math. ${SEE}`
+      );
+    case 'ellipsis':
+      return `"${snippet}" ${where} with a typed "...". Write \`\\ldots\`, and keep a full stop outside the math. ${SEE}`;
+    case 'abbreviation':
+      return (
+        `"${snippet}" ${where} with "${finding.word}". If that abbreviation is notation ` +
+        `(like \`\\text{h.c.}\`), add it to \`rules.N12.allow\`; if the "." is the sentence's ` +
+        `full stop, move it after the closing \`$\`. ${SEE}`
+      );
+    case 'sentence':
+      return finding.rowEnd
+        ? `"${snippet}" ${where} with "${finding.mark}" — the sentence's own punctuation. Drop it; the row break already separates the lines. ${SEE}`
+        : `"${snippet}" ${where} with "${finding.mark}" — the sentence's own punctuation. Move it after the closing \`$\` (inline math) or drop it (display math). ${SEE}`;
+  }
+}
+
+/** The last few characters of the math up to and including the mark, on one line. */
+function snippetOf(latex: string, index: number): string {
+  const from = Math.max(0, index - 24);
+  const text = latex.slice(from, index + 1).replace(/\s+/g, ' ').trimStart();
+  return from > 0 ? `…${text}` : text;
+}
+
+/**
+ * Scan every lesson body's math for the sentence's own trailing punctuation.
+ * One error per mark, located at the mark itself, so a per-place waiver
+ * (`{/* explico-allow N12: reason *\/}`) can cover it.
+ */
+export function gateMathPunctuation(
+  lessons: readonly NotationLessonInput[],
+  options: MathPunctuationOptions = {},
+): MathGateDiagnostic[] {
+  const allowed = allowedAbbreviations(options);
+  const diagnostics: MathGateDiagnostic[] = [];
+
+  for (const lesson of lessons) {
+    for (const span of mathSpans(lesson.body)) {
+      const equation = equationLinkFor(lesson.lessonId, span.latex);
+      for (const finding of findings(span.latex, allowed)) {
+        const at = span.offset + finding.index;
+        const snippet = snippetOf(span.latex, finding.index);
+        diagnostics.push({
+          context: 'body',
+          code: 'math-trailing-punctuation',
+          severity: 'error',
+          file: lesson.source.file,
+          line: lessonLine(lesson, at),
+          column: at - lesson.body.lastIndexOf('\n', at - 1),
+          token: snippet,
+          ...(equation === undefined ? {} : { equation }),
+          message: messageFor(finding, snippet),
+        });
+      }
+    }
+  }
+
+  return diagnostics;
+}
+
+/**
+ * The same check over every assessment item's `prompt` / `explanation`, which
+ * render through the same KaTeX pipeline as lesson math. No line number: the
+ * item is one JSON string field, so `file` names the item, the same
+ * `"<assessmentId> (<itemId>)"` shape `gate-math.ts` uses.
+ */
+export function gateMathPunctuationInAssessments(
+  assessments: readonly AssessmentLike[],
+  options: MathPunctuationOptions = {},
+): MathGateDiagnostic[] {
+  const allowed = allowedAbbreviations(options);
+  const diagnostics: MathGateDiagnostic[] = [];
+
+  for (const assessment of assessments) {
+    for (const item of assessment.items ?? []) {
+      for (const field of [item.prompt, item.explanation]) {
+        if (typeof field !== 'string') continue;
+        for (const span of mathSpans(field)) {
+          for (const finding of findings(span.latex, allowed)) {
+            const snippet = snippetOf(span.latex, finding.index);
+            diagnostics.push({
+              context: 'assessment',
+              code: 'math-trailing-punctuation',
+              severity: 'error',
+              file: `${assessment.id} (${item.id ?? '?'})`,
+              token: snippet,
+              message: messageFor(finding, snippet),
+            });
+          }
+        }
+      }
+    }
+  }
+
+  return diagnostics;
+}

package/src/reference/gate-math.ts

@@ -41,7 +41,14 @@ export interface EquationLink {
 }
 
 export interface MathGateDiagnostic {
-  readonly context: 'body' | 'formula' | 'assessment' | 'prose' | 'abbreviation' | 'structure';
+  readonly context:
+    | 'body'
+    | 'formula'
+    | 'assessment'
+    | 'prose'
+    | 'abbreviation'
+    | 'structure'
+    | 'course';
   /**
    * Body / component-slot math blocks the build (the render path needs every
    * glyph resolved for the KaTeX trust callback). `formula` joined it at D15:
@@ -60,13 +67,18 @@ export interface MathGateDiagnostic {
    * `gate-abbreviations.ts`) is the same "ships straight as an error" story
    * for a different shape of plain-text token — an unexplained acronym or
    * dotted abbreviation, rather than a mis-typed quantity/unit. `structure`
-   * (S1, `gate-heading-length.ts`) is unrelated to notation — it caps how
-   * long a heading or a keyed float's title may run — but reuses this same
+   * (S1, `gate-heading-length.ts`; S2, `gate-part-pages.ts`) is unrelated to
+   * notation — it caps how long a heading or a keyed float's title may run,
+   * and requires a part page's bridge and scope paragraphs — but reuses this same
    * diagnostic shape and the one `manifest.diagnostics.math` array rather
-   * than adding a second array the CLI has to know about.
+   * than adding a second array the CLI has to know about. `course`
+   * (`gate-course-rules.ts`) is a rule the *course* wrote in `courseRules`, not
+   * one the engine ships; its severity is the course's choice, so it is the one
+   * context whose severity is not fixed here.
    */
   readonly severity: 'error' | 'warning';
-  readonly code?: string;
+  /** Owned by exactly one rule or invariant in `src/rules/catalog.ts`. */
+  readonly code: string;
   readonly file: string;
   readonly line?: number;
   readonly column?: number;
@@ -83,6 +95,8 @@ const SEVERITY: Record<MathGateDiagnostic['context'], 'error' | 'warning'> = {
   prose: 'error',
   abbreviation: 'error',
   structure: 'error',
+  // Never read: `gateCourseRules` sets the severity the course chose.
+  course: 'error',
 };
 
 interface AssessmentItemLike {
@@ -114,13 +128,16 @@ function lineOf(text: string, index: number): number {
   return text.slice(0, index).split('\n').length;
 }
 
-interface MathSpan {
+export interface MathSpan {
   readonly latex: string;
   readonly line: number;
   readonly column: number;
+  /** Index of `latex` in the text it was cut from. */
+  readonly offset: number;
 }
 
-function mathSpans(body: string): MathSpan[] {
+/** Every `$$…$$` then `$…$` span in `body`, outside code, with its position. */
+export function mathSpans(body: string): MathSpan[] {
   const searchable = stripCode(body);
   const spans: MathSpan[] = [];
   for (const pattern of [DISPLAY_MATH, INLINE_MATH]) {
@@ -135,6 +152,7 @@ function mathSpans(body: string): MathSpan[] {
         latex,
         line: lineOf(body, latexOffset),
         column: latexOffset - lastNewline,
+        offset: latexOffset,
       });
     }
   }
@@ -193,6 +211,11 @@ export function equationLinkFor(
   return { key, href: `/${slug}/#${equationAnchorId({ key })}` };
 }
 
+/** Decorations a lesson suspends (`notation.suspend`, rule N22), by name. */
+export function suspendedOperators(lesson: NotationLessonInput): string[] {
+  return (lesson.suspended ?? []).map((item) => item.operator);
+}
+
 function resolveInto(
   latex: string,
   scope: readonly ScopeEntry[],
@@ -203,6 +226,7 @@ function resolveInto(
   fix: string,
   out: MathGateDiagnostic[],
   equation?: EquationLink,
+  suspended: readonly string[] = [],
 ): void {
   const at = equation === undefined ? {} : { equation };
   for (const diagnostic of lintDelimiterSizes(latex)) {
@@ -229,10 +253,18 @@ function resolveInto(
   let result;
   try {
     // D7: `\label{eq:…}` is an identity marker, not math KaTeX parses.
-    result = resolveMathGlyphs(stripEquationLabels(latex), [...scope]);
+    result = resolveMathGlyphs(stripEquationLabels(latex), [...scope], {
+      suspended,
+    });
   } catch (error) {
     out.push({
       context,
+      // The resolver's own code (`unparenthesized-function`, `invalid-latex`,
+      // …) is what the rule catalog keys on; anything else is a KaTeX crash.
+      code:
+        error instanceof GlyphResolutionError && typeof error.code === 'string'
+          ? error.code
+          : 'math-compile-error',
       severity: SEVERITY[context],
       file,
       line,
@@ -245,15 +277,35 @@ function resolveInto(
     });
     return;
   }
-  for (const { token } of result.unresolved) {
+  for (const item of result.unresolved) {
+    const { token } = item;
+    if ('suspended' in item && typeof item.suspended === 'string') {
+      out.push({
+        context,
+        code: 'suspended-operator-unresolved',
+        severity: SEVERITY[context],
+        file,
+        line,
+        token,
+        ...at,
+        message: `On this page ${JSON.stringify(item.suspended)} is suspended (notation.suspend), so ${JSON.stringify(token)} is not ${JSON.stringify(item.suspended)} applied to a symbol: it needs its own notation entry (latex: ${token}) or an \\explain{key}{${token}} wrap.`,
+      });
+      continue;
+    }
+    // Rule N4: a font makes a different symbol, never the plain letter.
+    const fontHint =
+      'font' in item && typeof item.font === 'string'
+        ? ` ${token} is a different symbol from the plain letter: give it its own entry (latex: ${token}) or wrap it in \\explain{key}{${token}}.`
+        : '';
     out.push({
       context,
+      code: 'unresolved-glyph',
       severity: SEVERITY[context],
       file,
       line,
       token,
       ...at,
-      message: `Unresolved notation ${JSON.stringify(token)} in ${context} math. ${fix}`,
+      message: `Unresolved notation ${JSON.stringify(token)} in ${context} math.${fontHint} ${fix}`,
     });
   }
 }
@@ -295,6 +347,9 @@ function reportUnknownScopeKeys(
   )) {
     out.push({
       context: 'formula',
+      // Same failure the resolver reports for a body `\explain{key}` that
+      // names nothing — one code, one owner (`notation-references`).
+      code: 'unknown-explicit-key',
       severity: SEVERITY.formula,
       file,
       token: key,
@@ -350,6 +405,9 @@ export function gateContentMath(input: {
   for (const lesson of notationInput.lessons) {
     const scope = pageScope(lesson, sharedByKey);
     const file = lesson.source.file;
+    // Suspension (rule N22) is lesson-body scope, like `notation.local`: a
+    // formula, an assessment and a lab keep the operator reading.
+    const suspended = suspendedOperators(lesson);
 
     // `mathSpans` indexes the body, which starts after the frontmatter, so
     // every span line is short by the frontmatter's height until corrected.
@@ -365,6 +423,7 @@ export function gateContentMath(input: {
         BODY_FIX,
         diagnostics,
         equationLinkFor(lesson.lessonId, span.latex),
+        suspended,
       );
     }