@rohal12/spindle 0.53.0 → 0.55.0

package/src/markup/validate.ts

@@ -0,0 +1,281 @@
+/**
+ * Markup validation: parse every passage when the story starts and report
+ * malformed markup, unknown macros, syntax errors in the code it runs and
+ * passage names that name no passage (see code-check.ts) with the passage,
+ * line and column they are at, instead of failing when the passage renders.
+ * Tooling runs it too (see src/tooling.ts).
+ */
+import { lineColumn, MarkupError, parseMarkup, tokenizeMarkup } from './parse';
+import type { Token } from './tokens';
+import { parseWidgetDef } from '../widgets/widget-def';
+import { codeAndText, parseOrError, withParseCache } from '../code-check';
+import { CodeSyntaxError } from '../js-lexer';
+import type { ParameterDef } from '../registry';
+
+/** A passage to validate. */
+export interface MarkupPassage {
+  name: string;
+  content: string;
+  tags?: string[];
+  /**
+   * Passage attributes; `data-source-file` and `data-source-line` (the
+   * line of its `::` header) place errors in the source file.
+   */
+  metadata?: Record<string, string>;
+}
+
+/** An error in a passage's markup. */
+export interface MarkupDiagnostic {
+  passage: string;
+  /** 1-based line within the passage's content. */
+  line: number;
+  /** 1-based column (UTF-16 code units). */
+  column: number;
+  message: string;
+  /** The source file and line, when the passage says where it came from. */
+  file?: string;
+  fileLine?: number;
+}
+
+export interface MarkupValidationOptions {
+  /**
+   * Whether a macro of this name exists. Widgets that passages define with
+   * `{widget}` and branch macros (`{else}`, `{case}`, …) always do.
+   */
+  isKnownMacro(name: string): boolean;
+  /** Names to suggest for a misspelt macro. */
+  macroNames?: Iterable<string>;
+  /** Whether a macro takes a body (default: the registered block macros). */
+  isBlockMacro?(name: string): boolean;
+  /**
+   * Report on only the passages for which this holds (default: all). The
+   * others are still read for the widgets they define.
+   */
+  only?(passage: MarkupPassage): boolean;
+  /**
+   * The declared parameters of a macro, whose `expression` and `statements`
+   * arguments are checked as code and whose `text` and `string` arguments
+   * as markup (see code-check.ts). Without them only the code of `{$…}`,
+   * `{do}`, branch conditions and attributes is checked.
+   */
+  parametersOf?(name: string): readonly ParameterDef[] | undefined;
+}
+
+/** Passages that hold no markup. */
+const NOT_MARKUP = new Set(['StoryVariables', 'StoryTransients']);
+const NOT_MARKUP_TAGS = ['script', 'stylesheet'];
+
+/** Branch macros, which their parent macro renders. */
+const BRANCH_MACROS = new Set(['elseif', 'else', 'case', 'default', 'next']);
+
+/** A diagnostic as one line of text. */
+export function formatDiagnostic(d: MarkupDiagnostic): string {
+  const source = d.file
+    ? ` (${d.file}${d.fileLine ? `:${d.fileLine}` : ''})`
+    : '';
+  return `Passage "${d.passage}", line ${d.line}, column ${d.column}${source}: ${d.message}`;
+}
+
+/** Edit distance, for "did you mean" suggestions. */
+function distance(a: string, b: string): number {
+  let prev = Array.from({ length: b.length + 1 }, (_, j) => j);
+  for (let i = 1; i <= a.length; i++) {
+    const row = [i];
+    for (let j = 1; j <= b.length; j++) {
+      row[j] = Math.min(
+        prev[j]! + 1,
+        row[j - 1]! + 1,
+        prev[j - 1]! + (a[i - 1] === b[j - 1] ? 0 : 1),
+      );
+    }
+    prev = row;
+  }
+  return prev[b.length]!;
+}
+
+/**
+ * " Did you mean …?" with the known name closest to `name`, written by
+ * `quote`, if one is close.
+ */
+function didYouMean(
+  name: string,
+  names: Iterable<string>,
+  quote: (name: string) => string,
+): string {
+  const lower = name.toLowerCase();
+  let best = '';
+  let bestDistance = Math.max(2, Math.floor(lower.length / 3)) + 1;
+  for (const candidate of names) {
+    const d = distance(lower, candidate.toLowerCase());
+    if (d < bestDistance) {
+      best = candidate;
+      bestDistance = d;
+    }
+  }
+  return best ? ` Did you mean ${quote(best)}?` : '';
+}
+
+/** What a passage argument may be, after an error in one. */
+const PASSAGE_ARGUMENT_HINT =
+  ' (a passage name is a quoted string or an expression)';
+
+/** A passage argument that is one bare word (`{goto Kitchen}`). */
+const BARE_WORD_RE = /^[A-Za-z][\w$]*$/;
+
+/** The name of the widget a `{widget}` with these arguments defines. */
+function widgetName(rawArgs: string): string | undefined {
+  try {
+    return parseWidgetDef(rawArgs).name.toLowerCase() || undefined;
+  } catch {
+    return undefined; // the {widget} macro reports it when it renders
+  }
+}
+
+/**
+ * Validate the markup of every passage. Passage names written out (links,
+ * quoted `passage` arguments) must name one of `passages`.
+ */
+export function validateMarkup(
+  passages: Iterable<MarkupPassage>,
+  options: MarkupValidationOptions,
+): MarkupDiagnostic[] {
+  const diagnostics: MarkupDiagnostic[] = [];
+  const hooks = options.isBlockMacro
+    ? { isBlock: options.isBlockMacro }
+    : undefined;
+  const report = (passage: MarkupPassage, offset: number, message: string) => {
+    const { line, column } = lineColumn(passage.content, offset);
+    const d: MarkupDiagnostic = {
+      passage: passage.name,
+      line,
+      column,
+      message,
+    };
+    const file = passage.metadata?.['data-source-file'];
+    if (file) {
+      d.file = file;
+      const header = Number(passage.metadata?.['data-source-line']);
+      if (header) d.fileLine = header + line;
+    }
+    diagnostics.push(d);
+  };
+
+  // Parse every passage first: the widgets they define are known macros.
+  const parsed: [MarkupPassage, Token[]][] = [];
+  const widgets = new Set<string>();
+  const passageNames = new Set<string>();
+  for (const passage of passages) {
+    passageNames.add(passage.name);
+    if (NOT_MARKUP.has(passage.name)) continue;
+    if (passage.tags?.some((tag) => NOT_MARKUP_TAGS.includes(tag))) continue;
+    const reported = !options.only || options.only(passage);
+    try {
+      // The tokens first: they don't depend on which macros take a body
+      const tokens = tokenizeMarkup(passage.content);
+      for (const token of tokens) {
+        if (token.type !== 'macro' || token.isClose) continue;
+        if (token.name.toLowerCase() !== 'widget') continue;
+        const name = widgetName(token.rawArgs);
+        if (name) widgets.add(name);
+      }
+      if (!reported) continue;
+      parseMarkup(passage.content, { hooks });
+      parsed.push([passage, tokens]);
+    } catch (err) {
+      if (!(err instanceof MarkupError)) throw err;
+      if (reported) {
+        report(passage, err.offset, err.reason);
+      }
+    }
+  }
+  const names = [...(options.macroNames ?? []), ...widgets];
+
+  /** Report the unknown macros among `tokens`, which start at `base`. */
+  const checkMacros = (
+    passage: MarkupPassage,
+    tokens: Token[],
+    base: number,
+    where: string,
+  ) => {
+    for (const token of tokens) {
+      if (token.type !== 'macro' || token.isClose) continue;
+      const lower = token.name.toLowerCase();
+      if (
+        BRANCH_MACROS.has(lower) ||
+        widgets.has(lower) ||
+        options.isKnownMacro(lower)
+      ) {
+        continue;
+      }
+      report(
+        passage,
+        base + token.start,
+        `${where}Unknown macro {${token.name}}.${didYouMean(token.name, names, (n) => `{${n}}`)}`,
+      );
+    }
+  };
+
+  const parametersOf = options.parametersOf ?? (() => undefined);
+
+  /**
+   * Check `tokens`, the tokens of `src`, which starts at `base` in the
+   * passage: their macros, the code in them and the markup in their text.
+   */
+  const checkTokens = (
+    passage: MarkupPassage,
+    src: string,
+    tokens: Token[],
+    base: number,
+    where: string,
+  ) => {
+    checkMacros(passage, tokens, base, where);
+    for (const piece of codeAndText(src, tokens, parametersOf)) {
+      const at = base + piece.offset;
+      if (piece.kind === 'passage') {
+        if (!passageNames.has(piece.name)) {
+          const hint = didYouMean(piece.name, passageNames, JSON.stringify);
+          report(
+            passage,
+            at,
+            `${where}No passage named ${JSON.stringify(piece.name)} in ${piece.label}.${hint}`,
+          );
+        }
+        continue;
+      }
+      if (piece.kind === 'code') {
+        const result = parseOrError(piece.code, piece.goal);
+        const code = piece.code.trim();
+        if (result instanceof CodeSyntaxError) {
+          const reason = result.reasonIn(passage.content, at);
+          report(
+            passage,
+            at + result.pos,
+            `${where}${reason} in ${piece.label}` +
+              (piece.passage ? PASSAGE_ARGUMENT_HINT : ''),
+          );
+        } else if (piece.passage && BARE_WORD_RE.test(code)) {
+          report(
+            passage,
+            at,
+            `${where}Unquoted passage name in ${piece.label}: write ${JSON.stringify(code)}${PASSAGE_ARGUMENT_HINT}`,
+          );
+        }
+        continue;
+      }
+      if (!piece.text.includes('{')) continue;
+      try {
+        parseMarkup(piece.text, { text: true, hooks });
+        const inner = tokenizeMarkup(piece.text, { text: true });
+        checkTokens(passage, piece.text, inner, at, piece.where);
+      } catch (err) {
+        if (!(err instanceof MarkupError)) throw err;
+        report(passage, at + err.offset, piece.where + err.reason);
+      }
+    }
+  };
+
+  for (const [passage, tokens] of parsed) {
+    withParseCache(() => checkTokens(passage, passage.content, tokens, 0, ''));
+  }
+  return diagnostics;
+}

package/src/parser.ts

@@ -1,4 +1,4 @@
-import { tokenize } from './markup/tokenizer';
+import { MarkupError, tokenizeMarkup } from './markup/parse';
 import { isCodeAttribute, splitSigilTemplate } from './markup/code-attributes';
 
 export interface Passage {
@@ -88,7 +88,7 @@ function elementMarkup(el: Element): string {
 }
 
 /**
- * An attribute value quoted so that the passage tokenizer reads it back:
+ * An attribute value quoted so that the passage parser reads it back:
  * in double quotes, else single ones, else with the double quotes in its
  * text written as `"`, which rendering decodes. Quotes inside markup in
  * the value (`{print "a"}`) are code, and don't end it.
@@ -97,7 +97,14 @@ function quoteAttribute(name: string, value: string): string {
   const code = isCodeAttribute(name);
   const probe = code ? 'on' : 'a';
   const readsBack = (quoted: string) => {
-    const tokens = tokenize(`<i ${probe}=${quoted}>`);
+    let tokens;
+    try {
+      tokens = tokenizeMarkup(`<i ${probe}=${quoted}>`);
+    } catch (err) {
+      // Malformed markup in the value, such as an unclosed quote
+      if (err instanceof MarkupError) return false;
+      throw err;
+    }
     const tag = tokens[0];
     return (
       tokens.length === 1 &&
@@ -129,7 +136,15 @@ function escapeTextQuotes(value: string, code: boolean): string {
       )
       .join('');
   }
-  return tokenize(value, { text: true })
+  let tokens;
+  try {
+    tokens = tokenizeMarkup(value, { text: true });
+  } catch (err) {
+    // Malformed markup: validation reports it when the story starts
+    if (err instanceof MarkupError) return escape(value);
+    throw err;
+  }
+  return tokens
     .map((token) => {
       const source = value.slice(token.start, token.end);
       return token.type === 'text' ? escape(source) : source;

package/src/registry.ts

@@ -69,9 +69,12 @@ export function isSubMacro(name: string): boolean {
 }
 
 /**
- * How a macro argument is read (see components/macros/macro-args.ts).
+ * How a macro argument is read (see components/macros/macro-args.ts). Every
+ * declared parameter names one: there is no default.
  * Quoted strings accept `\"`, `\'` and `\\` escapes.
- * - `expression`: code, as written (the default).
+ * - `expression`: code, as written.
+ * - `statements`: code run as statements (`{set}`), as written.
+ * - `passage`: a passage name: a quoted string or an expression, as written.
  * - `variable`: a variable reference such as `$name` or `"$name"`, as written.
  * - `string`: one quoted string; anything else leaves the argument unset.
  * - `text`: one quoted string, or text with any loose quotes stripped.
@@ -84,27 +87,59 @@ export function isSubMacro(name: string): boolean {
  * - `options`: keywords, the names of its `parameters`, each followed by a
  *   quoted string or a number unless it is a flag.
  */
-export type ParameterType =
-  | 'expression'
-  | 'variable'
-  | 'string'
-  | 'text'
-  | 'names'
-  | 'delay'
-  | 'number'
-  | 'flag'
-  | 'separator'
-  | 'options';
+export const PARAMETER_TYPES = [
+  'expression',
+  'statements',
+  'passage',
+  'variable',
+  'string',
+  'text',
+  'names',
+  'delay',
+  'number',
+  'flag',
+  'separator',
+  'options',
+] as const;
+
+export type ParameterType = (typeof PARAMETER_TYPES)[number];
 
 export interface ParameterDef {
   name: string;
   required?: boolean;
   description?: string;
-  type?: ParameterType;
+  type: ParameterType;
   /** The options of an `options` parameter. */
   parameters?: readonly ParameterDef[];
 }
 
+/**
+ * Throw if a parameter `macro` declares (or an option of one) has no type,
+ * or one that isn't a ParameterType: arguments are read by their type, and
+ * there is no default.
+ */
+export function checkParameterTypes(
+  macro: string,
+  parameters: readonly ParameterDef[],
+): void {
+  for (const param of parameters) {
+    const type: unknown = param.type;
+    if (!PARAMETER_TYPES.includes(type as ParameterType)) {
+      const problem =
+        type === undefined
+          ? 'has no type'
+          : `has the unknown type ${JSON.stringify(type)}`;
+      throw new Error(
+        `spindle: The parameter "${param.name}" of the macro {${macro}} ${problem}. ` +
+          `Give it one of the types ${PARAMETER_TYPES.join(', ')} ` +
+          '(see docs/custom-macros.md#parameter-types), ' +
+          'or declare no parameters and read props.rawArgs.',
+      );
+    }
+    if (param.parameters) checkParameterTypes(macro, param.parameters);
+  }
+}
+
 type ArgValue<T, D> = T extends 'flag' | 'separator'
   ? boolean
   : T extends 'names'

package/src/runtime-errors.ts

@@ -41,6 +41,16 @@ export function showRuntimeError(context: string, error: unknown): void {
   );
 }
 
+/**
+ * The error for `name`, a passage name that names no passage, used in the
+ * passage `from`.
+ */
+export function noPassageError(name: string, from: string): Error {
+  return new Error(
+    `No passage named ${JSON.stringify(name)} (in passage ${JSON.stringify(from)})`,
+  );
+}
+
 /** Remove the shown error `id`. */
 export function dismissRuntimeError(id: number): void {
   update(errors.filter((e) => e.id !== id));

package/src/store.ts

@@ -45,7 +45,7 @@ import {
 
 import { deepClone, mergeKeys, mergesWith, shareEqual } from './structural';
 import { shallowCopy } from './utils/object-path';
-import { showRuntimeError } from './runtime-errors';
+import { noPassageError, showRuntimeError } from './runtime-errors';
 import {
   snapshotPRNG,
   restorePRNG,
@@ -278,6 +278,9 @@ function persistSession(get: () => StoryState): void {
 const SESSION_ERROR_CONTEXT =
   'The game could not be saved for a page reload; a reload goes back to the last passage it could save:';
 
+/** What the page shows before an error naming a missing passage. */
+const NAVIGATION_ERROR_CONTEXT = 'The story could not go to another passage:';
+
 /**
  * Write the session, then run `after` (the rest of the operation: its
  * events and queued navigations), then throw the session's error if writing
@@ -1145,7 +1148,10 @@ export const useStoryStore = create<StoryState>()(
       }
 
       if (!storyData.passages.has(passageName)) {
-        console.error(`spindle: Passage "${passageName}" not found.`);
+        // Shown on the page too: code (Story.goto()) can name any passage
+        const error = noPassageError(passageName, get().currentPassage);
+        console.error(`spindle: ${error.message}`);
+        showRuntimeError(NAVIGATION_ERROR_CONTEXT, error);
         return;
       }
 

package/src/story-init.ts

@@ -1,7 +1,6 @@
 import { h, render } from 'preact';
 import { useStoryStore, recordStoryInitState } from './store';
-import { tokenize } from './markup/tokenizer';
-import { buildAST } from './markup/ast';
+import { parseMarkup } from './markup/parse';
 import { renderNodes } from './markup/render';
 import { setSaveTitlePassage } from './saves/save-manager';
 import { emit } from './event-emitter';
@@ -22,7 +21,7 @@ function unmountStoryInit(): void {
 }
 
 /**
- * Execute the StoryInit passage: tokenize, parse, and render all macros
+ * Execute the StoryInit passage: parse it and render all macros
  * into a detached DOM node so their side effects fire through the normal
  * Preact pipeline. This is macro-agnostic — any macro works in StoryInit.
  * Re-executing (on restart) first unmounts the previous StoryInit tree.
@@ -35,8 +34,7 @@ export function executeStoryInit() {
 
   const storyInit = state.storyData.passages.get('StoryInit');
   if (storyInit) {
-    const tokens = tokenize(storyInit.content);
-    const ast = buildAST(tokens);
+    const ast = parseMarkup(storyInit.content);
 
     // Mount into a persistent hidden container. It stays mounted until the
     // next execution (restart) — this lets async effects (useEffect,

package/src/story-variables.ts

@@ -1,10 +1,30 @@
 import type { Passage } from './parser';
-import { tokenize, type Token } from './markup/tokenizer';
+import type { Token } from './markup/tokens';
+import { MarkupError, tokenizeMarkup } from './markup/parse';
 import { isCodeAttribute, splitSigilTemplate } from './markup/code-attributes';
 import { errorMessage } from './utils/error-message';
-import { lexJs, scanStringLiteral, type JsGoal } from './js-lexer';
+import {
+  CodeSyntaxError,
+  lexJs,
+  scanStringLiteral,
+  type JsGoal,
+} from './js-lexer';
+import { parseOrError, withParseCache } from './code-check';
 import { createNamespace, variableNameError } from './utils/namespace';
 
+/**
+ * The tokens of markup, or none if it is malformed: validateMarkup reports
+ * that, with its position.
+ */
+function tokensOf(text: string, textMode: boolean): Token[] {
+  try {
+    return tokenizeMarkup(text, { text: textMode });
+  } catch (err) {
+    if (err instanceof MarkupError) return [];
+    throw err;
+  }
+}
+
 export type VarType = 'number' | 'string' | 'boolean' | 'array' | 'object';
 
 export interface FieldSchema {
@@ -217,7 +237,7 @@ function scanInterpolations(
   storeVarMacros: ReadonlySet<string> = NO_STORE_VAR_MACROS,
 ): void {
   if (!text.includes('{')) return;
-  collectTokenRefs(text, tokenize(text, { text: true }), storeVarMacros, onRef);
+  collectTokenRefs(text, tokensOf(text, true), storeVarMacros, onRef);
 }
 
 /**
@@ -231,6 +251,25 @@ function scanCode(
   code: string,
   onRef: RefCallback,
   goal: JsGoal = 'expression',
+): void {
+  const parsed = parseOrError(code, goal);
+  if (parsed instanceof CodeSyntaxError) {
+    scanCodeLeniently(code, onRef, goal);
+    return;
+  }
+  for (const ref of parsed.refs) {
+    if (ref.sigil !== '$') continue;
+    VAR_PATH_RE.lastIndex = ref.start;
+    onRef(VAR_PATH_RE.exec(code)![1]!);
+  }
+  for (const text of parsed.strings) scanInterpolations(text, onRef);
+}
+
+/** `scanCode` for code acorn can't parse: lexed leniently. */
+function scanCodeLeniently(
+  code: string,
+  onRef: RefCallback,
+  goal: JsGoal,
 ): void {
   /** Open template literals: their text so far, null in an interpolation. */
   const templates: { nesting: number; text: string | null }[] = [];
@@ -281,7 +320,9 @@ function collectPassageRefs(
   storeVarMacros: ReadonlySet<string>,
   onRef: RefCallback,
 ): void {
-  collectTokenRefs(content, tokenize(content), storeVarMacros, onRef);
+  withParseCache(() =>
+    collectTokenRefs(content, tokensOf(content, false), storeVarMacros, onRef),
+  );
 }
 
 /** Report the `$var` references in the tokens of `content`. */
@@ -313,7 +354,7 @@ function collectTokenRefs(
 
       if (token.name === 'do') {
         // A {do} body is JavaScript: scan its source text as code, however
-        // the markup tokenizer split it up.
+        // the markup tokens split it up.
         let close = t + 1;
         while (close < tokens.length) {
           const c = tokens[close]!;
@@ -331,8 +372,9 @@ function collectTokenRefs(
 }
 
 /**
- * Scan all passages for $var references, check against schema.
- * Returns list of error messages (empty = valid).
+ * Check the `$var` references of all passages against the schema, at story
+ * start. Returns the error messages (empty = valid). (The syntax of their
+ * code is checked with their markup: see markup/validate.ts.)
  *
  * `storeVarMacros` lists the input macros whose first argument names a bound
  * variable; it defaults to the built-in ones.
@@ -352,7 +394,6 @@ export function validatePassages(
     if (name === 'StoryVariables' || name === 'StoryTransients') continue;
 
     const forLocals = extractForLocals(passage.content);
-
     collectPassageRefs(passage.content, storeVarSet, (ref) => {
       const error = validateRef(ref, schema, forLocals);
       if (error) {

package/src/tooling.ts

@@ -0,0 +1,58 @@
+/**
+ * What the tooling entry point (`@rohal12/spindle/tooling`) bundles as
+ * dist/pkg/story-variables.js: the StoryVariables parser and the markup
+ * validator, with no Preact or DOM dependencies. pkg/tooling.js re-exports
+ * them, binding the validator to its macro metadata.
+ */
+import {
+  validateMarkup,
+  type MarkupDiagnostic,
+  type MarkupPassage,
+} from './markup/validate';
+import { isBlockMacro } from './markup/ast';
+import { blockWidgetNames } from './widgets/widget-def';
+import type { ParameterDef } from './registry';
+
+export { parseStoryVariables } from './story-variables';
+export { checkParameterTypes } from './registry';
+export { formatDiagnostic } from './markup/validate';
+export type { MarkupDiagnostic, MarkupPassage } from './markup/validate';
+
+/** What tooling knows about a macro (see MacroMetadata). */
+export interface ToolingMacro {
+  name: string;
+  block: boolean;
+  subMacros: string[];
+  /** Its declared parameters: their types tell which arguments are code. */
+  parameters?: readonly ParameterDef[];
+}
+
+/**
+ * Validate the markup of a story's passages as Spindle does when the story
+ * starts, against the given macros (built-in and user-defined): malformed
+ * markup, unknown macros and syntax errors in code, with their passage, line
+ * and column.
+ */
+export function validateStoryMarkup(
+  passages: Iterable<MarkupPassage>,
+  macros: Iterable<ToolingMacro>,
+): MarkupDiagnostic[] {
+  const list = [...passages];
+  const known = new Set<string>();
+  const parameters = new Map<string, readonly ParameterDef[] | undefined>();
+  const blocks = new Set(blockWidgetNames(list).map((n) => n.toLowerCase()));
+  for (const macro of macros) {
+    const name = macro.name.toLowerCase();
+    known.add(name);
+    parameters.set(name, macro.parameters);
+    for (const sub of macro.subMacros) known.add(sub.toLowerCase());
+    if (macro.block) blocks.add(name);
+  }
+  return validateMarkup(list, {
+    isKnownMacro: (name) => known.has(name),
+    macroNames: known,
+    parametersOf: (name) => parameters.get(name),
+    isBlockMacro: (name) =>
+      blocks.has(name.toLowerCase()) || isBlockMacro(name),
+  });
+}