@rohal12/spindle 0.53.0 → 0.54.0

package/src/markup/validate.ts

@@ -0,0 +1,219 @@
+/**
+ * Markup validation: parse every passage when the story starts and report
+ * malformed markup and unknown macros with the passage, line and column
+ * they are at, instead of failing when the passage renders. Tooling runs it
+ * too (see src/tooling.ts).
+ */
+import { lineColumn, MarkupError, parseMarkup, tokenizeMarkup } from './parse';
+import type { Token } from './tokens';
+import { isCodeAttribute } from './code-attributes';
+import { parseWidgetDef } from '../widgets/widget-def';
+
+/** A passage to validate. */
+export interface MarkupPassage {
+  name: string;
+  content: string;
+  tags?: string[];
+  /**
+   * Passage attributes; `data-source-file` and `data-source-line` (the
+   * line of its `::` header) place errors in the source file.
+   */
+  metadata?: Record<string, string>;
+}
+
+/** An error in a passage's markup. */
+export interface MarkupDiagnostic {
+  passage: string;
+  /** 1-based line within the passage's content. */
+  line: number;
+  /** 1-based column (UTF-16 code units). */
+  column: number;
+  message: string;
+  /** The source file and line, when the passage says where it came from. */
+  file?: string;
+  fileLine?: number;
+}
+
+export interface MarkupValidationOptions {
+  /**
+   * Whether a macro of this name exists. Widgets that passages define with
+   * `{widget}` and branch macros (`{else}`, `{case}`, …) always do.
+   */
+  isKnownMacro(name: string): boolean;
+  /** Names to suggest for a misspelt macro. */
+  macroNames?: Iterable<string>;
+  /** Whether a macro takes a body (default: the registered block macros). */
+  isBlockMacro?(name: string): boolean;
+  /**
+   * Report on only the passages for which this holds (default: all). The
+   * others are still read for the widgets they define.
+   */
+  only?(passage: MarkupPassage): boolean;
+}
+
+/** Passages that hold no markup. */
+const NOT_MARKUP = new Set(['StoryVariables', 'StoryTransients']);
+const NOT_MARKUP_TAGS = ['script', 'stylesheet'];
+
+/** Branch macros, which their parent macro renders. */
+const BRANCH_MACROS = new Set(['elseif', 'else', 'case', 'default', 'next']);
+
+/** A diagnostic as one line of text. */
+export function formatDiagnostic(d: MarkupDiagnostic): string {
+  const source = d.file
+    ? ` (${d.file}${d.fileLine ? `:${d.fileLine}` : ''})`
+    : '';
+  return `Passage "${d.passage}", line ${d.line}, column ${d.column}${source}: ${d.message}`;
+}
+
+/** Edit distance, for "did you mean" suggestions. */
+function distance(a: string, b: string): number {
+  let prev = Array.from({ length: b.length + 1 }, (_, j) => j);
+  for (let i = 1; i <= a.length; i++) {
+    const row = [i];
+    for (let j = 1; j <= b.length; j++) {
+      row[j] = Math.min(
+        prev[j]! + 1,
+        row[j - 1]! + 1,
+        prev[j - 1]! + (a[i - 1] === b[j - 1] ? 0 : 1),
+      );
+    }
+    prev = row;
+  }
+  return prev[b.length]!;
+}
+
+/** " Did you mean {name}?" for the closest known name, if one is close. */
+function suggestion(name: string, names: readonly string[]): string {
+  const lower = name.toLowerCase();
+  let best = '';
+  let bestDistance = Math.max(2, Math.floor(lower.length / 3)) + 1;
+  for (const candidate of names) {
+    const d = distance(lower, candidate.toLowerCase());
+    if (d < bestDistance) {
+      best = candidate;
+      bestDistance = d;
+    }
+  }
+  return best ? ` Did you mean {${best}}?` : '';
+}
+
+/** The name of the widget a `{widget}` with these arguments defines. */
+function widgetName(rawArgs: string): string | undefined {
+  try {
+    return parseWidgetDef(rawArgs).name.toLowerCase() || undefined;
+  } catch {
+    return undefined; // the {widget} macro reports it when it renders
+  }
+}
+
+/** Validate the markup of every passage. */
+export function validateMarkup(
+  passages: Iterable<MarkupPassage>,
+  options: MarkupValidationOptions,
+): MarkupDiagnostic[] {
+  const diagnostics: MarkupDiagnostic[] = [];
+  const hooks = options.isBlockMacro
+    ? { isBlock: options.isBlockMacro }
+    : undefined;
+  const report = (passage: MarkupPassage, offset: number, message: string) => {
+    const { line, column } = lineColumn(passage.content, offset);
+    const d: MarkupDiagnostic = {
+      passage: passage.name,
+      line,
+      column,
+      message,
+    };
+    const file = passage.metadata?.['data-source-file'];
+    if (file) {
+      d.file = file;
+      const header = Number(passage.metadata?.['data-source-line']);
+      if (header) d.fileLine = header + line;
+    }
+    diagnostics.push(d);
+  };
+
+  // Parse every passage first: the widgets they define are known macros.
+  const parsed: [MarkupPassage, Token[]][] = [];
+  const widgets = new Set<string>();
+  for (const passage of passages) {
+    if (NOT_MARKUP.has(passage.name)) continue;
+    if (passage.tags?.some((tag) => NOT_MARKUP_TAGS.includes(tag))) continue;
+    const reported = !options.only || options.only(passage);
+    try {
+      // The tokens first: they don't depend on which macros take a body
+      const tokens = tokenizeMarkup(passage.content);
+      for (const token of tokens) {
+        if (token.type !== 'macro' || token.isClose) continue;
+        if (token.name.toLowerCase() !== 'widget') continue;
+        const name = widgetName(token.rawArgs);
+        if (name) widgets.add(name);
+      }
+      if (!reported) continue;
+      parseMarkup(passage.content, { hooks });
+      parsed.push([passage, tokens]);
+    } catch (err) {
+      if (!(err instanceof MarkupError)) throw err;
+      if (reported) {
+        report(passage, err.offset, err.reason);
+      }
+    }
+  }
+  const names = [...(options.macroNames ?? []), ...widgets];
+
+  /** Report the unknown macros among `tokens`, which start at `base`. */
+  const checkMacros = (
+    passage: MarkupPassage,
+    tokens: Token[],
+    base: number,
+    where: string,
+  ) => {
+    for (const token of tokens) {
+      if (token.type !== 'macro' || token.isClose) continue;
+      const lower = token.name.toLowerCase();
+      if (
+        BRANCH_MACROS.has(lower) ||
+        widgets.has(lower) ||
+        options.isKnownMacro(lower)
+      ) {
+        continue;
+      }
+      report(
+        passage,
+        base + token.start,
+        `${where}Unknown macro {${token.name}}.${suggestion(token.name, names)}`,
+      );
+    }
+  };
+
+  for (const [passage, tokens] of parsed) {
+    checkMacros(passage, tokens, 0, '');
+    for (const token of tokens) {
+      if (token.type !== 'html') continue;
+      for (const [attr, value] of Object.entries(token.attributes)) {
+        if (isCodeAttribute(attr) || !value.includes('{')) continue;
+        // The value is the source text between its quotes
+        const found = passage.content.indexOf(value, token.start);
+        const at = found === -1 ? token.start : found;
+        const where = `In the ${attr} attribute of <${token.tag}>: `;
+        try {
+          parseMarkup(value, { text: true, hooks });
+          checkMacros(
+            passage,
+            tokenizeMarkup(value, { text: true }),
+            at,
+            where,
+          );
+        } catch (err) {
+          if (!(err instanceof MarkupError)) throw err;
+          report(
+            passage,
+            found === -1 ? at : at + err.offset,
+            where + err.reason,
+          );
+        }
+      }
+    }
+  }
+  return diagnostics;
+}

package/src/parser.ts

@@ -1,4 +1,4 @@
-import { tokenize } from './markup/tokenizer';
+import { MarkupError, tokenizeMarkup } from './markup/parse';
 import { isCodeAttribute, splitSigilTemplate } from './markup/code-attributes';
 
 export interface Passage {
@@ -88,7 +88,7 @@ function elementMarkup(el: Element): string {
 }
 
 /**
- * An attribute value quoted so that the passage tokenizer reads it back:
+ * An attribute value quoted so that the passage parser reads it back:
  * in double quotes, else single ones, else with the double quotes in its
  * text written as `"`, which rendering decodes. Quotes inside markup in
  * the value (`{print "a"}`) are code, and don't end it.
@@ -97,7 +97,14 @@ function quoteAttribute(name: string, value: string): string {
   const code = isCodeAttribute(name);
   const probe = code ? 'on' : 'a';
   const readsBack = (quoted: string) => {
-    const tokens = tokenize(`<i ${probe}=${quoted}>`);
+    let tokens;
+    try {
+      tokens = tokenizeMarkup(`<i ${probe}=${quoted}>`);
+    } catch (err) {
+      // Malformed markup in the value, such as an unclosed quote
+      if (err instanceof MarkupError) return false;
+      throw err;
+    }
     const tag = tokens[0];
     return (
       tokens.length === 1 &&
@@ -129,7 +136,15 @@ function escapeTextQuotes(value: string, code: boolean): string {
       )
       .join('');
   }
-  return tokenize(value, { text: true })
+  let tokens;
+  try {
+    tokens = tokenizeMarkup(value, { text: true });
+  } catch (err) {
+    // Malformed markup: validation reports it when the story starts
+    if (err instanceof MarkupError) return escape(value);
+    throw err;
+  }
+  return tokens
     .map((token) => {
       const source = value.slice(token.start, token.end);
       return token.type === 'text' ? escape(source) : source;

package/src/story-init.ts

@@ -1,7 +1,6 @@
 import { h, render } from 'preact';
 import { useStoryStore, recordStoryInitState } from './store';
-import { tokenize } from './markup/tokenizer';
-import { buildAST } from './markup/ast';
+import { parseMarkup } from './markup/parse';
 import { renderNodes } from './markup/render';
 import { setSaveTitlePassage } from './saves/save-manager';
 import { emit } from './event-emitter';
@@ -22,7 +21,7 @@ function unmountStoryInit(): void {
 }
 
 /**
- * Execute the StoryInit passage: tokenize, parse, and render all macros
+ * Execute the StoryInit passage: parse it and render all macros
  * into a detached DOM node so their side effects fire through the normal
  * Preact pipeline. This is macro-agnostic — any macro works in StoryInit.
  * Re-executing (on restart) first unmounts the previous StoryInit tree.
@@ -35,8 +34,7 @@ export function executeStoryInit() {
 
   const storyInit = state.storyData.passages.get('StoryInit');
   if (storyInit) {
-    const tokens = tokenize(storyInit.content);
-    const ast = buildAST(tokens);
+    const ast = parseMarkup(storyInit.content);
 
     // Mount into a persistent hidden container. It stays mounted until the
     // next execution (restart) — this lets async effects (useEffect,

package/src/story-variables.ts

@@ -1,10 +1,24 @@
 import type { Passage } from './parser';
-import { tokenize, type Token } from './markup/tokenizer';
+import type { Token } from './markup/tokens';
+import { MarkupError, tokenizeMarkup } from './markup/parse';
 import { isCodeAttribute, splitSigilTemplate } from './markup/code-attributes';
 import { errorMessage } from './utils/error-message';
 import { lexJs, scanStringLiteral, type JsGoal } from './js-lexer';
 import { createNamespace, variableNameError } from './utils/namespace';
 
+/**
+ * The tokens of markup, or none if it is malformed: validateMarkup reports
+ * that, with its position.
+ */
+function tokensOf(text: string, textMode: boolean): Token[] {
+  try {
+    return tokenizeMarkup(text, { text: textMode });
+  } catch (err) {
+    if (err instanceof MarkupError) return [];
+    throw err;
+  }
+}
+
 export type VarType = 'number' | 'string' | 'boolean' | 'array' | 'object';
 
 export interface FieldSchema {
@@ -217,7 +231,7 @@ function scanInterpolations(
   storeVarMacros: ReadonlySet<string> = NO_STORE_VAR_MACROS,
 ): void {
   if (!text.includes('{')) return;
-  collectTokenRefs(text, tokenize(text, { text: true }), storeVarMacros, onRef);
+  collectTokenRefs(text, tokensOf(text, true), storeVarMacros, onRef);
 }
 
 /**
@@ -281,7 +295,7 @@ function collectPassageRefs(
   storeVarMacros: ReadonlySet<string>,
   onRef: RefCallback,
 ): void {
-  collectTokenRefs(content, tokenize(content), storeVarMacros, onRef);
+  collectTokenRefs(content, tokensOf(content, false), storeVarMacros, onRef);
 }
 
 /** Report the `$var` references in the tokens of `content`. */
@@ -313,7 +327,7 @@ function collectTokenRefs(
 
       if (token.name === 'do') {
         // A {do} body is JavaScript: scan its source text as code, however
-        // the markup tokenizer split it up.
+        // the markup tokens split it up.
         let close = t + 1;
         while (close < tokens.length) {
           const c = tokens[close]!;

package/src/tooling.ts

@@ -0,0 +1,50 @@
+/**
+ * What the tooling entry point (`@rohal12/spindle/tooling`) bundles as
+ * dist/pkg/story-variables.js: the StoryVariables parser and the markup
+ * validator, with no Preact or DOM dependencies. pkg/tooling.js re-exports
+ * them, binding the validator to its macro metadata.
+ */
+import {
+  validateMarkup,
+  type MarkupDiagnostic,
+  type MarkupPassage,
+} from './markup/validate';
+import { isBlockMacro } from './markup/ast';
+import { blockWidgetNames } from './widgets/widget-def';
+
+export { parseStoryVariables } from './story-variables';
+export { formatDiagnostic } from './markup/validate';
+export type { MarkupDiagnostic, MarkupPassage } from './markup/validate';
+
+/** What tooling knows about a macro (see MacroMetadata). */
+export interface ToolingMacro {
+  name: string;
+  block: boolean;
+  subMacros: string[];
+}
+
+/**
+ * Validate the markup of a story's passages as Spindle does when the story
+ * starts, against the given macros (built-in and user-defined): malformed
+ * markup and unknown macros, with their passage, line and column.
+ */
+export function validateStoryMarkup(
+  passages: Iterable<MarkupPassage>,
+  macros: Iterable<ToolingMacro>,
+): MarkupDiagnostic[] {
+  const list = [...passages];
+  const known = new Set<string>();
+  const blocks = new Set(blockWidgetNames(list).map((n) => n.toLowerCase()));
+  for (const macro of macros) {
+    const name = macro.name.toLowerCase();
+    known.add(name);
+    for (const sub of macro.subMacros) known.add(sub.toLowerCase());
+    if (macro.block) blocks.add(name);
+  }
+  return validateMarkup(list, {
+    isKnownMacro: (name) => known.has(name),
+    macroNames: known,
+    isBlockMacro: (name) =>
+      blocks.has(name.toLowerCase()) || isBlockMacro(name),
+  });
+}

package/src/types-drift-check.ts

@@ -21,7 +21,19 @@ import type {
   MacroProps as PublishedMacroProps,
   ASTNode as PublishedASTNode,
 } from '../types/index';
-import type { parseStoryVariables as PublishedParse } from '../types/tooling';
+import type {
+  parseStoryVariables as PublishedParse,
+  formatDiagnostic as PublishedFormatDiagnostic,
+  MarkupDiagnostic as PublishedMarkupDiagnostic,
+  MarkupPassage as PublishedMarkupPassage,
+  validateMarkup as PublishedValidateMarkup,
+} from '../types/tooling';
+import type {
+  formatDiagnostic as SourceFormatDiagnostic,
+  MarkupDiagnostic as SourceMarkupDiagnostic,
+  MarkupPassage as SourceMarkupPassage,
+  validateStoryMarkup as SourceValidateStoryMarkup,
+} from './tooling';
 import type { bootStory as PublishedBootStory } from '../types/headless';
 
 /** What the source declares, by published name. */
@@ -30,6 +42,11 @@ interface Source {
   // Tooling entry point (`@rohal12/spindle/tooling`): the parser that
   // dist/pkg/tooling.js re-exports.
   parseStoryVariables: typeof SourceParse;
+  // Its markup validator (pkg/tooling.js binds validateStoryMarkup to the
+  // macro metadata, so validateMarkup itself takes only the passages).
+  MarkupPassage: SourceMarkupPassage;
+  MarkupDiagnostic: SourceMarkupDiagnostic;
+  formatDiagnostic: typeof SourceFormatDiagnostic;
   // Custom macro API: the MacroContext/MacroDefinition defineMacro()
   // actually passes and accepts.
   MacroContext: SourceMacroContext;
@@ -44,6 +61,9 @@ interface Source {
 interface Published {
   StoryAPI: PublishedAPI;
   parseStoryVariables: typeof PublishedParse;
+  MarkupPassage: PublishedMarkupPassage;
+  MarkupDiagnostic: PublishedMarkupDiagnostic;
+  formatDiagnostic: typeof PublishedFormatDiagnostic;
   MacroContext: PublishedMacroContext;
   MacroDefinition: PublishedMacroDefinition;
   MacroProps: PublishedMacroProps;
@@ -90,3 +110,10 @@ const _exampleMacro: PublishedMacroDefinition = {
     );
   },
 };
+
+// pkg/tooling.js's validateMarkup is validateStoryMarkup bound to the macro
+// metadata.
+declare const validateStoryMarkup: typeof SourceValidateStoryMarkup;
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+const _validateMarkup: typeof PublishedValidateMarkup = (passages) =>
+  validateStoryMarkup(passages, []);

package/src/widgets/widget-def.ts

@@ -0,0 +1,60 @@
+/**
+ * Reading widget definitions without rendering anything: the `{widget}`
+ * macro, startup and markup validation (also in tooling) share these.
+ */
+import { parseMacroArgs } from '../components/macros/macro-args';
+import type { MacroArgs } from '../registry';
+
+/** A {widget} definition's name, then its `@` parameters. */
+export const WIDGET_PARAMETERS = [
+  { name: 'name', type: 'text', required: true },
+  { name: 'parameters', type: 'expression' },
+] as const;
+
+export interface WidgetDef {
+  name: string;
+  params: string[];
+}
+
+/**
+ * The widget a {widget} definition's arguments declare: its name and its
+ * parameters, the words after it that start with `@` (docs/widgets.md).
+ * Other words, such as `$name`, are not parameters.
+ */
+export function widgetDef({
+  name = '',
+  parameters = '',
+}: MacroArgs<typeof WIDGET_PARAMETERS>): WidgetDef {
+  return {
+    name,
+    params: parameters.split(/\s+/).filter((word) => word.startsWith('@')),
+  };
+}
+
+/** Read the arguments of a {widget} definition (see widgetDef). */
+export function parseWidgetDef(rawArgs: string): WidgetDef {
+  return widgetDef(parseMacroArgs(rawArgs, WIDGET_PARAMETERS));
+}
+
+const BLOCK_WIDGET = /\{widget\s+["']?(\w+)["']?[^}]*\}([\s\S]*?)\{\/widget\}/g;
+
+/**
+ * The names of the block widgets (those whose body renders `{@children}`)
+ * that StoryInit and the passages tagged `widget` define. They must be
+ * known as block macros before any passage is parsed, so that passages
+ * invoking them nest their content whatever the passage order.
+ */
+export function blockWidgetNames(
+  passages: Iterable<{ name: string; tags?: string[]; content: string }>,
+): string[] {
+  const names: string[] = [];
+  for (const passage of passages) {
+    if (passage.name !== 'StoryInit' && !passage.tags?.includes('widget')) {
+      continue;
+    }
+    for (const match of passage.content.matchAll(BLOCK_WIDGET)) {
+      if (/\{@children\}/.test(match[2]!)) names.push(match[1]!);
+    }
+  }
+  return names;
+}

package/types/tooling.d.ts

@@ -99,3 +99,45 @@ export declare function parseStoryVariables(
   content: string,
   sigil?: '$' | '%',
 ): Map<string, VariableSchema>;
+
+/** A passage to validate. */
+export interface MarkupPassage {
+  name: string;
+  content: string;
+  tags?: string[];
+  /**
+   * Passage attributes; `data-source-file` and `data-source-line` (the
+   * line of its `::` header) place errors in the source file.
+   */
+  metadata?: Record<string, string>;
+}
+
+/** An error in a passage's markup. */
+export interface MarkupDiagnostic {
+  passage: string;
+  /** 1-based line within the passage's content. */
+  line: number;
+  /** 1-based column (UTF-16 code units). */
+  column: number;
+  message: string;
+  /** The source file and line, when the passage says where it came from. */
+  file?: string;
+  fileLine?: number;
+}
+
+/**
+ * Validate the markup of a story's passages as Spindle does when the story
+ * starts: malformed markup (unclosed or mismatched macros, tags, links,
+ * braces and attribute values) and unknown macros, checked against the
+ * built-in macros, those registered with `defineMacro` and the widgets the
+ * passages define. Spindle refuses to start a story with any of these.
+ */
+export declare function validateMarkup(
+  passages: Iterable<MarkupPassage>,
+): MarkupDiagnostic[];
+
+/**
+ * A diagnostic as one line of text, as Spindle shows it:
+ * `Passage "Start", line 3, column 5 (story.twee:12): Unclosed {if}: …`.
+ */
+export declare function formatDiagnostic(diagnostic: MarkupDiagnostic): string;