@rohal12/spindle 0.53.0 → 0.54.0

package/dist/pkg/tooling.js

@@ -2,7 +2,9 @@ import { readFileSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 
-export { parseStoryVariables } from './story-variables.js';
+import { validateStoryMarkup } from './story-variables.js';
+
+export { parseStoryVariables, formatDiagnostic } from './story-variables.js';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
@@ -44,3 +46,13 @@ export function defineMacro(config) {
 export function getMacroRegistry() {
   return Array.from(metadata.values());
 }
+
+/**
+ * Validate the markup of a story's passages as Spindle does when the story
+ * starts: malformed markup and unknown macros (checked against the built-in
+ * macros and those registered with defineMacro), with passage, line and
+ * column.
+ */
+export function validateMarkup(passages) {
+  return validateStoryMarkup(passages, metadata.values());
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rohal12/spindle",
-  "version": "0.53.0",
+  "version": "0.54.0",
   "type": "module",
   "description": "A Preact-based story format for Twine 2.",
   "license": "Unlicense",
@@ -94,6 +94,7 @@
     "js-yaml": "^4.1.1",
     "jscpd": "5.4.0",
     "lint-staged": "^16.4.0",
+    "peggy": "5.1.0",
     "playwright": "^1.58.2",
     "prettier": "^3.8.1",
     "typescript": "^6.0.2",

package/src/components/Passage.tsx

@@ -5,8 +5,8 @@ import {
   useRef,
   useState,
 } from 'preact/hooks';
-import { tokenize } from '../markup/tokenizer';
-import { buildAST, type ASTNode } from '../markup/ast';
+import { parseMarkup } from '../markup/parse';
+import type { ASTNode } from '../markup/ast';
 import { renderNodes, NobrContext } from '../markup/render';
 import { useStoryFields } from '../hooks/use-story-fields';
 import type { Passage as PassageData } from '../parser';
@@ -29,7 +29,7 @@ const astCache = new WeakMap<
 function parsePassage(passage: PassageData): ASTNode[] {
   const cached = astCache.get(passage);
   if (cached && cached.content === passage.content) return cached.ast;
-  const ast = buildAST(tokenize(passage.content));
+  const ast = parseMarkup(passage.content);
   astCache.set(passage, { content: passage.content, ast });
   return ast;
 }

package/src/components/PassageDialog.tsx

@@ -1,7 +1,6 @@
 import { createContext } from 'preact';
 import { useCallback, useLayoutEffect, useMemo, useRef } from 'preact/hooks';
-import { tokenize } from '../markup/tokenizer';
-import { buildAST } from '../markup/ast';
+import { parseMarkup } from '../markup/parse';
 import { renderNodes, NobrContext } from '../markup/render';
 import { useStoryFields } from '../hooks/use-story-fields';
 import { emitFromRender } from '../event-emitter';
@@ -53,8 +52,7 @@ export function PassageDialog({
       return <div class="error">Dialog: no content available</div>;
     }
     try {
-      const tokens = tokenize(markup);
-      const ast = buildAST(tokens);
+      const ast = parseMarkup(markup);
       const nodes = renderNodes(ast, nobr ? { nobr: true } : undefined);
       return nobr ? (
         <NobrContext.Provider value={true}>{nodes}</NobrContext.Provider>

package/src/components/StoryInterface.tsx

@@ -1,7 +1,6 @@
 import { useMemo } from 'preact/hooks';
 import { useStoryFields } from '../hooks/use-story-fields';
-import { tokenize } from '../markup/tokenizer';
-import { buildAST } from '../markup/ast';
+import { parseMarkup } from '../markup/parse';
 import { renderInlineNodes, NobrContext } from '../markup/render';
 import { errorMessage } from '../utils/error-message';
 
@@ -18,8 +17,7 @@ export function StoryInterface() {
 
   const rendered = useMemo(() => {
     try {
-      const tokens = tokenize(markup);
-      const ast = buildAST(tokens);
+      const ast = parseMarkup(markup);
       return <>{renderInlineNodes(ast)}</>;
     } catch (err) {
       return (

package/src/components/macros/Do.tsx

@@ -5,7 +5,7 @@ defineMacro({
   name: 'do',
   block: true,
   render({ children = [] }, ctx) {
-    // The tokenizer keeps the body verbatim as a single text node
+    // The parser keeps the body verbatim as a single text node
     const code = ctx.collectText(children);
 
     ctx.hooks.useLayoutEffect(() => {

package/src/components/macros/Include.tsx

@@ -1,7 +1,6 @@
 import { useStoryStore } from '../../store';
 import { useStoryFields } from '../../hooks/use-story-fields';
-import { tokenize } from '../../markup/tokenizer';
-import { buildAST } from '../../markup/ast';
+import { parseMarkup } from '../../markup/parse';
 import { NobrContext } from '../../markup/render';
 import { defineMacro } from '../../define-macro';
 import { evaluatePassageName } from './macro-args';
@@ -28,7 +27,7 @@ defineMacro({
     // included passage changes, while re-renders of the same passage keep
     // its macros mounted (#175).
     const ast = ctx.hooks.useMemo(
-      () => (passage ? buildAST(tokenize(passage.content)) : null),
+      () => (passage ? parseMarkup(passage.content) : null),
       [passage],
     );
 

package/src/components/macros/VarDisplay.tsx

@@ -3,7 +3,7 @@ import { useContext } from 'preact/hooks';
 import { LocalsValuesContext } from '../../markup/render';
 import { useInterpolate } from '../../hooks/use-interpolate';
 import { ownValue, variableNameError } from '../../utils/namespace';
-import { SCOPE_SIGILS } from '../../markup/tokenizer';
+import { SCOPE_SIGILS } from '../../markup/tokens';
 import type { VariableNode } from '../../markup/ast';
 import { display, wrapContent } from './display';
 

package/src/components/macros/Widget.tsx

@@ -4,39 +4,11 @@ import { registerBlockMacro, type ASTNode } from '../../markup/ast';
 import { defineMacro } from '../../define-macro';
 import { checkVariableName } from '../../utils/namespace';
 import { MacroError } from './MacroError';
-import { parseMacroArgs } from './macro-args';
-import type { MacroArgs } from '../../registry';
-
-/** A {widget} definition's name, then its `@` parameters. */
-const WIDGET_PARAMETERS = [
-  { name: 'name', type: 'text', required: true },
-  { name: 'parameters', type: 'expression' },
-] as const;
-
-interface WidgetDef {
-  name: string;
-  params: string[];
-}
-
-/**
- * The widget a {widget} definition's arguments declare: its name and its
- * parameters, the words after it that start with `@` (docs/widgets.md).
- * Other words, such as `$name`, are not parameters.
- */
-function widgetDef({
-  name = '',
-  parameters = '',
-}: MacroArgs<typeof WIDGET_PARAMETERS>): WidgetDef {
-  return {
-    name,
-    params: parameters.split(/\s+/).filter((word) => word.startsWith('@')),
-  };
-}
-
-/** Read the arguments of a {widget} definition (see widgetDef). */
-export function parseWidgetDef(rawArgs: string): WidgetDef {
-  return widgetDef(parseMacroArgs(rawArgs, WIDGET_PARAMETERS));
-}
+import {
+  WIDGET_PARAMETERS,
+  widgetDef,
+  type WidgetDef,
+} from '../../widgets/widget-def';
 
 /**
  * Register the widget a definition declares, with its body. Widgets whose

package/src/index.tsx

@@ -1,6 +1,6 @@
 import { render } from 'preact';
 import { App } from './components/App';
-import { parseStoryData } from './parser';
+import { parseStoryData, type StoryData } from './parser';
 import { useStoryStore, enterRuntimePhase } from './store';
 import {
   installStoryAPI,
@@ -16,10 +16,17 @@ import {
   validatePassages,
   extractDefaults,
 } from './story-variables';
-import { getMacroRegistry } from './registry';
-import { tokenize } from './markup/tokenizer';
-import { buildAST, registerBlockMacro } from './markup/ast';
-import { parseWidgetDef, registerWidgetDef } from './components/macros/Widget';
+import { getMacro, getMacroRegistry, isSubMacro } from './registry';
+import { getWidget } from './widgets/widget-registry';
+import {
+  formatDiagnostic,
+  validateMarkup,
+  type MarkupPassage,
+} from './markup/validate';
+import { blockWidgetNames, parseWidgetDef } from './widgets/widget-def';
+import { parseMarkup } from './markup/parse';
+import { registerBlockMacro } from './markup/ast';
+import { registerWidgetDef } from './components/macros/Widget';
 import { errorMessage } from './utils/error-message';
 import type { ASTNode } from './markup/ast';
 import './macros/register-builtins';
@@ -45,10 +52,36 @@ function renderErrors(root: HTMLElement, errors: string[]) {
   root.appendChild(container);
 }
 
+/** Show the validation errors instead of the story, and stop booting. */
+function stopWithErrors(errors: string[]): never {
+  const root = document.getElementById('root');
+  if (root) renderErrors(root, errors);
+  throw new Error(
+    `spindle: ${errors.length} validation error(s):\n${errors.join('\n')}`,
+  );
+}
+
+/**
+ * The markup errors (malformed markup, unknown macros) of the passages for
+ * which `only` holds, with the macros and widgets known now.
+ */
+function markupErrors(
+  storyData: StoryData,
+  only: (passage: MarkupPassage) => boolean,
+): string[] {
+  return validateMarkup(storyData.passages.values(), {
+    isKnownMacro: (name) =>
+      !!getMacro(name) || isSubMacro(name) || !!getWidget(name),
+    macroNames: getMacroRegistry().map((m) => m.name),
+    only,
+  }).map(formatDiagnostic);
+}
+
 /**
  * Boot Spindle in the current document: parse `<tw-storydata>`, install the
- * `Story` API, run author JavaScript, validate `StoryVariables`, run
- * `StoryInit`, and render into `#root`. Dispatches `:storyready` when the
+ * `Story` API, run author JavaScript, validate `StoryVariables` and
+ * `StoryInit`'s markup, run `StoryInit`, validate the other passages' markup
+ * (with the macros StoryInit defined), and render into `#root`. Dispatches `:storyready` when the
  * first passage is shown. Call once per page (module state is global).
  * The browser story format calls it from `main.tsx`.
  */
@@ -101,7 +134,18 @@ export function boot() {
   const storeVarMacros = getMacroRegistry()
     .filter((m) => m.storeVar)
     .map((m) => m.name);
+  // Pass 1: Register the block widgets as block macros BEFORE any passage
+  // is parsed (validation and StoryInit included), so that passages
+  // invoking block widgets and widget bodies using other block widgets
+  // parse correctly regardless of passage or definition order.
+  for (const name of blockWidgetNames(storyData.passages.values())) {
+    registerBlockMacro(name);
+  }
+
   const errors = validatePassages(storyData.passages, schema, storeVarMacros);
+  // StoryInit's markup must be valid before it runs; the other passages are
+  // validated once it has run, as it may define macros (see below).
+  errors.push(...markupErrors(storyData, (p) => p.name === 'StoryInit'));
 
   // Parse StoryTransients (optional — no error if missing)
   let transientDefaults: Record<string, unknown> = {};
@@ -124,13 +168,7 @@ export function boot() {
     transientDefaults = extractDefaults(transientSchema);
   }
 
-  if (errors.length > 0) {
-    const root = document.getElementById('root');
-    if (root) renderErrors(root, errors);
-    throw new Error(
-      `spindle: ${errors.length} validation error(s):\n${errors.join('\n')}`,
-    );
-  }
+  if (errors.length > 0) stopWithErrors(errors);
 
   defaults = extractDefaults(schema);
   setDeclaredVariables(Object.keys(defaults), Object.keys(transientDefaults));
@@ -140,36 +178,18 @@ export function boot() {
   // Enter runtime phase — handlers registered from here on are cleaned on restart
   enterRuntimePhase();
 
-  // Pass 1: Pre-scan StoryInit and all widget passages to discover block
-  // widgets. Register them as block macros BEFORE any tokenize/buildAST
-  // calls (StoryInit's included), so that passages invoking block widgets
-  // and widget bodies using other block widgets parse correctly regardless
-  // of passage or definition order.
-  const blockWidgetPattern =
-    /\{widget\s+["']?(\w+)["']?[^}]*\}([\s\S]*?)\{\/widget\}/g;
-  for (const [passageName, passage] of storyData.passages) {
-    if (passageName === 'StoryInit' || passage.tags.includes('widget')) {
-      let match;
-      while ((match = blockWidgetPattern.exec(passage.content)) !== null) {
-        const name = match[1]!;
-        const body = match[2]!;
-        if (/\{@children\}/.test(body)) {
-          registerBlockMacro(name);
-        }
-      }
-      blockWidgetPattern.lastIndex = 0;
-    }
-  }
-
   // Run StoryInit, restore the session if the page was refreshed, and fire
   // storyinit after all state is settled (defaults + StoryInit + session)
   initializeStory(loadSession(storyData.ifid));
 
+  // Every other passage's markup, now that StoryInit may have defined macros
+  const markup = markupErrors(storyData, (p) => p.name !== 'StoryInit');
+  if (markup.length > 0) stopWithErrors(markup);
+
   // Pass 2: Full parse and register widgets from passages tagged "widget"
   for (const [, passage] of storyData.passages) {
     if (passage.tags.includes('widget')) {
-      const widgetTokens = tokenize(passage.content);
-      const widgetAST = buildAST(widgetTokens);
+      const widgetAST = parseMarkup(passage.content);
       for (const node of widgetAST) {
         if (node.type === 'macro' && node.name === 'widget' && node.rawArgs) {
           // Read and registered as the {widget} macro does

package/src/interpolation.ts

@@ -3,7 +3,7 @@
  * and macro labels resolve the inline markup of passage text (variables,
  * expressions, macros, widgets) to a string.
  *
- * Such a value is parsed with the passage tokenizer in text mode, so `{…}`
+ * Such a value is parsed with the passage parser in text mode, so `{…}`
  * means the same as in passage text, and its AST is evaluated directly to a
  * string rather than rendered: the result is ready during the render that
  * needs it (so it goes through the usual attribute handling, boolean
@@ -17,11 +17,10 @@ import { evaluate } from './expression';
 import {
   SCOPE_SIGILS,
   SIGIL_SCOPES,
-  tokenize,
   type VariableScope,
-} from './markup/tokenizer';
+} from './markup/tokens';
 import type { Sigil } from './js-lexer';
-import { buildAST } from './markup/ast';
+import { parseMarkup } from './markup/parse';
 import type { ASTNode, MacroNode, TextNode } from './markup/ast';
 import { getMacro, getMacroText, isSubMacro } from './registry';
 import type { MacroTextContext } from './registry';
@@ -76,7 +75,7 @@ export function parseText(template: string): ParsedText {
   let parsed = parseCache.get(template);
   if (parsed === undefined) {
     try {
-      parsed = { nodes: buildAST(tokenize(template, { text: true })) };
+      parsed = { nodes: parseMarkup(template, { text: true }) };
     } catch (error) {
       parsed = { error };
     }

package/src/markup/ast.ts

@@ -1,11 +1,8 @@
-import {
-  withSelectors,
-  type HtmlToken,
-  type MacroToken,
-  type Selectors,
-  type Token,
-  type VariableScope,
-} from './tokenizer';
+/**
+ * The passage AST (built by parse.ts), and the registry of block macros:
+ * those whose body the parser nests up to their `{/name}`.
+ */
+import type { Selectors, VariableScope } from './tokens';
 import { NameSet } from '../utils/macro-names';
 
 export interface TextNode {
@@ -69,7 +66,12 @@ const BLOCK_MACROS = new NameSet([
   'nobr',
 ]);
 
-/** Register a custom macro as a block macro so the AST builder nests children. */
+/** Whether a macro takes a body closed by `{/name}`. */
+export function isBlockMacro(name: string): boolean {
+  return BLOCK_MACROS.has(name);
+}
+
+/** Register a custom macro as a block macro so the parser nests children. */
 export function registerBlockMacro(name: string): void {
   BLOCK_MACROS.add(name);
 }
@@ -78,220 +80,3 @@ export function registerBlockMacro(name: string): void {
 export function unregisterBlockMacro(name: string): void {
   BLOCK_MACROS.delete(name);
 }
-
-/** Map from branch macro name → required parent macro name */
-const BRANCH_PARENT: Record<string, string> = {
-  elseif: 'if',
-  else: 'if',
-  case: 'switch',
-  default: 'switch',
-  next: 'timed',
-};
-
-/** Block macros that use the branches[] array */
-const BRANCHING_BLOCK_MACROS = new Set(['if', 'switch', 'timed']);
-
-/**
- * Quote a bracket-link value as a {link} argument, escaping backslashes and
- * double quotes so MacroLink reads the value back unchanged (#200).
- */
-function quoteArg(value: string): string {
-  return `"${value.replace(/[\\"]/g, '\\$&')}"`;
-}
-
-/** A new branch for the macro token opening it, with its selectors. */
-function newBranch(token: MacroToken): Branch {
-  return withSelectors<Branch>({ rawArgs: token.rawArgs, children: [] }, token);
-}
-
-/** A macro or HTML element, or a token opening or closing one. */
-type Tagged = MacroNode | HtmlNode | MacroToken | HtmlToken;
-
-/** The name of a macro or HTML element. */
-function tagName(tagged: Tagged): string {
-  return tagged.type === 'html' ? tagged.tag : tagged.name;
-}
-
-/** The tag closing a macro or HTML element, as written in markup. */
-function closerOf(tagged: Tagged): string {
-  return tagged.type === 'html' ? `</${tagged.tag}>` : `{/${tagged.name}}`;
-}
-
-/**
- * Build an AST from a token array. Block macros are nested into trees
- * using a stack. Throws on unclosed or mismatched macros.
- */
-export function buildAST(tokens: Token[]): ASTNode[] {
-  const root: ASTNode[] = [];
-
-  // Stack entries: the node being built and its token start position
-  const stack: { node: MacroNode | HtmlNode; start: number }[] = [];
-
-  function current(): ASTNode[] {
-    if (stack.length === 0) return root;
-    const top = stack[stack.length - 1]!.node;
-    // For if-blocks, append to the last branch's children
-    if (top.type === 'macro' && top.branches && top.branches.length > 0) {
-      return top.branches[top.branches.length - 1]!.children;
-    }
-    return top.children;
-  }
-
-  /**
-   * Pop the node on top of the stack, which the closing `token` (`</b>`,
-   * `{/if}`) must close. Names match ignoring case.
-   */
-  function close(token: MacroToken | HtmlToken) {
-    const found = closerOf(token);
-    if (stack.length === 0) {
-      throw new Error(
-        `Unexpected closing ${found} (at character ${token.start})`,
-      );
-    }
-
-    const top = stack[stack.length - 1]!;
-    if (
-      top.node.type !== token.type ||
-      tagName(top.node).toLowerCase() !== tagName(token).toLowerCase()
-    ) {
-      throw new Error(
-        `Expected ${closerOf(top.node)} but found ${found} (at character ${token.start})`,
-      );
-    }
-
-    stack.pop();
-    current().push(top.node);
-  }
-
-  for (const token of tokens) {
-    switch (token.type) {
-      case 'text':
-        current().push({ type: 'text', value: token.value });
-        break;
-
-      case 'link': {
-        const rawArgs = `${quoteArg(token.display)} ${quoteArg(token.target)}`;
-        current().push(
-          withSelectors<MacroNode>(
-            { type: 'macro', name: 'link', rawArgs, children: [] },
-            token,
-          ),
-        );
-        break;
-      }
-
-      case 'variable':
-        current().push(
-          withSelectors<VariableNode>(
-            { type: 'variable', name: token.name, scope: token.scope },
-            token,
-          ),
-        );
-        break;
-
-      case 'expression':
-        current().push(
-          withSelectors<ExpressionNode>(
-            { type: 'expression', expression: token.expression },
-            token,
-          ),
-        );
-        break;
-
-      case 'html': {
-        const htmlNode: HtmlNode = {
-          type: 'html',
-          tag: token.tag,
-          attributes: token.attributes,
-          children: [],
-        };
-        if (token.isSelfClose) {
-          // Self-closing HTML tag (br, hr, img, etc.)
-          current().push(htmlNode);
-        } else if (token.isClose) {
-          // Closing HTML tag — pop from stack
-          close(token);
-        } else {
-          // Opening HTML tag — push onto stack
-          stack.push({ node: htmlNode, start: token.start });
-        }
-        break;
-      }
-
-      case 'macro': {
-        if (token.isClose) {
-          // Closing tag — pop from stack
-          close(token);
-          break;
-        }
-
-        // Normalize macro name to lowercase — registration is lowercase,
-        // but the tokenizer preserves original casing from passage markup.
-        const name = token.name.toLowerCase();
-
-        // Handle branch macros (elseif/else, case/default, next)
-        const expectedParent = Object.prototype.hasOwnProperty.call(
-          BRANCH_PARENT,
-          name,
-        )
-          ? BRANCH_PARENT[name]
-          : undefined;
-        if (expectedParent) {
-          const topNode =
-            stack.length > 0 ? stack[stack.length - 1]!.node : null;
-          if (
-            !topNode ||
-            topNode.type !== 'macro' ||
-            topNode.name !== expectedParent
-          ) {
-            throw new Error(
-              `{${token.name}} without matching {${expectedParent}} (at character ${token.start})`,
-            );
-          }
-
-          topNode.branches!.push(newBranch(token));
-          break;
-        }
-
-        const node: MacroNode = {
-          type: 'macro',
-          name,
-          rawArgs: token.rawArgs,
-          children: [],
-        };
-
-        if (!BLOCK_MACROS.has(name)) {
-          // Self-closing macro (set, print, etc.)
-          current().push(withSelectors(node, token));
-          break;
-        }
-
-        // Block macro — push onto stack. Branching blocks: className/id
-        // goes on the first branch, not the node
-        if (BRANCHING_BLOCK_MACROS.has(name))
-          node.branches = [newBranch(token)];
-        else withSelectors(node, token);
-        stack.push({ node, start: token.start });
-        break;
-      }
-
-      default: {
-        const _exhaustive: never = token;
-        throw new Error(`Unknown token type: ${(_exhaustive as Token).type}`);
-      }
-    }
-  }
-
-  if (stack.length > 0) {
-    const unclosed = stack[stack.length - 1]!;
-    const label =
-      unclosed.node.type === 'html'
-        ? `<${unclosed.node.tag}>`
-        : `{${unclosed.node.name}} macro`;
-    throw new Error(
-      `Unclosed ${label} (opened at character ${unclosed.start})`,
-    );
-  }
-
-  return root;
-}

package/src/markup/code-attributes.ts

@@ -1,4 +1,5 @@
-import { createScanMemo, isSigil, scanBalancedBrace } from './tokenizer';
+import { isSigil } from './tokens';
+import { defaultCodeEnd } from './code-end';
 
 /**
  * Attributes whose value is code with braces of its own, not text: event
@@ -29,7 +30,6 @@ export type SigilPart =
  */
 export function splitSigilTemplate(template: string): SigilPart[] {
   const parts: SigilPart[] = [];
-  const memo = createScanMemo();
   let text = '';
   let i = 0;
 
@@ -42,7 +42,7 @@ export function splitSigilTemplate(template: string): SigilPart[] {
     text += template.slice(i, brace);
     i = brace + 1;
     const end = isSigil(template[i])
-      ? scanBalancedBrace(template, i, memo)
+      ? defaultCodeEnd.closeBrace(template, i, i)
       : -1;
     if (end === -1) {
       text += '{';