@rohal12/spindle 0.53.0 → 0.55.0

package/dist/pkg/tooling.js

@@ -2,7 +2,9 @@ import { readFileSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 
-export { parseStoryVariables } from './story-variables.js';
+import { checkParameterTypes, validateStoryMarkup } from './story-variables.js';
+
+export { parseStoryVariables, formatDiagnostic } from './story-variables.js';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
@@ -22,8 +24,10 @@ for (const m of builtins) metadata.set(m.name.toLowerCase(), m);
  * Metadata-only defineMacro for tooling.
  * Captures macro metadata without creating Preact components.
  * LSP servers call this to register user-defined macros discovered in story scripts.
+ * Like Story.defineMacro(), it throws if a declared parameter has no type.
  */
 export function defineMacro(config) {
+  checkParameterTypes(config.name, config.parameters ?? []);
   const name = config.name.toLowerCase();
   metadata.set(name, {
     name: config.name,
@@ -44,3 +48,13 @@ export function defineMacro(config) {
 export function getMacroRegistry() {
   return Array.from(metadata.values());
 }
+
+/**
+ * Validate the markup of a story's passages as Spindle does when the story
+ * starts: malformed markup and unknown macros (checked against the built-in
+ * macros and those registered with defineMacro), with passage, line and
+ * column.
+ */
+export function validateMarkup(passages) {
+  return validateStoryMarkup(passages, metadata.values());
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rohal12/spindle",
-  "version": "0.53.0",
+  "version": "0.55.0",
   "type": "module",
   "description": "A Preact-based story format for Twine 2.",
   "license": "Unlicense",
@@ -67,6 +67,7 @@
     "prepare": "husky"
   },
   "dependencies": {
+    "acorn": "^8.19.0",
     "devalue": "^6",
     "immer": "^11.1.4",
     "micromark": "^4.0.2",
@@ -94,6 +95,7 @@
     "js-yaml": "^4.1.1",
     "jscpd": "5.4.0",
     "lint-staged": "^16.4.0",
+    "peggy": "5.1.0",
     "playwright": "^1.58.2",
     "prettier": "^3.8.1",
     "typescript": "^6.0.2",

package/src/code-check.ts

@@ -0,0 +1,314 @@
+/**
+ * The code and the markup-holding text in a passage's tokens, for the
+ * story-start check (markup/validate.ts): every piece of code a passage
+ * runs is parsed when the story starts, and each syntax error is reported
+ * with its line and column in the passage.
+ *
+ * Code is found where passages run it:
+ * - `{$…}` expressions, and `{do}` bodies (statements);
+ * - the conditions of `{if}`, `{elseif}` and `{case}`;
+ * - macro arguments whose declared parameter type is `expression`,
+ *   `statements` or `passage`, built-in and custom macros alike; the
+ *   condition and `run` action of `{watch}`, code in quoted strings;
+ * - the `{$…}` references in attributes holding code (`onclick`).
+ *
+ * Text that may hold markup is found in quoted labels (arguments of type
+ * `text` and `string`) and HTML attribute values; the check reads its
+ * markup in turn.
+ *
+ * Passage names written out are found in links (`[[Go->Hall]]`), in
+ * `passage` arguments that are one quoted string (`{goto "Hall"}`), in the
+ * passage of `{link}`, the `goto` and `dialog` actions of `{watch}` and the
+ * body of `{dialog}`; the check looks each one up.
+ *
+ * The arguments of macros that declare no parameters may be anything:
+ * they are not checked.
+ */
+import type { Token } from './markup/tokens';
+import { isCodeAttribute, splitSigilTemplate } from './markup/code-attributes';
+import { CodeSyntaxError, parseCode, type JsGoal } from './js-lexer';
+import type { ParsedCode } from './js-lexer';
+import type { ParameterDef } from './registry';
+import { parseMacroArgs } from './components/macros/macro-args';
+import { readWholeQuoted } from './components/macros/arg-utils';
+
+/** While a pass runs: the code it parsed so far, by goal and source. */
+let parses: Map<string, ParsedCode | CodeSyntaxError> | null = null;
+
+/**
+ * Run `fn` with a cache of parsed code, so that a pass parses each piece of
+ * code once however often it meets it.
+ */
+export function withParseCache<T>(fn: () => T): T {
+  const outer = parses;
+  parses = new Map();
+  try {
+    return fn();
+  } finally {
+    parses = outer;
+  }
+}
+
+/** `parseCode`, with the error it throws for code that is not well-formed. */
+export function parseOrError(
+  code: string,
+  goal: JsGoal,
+): ParsedCode | CodeSyntaxError {
+  const key = goal[0] + code;
+  let result = parses?.get(key);
+  if (result) return result;
+  try {
+    result = parseCode(code, goal);
+  } catch (error) {
+    if (!(error instanceof CodeSyntaxError)) throw error;
+    result = error;
+  }
+  parses?.set(key, result);
+  return result;
+}
+
+/** A piece of code in markup, at `offset` in it. */
+export interface CodePiece {
+  kind: 'code';
+  code: string;
+  offset: number;
+  goal: JsGoal;
+  /** The markup it is in, for the error: `{print $a +}`. */
+  label: string;
+  /** Whether it names a passage: a `passage` argument. */
+  passage?: boolean;
+}
+
+/** A passage name written out in markup, at `offset` in it. */
+export interface PassagePiece {
+  kind: 'passage';
+  name: string;
+  offset: number;
+  /** The markup it is in, for the error: `[[Go->Hall]]`. */
+  label: string;
+}
+
+/** Text in markup that may hold markup of its own, at `offset` in it. */
+export interface TextPiece {
+  kind: 'text';
+  text: string;
+  offset: number;
+  /** Where it is, for the error: `In the label of {button}: `. */
+  where: string;
+}
+
+/** Macros whose whole argument text is one expression: branch conditions. */
+const CONDITION_MACROS = new Set(['if', 'elseif', 'case']);
+
+/** Code inside the quoted strings of a macro's arguments, by parameter. */
+const CODE_IN_STRINGS: Record<string, Record<string, JsGoal>> = {
+  watch: { condition: 'expression', run: 'statements' },
+};
+
+/** The `string` parameters of a macro that name a passage. */
+const PASSAGE_STRINGS: Record<string, readonly string[]> = {
+  link: ['passage'],
+  watch: ['goto', 'dialog'],
+};
+
+/** Block macros whose body is the name of a passage. */
+const PASSAGE_BODIES = new Set(['dialog']);
+
+type Piece = CodePiece | TextPiece | PassagePiece;
+
+/** The goal of the code an argument of this type holds, if it is code. */
+function codeGoal(param: ParameterDef): JsGoal | undefined {
+  if (param.type === 'expression' || param.type === 'passage') {
+    return 'expression';
+  }
+  if (param.type === 'statements') return 'statements';
+  return undefined;
+}
+
+/**
+ * The code and the markup-holding text in `tokens`, the tokens of `src`.
+ * `parametersOf` gives the declared parameters of a macro, if it has any.
+ */
+export function* codeAndText(
+  src: string,
+  tokens: readonly Token[],
+  parametersOf: (macro: string) => readonly ParameterDef[] | undefined,
+): Generator<Piece> {
+  /** The index of the closing tag of the `name` macro opened at `t`. */
+  const closeOf = (t: number, name: string) =>
+    tokens.findIndex(
+      (c, k) =>
+        k > t &&
+        c.type === 'macro' &&
+        c.isClose &&
+        c.name.toLowerCase() === name,
+    );
+  for (let t = 0; t < tokens.length; t++) {
+    const token = tokens[t]!;
+    if (token.type === 'link') {
+      yield {
+        kind: 'passage',
+        name: token.target,
+        offset: locate(src, token.target, token.start),
+        label: src.slice(token.start, token.end),
+      };
+    } else if (token.type === 'expression') {
+      yield {
+        kind: 'code',
+        code: token.expression,
+        offset: token.end - 1 - token.expression.length,
+        goal: 'expression',
+        label: `{${token.expression}}`,
+      };
+    } else if (token.type === 'html') {
+      for (const [name, value] of Object.entries(token.attributes)) {
+        // The value is the source text between its quotes
+        const at = locate(src, value, token.start);
+        if (!isCodeAttribute(name)) {
+          yield {
+            kind: 'text',
+            text: value,
+            offset: at,
+            where: `In the ${name} attribute of <${token.tag}>: `,
+          };
+          continue;
+        }
+        let from = 0;
+        for (const part of splitSigilTemplate(value)) {
+          if (!('expr' in part)) continue;
+          const k = locate(value, part.expr, from);
+          from = k + part.expr.length;
+          yield {
+            kind: 'code',
+            code: part.expr,
+            offset: at + k,
+            goal: 'expression',
+            label: `{${part.expr}} in the ${name} attribute of <${token.tag}>`,
+          };
+        }
+      }
+    } else if (token.type === 'macro' && !token.isClose) {
+      const name = token.name.toLowerCase();
+      if (name === 'do') {
+        const close = closeOf(t, name);
+        if (close < 0) continue;
+        const body = src.slice(token.end, tokens[close]!.start);
+        yield {
+          kind: 'code',
+          code: body,
+          offset: token.end,
+          goal: 'statements',
+          label: '{do}',
+        };
+        t = close;
+        continue;
+      }
+      const args = token.rawArgs;
+      if (PASSAGE_BODIES.has(name)) {
+        // The body names the passage when it is only text
+        const close = closeOf(t, name);
+        const body = tokens.slice(t + 1, close);
+        if (close > t + 1 && body.every((b) => b.type === 'text')) {
+          const text = src.slice(token.end, tokens[close]!.start);
+          const passage = text.trim().replace(/^["']|["']$/g, '');
+          yield {
+            kind: 'passage',
+            name: passage,
+            offset: locate(src, passage, token.end),
+            label: src.slice(token.start, token.end),
+          };
+        }
+      }
+      if (!args) continue;
+      const label = `{${token.name} ${args}}`;
+      const argsAt = locate(src, args, token.start + 1);
+      if (CONDITION_MACROS.has(name)) {
+        yield {
+          kind: 'code',
+          code: args,
+          offset: argsAt,
+          goal: 'expression',
+          label,
+        };
+        continue;
+      }
+      const params = parametersOf(name);
+      if (params) yield* argPieces(args, argsAt, params, token.name, label);
+    }
+  }
+}
+
+/** The code and text in the arguments `args` (at `offset`) of `macro`. */
+function* argPieces(
+  args: string,
+  offset: number,
+  params: readonly ParameterDef[],
+  macro: string,
+  label: string,
+): Generator<Piece> {
+  const values = parseMacroArgs(args, params) as Record<string, unknown>;
+  const inStrings = CODE_IN_STRINGS[macro.toLowerCase()] ?? {};
+  const passageStrings = PASSAGE_STRINGS[macro.toLowerCase()] ?? [];
+  let cursor = 0;
+  /** The pieces of the parameters `list`, with their values in `from`. */
+  function* visit(
+    list: readonly ParameterDef[],
+    from: Record<string, unknown>,
+  ): Generator<Piece> {
+    for (const param of list) {
+      const value = from[param.name];
+      if (param.type === 'options' && param.parameters) {
+        yield* visit(
+          param.parameters,
+          (value ?? {}) as Record<string, unknown>,
+        );
+        continue;
+      }
+      if (typeof value !== 'string' || !value.trim()) continue;
+      const at = locate(args, value, cursor);
+      if (at >= cursor) cursor = at + value.length;
+      const goal = codeGoal(param) ?? inStrings[param.name];
+      if (goal) {
+        const piece: CodePiece = {
+          kind: 'code',
+          code: value,
+          offset: offset + at,
+          goal,
+          label,
+        };
+        if (param.type === 'passage') piece.passage = true;
+        yield piece;
+      }
+      // A passage written out: a quoted `passage` argument, or a string
+      const name =
+        param.type === 'passage'
+          ? readWholeQuoted(value.trim())
+          : passageStrings.includes(param.name)
+            ? value
+            : null;
+      if (name !== null) {
+        yield { kind: 'passage', name, offset: offset + at, label };
+      }
+      if (!goal && (param.type === 'text' || param.type === 'string')) {
+        yield {
+          kind: 'text',
+          text: value,
+          offset: offset + at,
+          where: `In the ${param.name} of {${macro}}: `,
+        };
+      }
+    }
+  }
+  yield* visit(params, values);
+}
+
+/**
+ * Index of `part` in `text` from `from` on, else anywhere, else `from`: the
+ * place to report an error in `part` at.
+ */
+function locate(text: string, part: string, from: number): number {
+  const at = text.indexOf(part, from);
+  if (at >= 0) return at;
+  const anywhere = text.indexOf(part);
+  return anywhere >= 0 ? anywhere : from;
+}

package/src/components/Passage.tsx

@@ -5,8 +5,8 @@ import {
   useRef,
   useState,
 } from 'preact/hooks';
-import { tokenize } from '../markup/tokenizer';
-import { buildAST, type ASTNode } from '../markup/ast';
+import { parseMarkup } from '../markup/parse';
+import type { ASTNode } from '../markup/ast';
 import { renderNodes, NobrContext } from '../markup/render';
 import { useStoryFields } from '../hooks/use-story-fields';
 import type { Passage as PassageData } from '../parser';
@@ -29,7 +29,7 @@ const astCache = new WeakMap<
 function parsePassage(passage: PassageData): ASTNode[] {
   const cached = astCache.get(passage);
   if (cached && cached.content === passage.content) return cached.ast;
-  const ast = buildAST(tokenize(passage.content));
+  const ast = parseMarkup(passage.content);
   astCache.set(passage, { content: passage.content, ast });
   return ast;
 }

package/src/components/PassageDialog.tsx

@@ -1,7 +1,6 @@
 import { createContext } from 'preact';
 import { useCallback, useLayoutEffect, useMemo, useRef } from 'preact/hooks';
-import { tokenize } from '../markup/tokenizer';
-import { buildAST } from '../markup/ast';
+import { parseMarkup } from '../markup/parse';
 import { renderNodes, NobrContext } from '../markup/render';
 import { useStoryFields } from '../hooks/use-story-fields';
 import { emitFromRender } from '../event-emitter';
@@ -53,8 +52,7 @@ export function PassageDialog({
       return <div class="error">Dialog: no content available</div>;
     }
     try {
-      const tokens = tokenize(markup);
-      const ast = buildAST(tokens);
+      const ast = parseMarkup(markup);
       const nodes = renderNodes(ast, nobr ? { nobr: true } : undefined);
       return nobr ? (
         <NobrContext.Provider value={true}>{nodes}</NobrContext.Provider>

package/src/components/StoryInterface.tsx

@@ -1,7 +1,6 @@
 import { useMemo } from 'preact/hooks';
 import { useStoryFields } from '../hooks/use-story-fields';
-import { tokenize } from '../markup/tokenizer';
-import { buildAST } from '../markup/ast';
+import { parseMarkup } from '../markup/parse';
 import { renderInlineNodes, NobrContext } from '../markup/render';
 import { errorMessage } from '../utils/error-message';
 
@@ -18,8 +17,7 @@ export function StoryInterface() {
 
   const rendered = useMemo(() => {
     try {
-      const tokens = tokenize(markup);
-      const ast = buildAST(tokens);
+      const ast = parseMarkup(markup);
       return <>{renderInlineNodes(ast)}</>;
     } catch (err) {
       return (

package/src/components/macros/Do.tsx

@@ -5,7 +5,7 @@ defineMacro({
   name: 'do',
   block: true,
   render({ children = [] }, ctx) {
-    // The tokenizer keeps the body verbatim as a single text node
+    // The parser keeps the body verbatim as a single text node
     const code = ctx.collectText(children);
 
     ctx.hooks.useLayoutEffect(() => {

package/src/components/macros/Goto.tsx

@@ -1,13 +1,15 @@
 import { useStoryStore } from '../../store';
 import { defineMacro } from '../../define-macro';
 import { evaluatePassageName } from './macro-args';
-import { logMacroError } from './MacroError';
+import { logMacroError, MacroError } from './MacroError';
 
 defineMacro({
   name: 'goto',
   merged: true,
-  parameters: [{ name: 'passage', type: 'expression', required: true }],
+  parameters: [{ name: 'passage', type: 'passage', required: true }],
   render(_props, ctx) {
+    // Boxed: anything can be thrown, including null and other falsy values
+    const [failure, setFailure] = ctx.hooks.useState<{ error: unknown }>();
     ctx.hooks.useLayoutEffect(() => {
       // Reported like an error in {do}: thrown out of an effect, it would
       // abort the rest of the render's effects (e.g. the session could not
@@ -15,14 +17,23 @@ defineMacro({
       // this passage's, taken before navigating away from it
       const location = ctx.sourceLocation();
       try {
-        useStoryStore
-          .getState()
-          .navigate(evaluatePassageName(ctx.args.passage, ctx.evaluate!));
-      } catch (err) {
-        logMacroError('goto', err, location);
+        const store = useStoryStore.getState();
+        store.navigate(
+          evaluatePassageName(ctx.args.passage, ctx.evaluate!, store),
+        );
+      } catch (error) {
+        logMacroError('goto', error, location);
+        // Shown in place, as other macros show their errors, unless it
+        // navigated away
+        setFailure({ error });
       }
     }, []);
 
-    return null;
+    return failure ? (
+      <MacroError
+        macro="goto"
+        error={failure.error}
+      />
+    ) : null;
   },
 });

package/src/components/macros/Include.tsx

@@ -1,10 +1,10 @@
 import { useStoryStore } from '../../store';
 import { useStoryFields } from '../../hooks/use-story-fields';
-import { tokenize } from '../../markup/tokenizer';
-import { buildAST } from '../../markup/ast';
+import { parseMarkup } from '../../markup/parse';
 import { NobrContext } from '../../markup/render';
 import { defineMacro } from '../../define-macro';
 import { evaluatePassageName } from './macro-args';
+import { MacroError } from './MacroError';
 
 defineMacro({
   name: 'include',
@@ -14,13 +14,23 @@ defineMacro({
   // passage names and expressions containing the word stay intact (#201).
   parameters: [
     { name: 'inline', type: 'flag' },
-    { name: 'passage', type: 'expression', required: true },
+    { name: 'passage', type: 'passage', required: true },
   ],
   render(_props, ctx) {
     const { storyData } = useStoryFields('storyData');
 
     const { inline } = ctx.args;
-    const passageName = evaluatePassageName(ctx.args.passage, ctx.evaluate!);
+    let passageName = '';
+    let failure: { error: unknown } | undefined;
+    try {
+      passageName = evaluatePassageName(
+        ctx.args.passage,
+        ctx.evaluate!,
+        useStoryStore.getState(),
+      );
+    } catch (error) {
+      failure = { error };
+    }
 
     const passage = storyData?.passages.get(passageName);
     // Parse once per included passage: the renderer keys children by AST
@@ -28,20 +38,22 @@ defineMacro({
     // included passage changes, while re-renders of the same passage keep
     // its macros mounted (#175).
     const ast = ctx.hooks.useMemo(
-      () => (passage ? buildAST(tokenize(passage.content)) : null),
+      () => (passage ? parseMarkup(passage.content) : null),
       [passage],
     );
 
     if (!storyData) return null;
-
-    if (passage) {
-      useStoryStore.getState().trackRender(passageName);
-    }
-    if (!passage || !ast) {
+    if (failure) {
       return (
-        <span class="error">{`{include${ctx.sourceLocation()}: passage "${passageName}" not found}`}</span>
+        <MacroError
+          macro="include"
+          error={failure.error}
+        />
       );
     }
+    if (!passage || !ast) return null;
+
+    useStoryStore.getState().trackRender(passageName);
 
     const nobr = passage.tags.includes('nobr');
     const content = inline

package/src/components/macros/Set.tsx

@@ -3,7 +3,7 @@ import { useRunOnce } from './MacroError';
 
 defineMacro({
   name: 'set',
-  parameters: [{ name: 'code', type: 'expression', required: true }],
+  parameters: [{ name: 'code', type: 'statements', required: true }],
   render({ rawArgs }, ctx) {
     return useRunOnce('set', rawArgs, () => ctx.mutate(rawArgs));
   },

package/src/components/macros/VarDisplay.tsx

@@ -3,7 +3,7 @@ import { useContext } from 'preact/hooks';
 import { LocalsValuesContext } from '../../markup/render';
 import { useInterpolate } from '../../hooks/use-interpolate';
 import { ownValue, variableNameError } from '../../utils/namespace';
-import { SCOPE_SIGILS } from '../../markup/tokenizer';
+import { SCOPE_SIGILS } from '../../markup/tokens';
 import type { VariableNode } from '../../markup/ast';
 import { display, wrapContent } from './display';
 

package/src/components/macros/Widget.tsx

@@ -4,39 +4,11 @@ import { registerBlockMacro, type ASTNode } from '../../markup/ast';
 import { defineMacro } from '../../define-macro';
 import { checkVariableName } from '../../utils/namespace';
 import { MacroError } from './MacroError';
-import { parseMacroArgs } from './macro-args';
-import type { MacroArgs } from '../../registry';
-
-/** A {widget} definition's name, then its `@` parameters. */
-const WIDGET_PARAMETERS = [
-  { name: 'name', type: 'text', required: true },
-  { name: 'parameters', type: 'expression' },
-] as const;
-
-interface WidgetDef {
-  name: string;
-  params: string[];
-}
-
-/**
- * The widget a {widget} definition's arguments declare: its name and its
- * parameters, the words after it that start with `@` (docs/widgets.md).
- * Other words, such as `$name`, are not parameters.
- */
-function widgetDef({
-  name = '',
-  parameters = '',
-}: MacroArgs<typeof WIDGET_PARAMETERS>): WidgetDef {
-  return {
-    name,
-    params: parameters.split(/\s+/).filter((word) => word.startsWith('@')),
-  };
-}
-
-/** Read the arguments of a {widget} definition (see widgetDef). */
-export function parseWidgetDef(rawArgs: string): WidgetDef {
-  return widgetDef(parseMacroArgs(rawArgs, WIDGET_PARAMETERS));
-}
+import {
+  WIDGET_PARAMETERS,
+  widgetDef,
+  type WidgetDef,
+} from '../../widgets/widget-def';
 
 /**
  * Register the widget a definition declares, with its body. Widgets whose

package/src/components/macros/macro-args.ts

@@ -16,6 +16,8 @@
  */
 import { parseDelay } from '../../utils/parse-delay';
 import type { MacroArgs, ParameterDef } from '../../registry';
+import type { StoryState } from '../../store';
+import { noPassageError } from '../../runtime-errors';
 import {
   endsWithOperator,
   isWhitespace,
@@ -177,8 +179,9 @@ function findSeparator(src: string, word: string): Span | null {
 }
 
 const takesRest = (param: ParameterDef) =>
-  param.type === undefined ||
   param.type === 'expression' ||
+  param.type === 'statements' ||
+  param.type === 'passage' ||
   param.type === 'text';
 
 /** Read the parameters of one group from `src` (`null`: no text) into `args`. */
@@ -262,18 +265,24 @@ export function parseMacroArgs<const P extends readonly ParameterDef[]>(
 }
 
 /**
- * The passage a passage-name argument names: the value of its expression
- * or, when that can't be evaluated (`{goto Bob's room}`), its text.
+ * The passage a `passage` argument names (`{goto "Hall"}`, `{goto $room}`):
+ * the value of its expression. Throws what evaluating it throws, and if no
+ * passage of the story `state` holds has that name, an error naming it and
+ * the current passage.
  */
 export function evaluatePassageName(
   expr: string | undefined,
   evaluate: (expr: string) => unknown,
+  {
+    storyData,
+    currentPassage,
+  }: Pick<StoryState, 'storyData' | 'currentPassage'>,
 ): string {
-  try {
-    return String(evaluate(expr ?? ''));
-  } catch {
-    return readText(expr ?? '');
+  const name = String(evaluate(expr ?? ''));
+  if (storyData && !storyData.passages.has(name)) {
+    throw noPassageError(name, currentPassage);
   }
+  return name;
 }
 
 /**