@rohal12/spindle 0.53.0 → 0.55.0

package/src/markup/ast.ts

@@ -1,11 +1,8 @@
-import {
-  withSelectors,
-  type HtmlToken,
-  type MacroToken,
-  type Selectors,
-  type Token,
-  type VariableScope,
-} from './tokenizer';
+/**
+ * The passage AST (built by parse.ts), and the registry of block macros:
+ * those whose body the parser nests up to their `{/name}`.
+ */
+import type { Selectors, VariableScope } from './tokens';
 import { NameSet } from '../utils/macro-names';
 
 export interface TextNode {
@@ -69,7 +66,12 @@ const BLOCK_MACROS = new NameSet([
   'nobr',
 ]);
 
-/** Register a custom macro as a block macro so the AST builder nests children. */
+/** Whether a macro takes a body closed by `{/name}`. */
+export function isBlockMacro(name: string): boolean {
+  return BLOCK_MACROS.has(name);
+}
+
+/** Register a custom macro as a block macro so the parser nests children. */
 export function registerBlockMacro(name: string): void {
   BLOCK_MACROS.add(name);
 }
@@ -78,220 +80,3 @@ export function registerBlockMacro(name: string): void {
 export function unregisterBlockMacro(name: string): void {
   BLOCK_MACROS.delete(name);
 }
-
-/** Map from branch macro name → required parent macro name */
-const BRANCH_PARENT: Record<string, string> = {
-  elseif: 'if',
-  else: 'if',
-  case: 'switch',
-  default: 'switch',
-  next: 'timed',
-};
-
-/** Block macros that use the branches[] array */
-const BRANCHING_BLOCK_MACROS = new Set(['if', 'switch', 'timed']);
-
-/**
- * Quote a bracket-link value as a {link} argument, escaping backslashes and
- * double quotes so MacroLink reads the value back unchanged (#200).
- */
-function quoteArg(value: string): string {
-  return `"${value.replace(/[\\"]/g, '\\$&')}"`;
-}
-
-/** A new branch for the macro token opening it, with its selectors. */
-function newBranch(token: MacroToken): Branch {
-  return withSelectors<Branch>({ rawArgs: token.rawArgs, children: [] }, token);
-}
-
-/** A macro or HTML element, or a token opening or closing one. */
-type Tagged = MacroNode | HtmlNode | MacroToken | HtmlToken;
-
-/** The name of a macro or HTML element. */
-function tagName(tagged: Tagged): string {
-  return tagged.type === 'html' ? tagged.tag : tagged.name;
-}
-
-/** The tag closing a macro or HTML element, as written in markup. */
-function closerOf(tagged: Tagged): string {
-  return tagged.type === 'html' ? `</${tagged.tag}>` : `{/${tagged.name}}`;
-}
-
-/**
- * Build an AST from a token array. Block macros are nested into trees
- * using a stack. Throws on unclosed or mismatched macros.
- */
-export function buildAST(tokens: Token[]): ASTNode[] {
-  const root: ASTNode[] = [];
-
-  // Stack entries: the node being built and its token start position
-  const stack: { node: MacroNode | HtmlNode; start: number }[] = [];
-
-  function current(): ASTNode[] {
-    if (stack.length === 0) return root;
-    const top = stack[stack.length - 1]!.node;
-    // For if-blocks, append to the last branch's children
-    if (top.type === 'macro' && top.branches && top.branches.length > 0) {
-      return top.branches[top.branches.length - 1]!.children;
-    }
-    return top.children;
-  }
-
-  /**
-   * Pop the node on top of the stack, which the closing `token` (`</b>`,
-   * `{/if}`) must close. Names match ignoring case.
-   */
-  function close(token: MacroToken | HtmlToken) {
-    const found = closerOf(token);
-    if (stack.length === 0) {
-      throw new Error(
-        `Unexpected closing ${found} (at character ${token.start})`,
-      );
-    }
-
-    const top = stack[stack.length - 1]!;
-    if (
-      top.node.type !== token.type ||
-      tagName(top.node).toLowerCase() !== tagName(token).toLowerCase()
-    ) {
-      throw new Error(
-        `Expected ${closerOf(top.node)} but found ${found} (at character ${token.start})`,
-      );
-    }
-
-    stack.pop();
-    current().push(top.node);
-  }
-
-  for (const token of tokens) {
-    switch (token.type) {
-      case 'text':
-        current().push({ type: 'text', value: token.value });
-        break;
-
-      case 'link': {
-        const rawArgs = `${quoteArg(token.display)} ${quoteArg(token.target)}`;
-        current().push(
-          withSelectors<MacroNode>(
-            { type: 'macro', name: 'link', rawArgs, children: [] },
-            token,
-          ),
-        );
-        break;
-      }
-
-      case 'variable':
-        current().push(
-          withSelectors<VariableNode>(
-            { type: 'variable', name: token.name, scope: token.scope },
-            token,
-          ),
-        );
-        break;
-
-      case 'expression':
-        current().push(
-          withSelectors<ExpressionNode>(
-            { type: 'expression', expression: token.expression },
-            token,
-          ),
-        );
-        break;
-
-      case 'html': {
-        const htmlNode: HtmlNode = {
-          type: 'html',
-          tag: token.tag,
-          attributes: token.attributes,
-          children: [],
-        };
-        if (token.isSelfClose) {
-          // Self-closing HTML tag (br, hr, img, etc.)
-          current().push(htmlNode);
-        } else if (token.isClose) {
-          // Closing HTML tag — pop from stack
-          close(token);
-        } else {
-          // Opening HTML tag — push onto stack
-          stack.push({ node: htmlNode, start: token.start });
-        }
-        break;
-      }
-
-      case 'macro': {
-        if (token.isClose) {
-          // Closing tag — pop from stack
-          close(token);
-          break;
-        }
-
-        // Normalize macro name to lowercase — registration is lowercase,
-        // but the tokenizer preserves original casing from passage markup.
-        const name = token.name.toLowerCase();
-
-        // Handle branch macros (elseif/else, case/default, next)
-        const expectedParent = Object.prototype.hasOwnProperty.call(
-          BRANCH_PARENT,
-          name,
-        )
-          ? BRANCH_PARENT[name]
-          : undefined;
-        if (expectedParent) {
-          const topNode =
-            stack.length > 0 ? stack[stack.length - 1]!.node : null;
-          if (
-            !topNode ||
-            topNode.type !== 'macro' ||
-            topNode.name !== expectedParent
-          ) {
-            throw new Error(
-              `{${token.name}} without matching {${expectedParent}} (at character ${token.start})`,
-            );
-          }
-
-          topNode.branches!.push(newBranch(token));
-          break;
-        }
-
-        const node: MacroNode = {
-          type: 'macro',
-          name,
-          rawArgs: token.rawArgs,
-          children: [],
-        };
-
-        if (!BLOCK_MACROS.has(name)) {
-          // Self-closing macro (set, print, etc.)
-          current().push(withSelectors(node, token));
-          break;
-        }
-
-        // Block macro — push onto stack. Branching blocks: className/id
-        // goes on the first branch, not the node
-        if (BRANCHING_BLOCK_MACROS.has(name))
-          node.branches = [newBranch(token)];
-        else withSelectors(node, token);
-        stack.push({ node, start: token.start });
-        break;
-      }
-
-      default: {
-        const _exhaustive: never = token;
-        throw new Error(`Unknown token type: ${(_exhaustive as Token).type}`);
-      }
-    }
-  }
-
-  if (stack.length > 0) {
-    const unclosed = stack[stack.length - 1]!;
-    const label =
-      unclosed.node.type === 'html'
-        ? `<${unclosed.node.tag}>`
-        : `{${unclosed.node.name}} macro`;
-    throw new Error(
-      `Unclosed ${label} (opened at character ${unclosed.start})`,
-    );
-  }
-
-  return root;
-}

package/src/markup/code-attributes.ts

@@ -1,4 +1,5 @@
-import { createScanMemo, isSigil, scanBalancedBrace } from './tokenizer';
+import { isSigil } from './tokens';
+import { defaultCodeEnd } from './code-end';
 
 /**
  * Attributes whose value is code with braces of its own, not text: event
@@ -29,7 +30,6 @@ export type SigilPart =
  */
 export function splitSigilTemplate(template: string): SigilPart[] {
   const parts: SigilPart[] = [];
-  const memo = createScanMemo();
   let text = '';
   let i = 0;
 
@@ -42,7 +42,7 @@ export function splitSigilTemplate(template: string): SigilPart[] {
     text += template.slice(i, brace);
     i = brace + 1;
     const end = isSigil(template[i])
-      ? scanBalancedBrace(template, i, memo)
+      ? defaultCodeEnd.closeBrace(template, i, i)
       : -1;
     if (end === -1) {
       text += '{';

package/src/markup/code-end.ts

@@ -0,0 +1,121 @@
+import { findCodeEnd } from '../js-lexer';
+
+/**
+ * Where the JavaScript inside markup ends. The grammar (spindle.peggy) leaves
+ * this to these two functions, so the JavaScript reader can be swapped (for
+ * example for acorn's `parseExpressionAt`) without touching the grammar.
+ */
+export interface CodeEnd {
+  /**
+   * Index of the `}` ending the code that starts at `codeStart` (macro
+   * arguments, a `{$…}` expression), or -1 if there is none. Code that is
+   * not well-formed JavaScript is read leniently from `lenientStart`.
+   */
+  closeBrace(input: string, codeStart: number, lenientStart: number): number;
+  /**
+   * Index of the `{` of the `{/name}` ending the raw JavaScript body (of
+   * `{do}`) that starts at `bodyStart`, or -1 if there is none.
+   */
+  rawBodyEnd(input: string, bodyStart: number, name: string): number;
+}
+
+/**
+ * A quote directly after a letter/digit is an apostrophe (don't), not a
+ * string; after a backslash it is an escaped attribute delimiter (\").
+ */
+const NON_STRING_QUOTE_PREFIX = /[\p{L}\p{N}_\\]/u;
+
+/**
+ * Skip the literal opening at `i`: a '…' or "…" string, which must close on
+ * its line, or a `…` template, whose `${…}` parts are read as braces.
+ * Returns the index just past its closing quote or backtick, or -1.
+ */
+function skipLiteral(input: string, i: number): number {
+  const quote = input[i];
+  const template = quote === '`';
+  for (let j = i + 1; j < input.length; ) {
+    const c = input[j];
+    if (c === '\\') j += 2;
+    else if (c === quote) return j + 1;
+    else if (!template && c === '\n') return -1;
+    else if (template && c === '$' && input[j + 1] === '{') {
+      const close = lenientClose(input, j + 2);
+      if (close === -1) return -1;
+      j = close + 1;
+    } else j++;
+  }
+  return -1;
+}
+
+/**
+ * The closing `}` of braces read leniently from `i` (just past a `{`): braces
+ * count except inside string and template literals, and a quote that can't
+ * start a string (an apostrophe, or one not closed on its line) is text, as
+ * is a backtick without a closing one.
+ */
+export function lenientClose(input: string, i: number): number {
+  // Results per start, for the last input: unclosed nested template
+  // literals (`` {$a`${$a`${… ``) would otherwise take exponential time.
+  if (memoInput !== input) {
+    memoInput = input;
+    memo.clear();
+  }
+  let end = memo.get(i);
+  if (end === undefined) {
+    end = scanLenient(input, i);
+    memo.set(i, end);
+  }
+  return end;
+}
+
+let memoInput = '';
+const memo = new Map<number, number>();
+
+function scanLenient(input: string, i: number): number {
+  let depth = 0;
+  while (i < input.length) {
+    const c = input[i]!;
+    if (c === '{') {
+      depth++;
+      i++;
+    } else if (c === '}') {
+      if (depth === 0) return i;
+      depth--;
+      i++;
+    } else if (
+      c === '`' ||
+      ((c === '"' || c === "'") &&
+        !(i > 0 && NON_STRING_QUOTE_PREFIX.test(input[i - 1]!)))
+    ) {
+      const close = skipLiteral(input, i);
+      i = close === -1 ? i + 1 : close;
+    } else {
+      i++;
+    }
+  }
+  return -1;
+}
+
+/** The JavaScript reader Spindle uses today: js-lexer, then a lenient scan. */
+export const defaultCodeEnd: CodeEnd = {
+  closeBrace(input, codeStart, lenientStart) {
+    const end = findCodeEnd(input, codeStart);
+    return end !== -1 ? end : lenientClose(input, lenientStart);
+  },
+  rawBodyEnd(input, bodyStart, name) {
+    const closer = `\\{/${name}\\s*\\}`;
+    const first = new RegExp(closer, 'gi');
+    first.lastIndex = bodyStart;
+    const firstAt = first.exec(input)?.index ?? -1;
+    if (firstAt === -1) return -1;
+    const at = new RegExp(closer, 'iy');
+    const end = findCodeEnd(input, bodyStart, {
+      goal: 'statements',
+      stop: (k) => {
+        at.lastIndex = k;
+        return at.test(input);
+      },
+    });
+    return end === -1 ? firstAt : end;
+  },
+};

package/src/markup/parse.ts

@@ -0,0 +1,118 @@
+/**
+ * Passage markup parser: the Peggy grammar (spindle.peggy) with Spindle's
+ * hooks plugged in. It builds the AST directly and reports malformed markup
+ * as a MarkupError carrying its line and column.
+ */
+import { parse as pegParse } from './spindle.peggy';
+import { isBlockMacro, type ASTNode } from './ast';
+import type { Token } from './tokens';
+import { isCodeAttribute } from './code-attributes';
+import { defaultCodeEnd, type CodeEnd } from './code-end';
+
+/** Macros whose body is JavaScript source, kept verbatim. */
+const RAW_BODY_MACROS = new Set(['do']);
+
+/** What the grammar leaves to code. */
+export interface MarkupHooks extends CodeEnd {
+  /** Whether a macro takes a body closed by `{/name}`. */
+  isBlock(name: string): boolean;
+  /** Whether a macro's body is JavaScript, kept verbatim (`{do}`). */
+  isRaw(name: string): boolean;
+  /** Whether an attribute's value is code (`onclick`), not markup. */
+  isCodeAttribute(name: string): boolean;
+}
+
+const defaultHooks: MarkupHooks = {
+  ...defaultCodeEnd,
+  isBlock: isBlockMacro,
+  isRaw: (name) => RAW_BODY_MACROS.has(name),
+  isCodeAttribute,
+};
+
+/** Malformed markup, with where it starts (1-based line and column). */
+export class MarkupError extends Error {
+  constructor(
+    /** What is wrong, without the position. */
+    readonly reason: string,
+    /** Where it is, from the start of the markup (0-based). */
+    readonly offset: number,
+    readonly line: number,
+    readonly column: number,
+  ) {
+    super(`${reason} (line ${line}, column ${column})`);
+    this.name = 'MarkupError';
+  }
+}
+
+export interface ParseMarkupOptions {
+  /**
+   * Text mode, for markup that becomes a string (HTML attribute values,
+   * macro labels): only `{…}` markup and brace escapes are recognized, while
+   * `[[` and `<` are text. With no markdown to pair up the backslashes of a
+   * run before a brace, they are paired up here: `\\{` is one backslash
+   * before a live brace, `\\\{` one before a literal one.
+   */
+  text?: boolean;
+  /** Replace some hooks, e.g. which macros take a body. */
+  hooks?: Partial<MarkupHooks>;
+}
+
+/** 1-based line and column of `offset` in `text`. */
+export function lineColumn(
+  text: string,
+  offset: number,
+): { line: number; column: number } {
+  let line = 1;
+  let lineStart = 0;
+  for (let i = text.indexOf('\n'); i !== -1 && i < offset; ) {
+    line++;
+    lineStart = i + 1;
+    i = text.indexOf('\n', lineStart);
+  }
+  return { line, column: offset - lineStart + 1 };
+}
+
+function run(
+  source: string,
+  startRule: 'Markup' | 'Tokens',
+  options: ParseMarkupOptions,
+): unknown {
+  const hooks = options.hooks
+    ? { ...defaultHooks, ...options.hooks }
+    : defaultHooks;
+  try {
+    return pegParse(source, { startRule, text: options.text === true, hooks });
+  } catch (err) {
+    const location = (err as { location?: { start: { offset: number } } })
+      .location;
+    if (err instanceof Error && err.name === 'SyntaxError' && location) {
+      const { offset } = location.start;
+      const { line, column } = lineColumn(source, offset);
+      throw new MarkupError(err.message, offset, line, column);
+    }
+    throw err;
+  }
+}
+
+/**
+ * Parse markup into its AST: macros with their bodies and branches, HTML
+ * elements with their children. Throws a MarkupError for malformed markup.
+ */
+export function parseMarkup(
+  source: string,
+  options: ParseMarkupOptions = {},
+): ASTNode[] {
+  return run(source, 'Markup', options) as ASTNode[];
+}
+
+/**
+ * The flat tokens of markup, without nesting, so unclosed or mismatched
+ * macros and elements are no error here. Throws a MarkupError for a
+ * malformed tag, such as an unclosed `{`, `[[` or attribute value.
+ */
+export function tokenizeMarkup(
+  source: string,
+  options: ParseMarkupOptions = {},
+): Token[] {
+  return run(source, 'Tokens', options) as Token[];
+}

package/src/markup/render.tsx

@@ -297,7 +297,7 @@ function convertDomNode(
     }
 
     // Convert attributes, as author HTML's (see splitAttributes): markdown
-    // output holds raw HTML the passage tokenizer didn't take as a tag
+    // output holds raw HTML from text nodes (a custom macro's, or comments)
     const svg = el.namespaceURI === SVG_NAMESPACE;
     const attributes: Attribute[] = [];
     let placeholders: Record<string, ASTNode[]> | undefined;

package/src/markup/spindle.d.peggy.ts

@@ -0,0 +1,24 @@
+// Types of the parser spindle.peggy compiles to (see scripts/peggy.ts).
+
+/** A position in the input; only `offset` is used by parse.ts. */
+export interface Location {
+  offset: number;
+  line: number;
+  column: number;
+}
+
+/** What the generated parser throws for malformed markup. */
+export declare class SyntaxError extends globalThis.SyntaxError {
+  location: { start: Location; end: Location };
+}
+
+export interface ParseOptions {
+  startRule: 'Markup' | 'Tokens';
+  /** Text mode (attribute values, labels). */
+  text: boolean;
+  /** What the grammar leaves to code (see parse.ts MarkupHooks). */
+  hooks: unknown;
+}
+
+/** Parse markup: the AST for `Markup`, the tokens for `Tokens`. */
+export declare function parse(input: string, options: ParseOptions): unknown;