@rohal12/spindle 0.51.3 → 0.52.0

package/src/markup/tokenizer.ts

@@ -1,3 +1,6 @@
+import { createJsScanCache, findCodeEnd, type JsScanCache } from '../js-lexer';
+import { isCodeAttribute } from './code-attributes';
+
 export interface TextToken {
   type: 'text';
   value: string;
@@ -84,6 +87,17 @@ const HTML_VOID_TAGS = new Set([
   'wbr',
 ]);
 
+/** Variable sigils: story ($), temporary (_), local (@), transient (%). */
+const SIGIL_CHARS = new Set(['$', '_', '@', '%']);
+
+/**
+ * Characters other than a sigil that open an expression after `{`:
+ * `{(Math.max($a, 0))}`, `{!$done}`. Other characters that can start an
+ * expression (quotes, digits, `[`, `-`) are left out, because braces around
+ * them are common as literal text (JSON, regex quantifiers, `{[[link]]}`).
+ */
+const EXPRESSION_START = new Set(['(', '!']);
+
 /** Macros whose body is JavaScript source, kept verbatim instead of tokenized. */
 const RAW_BODY_MACROS = new Set(['do']);
 
@@ -172,11 +186,8 @@ function parseSelectors(
       if (/[a-zA-Z0-9_-]/.test(input[i]!)) {
         name += input[i];
         i++;
-      } else if (
-        input[i] === '{' &&
-        (input[i + 1] === '$' || input[i + 1] === '_' || input[i + 1] === '@')
-      ) {
-        // Consume interpolation: {$var}, {_var}, {@var} (with optional dot paths)
+      } else if (input[i] === '{' && SIGIL_CHARS.has(input[i + 1]!)) {
+        // Consume interpolation: {$var}, {_var}, {@var}, {%var} (with optional dot paths)
         const braceStart = i;
         i += 2; // skip { and prefix
         while (i < input.length && /[\w.]/.test(input[i]!)) i++;
@@ -211,12 +222,69 @@ function parseSelectors(
 function parseHtmlAttributes(
   input: string,
   j: number,
+  memo: ScanMemo,
 ): { attributes: Record<string, string>; endIdx: number } {
   const attributes: Record<string, string> = {};
+  // As in HTML, the first of attributes with the same (case-insensitive)
+  // name wins. Defined as own properties so `__proto__` is kept too.
+  const seen = new Set<string>();
+  const endIdx = scanAttributes(input, j, memo, (name, value) => {
+    const lower = name.toLowerCase();
+    if (seen.has(lower)) return;
+    seen.add(lower);
+    Object.defineProperty(attributes, name, {
+      value,
+      enumerable: true,
+      writable: true,
+      configurable: true,
+    });
+  });
+  return { attributes, endIdx };
+}
 
+/**
+ * Index just past the attributes of a tag that start at `j` (after its
+ * name), where its `>` or `/>` goes. Pass the same memo to repeated scans of
+ * one input to share their work.
+ */
+export function scanTagAttributes(
+  input: string,
+  j: number,
+  memo: ScanMemo = createScanMemo(),
+): number {
+  return scanAttributes(input, j, memo);
+}
+
+/**
+ * Scan the attributes of a tag from position j, handing each to `add` when
+ * given. Returns the position after the last attribute.
+ *
+ * Where the attributes end depends only on where an attribute starts, so
+ * without `add` the result is recorded for every attribute start passed (in
+ * `memo.tag`) and a recorded one is used instead of scanning on. An unquoted
+ * value takes in a `<`, so in `<a x=<a x=<a x=…` each opener's scan would
+ * otherwise run over all the openers after it, taking quadratic time. A tag
+ * whose attributes end in its `>` is consumed, so collecting the attributes
+ * of those (with `add`, unrecorded) reads each character once.
+ */
+function scanAttributes(
+  input: string,
+  j: number,
+  memo: ScanMemo,
+  add?: (name: string, value: string) => void,
+): number {
+  const passed: number[] = [];
   while (j < input.length) {
     // Skip whitespace
     while (j < input.length && /\s/.test(input[j]!)) j++;
+    if (!add) {
+      const known = memo.tag.get(j);
+      if (known !== undefined) {
+        j = known;
+        break;
+      }
+      passed.push(j);
+    }
     // End of tag?
     if (
       j >= input.length ||
@@ -231,39 +299,267 @@ function parseHtmlAttributes(
     const attrName = input.slice(attrStart, j);
     if (!attrName) break;
 
-    // Check for = value
-    if (input[j] === '=') {
-      j++; // skip =
+    // Check for = value. HTML allows whitespace on either side of the =
+    // (`id = "x"`); whitespace not followed by = ends a boolean attribute.
+    let eqIdx = j;
+    while (eqIdx < input.length && /\s/.test(input[eqIdx]!)) eqIdx++;
+    if (input[eqIdx] === '=') {
+      j = eqIdx + 1; // skip =
+      while (j < input.length && /\s/.test(input[j]!)) j++;
       if (input[j] === '"' || input[j] === "'") {
         const quote = input[j]!;
         j++; // skip opening quote
         const valStart = j;
-        while (j < input.length) {
-          if (input[j] === '{') {
-            // Skip a whole {…} interpolation so quotes inside it don't end the value
-            const closeIdx = scanBalancedBrace(input, j + 1);
-            if (closeIdx !== -1) {
-              j = closeIdx + 1;
-              continue;
-            }
-          } else if (input[j] === quote) break;
-          j++;
-        }
-        attributes[attrName] = input.slice(valStart, j);
+        j = scanQuotedValue(input, j, quote, isCodeAttribute(attrName), memo);
+        add?.(attrName, input.slice(valStart, j));
         if (j < input.length) j++; // skip closing quote
       } else {
-        // Unquoted value
+        // Unquoted value, up to whitespace or `>`. It takes in a `<`, so in
+        // `<a:=<a:=<a:=…` it holds every tag after it: the last run of such
+        // characters found is kept, as each tag's value ends where it does.
         const valStart = j;
-        while (j < input.length && /[^\s>]/.test(input[j]!)) j++;
-        attributes[attrName] = input.slice(valStart, j);
+        const run = memo.unquoted;
+        if (j < run.from || j > run.to) {
+          run.from = j;
+          while (j < input.length && /[^\s>]/.test(input[j]!)) j++;
+          run.to = j;
+        }
+        j = run.to;
+        add?.(attrName, input.slice(valStart, j));
       }
     } else {
       // Boolean attribute
-      attributes[attrName] = '';
+      add?.(attrName, '');
     }
   }
+  for (const at of passed) memo.tag.set(at, j);
+  return j;
+}
+
+/**
+ * Index of the quote ending the attribute value that starts at `j`, or the
+ * end of the input. A `{…}` interpolation in it is skipped whole, so quotes
+ * inside it don't end the value. A code attribute's value is no markup
+ * (`isCodeAttribute`): only `{` and a sigil open a reference, other braces
+ * and backslashes are text, as `splitSigilTemplate` reads them.
+ *
+ * The value reads the same from just past an interpolation, however the scan
+ * got there, so the end is recorded there (in `memo.value`), and a recorded
+ * one is used. An unclosed value that skips interpolations to the end of
+ * the input (`<a x="}<a x={<a x="}…`) is then read once, not once per tag.
+ */
+function scanQuotedValue(
+  input: string,
+  j: number,
+  quote: string,
+  code: boolean,
+  memo: ScanMemo,
+): number {
+  const variant = (quote === '"' ? 0 : 2) + (code ? 1 : 0);
+  const passed: number[] = [];
+  let checkpoint = true;
+  while (j < input.length) {
+    if (checkpoint) {
+      const known = memo.value.get(j * 4 + variant);
+      if (known !== undefined) {
+        j = known;
+        break;
+      }
+      passed.push(j * 4 + variant);
+      checkpoint = false;
+    }
+    if (!code && input[j] === '\\') {
+      // A brace after an odd backslash run is escaped (`\{`) and opens
+      // no interpolation, as in passage text.
+      let k = j + 1;
+      while (input[k] === '\\') k++;
+      const brace = input[k] === '{' || input[k] === '}';
+      j = brace && (k - j) % 2 === 1 ? k + 1 : k;
+      continue;
+    }
+    if (input[j] === '{') {
+      const closeIdx = !code
+        ? scanBlockClose(input, j, memo)
+        : SIGIL_CHARS.has(input[j + 1]!)
+          ? scanBalancedBrace(input, j + 1, memo)
+          : -1;
+      if (closeIdx !== -1) {
+        j = closeIdx + 1;
+        checkpoint = true;
+        continue;
+      }
+    } else if (input[j] === quote) break;
+    j++;
+  }
+  for (const at of passed) memo.value.set(at, j);
+  return j;
+}
+
+/**
+ * Results of the brace and template scans of one input, shared by repeated
+ * scans of it. A scan's result depends only on the input and where it
+ * starts, so caching it is exact. Without the cache, each unclosed template
+ * literal was scanned once as a template and again as plain text, so nested
+ * unclosed ones (`` {$a`${$a`${… ``) took time exponential in their depth;
+ * and each unclosed `{` was scanned to the end of the input, so many of them
+ * took quadratic time. A memo must only be reused for scans of the same input
+ * string.
+ */
+export interface ScanMemo {
+  /** `scanBalancedBrace` results, by lenient start. */
+  code: Map<number, number>;
+  /** JavaScript lexer results. */
+  js: JsScanCache;
+  /** Lenient brace scan results. */
+  brace: Map<number, number>;
+  /** Lenient template literal scan results. */
+  template: Map<number, number>;
+  /** Lenient template literal scan results, by a point in its text. */
+  templateText: Map<number, number>;
+  /** The last macro name run found: no whitespace or } in [from, to). */
+  name: { from: number; to: number };
+  /** The last unquoted attribute value run: no whitespace or > in [from, to). */
+  unquoted: { from: number; to: number };
+  /** Link scan results (`scanLinkClose`). */
+  link: Map<number, number>;
+  /** Where the attributes of a tag end, by attribute start (`scanAttributes`). */
+  tag: Map<number, number>;
+  /** Where quoted attribute values end, by checkpoint (`scanQuotedValue`). */
+  value: Map<number, number>;
+  /**
+   * The last search for a raw-body closer, by macro name: the first one
+   * from `from` on is at `at` (-1 for none).
+   */
+  rawClose: Map<string, { from: number; at: number }>;
+}
+
+export function createScanMemo(): ScanMemo {
+  return {
+    code: new Map(),
+    js: createJsScanCache(),
+    brace: new Map(),
+    template: new Map(),
+    templateText: new Map(),
+    name: { from: 0, to: -1 },
+    unquoted: { from: 0, to: -1 },
+    link: new Map(),
+    tag: new Map(),
+    value: new Map(),
+    rawClose: new Map(),
+  };
+}
+
+/**
+ * Find the } closing the code that starts at position i: a `{$…}`
+ * expression from its sigil on, an attribute interpolation. The code is
+ * lexed as JavaScript, so only a } in code outside the brackets it opened
+ * counts — not one in a string, template or regex literal or a comment —
+ * and `/` after an operand divides.
+ *
+ * Text that is not well-formed JavaScript (an apostrophe, an unterminated
+ * literal or comment, unbalanced brackets) is scanned leniently instead, as
+ * prose-like macro arguments are: braces count except inside string and
+ * template literals, and a quote that can't start a string (apostrophe,
+ * escaped, not closed on its line) is text.
+ *
+ * Returns the index of the closing } or -1 if there is none. Pass the same
+ * memo to repeated scans of one input to share their work.
+ */
+export function scanBalancedBrace(
+  input: string,
+  i: number,
+  memo: ScanMemo = createScanMemo(),
+): number {
+  return scanClose(input, i, i, memo);
+}
+
+/**
+ * Index of the } closing the macro whose content (name, then arguments)
+ * starts at `contentStart`, or -1. The name runs up to whitespace or the }
+ * (as `parseMacroContent` reads it); the arguments after it are code. The
+ * lenient scan covers the whole content, as it always has.
+ */
+function scanMacroClose(
+  input: string,
+  contentStart: number,
+  memo: ScanMemo,
+): number {
+  const run = memo.name;
+  if (contentStart < run.from || contentStart > run.to) {
+    let k = contentStart;
+    while (k < input.length && input[k] !== '}' && !/\s/.test(input[k]!)) k++;
+    run.from = contentStart;
+    run.to = k;
+  }
+  return scanClose(input, run.to, contentStart, memo);
+}
+
+/**
+ * Index of the } closing the `{…}` block opened at `open`, read as passage
+ * text reads it: a macro (`{name args}`, `{/name}`) has its arguments lexed
+ * after its name, an expression (`{$…}`, `{(…)}`, `{!…}`) from its first
+ * character, after any `.class#id` selectors. Any other block is scanned as
+ * code from just past the {. Returns -1 if it is unclosed.
+ */
+function scanBlockClose(input: string, open: number, memo: ScanMemo): number {
+  let at = open + 1;
+  const c = input[at];
+  if (c === '.' || c === '#') {
+    at = parseSelectors(input, at).endIdx;
+    if (input[at] === ' ') at++;
+  }
+  const first = input[at];
+  if (first !== undefined && (first === '/' || /[a-zA-Z]/.test(first))) {
+    return scanMacroClose(input, at, memo);
+  }
+  return scanBalancedBrace(input, at, memo);
+}
 
-  return { attributes, endIdx: j };
+/** Lex the code from `codeStart`, else scan leniently from `lenientStart`. */
+function scanClose(
+  input: string,
+  codeStart: number,
+  lenientStart: number,
+  memo: ScanMemo,
+): number {
+  let end = memo.code.get(lenientStart);
+  if (end === undefined) {
+    end = findCodeEnd(input, codeStart, { cache: memo.js });
+    if (end === -1) end = scanBraceLenient(input, lenientStart, memo);
+    memo.code.set(lenientStart, end);
+  }
+  return end;
+}
+
+/**
+ * Index of the ]] closing the link whose text starts at `from`, or -1. A
+ * [[ inside opens a nested pair. Each nested [[ starts a scan that reads the
+ * rest the same way, so its result is recorded too: unclosed [[s don't each
+ * scan to the end of the input.
+ */
+function scanLinkClose(input: string, from: number, memo: ScanMemo): number {
+  const known = memo.link.get(from);
+  if (known !== undefined) return known;
+  const opens = [from];
+  let i = from;
+  while (i < input.length) {
+    if (input[i] === '[' && input[i + 1] === '[') {
+      i += 2;
+      const inner = memo.link.get(i);
+      if (inner === undefined) opens.push(i);
+      else if (inner === -1)
+        break; // nor does this one close
+      else i = inner + 2;
+    } else if (input[i] === ']' && input[i + 1] === ']') {
+      memo.link.set(opens.pop()!, i);
+      if (!opens.length) return i;
+      i += 2;
+    } else {
+      i++;
+    }
+  }
+  for (const open of opens) memo.link.set(open, -1);
+  return -1;
 }
 
 /**
@@ -284,76 +580,188 @@ function skipQuoted(input: string, i: number): number {
   return -1;
 }
 
-/**
- * Skip a `…` template literal opening at i, including ${…} parts.
- * Returns the index just past the closing backtick, or -1 if unclosed.
- */
-function skipTemplate(input: string, i: number): number {
-  let j = i + 1;
-  while (j < input.length) {
-    const c = input[j];
-    if (c === '\\') {
-      j += 2;
-    } else if (c === '`') {
-      return j + 1;
-    } else if (c === '$' && input[j + 1] === '{') {
-      const closeIdx = scanBalancedBrace(input, j + 2);
-      if (closeIdx === -1) return -1;
-      j = closeIdx + 1;
-    } else {
-      j++;
-    }
-  }
-  return -1;
-}
-
 /**
  * A quote directly after a letter/digit is an apostrophe (don't), not a
  * string; after a backslash it is an escaped attribute delimiter (\").
  */
 const NON_STRING_QUOTE_PREFIX = /[\p{L}\p{N}_\\]/u;
 
+/**
+ * A brace or template literal scan in progress: braces from `start` (just
+ * past a {) to their closing }, or a template literal from its backtick at
+ * `start` to the closing one.
+ */
+interface LenientScan {
+  template: boolean;
+  start: number;
+  /**
+   * For braces, per brace depth from the outermost: the checkpoints at that
+   * depth whose scans end where the depth does.
+   */
+  levels: number[][];
+  /**
+   * For a template literal, the points in its text it passed (just past its
+   * backtick, an escape or an interpolation), whose scans end where it does.
+   */
+  passed?: number[];
+}
+
 /**
  * Scan for the balanced closing } starting at position i (just past the {).
  * Braces inside string and template literals are ignored. A quote that
- * can't start a string (apostrophe, escaped, unterminated) counts as text.
+ * can't start a string (apostrophe, escaped, unterminated) counts as text,
+ * and so does a backtick without a closing one.
  * Returns the index of the closing } or -1 if unbalanced.
+ *
+ * Template literals and their ${…} parts are scans on a stack, not
+ * recursive calls, so deep nesting can't overflow the call stack. How the
+ * text reads depends only on where a scan is, so a scan passing a point —
+ * just past a {, a string or a template literal — goes on as a scan
+ * starting there would: its result there is recorded too, and a recorded
+ * result is used instead of scanning on. Scans from many starts in one
+ * input then take about linear time.
  */
-export function scanBalancedBrace(input: string, i: number): number {
-  let depth = 1;
-  while (i < input.length) {
-    const c = input[i]!;
-    if (c === '{') {
-      depth++;
-    } else if (c === '}') {
-      if (--depth === 0) return i;
-    } else if (
-      (c === '"' || c === "'") &&
-      !(i > 0 && NON_STRING_QUOTE_PREFIX.test(input[i - 1]!))
-    ) {
-      const end = skipQuoted(input, i);
-      if (end !== -1) {
-        i = end;
+function scanBraceLenient(input: string, i: number, memo: ScanMemo): number {
+  const known = memo.brace.get(i);
+  if (known !== undefined) return known;
+  const stack: LenientScan[] = [{ template: false, start: i, levels: [[]] }];
+  for (;;) {
+    const scan = stack[stack.length - 1]!;
+    let end: number | undefined; // set when `scan` is done
+    if (scan.template) {
+      // Template text reads the same from such a point, however the scan got
+      // there: in `` `\`\`\`… `` each backtick a scan from an earlier one
+      // reads as escaped starts a template that reads the same rest.
+      let point = true;
+      while (end === undefined && i < input.length) {
+        if (point) {
+          const known = memo.templateText.get(i);
+          if (known !== undefined) {
+            end = known;
+            break;
+          }
+          scan.passed!.push(i);
+          point = false;
+        }
+        const c = input[i];
+        if (c === '\\') {
+          i += 2;
+          point = true;
+        } else if (c === '`') {
+          end = i + 1;
+        } else if (c === '$' && input[i + 1] === '{') {
+          const inner = memo.brace.get(i + 2);
+          if (inner === undefined) break; // scan the ${…} first
+          if (inner === -1) end = -1;
+          else {
+            i = inner + 1;
+            point = true;
+          }
+        } else {
+          i++;
+        }
+      }
+      if (end === undefined && i < input.length) {
+        stack.push({ template: false, start: i + 2, levels: [[]] });
+        i += 2;
         continue;
       }
-    } else if (c === '`') {
-      const end = skipTemplate(input, i);
-      if (end !== -1) {
-        i = end;
+      end ??= -1;
+    } else {
+      const { levels } = scan;
+      /** The } at `close` ends the innermost level. */
+      const closeLevel = (close: number) => {
+        for (const at of levels.pop()!) memo.brace.set(at, close);
+        if (!levels.length) end = close;
+      };
+      while (end === undefined && i < input.length) {
+        const c = input[i]!;
+        let at = -1; // a checkpoint, if one starts here
+        if (c === '{') {
+          levels.push([]);
+          at = ++i;
+        } else if (c === '}') {
+          closeLevel(i++);
+        } else if (
+          (c === '"' || c === "'") &&
+          !(i > 0 && NON_STRING_QUOTE_PREFIX.test(input[i - 1]!))
+        ) {
+          const close = skipQuoted(input, i);
+          if (close === -1) i++;
+          else at = i = close;
+        } else if (c === '`') {
+          const close = memo.template.get(i);
+          if (close === undefined) break; // scan the template first
+          if (close === -1) i++;
+          else at = i = close;
+        } else {
+          i++;
+        }
+        if (at === -1) continue;
+        const known = memo.brace.get(at);
+        if (known === undefined) {
+          levels[levels.length - 1]!.push(at);
+        } else if (known === -1) {
+          // The innermost level never closes, so neither do the others
+          i = input.length;
+        } else {
+          closeLevel(known);
+          i = known + 1;
+        }
+      }
+      if (end === undefined && i < input.length) {
+        stack.push({ template: true, start: i, levels: [], passed: [] });
+        i++;
         continue;
       }
+      if (end === undefined) {
+        end = -1;
+        for (const level of levels) {
+          for (const at of level) memo.brace.set(at, -1);
+        }
+      }
+    }
+    // `scan` is done: hand its result to the scan that started it
+    stack.pop();
+    (scan.template ? memo.template : memo.brace).set(scan.start, end);
+    for (const at of scan.passed ?? []) memo.templateText.set(at, end);
+    const parent = stack[stack.length - 1];
+    if (!parent) return end;
+    if (parent.template) {
+      // An unclosed ${…} leaves the template unclosed: go on to its end
+      i = end === -1 ? input.length : end + 1;
+    } else {
+      // An unclosed template's backtick is text
+      i = end === -1 ? scan.start + 1 : end;
     }
-    i++;
   }
-  return -1;
+}
+
+/**
+ * Options for {@link tokenize}.
+ */
+export interface TokenizeOptions {
+  /**
+   * Text mode, for markup that becomes a string (HTML attribute values,
+   * macro labels): only `{…}` markup and brace escapes are recognized, while
+   * `[[` and `<` are text. With no markdown to pair up the backslashes of a
+   * run before a brace, they are paired up here: `\\{` is one backslash
+   * before a live brace, `\\\{` one before a literal one.
+   */
+  text?: boolean;
 }
 
 /**
  * Single-pass tokenizer for Twine passage content.
  * Recognizes: [[links]], {$variable}, {_temporary}, {macroName args}
  */
-export function tokenize(input: string): Token[] {
+export function tokenize(
+  input: string,
+  options: TokenizeOptions = {},
+): Token[] {
+  const textMode = options.text === true;
   const tokens: Token[] = [];
+  const memo = createScanMemo();
   let i = 0;
   let textStart = 0;
 
@@ -369,20 +777,43 @@ export function tokenize(input: string): Token[] {
   }
 
   /**
-   * After an opening raw-body macro ({do}), emit everything up to the
-   * first {/name} as a single text token so JavaScript source is not
-   * parsed as markup. A literal "{/do}" inside the code would end it early.
-   * Without a closer, nothing is consumed and the AST builder reports it.
+   * After an opening raw-body macro ({do}), emit everything up to its
+   * {/name} as a single text token so JavaScript source is not parsed as
+   * markup. The body is lexed as JavaScript statements: a {/do} in code ends
+   * it, one inside a string, template or regex literal or a comment does
+   * not. If the body is not well-formed JavaScript up to a {/do} in code,
+   * the first {/do} ends it. Without a closer, nothing is consumed and the
+   * AST builder reports it.
    */
   function consumeRawBody(name: string, isClose: boolean) {
     const lower = name.toLowerCase();
     if (isClose || !RAW_BODY_MACROS.has(lower)) return;
-    const closeRe = new RegExp(`\\{/${lower}\\s*\\}`, 'gi');
-    closeRe.lastIndex = i;
-    const m = closeRe.exec(input);
-    if (!m) return;
-    const closeStart = m.index;
-    const closeEnd = closeStart + m[0].length;
+    const closer = `\\{/${lower}\\s*\\}`;
+    // The first closer from here on. Without one from `from` on there is
+    // none later either, so many unclosed {do}s don't each search the rest.
+    let last = memo.rawClose.get(lower);
+    if (!last || i < last.from || (last.at >= 0 && i > last.at)) {
+      const firstRe = new RegExp(closer, 'gi');
+      firstRe.lastIndex = i;
+      last = { from: i, at: firstRe.exec(input)?.index ?? -1 };
+      memo.rawClose.set(lower, last);
+    }
+    const first = last.at;
+    if (first === -1) return;
+    const atRe = new RegExp(closer, 'iy');
+    const closerAt = (k: number) => {
+      atRe.lastIndex = k;
+      return atRe.test(input);
+    };
+    let closeStart = findCodeEnd(input, i, {
+      goal: 'statements',
+      stop: closerAt,
+      stopKey: lower,
+      cache: memo.js,
+    });
+    if (closeStart === -1) closeStart = first;
+    atRe.lastIndex = closeStart;
+    const closeEnd = closeStart + atRe.exec(input)![0].length;
     flushText(closeStart);
     tokens.push({
       type: 'macro',
@@ -394,18 +825,65 @@ export function tokenize(input: string): Token[] {
     textStart = closeEnd;
   }
 
+  /**
+   * Push an expression token for the `{…}` block opened at `start` whose
+   * expression starts at `exprStart`, flushing the text before it first
+   * when `flush`. Returns false, consuming nothing, if the block is unclosed.
+   */
+  function pushExpression(
+    exprStart: number,
+    start: number,
+    flush: boolean,
+    className?: string,
+    id?: string,
+  ): boolean {
+    const closeIdx = scanBalancedBrace(input, exprStart, memo);
+    if (closeIdx === -1) return false;
+    if (flush) flushText(start);
+    const token: ExpressionToken = {
+      type: 'expression',
+      expression: input.slice(exprStart, closeIdx),
+      start,
+      end: closeIdx + 1,
+    };
+    if (className) token.className = className;
+    if (id) token.id = id;
+    tokens.push(token);
+    i = textStart = closeIdx + 1;
+    return true;
+  }
+
   while (i < input.length) {
-    // Handle escaped braces: \{ and \}
-    if (input[i] === '\\' && (input[i + 1] === '{' || input[i + 1] === '}')) {
-      flushText(i);
-      tokens.push({ type: 'text', value: input[i + 1]!, start: i, end: i + 2 });
-      i += 2;
-      textStart = i;
+    // Escaped braces: \{ and \}. Count the whole backslash run so \\{ is a
+    // backslash pair before a live brace. In an odd run the last backslash
+    // escapes the brace; the even rest stays text, which markdown collapses
+    // pair by pair like any other \\ in the passage.
+    if (input[i] === '\\') {
+      let k = i + 1;
+      while (input[k] === '\\') k++;
+      const next = input[k];
+      if (textMode && (next === '{' || next === '}')) {
+        const escaped = (k - i) % 2 === 1;
+        const end = escaped ? k + 1 : k;
+        const value = '\\'.repeat((k - i) >> 1) + (escaped ? next : '');
+        flushText(i);
+        if (value) tokens.push({ type: 'text', value, start: i, end });
+        i = textStart = end;
+        continue;
+      }
+      if ((next === '{' || next === '}') && (k - i) % 2 === 1) {
+        flushText(k - 1);
+        tokens.push({ type: 'text', value: next, start: k - 1, end: k + 1 });
+        i = k + 1;
+        textStart = i;
+        continue;
+      }
+      i = k;
       continue;
     }
 
     // Check for [[ link
-    if (input[i] === '[' && input[i + 1] === '[') {
+    if (!textMode && input[i] === '[' && input[i + 1] === '[') {
       flushText(i);
       const start = i;
       i += 2;
@@ -423,30 +901,18 @@ export function tokenize(input: string): Token[] {
       }
 
       // Find closing ]]
-      let depth = 1;
       const innerStart = i;
-      while (i < input.length && depth > 0) {
-        if (input[i] === '[' && input[i + 1] === '[') {
-          depth++;
-          i += 2;
-        } else if (input[i] === ']' && input[i + 1] === ']') {
-          depth--;
-          if (depth === 0) break;
-          i += 2;
-        } else {
-          i++;
-        }
-      }
+      const closeIdx = scanLinkClose(input, innerStart, memo);
 
-      if (depth !== 0) {
+      if (closeIdx === -1) {
         // Unclosed link — treat as text
         i = start + 2;
         textStart = start;
         continue;
       }
 
-      const inner = input.slice(innerStart, i);
-      i += 2; // skip ]]
+      const inner = input.slice(innerStart, closeIdx);
+      i = closeIdx + 2; // skip ]]
 
       const { display, target } = parseLink(inner);
       const linkToken: LinkToken = {
@@ -504,7 +970,7 @@ export function tokenize(input: string): Token[] {
             continue;
           }
           // Complex expression — scan for balanced closing }
-          const closeIdx$ = scanBalancedBrace(input, nameStart);
+          const closeIdx$ = scanBalancedBrace(input, nameStart - 1, memo);
           if (closeIdx$ !== -1) {
             const expression = input.slice(afterSelectors, closeIdx$);
             i = closeIdx$ + 1;
@@ -549,7 +1015,7 @@ export function tokenize(input: string): Token[] {
             continue;
           }
           // Complex expression — scan for balanced closing }
-          const closeIdx_ = scanBalancedBrace(input, nameStart);
+          const closeIdx_ = scanBalancedBrace(input, nameStart - 1, memo);
           if (closeIdx_ !== -1) {
             const expression = input.slice(afterSelectors, closeIdx_);
             i = closeIdx_ + 1;
@@ -594,7 +1060,7 @@ export function tokenize(input: string): Token[] {
             continue;
           }
           // Complex expression — scan for balanced closing }
-          const closeIdx_at = scanBalancedBrace(input, nameStart);
+          const closeIdx_at = scanBalancedBrace(input, nameStart - 1, memo);
           if (closeIdx_at !== -1) {
             const expression = input.slice(afterSelectors, closeIdx_at);
             i = closeIdx_at + 1;
@@ -639,7 +1105,7 @@ export function tokenize(input: string): Token[] {
             continue;
           }
           // Complex expression — scan for balanced closing }
-          const closeIdx_pct = scanBalancedBrace(input, nameStart);
+          const closeIdx_pct = scanBalancedBrace(input, nameStart - 1, memo);
           if (closeIdx_pct !== -1) {
             const expression = input.slice(afterSelectors, closeIdx_pct);
             i = closeIdx_pct + 1;
@@ -661,11 +1127,18 @@ export function tokenize(input: string): Token[] {
           continue;
         }
 
+        if (
+          EXPRESSION_START.has(charAfter!) &&
+          pushExpression(afterSelectors, start, false, className, id)
+        ) {
+          continue;
+        }
+
         if (charAfter !== undefined && /[a-zA-Z]/.test(charAfter)) {
           // {.class#id macroName args}
           // Scan to closing }, tracking brace nesting and string literals
           const contentStart = afterSelectors;
-          const closeIdx = scanBalancedBrace(input, contentStart);
+          const closeIdx = scanMacroClose(input, contentStart, memo);
 
           if (closeIdx === -1) {
             i = start + 1;
@@ -720,7 +1193,7 @@ export function tokenize(input: string): Token[] {
           continue;
         }
         // Complex expression — scan for balanced closing }
-        const closeIdx = scanBalancedBrace(input, nameStart);
+        const closeIdx = scanBalancedBrace(input, nameStart - 1, memo);
         if (closeIdx !== -1) {
           const expression = input.slice(start + 1, closeIdx);
           i = closeIdx + 1;
@@ -760,7 +1233,7 @@ export function tokenize(input: string): Token[] {
           continue;
         }
         // Complex expression — scan for balanced closing }
-        const closeIdx = scanBalancedBrace(input, nameStart);
+        const closeIdx = scanBalancedBrace(input, nameStart - 1, memo);
         if (closeIdx !== -1) {
           const expression = input.slice(start + 1, closeIdx);
           i = closeIdx + 1;
@@ -800,7 +1273,7 @@ export function tokenize(input: string): Token[] {
           continue;
         }
         // Complex expression — scan for balanced closing }
-        const closeIdx = scanBalancedBrace(input, nameStart);
+        const closeIdx = scanBalancedBrace(input, nameStart - 1, memo);
         if (closeIdx !== -1) {
           const expression = input.slice(start + 1, closeIdx);
           i = closeIdx + 1;
@@ -840,7 +1313,7 @@ export function tokenize(input: string): Token[] {
           continue;
         }
         // Complex expression — scan for balanced closing }
-        const closeIdx = scanBalancedBrace(input, nameStart);
+        const closeIdx = scanBalancedBrace(input, nameStart - 1, memo);
         if (closeIdx !== -1) {
           const expression = input.slice(start + 1, closeIdx);
           i = closeIdx + 1;
@@ -859,6 +1332,13 @@ export function tokenize(input: string): Token[] {
         continue;
       }
 
+      // {(expr)} or {!expr}: an expression that doesn't start with a variable
+      if (EXPRESSION_START.has(nextChar!)) {
+        if (pushExpression(i + 1, start, true)) continue;
+        i++;
+        continue;
+      }
+
       // {macro ...} or {/macro} — but not bare { that's just text
       // Must start with a letter or /
       if (
@@ -870,7 +1350,7 @@ export function tokenize(input: string): Token[] {
         // Scan to closing }, tracking brace nesting (object literals)
         // and string literals
         const contentStart = i + 1;
-        const closeIdx = scanBalancedBrace(input, contentStart);
+        const closeIdx = scanMacroClose(input, contentStart, memo);
 
         if (closeIdx === -1) {
           // Unclosed macro — treat as text
@@ -902,7 +1382,7 @@ export function tokenize(input: string): Token[] {
     }
 
     // Check for < — HTML tag
-    if (input[i] === '<') {
+    if (!textMode && input[i] === '<') {
       const start = i;
       let j = i + 1;
 
@@ -944,9 +1424,10 @@ export function tokenize(input: string): Token[] {
             continue;
           }
         } else {
-          // Opening or self-closing tag: parse attributes
-          const parsed = parseHtmlAttributes(input, j);
-          j = parsed.endIdx;
+          // Opening or self-closing tag: find where its attributes end, and
+          // read them only if the tag closes there
+          const attrsStart = j;
+          j = scanAttributes(input, attrsStart, memo);
 
           let isSelfClose = HTML_VOID_TAGS.has(tagLower);
           if (input[j] === '/') {
@@ -960,7 +1441,8 @@ export function tokenize(input: string): Token[] {
             tokens.push({
               type: 'html',
               tag,
-              attributes: parsed.attributes,
+              attributes: parseHtmlAttributes(input, attrsStart, memo)
+                .attributes,
               isClose: false,
               isSelfClose,
               start,