@rohal12/spindle 0.57.0 → 0.58.0

package/dist/pkg/tooling.js

@@ -2,9 +2,36 @@ import { readFileSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 
-import { checkParameterTypes, validateStoryMarkup } from './story-variables.js';
+import {
+  checkParameterTypes,
+  collectStoryPassageReferences,
+  validateStoryMarkup,
+} from './story-variables.js';
 
-export { parseStoryVariables, formatDiagnostic } from './story-variables.js';
+export {
+  parseStoryVariables,
+  formatDiagnostic,
+  findCodeEnd,
+  lexJs,
+  lexTemplate,
+  scanStringLiteral,
+  MarkupError,
+  parseSelectors,
+  tokenizeMarkup,
+  tokenizeMarkupTolerant,
+  transform,
+  passageTarget,
+  evaluatePassageName,
+  SIGIL_SCOPES,
+  isSigil,
+  endsWithOperator,
+  readQuoted,
+  splitArgs,
+  splitTopLevel,
+  stripLooseQuotes,
+  unescapeQuoted,
+  splitIncludeFlag,
+} from './story-variables.js';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
@@ -58,3 +85,13 @@ export function getMacroRegistry() {
 export function validateMarkup(passages, options) {
   return validateStoryMarkup(passages, metadata.values(), options);
 }
+
+/**
+ * The passages the markup of `source` names, with where each is written:
+ * `[[…]]` links and the passage of `{goto}`, `{include}`, `{link}`,
+ * `{watch}` and `{dialog}`, as the macros registered here declare them.
+ * Malformed tags are skipped, so half-typed markup reads.
+ */
+export function collectPassageReferences(source) {
+  return collectStoryPassageReferences(source, metadata.values());
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rohal12/spindle",
-  "version": "0.57.0",
+  "version": "0.58.0",
   "type": "module",
   "description": "A Preact-based story format for Twine 2.",
   "license": "Unlicense",

package/src/code-check.ts

@@ -32,8 +32,9 @@ import type { ParameterDef } from './registry';
 import {
   MacroArgumentError,
   parseMacroArgs,
+  passageTarget,
+  type PassageTarget,
 } from './components/macros/macro-args';
-import { readWholeQuoted } from './components/macros/arg-utils';
 
 /** While a pass runs: the code it parsed so far, by goal and source. */
 let parses: Map<string, ParsedCode | CodeSyntaxError> | null = null;
@@ -80,6 +81,10 @@ export interface CodePiece {
   label: string;
   /** Whether it names a passage: a `passage` argument. */
   passage?: boolean;
+  /** Whether it is the code in a quoted string, as in `{watch}`. */
+  inString?: boolean;
+  /** The macro it is the argument of, for a `passage` argument. */
+  macro?: string;
 }
 
 /** A passage name written out in markup, at `offset` in it. */
@@ -89,6 +94,10 @@ export interface PassagePiece {
   offset: number;
   /** The markup it is in, for the error: `[[Go->Hall]]`. */
   label: string;
+  /** How much of the markup is the name, as written. */
+  length: number;
+  /** The macro it is the argument of; `link` for `[[…]]` links. */
+  macro: string;
 }
 
 /** Macro arguments that don't have their parameters' forms. */
@@ -113,14 +122,24 @@ export interface TextPiece {
 const CONDITION_MACROS = new Set(['if', 'elseif', 'case']);
 
 /** Code inside the quoted strings of a macro's arguments, by parameter. */
-const CODE_IN_STRINGS: Record<string, Record<string, JsGoal>> = {
-  watch: { condition: 'expression', run: 'statements' },
-};
+const CODE_IN_STRINGS: ReadonlyMap<string, Record<string, JsGoal>> = new Map([
+  ['watch', { condition: 'expression', run: 'statements' }],
+]);
 
 /** The `string` parameters of a macro that name a passage. */
-const PASSAGE_STRINGS: Record<string, readonly string[]> = {
-  watch: ['goto', 'dialog'],
-};
+const PASSAGE_STRINGS: ReadonlyMap<string, readonly string[]> = new Map([
+  ['watch', ['goto', 'dialog']],
+]);
+
+/**
+ * The `text` and `string` parameters of a macro that the runtime keeps as
+ * written, not as markup: an option's value, the name of a watcher.
+ */
+const LITERAL_STRINGS: ReadonlyMap<string, readonly string[]> = new Map([
+  ['option', ['value']],
+  ['watch', ['name']],
+  ['unwatch', ['name']],
+]);
 
 /** Block macros whose body is the name of a passage. */
 const PASSAGE_BODIES = new Set(['dialog']);
@@ -161,6 +180,8 @@ export function* codeAndText(
         kind: 'passage',
         name: token.target,
         offset: locate(src, token.target, token.start),
+        length: token.target.length,
+        macro: 'link',
         label: src.slice(token.start, token.end),
       };
     } else if (token.type === 'expression') {
@@ -226,6 +247,8 @@ export function* codeAndText(
             kind: 'passage',
             name: passage,
             offset: locate(src, passage, token.end),
+            length: passage.length,
+            macro: name,
             label: src.slice(token.start, token.end),
           };
         }
@@ -250,7 +273,7 @@ export function* codeAndText(
 }
 
 /** The code and text in the arguments `args` (at `offset`) of `macro`. */
-function* argPieces(
+export function* argPieces(
   args: string,
   offset: number,
   params: readonly ParameterDef[],
@@ -265,8 +288,10 @@ function* argPieces(
     yield { kind: 'argument-error', message: error.message, offset, label };
     return;
   }
-  const inStrings = CODE_IN_STRINGS[macro.toLowerCase()] ?? {};
-  const passageStrings = PASSAGE_STRINGS[macro.toLowerCase()] ?? [];
+  const key = macro.toLowerCase();
+  const inStrings = CODE_IN_STRINGS.get(key) ?? {};
+  const passageStrings = PASSAGE_STRINGS.get(key) ?? [];
+  const literalStrings = LITERAL_STRINGS.get(key) ?? [];
   let cursor = 0;
   /** The pieces of the parameters `list`, with their values in `from`. */
   function* visit(
@@ -294,20 +319,36 @@ function* argPieces(
           goal,
           label,
         };
-        if (param.type === 'passage') piece.passage = true;
+        if (param.type === 'passage') {
+          piece.passage = true;
+          piece.macro = macro.toLowerCase();
+        }
+        if (!codeGoal(param)) piece.inString = true;
         yield piece;
       }
       // A passage written out: a quoted `passage` argument, or a string
-      const name =
+      const written = param.type === 'passage' ? value.trim() : value;
+      const target =
         param.type === 'passage'
-          ? readWholeQuoted(value.trim())
+          ? passageTarget(value)
           : passageStrings.includes(param.name)
-            ? value
+            ? ({ kind: 'name', name: value } as const)
             : null;
-      if (name !== null) {
-        yield { kind: 'passage', name, offset: offset + at, label };
+      if (target?.kind === 'name') {
+        yield {
+          kind: 'passage',
+          name: target.name,
+          offset: offset + at + value.indexOf(written),
+          length: written.length,
+          macro: macro.toLowerCase(),
+          label,
+        };
       }
-      if (!goal && (param.type === 'text' || param.type === 'string')) {
+      if (
+        !goal &&
+        (param.type === 'text' || param.type === 'string') &&
+        !literalStrings.includes(param.name)
+      ) {
         yield {
           kind: 'text',
           text: value,
@@ -330,3 +371,52 @@ function locate(text: string, part: string, from: number): number {
   const anywhere = text.indexOf(part);
   return anywhere >= 0 ? anywhere : from;
 }
+
+/** A passage a piece of markup names, and where. */
+export interface PassageReference {
+  /** The macro it is the argument of; `link` for `[[…]]` links. */
+  macro: string;
+  /** The name written out, or the expression whose value is the name. */
+  target: PassageTarget;
+  /** Where the name is written, as it is written (quotes included). */
+  start: number;
+  end: number;
+}
+
+/**
+ * The passages the markup `src` (with the `tokens` it tokenizes to) names:
+ * `[[…]]` links, the passage argument of `{goto}`, `{include}` and `{link}`
+ * (and of macros that declare one), the `goto` and `dialog` actions of
+ * `{watch}` and the body of `{dialog}`, in source order. A name written out
+ * is a literal; the others are expressions, which name a passage when they
+ * run.
+ */
+export function collectPassageReferences(
+  src: string,
+  tokens: readonly Token[],
+  parametersOf: (macro: string) => readonly ParameterDef[] | undefined,
+): PassageReference[] {
+  const refs: PassageReference[] = [];
+  for (const piece of codeAndText(src, tokens, parametersOf)) {
+    if (piece.kind === 'passage') {
+      refs.push({
+        macro: piece.macro,
+        target: { kind: 'name', name: piece.name },
+        start: piece.offset,
+        end: piece.offset + piece.length,
+      });
+    } else if (piece.kind === 'code' && piece.passage) {
+      refs.push({
+        macro: piece.macro!,
+        target: passageTarget(piece.code),
+        start: piece.offset,
+        end: piece.offset + piece.code.length,
+      });
+    }
+  }
+  return refs.filter(
+    (ref, i) =>
+      refs.findIndex((r) => r.start === ref.start && r.macro === ref.macro) ===
+      i,
+  );
+}

package/src/components/macros/Include.tsx

@@ -4,18 +4,14 @@ import { parseMarkup } from '../../markup/parse';
 import { NobrContext } from '../../markup/render';
 import { defineMacro } from '../../define-macro';
 import { evaluatePassageName } from './macro-args';
+import { INCLUDE_PARAMETERS } from './include-args';
 import { MacroError } from './MacroError';
 
 defineMacro({
   name: 'include',
   interpolate: true,
   merged: true,
-  // A standalone `inline` flag only counts outside quotes and brackets, so
-  // passage names and expressions containing the word stay intact (#201).
-  parameters: [
-    { name: 'inline', type: 'flag' },
-    { name: 'passage', type: 'passage', required: true },
-  ],
+  parameters: INCLUDE_PARAMETERS,
   render(_props, ctx) {
     const { storyData } = useStoryFields('storyData');
 

package/src/components/macros/MacroLink.tsx

@@ -34,6 +34,8 @@ defineMacro({
     const runBody = useDetachedBody();
 
     const perform = () => {
+      // A link that failed to resolve is an error, not a control
+      if (failure) return;
       runBody(children);
       if (passage) {
         useStoryStore.getState().navigate(passage);
@@ -51,6 +53,7 @@ defineMacro({
       authorId: ctx.id,
       label: display,
       target: passage ?? undefined,
+      disabled: failure ? true : undefined,
       perform,
     });
 

package/src/components/macros/arg-utils.ts

@@ -69,6 +69,43 @@ export function readWholeQuoted(src: string): string | null {
   return quoted && quoted.end === src.length ? quoted.value : null;
 }
 
+const JS_SIMPLE_ESCAPES: Record<string, string> = {
+  n: '\n',
+  t: '\t',
+  r: '\r',
+  b: '\b',
+  f: '\f',
+  v: '\v',
+  '0': '\0',
+};
+
+/** Undo every escape of a JavaScript string literal's body. */
+function decodeJsEscapes(body: string): string {
+  return body.replace(
+    /\\(?:u\{([0-9a-fA-F]+)\}|u([0-9a-fA-F]{4})|x([0-9a-fA-F]{2})|(\r\n|[\n\r\u2028\u2029])|([\s\S]))/g,
+    (_, braced, unicode, hex, lineBreak, other: string | undefined) => {
+      if (lineBreak !== undefined) return '';
+      const code = braced ?? unicode ?? hex;
+      if (code !== undefined) {
+        const point = parseInt(code, 16);
+        return point <= 0x10ffff ? String.fromCodePoint(point) : '';
+      }
+      return JS_SIMPLE_ESCAPES[other!] ?? other!;
+    },
+  );
+}
+
+/**
+ * The value of `src` when it is exactly one `"…"` or `'…'` string, read as
+ * the JavaScript literal it is (`"\u0048all"` is `Hall`), otherwise `null`.
+ * `readWholeQuoted` keeps other escapes as written, for labels and code.
+ */
+export function readWholeJsString(src: string): string | null {
+  const quoted = readQuoted(src, 0);
+  if (!quoted || quoted.end !== src.length) return null;
+  return decodeJsEscapes(src.slice(1, -1));
+}
+
 /**
  * Strip an optional quote from each end of an unquoted-or-loosely-quoted
  * argument (`"Red`, `Red'`), leaving at least one character. This is the

package/src/components/macros/include-args.ts

@@ -0,0 +1,23 @@
+import { parseMacroArgs } from './macro-args';
+
+/**
+ * The parameters of `{include}`. A standalone `inline` flag only counts
+ * outside quotes and brackets, so passage names and expressions containing
+ * the word stay intact (#201).
+ */
+export const INCLUDE_PARAMETERS = [
+  { name: 'inline', type: 'flag' },
+  { name: 'passage', type: 'passage', required: true },
+] as const;
+
+/**
+ * The arguments of `{include}`: whether it has the `inline` flag, as its
+ * first or last word, and the passage expression that is left.
+ */
+export function splitIncludeFlag(rawArgs: string): {
+  inline: boolean;
+  passage: string | undefined;
+} {
+  const { inline, passage } = parseMacroArgs(rawArgs, INCLUDE_PARAMETERS);
+  return { inline, passage };
+}

package/src/components/macros/macro-args.ts

@@ -22,6 +22,7 @@ import {
   endsWithOperator,
   isWhitespace,
   readQuoted,
+  readWholeJsString,
   readWholeQuoted,
   stripLooseQuotes,
   topLevelIndices,
@@ -56,7 +57,9 @@ export function readText(src: string): string {
  */
 export class MacroArgumentError extends Error {
   constructor(param: ParameterDef, src: string) {
-    super(`The ${param.name} must be a quoted string ("…" or '…'), not ${src}`);
+    super(
+      `The ${param.name} must be a quoted string ("…" or '…'), ${src ? `not ${src}` : 'but it has no value'}`,
+    );
     this.name = 'MacroArgumentError';
   }
 }
@@ -90,6 +93,7 @@ function unset(param: ParameterDef): unknown {
 
 const WORD_RE = /\w+/y;
 const DIGITS_RE = /\d+/y;
+const WORD_OR_PUNCT_RE = /\S+/y;
 
 /** Match the sticky regex `re` at `pos`, returning the matched text. */
 function matchAt(re: RegExp, src: string, pos: number): string | null {
@@ -100,7 +104,8 @@ function matchAt(re: RegExp, src: string, pos: number): string | null {
 /**
  * Read keyword options (`goto "X" priority 5 once`). A keyword takes the
  * quoted string or digit run after it as its value; keywords that aren't
- * declared, and their values, are skipped.
+ * declared, and their values, are skipped. A keyword of a `string` parameter
+ * must be followed by a quoted string.
  */
 function readOptions(
   params: readonly ParameterDef[],
@@ -119,10 +124,11 @@ function readOptions(
 
     // A value is a quoted string or a digit run, after whitespace.
     let val: string | undefined;
+    let quoted: ReturnType<typeof readQuoted> = null;
     let j = i;
     while (j < src.length && isWhitespace(src[j]!)) j++;
     if (j > i) {
-      const quoted = readQuoted(src, j);
+      quoted = readQuoted(src, j);
       const digits = quoted ? null : matchAt(DIGITS_RE, src, j);
       if (quoted) {
         val = quoted.value;
@@ -134,6 +140,13 @@ function readOptions(
     }
 
     const param = params.find((p) => p.name === key);
+    // A keyword that takes a string was given anything else, or nothing
+    if (param?.type === 'string' && (!quoted || val === undefined)) {
+      throw new MacroArgumentError(
+        param,
+        matchAt(WORD_OR_PUNCT_RE, src, j) ?? '',
+      );
+    }
     if (param?.type === 'flag') options[key] = true;
     else if (param?.type === 'number') options[key] = Number(val ?? 0);
     else if (param) options[key] = val;
@@ -288,6 +301,25 @@ export function parseMacroArgs<const P extends readonly ParameterDef[]>(
   return args as MacroArgs<P>;
 }
 
+/** What a `passage` argument is: a quoted name, or an expression. */
+export type PassageTarget =
+  | { kind: 'name'; name: string }
+  | { kind: 'expression'; expression: string };
+
+/**
+ * Read a `passage` argument as written (`{goto "Hall"}`, `{goto $room}`): a
+ * quoted string is the name its JavaScript literal has (`"\u0048all"` is
+ * `Hall`), anything else an expression, whose value is the name when it
+ * runs (see evaluatePassageName).
+ */
+export function passageTarget(arg: string): PassageTarget {
+  const expression = arg.trim();
+  const name = readWholeJsString(expression);
+  return name === null
+    ? { kind: 'expression', expression }
+    : { kind: 'name', name };
+}
+
 /**
  * The passage a `passage` argument names (`{goto "Hall"}`, `{goto $room}`):
  * the value of its expression. Throws what evaluating it throws, and if no

package/src/components/macros/option-utils.ts

@@ -20,9 +20,15 @@ const OPTION_PARAMETERS = [
 ] as const;
 
 /** The parameters of the built-in sub-macros that take arguments. */
-export const SUB_MACRO_PARAMETERS: Record<string, readonly ParameterDef[]> = {
-  option: OPTION_PARAMETERS,
-};
+const SUB_MACRO_PARAMETERS: ReadonlyMap<string, readonly ParameterDef[]> =
+  new Map([['option', OPTION_PARAMETERS]]);
+
+/** The parameters of the built-in sub-macro `name`, if it takes any. */
+export function subMacroParameters(
+  name: string,
+): readonly ParameterDef[] | undefined {
+  return SUB_MACRO_PARAMETERS.get(name.toLowerCase());
+}
 
 /**
  * Walk AST children to find {option} macro nodes, returning their values.

package/src/expression.ts

@@ -2,13 +2,12 @@ import type { StoryState } from './store';
 import { useStoryStore } from './store';
 import type { Passage } from './parser';
 import { random, randomInt } from './prng';
-import { parseCode, type JsGoal, type Sigil } from './js-lexer';
+import { clearTransformCache, transform } from './transform';
 import {
   EMPTY_NAMESPACE,
   asNamespace,
   countOf,
   isNamespace,
-  variableNameError,
   type Counts,
   type Namespace,
 } from './utils/namespace';
@@ -36,60 +35,11 @@ type CompiledExpression = (
   transient: Record<string, unknown>,
 ) => unknown;
 
+export { transform };
+
 const FN_CACHE_MAX = 500;
 const fnCache = new Map<string, CompiledExpression>();
 
-const NAMESPACES: Record<Sigil, string> = {
-  $: 'variables',
-  _: 'temporary',
-  '@': 'locals',
-  '%': 'transient',
-};
-
-/** Ends with an identifier character. */
-const IDENT_END_RE = /[\p{ID_Continue}$\u200c\u200d]$/u;
-
-/**
- * Transform expression: $var → variables["var"], _var → temporary["var"],
- * @var → locals["var"], %var → transient["var"].
- * The lexer (`lexJs`) finds the references: only in code (string, template
- * and regex literal text and comments are left untouched), only where an
- * identifier starts (not in `a$b`, nor as the property name in `obj._x`),
- * and `%var` only where an operand is expected, since `%` is also the
- * modulo operator. `goal` tells whether `expr` is an expression or a list of
- * statements, as that decides whether a leading `{` opens an object literal
- * or a block. A reference to a variable named `__proto__` throws a
- * SyntaxError: no namespace can hold one (see utils/namespace.ts).
- * Exported for tests.
- */
-export function transform(expr: string, goal: JsGoal = 'expression'): string {
-  const key = goal === 'statements' ? 's' + expr : 'e' + expr;
-  const cached = transformCache.get(key);
-  if (cached !== undefined) return cached;
-  const { refs } = parseCode(expr, goal);
-  let result = '';
-  let at = 0;
-  for (const ref of refs) {
-    const error = variableNameError(ref.name, ref.sigil + ref.name);
-    if (error) throw new SyntaxError(`spindle: ${error}`);
-    result += expr.slice(at, ref.start);
-    if (ref.shorthand) result += `${ref.name}: `;
-    // `typeof%x` needs a space once `%x` turns into an identifier.
-    else if (IDENT_END_RE.test(result.slice(-2))) result += ' ';
-    result += `${NAMESPACES[ref.sigil]}["${ref.name}"]`;
-    at = ref.end;
-  }
-  result += expr.slice(at);
-  if (transformCache.size >= FN_CACHE_MAX) {
-    transformCache.delete(transformCache.keys().next().value!);
-  }
-  transformCache.set(key, result);
-  return result;
-}
-
-/** Transformed code by goal and source. */
-const transformCache = new Map<string, string>();
-
 const preamble =
   'const {currentPassage,previousPassage,visited,hasVisited,hasVisitedAny,hasVisitedAll,rendered,hasRendered,hasRenderedAny,hasRenderedAll,random,randomInt}=__fns;';
 
@@ -269,7 +219,7 @@ export function execute(code: string, ...namespaces: Namespaces): void {
 /** Clear the compiled expression cache. Useful for testing and HMR. */
 export function clearExpressionCache(): void {
   fnCache.clear();
-  transformCache.clear();
+  clearTransformCache();
   cachedFns = null;
   cachedVisitCounts = null;
   cachedRenderCounts = null;

package/src/markup/grammar-shared.ts

@@ -0,0 +1,8 @@
+/**
+ * What the grammar (spindle.peggy) imports from the rest of Spindle, so that
+ * it holds no second copy: the generated parser imports this module's
+ * default export as `shared` (see scripts/peggy.ts).
+ */
+import { SIGIL_SCOPES, withSelectors } from './tokens';
+
+export default { SIGIL_SCOPES, withSelectors };

package/src/markup/parse.ts

@@ -5,7 +5,7 @@
  */
 import { parse as pegParse } from './spindle.peggy';
 import { isBlockMacro, type ASTNode } from './ast';
-import type { Token } from './tokens';
+import type { Selectors, Token } from './tokens';
 import { isCodeAttribute } from './code-attributes';
 import { defaultCodeEnd, type CodeEnd } from './code-end';
 
@@ -74,7 +74,7 @@ export function lineColumn(
 
 function run(
   source: string,
-  startRule: 'Markup' | 'Tokens',
+  startRule: 'Markup' | 'Tokens' | 'SelectorsPrefix',
   options: ParseMarkupOptions,
 ): unknown {
   const hooks = options.hooks
@@ -116,3 +116,90 @@ export function tokenizeMarkup(
 ): Token[] {
   return run(source, 'Tokens', options) as Token[];
 }
+
+/**
+ * The `.class#id` selectors that start `source` at `at`, with the index just
+ * past them and the one space that may follow (`end`; `at` if there are
+ * none). A selector name may hold `{$name}` interpolations, as in markup.
+ */
+export function parseSelectors(
+  source: string,
+  at = 0,
+): Selectors & { end: number } {
+  const found = run(source.slice(at), 'SelectorsPrefix', {}) as Selectors & {
+    end: number;
+  };
+  return { ...found, end: at + found.end };
+}
+
+/** The tokens of markup that may be malformed, and its errors. */
+export interface TolerantTokens {
+  tokens: Token[];
+  /** One for each malformed tag, in source order, with offsets in the source. */
+  errors: MarkupError[];
+}
+
+/**
+ * `tokenizeMarkup` for half-typed markup: the tokens it can read, and every
+ * error, instead of throwing at the first. At a malformed tag, the tokens
+ * before it are kept, its first character is read as text and tokenizing
+ * resumes after it, so what follows is read as if the tag had not been
+ * started. Tokens and errors have offsets in `source`; for well-formed
+ * markup the tokens are those of `tokenizeMarkup` and there are no errors.
+ */
+export function tokenizeMarkupTolerant(
+  source: string,
+  options: ParseMarkupOptions = {},
+): TolerantTokens {
+  const tokens: Token[] = [];
+  const errors: MarkupError[] = [];
+  /** `text` shifted by `base`, tokenized: the tokens or the error. */
+  const attempt = (text: string, base: number): Token[] | MarkupError => {
+    try {
+      return tokenizeMarkup(text, options).map((token) => shift(token, base));
+    } catch (error) {
+      if (!(error instanceof MarkupError)) throw error;
+      return error;
+    }
+  };
+  let base = 0;
+  while (base <= source.length) {
+    const rest = source.slice(base);
+    const result = attempt(rest, base);
+    if (Array.isArray(result)) {
+      tokens.push(...result);
+      break;
+    }
+    const at = base + result.offset;
+    const { line, column } = lineColumn(source, at);
+    errors.push(new MarkupError(result.reason, at, line, column));
+    // The longest prefix that tokenizes: the error may be inside a tag
+    // (an attribute value) of which the prefix is itself malformed
+    let cut = result.offset;
+    let prefix = attempt(rest.slice(0, cut), base);
+    while (!Array.isArray(prefix)) {
+      cut = prefix.offset;
+      prefix = attempt(rest.slice(0, cut), base);
+    }
+    tokens.push(...prefix);
+    // The character that started the damage is text
+    if (cut < rest.length) {
+      const start = base + cut;
+      tokens.push({
+        type: 'text',
+        value: rest[cut]!,
+        start,
+        end: start + 1,
+      });
+    }
+    base += cut + 1;
+  }
+  return { tokens, errors };
+}
+
+/** `token` with its offsets moved `by` on. */
+function shift(token: Token, by: number): Token {
+  return by === 0
+    ? token
+    : { ...token, start: token.start + by, end: token.end + by };
+}