@rohal12/spindle 0.57.0 → 0.59.0

package/src/components/macros/macro-args.ts

@@ -18,6 +18,7 @@ import { parseDelay } from '../../utils/parse-delay';
 import type { MacroArgs, ParameterDef } from '../../registry';
 import type { StoryState } from '../../store';
 import { noPassageError } from '../../runtime-errors';
+import { stringLiteralValue } from '../../js-lexer';
 import {
   endsWithOperator,
   isWhitespace,
@@ -29,6 +30,20 @@ import {
 
 type Span = [start: number, end: number];
 
+/**
+ * Where parseMacroArgs read each argument it set from text (not flags or
+ * separators), by parameter, options too: the start and end of its text in
+ * the raw arguments, quotes included.
+ */
+export type ArgSpans = Map<ParameterDef, Span>;
+
+/** Where a group's text is in the raw arguments, and the spans read so far. */
+interface Place {
+  /** The index in the raw arguments of the text being read. */
+  at: number;
+  spans: ArgSpans | undefined;
+}
+
 /** The terms of `src`: the runs of text between its depth-0 whitespace. */
 function terms(src: string): Span[] {
   const spans: Span[] = [];
@@ -56,12 +71,16 @@ export function readText(src: string): string {
  */
 export class MacroArgumentError extends Error {
   constructor(param: ParameterDef, src: string) {
-    super(`The ${param.name} must be a quoted string ("…" or '…'), not ${src}`);
+    super(
+      `The ${param.name} must be a quoted string ("…" or '…'), ${src ? `not ${src}` : 'but it has no value'}`,
+    );
     this.name = 'MacroArgumentError';
   }
 }
 
-function readValue(param: ParameterDef, src: string): unknown {
+/** Read `src`, at `place`, as `param` says. */
+function readValue(param: ParameterDef, src: string, place: Place): unknown {
+  place.spans?.set(param, [place.at, place.at + src.length]);
   switch (param.type) {
     case 'string': {
       const value = readWholeQuoted(src);
@@ -77,7 +96,7 @@ function readValue(param: ParameterDef, src: string): unknown {
     case 'number':
       return Number(src);
     case 'options':
-      return readOptions(param.parameters ?? [], src);
+      return readOptions(param.parameters ?? [], src, place);
     default:
       return src;
   }
@@ -90,6 +109,7 @@ function unset(param: ParameterDef): unknown {
 
 const WORD_RE = /\w+/y;
 const DIGITS_RE = /\d+/y;
+const WORD_OR_PUNCT_RE = /\S+/y;
 
 /** Match the sticky regex `re` at `pos`, returning the matched text. */
 function matchAt(re: RegExp, src: string, pos: number): string | null {
@@ -99,12 +119,15 @@ function matchAt(re: RegExp, src: string, pos: number): string | null {
 
 /**
  * Read keyword options (`goto "X" priority 5 once`). A keyword takes the
- * quoted string or digit run after it as its value; keywords that aren't
- * declared, and their values, are skipped.
+ * quoted string (with or without whitespace before it) or digit run after
+ * it as its value; keywords that aren't declared, and their values, are
+ * skipped. A keyword of a `string` parameter must be followed by a quoted
+ * string.
  */
 function readOptions(
   params: readonly ParameterDef[],
   src: string,
+  { at, spans }: Place,
 ): Record<string, unknown> {
   const options: Record<string, unknown> = {};
   let i = 0;
@@ -117,23 +140,33 @@ function readOptions(
     }
     i += key.length;
 
-    // A value is a quoted string or a digit run, after whitespace.
+    // A value is a quoted string, which may follow the keyword directly
+    // (`goto"Hall"`: a quote can't be part of one), or a digit run after
+    // whitespace.
     let val: string | undefined;
     let j = i;
     while (j < src.length && isWhitespace(src[j]!)) j++;
-    if (j > i) {
-      const quoted = readQuoted(src, j);
-      const digits = quoted ? null : matchAt(DIGITS_RE, src, j);
-      if (quoted) {
-        val = quoted.value;
-        i = quoted.end;
-      } else if (digits) {
-        val = digits;
-        i = j + digits.length;
-      }
+    const quoted = readQuoted(src, j);
+    const digits = quoted || j === i ? null : matchAt(DIGITS_RE, src, j);
+    if (quoted) {
+      val = quoted.value;
+      i = quoted.end;
+    } else if (digits) {
+      val = digits;
+      i = j + digits.length;
     }
 
     const param = params.find((p) => p.name === key);
+    if (param && param.type !== 'flag' && val !== undefined) {
+      spans?.set(param, [at + j, at + i]);
+    }
+    // A keyword that takes a string was given anything else, or nothing
+    if (param?.type === 'string' && !quoted) {
+      throw new MacroArgumentError(
+        param,
+        matchAt(WORD_OR_PUNCT_RE, src, j) ?? '',
+      );
+    }
     if (param?.type === 'flag') options[key] = true;
     else if (param?.type === 'number') options[key] = Number(val ?? 0);
     else if (param) options[key] = val;
@@ -145,21 +178,21 @@ function readOptions(
 const LEADING_OPERATOR_RE = /^(?:[-+*/&|^=<>?:,.]|%(?![A-Za-z_]))/;
 
 /**
- * The rest of `src` without the flag `word` as its last or first term, or
- * `null` if it has no such flag. Next to a binary operator the word is an
- * operand (`"a" + inline`), not a flag; the closing `/` of a regex literal
- * is not one.
+ * Where the rest of `src` is without the flag `word` as its last or first
+ * term, or `null` if it has no such flag. Next to a binary operator the
+ * word is an operand (`"a" + inline`), not a flag; the closing `/` of a
+ * regex literal is not one.
  */
-function takeFlag(src: string, word: string): string | null {
+function takeFlag(src: string, word: string): Span | null {
   const spans = terms(src);
   if (spans.length < 2) return null;
   if (slice(src, spans[spans.length - 1]!) === word) {
-    const rest = src.slice(0, spans[spans.length - 2]![1]);
-    if (!endsWithOperator(rest)) return rest;
+    const rest: Span = [0, spans[spans.length - 2]![1]];
+    if (!endsWithOperator(slice(src, rest))) return rest;
   }
   if (slice(src, spans[0]!) === word) {
-    const rest = src.slice(spans[1]![0]);
-    if (!LEADING_OPERATOR_RE.test(rest)) return rest;
+    const rest: Span = [spans[1]![0], src.length];
+    if (!LEADING_OPERATOR_RE.test(slice(src, rest))) return rest;
   }
   return null;
 }
@@ -198,17 +231,27 @@ const takesRest = (param: ParameterDef) =>
   param.type === 'passage' ||
   param.type === 'text';
 
-/** Read the parameters of one group from `src` (`null`: no text) into `args`. */
+/** The text of `src` without the whitespace around it, and where it is. */
+function trimmed(src: string, at: number): [text: string, at: number] {
+  const text = src.trim();
+  return [text, text ? at + src.indexOf(text) : at];
+}
+
+/**
+ * Read the parameters of one group from `src` (`null`: no text), which is
+ * at `at` in the raw arguments, into `args`.
+ */
 function readGroup(
   params: readonly ParameterDef[],
   src: string | null,
   args: Record<string, unknown>,
+  { at: srcAt, spans }: Place,
 ): void {
-  let text = src?.trim() ?? '';
+  let [text, at] = trimmed(src ?? '', srcAt);
   for (const flag of params.filter((p) => p.type === 'flag')) {
     const rest = src === null ? null : takeFlag(text, flag.name);
     args[flag.name] = rest !== null;
-    text = rest ?? text;
+    if (rest) [text, at] = trimmed(slice(text, rest), at + rest[0]);
   }
   const positional = params.filter((p) => p.type !== 'flag');
   if (positional.length === 0) return;
@@ -218,27 +261,29 @@ function readGroup(
     (p, i) => takesRest(p) && !positional.slice(i + 1).some(takesRest),
   );
   const restIndex = lastText < 0 ? positional.length - 1 : lastText;
-  const read = (param: ParameterDef, value: string) =>
-    value === '' ? unset(param) : readValue(param, value);
+  /** Read `param` from the text from `start` to `end`. */
+  const read = (param: ParameterDef, [start, end]: Span) =>
+    start === end
+      ? unset(param)
+      : readValue(param, text.slice(start, end), { at: at + start, spans });
 
   // One parameter reads the whole text: no need to split it.
   if (positional.length === 1) {
-    args[positional[0]!.name] = read(positional[0]!, text);
+    args[positional[0]!.name] = read(positional[0]!, [0, text.length]);
     return;
   }
 
-  const spans = terms(text);
+  const parts = terms(text);
   let from = 0;
-  let to = spans.length;
+  let to = parts.length;
   for (const param of positional.slice(0, restIndex)) {
-    args[param.name] =
-      from < to ? read(param, slice(text, spans[from++]!)) : unset(param);
+    args[param.name] = from < to ? read(param, parts[from++]!) : unset(param);
   }
   // Optional ones at the end: a term that doesn't have their form (a
   // `string` that isn't quoted) belongs to the rest, as in {meter $hp 100}
-  const readTrailing = (param: ParameterDef, value: string) => {
+  const readTrailing = (param: ParameterDef, span: Span) => {
     try {
-      return read(param, value);
+      return read(param, span);
     } catch (error) {
       if (error instanceof MacroArgumentError) return undefined;
       throw error;
@@ -246,17 +291,17 @@ function readGroup(
   };
   for (const param of positional.slice(restIndex + 1).reverse()) {
     const value =
-      from < to && !endsWithOperator(text.slice(0, spans[to - 1]![0]))
-        ? readTrailing(param, slice(text, spans[to - 1]!))
+      from < to && !endsWithOperator(text.slice(0, parts[to - 1]![0]))
+        ? readTrailing(param, parts[to - 1]!)
         : undefined;
     args[param.name] = value ?? unset(param);
     if (value !== undefined) to--;
+    // A term it didn't take
+    else spans?.delete(param);
   }
   const rest = positional[restIndex]!;
   args[rest.name] =
-    from < to
-      ? read(rest, text.slice(spans[from]![0], spans[to - 1]![1]))
-      : unset(rest);
+    from < to ? read(rest, [parts[from]![0], parts[to - 1]![1]]) : unset(rest);
 }
 
 /** The duration a timing macro ({repeat}, {type}, {timed}/{next}) waits. */
@@ -266,28 +311,56 @@ export const DELAY_PARAMETER = {
   required: true,
 } as const;
 
-/** Read `rawArgs` into the declared `parameters` (see above). */
+/**
+ * Read `rawArgs` into the declared `parameters` (see above). With `spans`,
+ * note there where each argument set was read.
+ */
 export function parseMacroArgs<const P extends readonly ParameterDef[]>(
   rawArgs: string,
   parameters: P,
+  spans?: ArgSpans,
 ): MacroArgs<P> {
   const args: Record<string, unknown> = {};
   // The text after the last separator read, null once one is missing
-  let rest: string | null = rawArgs.trim();
+  let [rest, restAt]: [string | null, number] = trimmed(rawArgs, 0);
   let groupStart = 0;
   parameters.forEach((separator, i) => {
     if (separator.type !== 'separator') return;
     const at = rest === null ? null : findSeparator(rest, separator.name);
     args[separator.name] = at !== null;
     const before = at ? rest!.slice(0, at[0]) : rest;
-    readGroup(parameters.slice(groupStart, i), before, args);
+    readGroup(parameters.slice(groupStart, i), before, args, {
+      at: restAt,
+      spans,
+    });
     rest = at ? rest!.slice(at[1]) : null;
+    if (at) restAt += at[1];
     groupStart = i + 1;
   });
-  readGroup(parameters.slice(groupStart), rest, args);
+  readGroup(parameters.slice(groupStart), rest, args, { at: restAt, spans });
   return args as MacroArgs<P>;
 }
 
+/** What a `passage` argument is: a quoted name, or an expression. */
+export type PassageTarget =
+  | { kind: 'name'; name: string }
+  | { kind: 'expression'; expression: string };
+
+/**
+ * Read a `passage` argument as written (`{goto "Hall"}`, `{goto $room}`): a
+ * string literal is the name JavaScript reads from it (`"\u0048all"` is
+ * `Hall`), anything else (a literal that is not well-formed too) an
+ * expression, whose value is the name when it
+ * runs (see evaluatePassageName).
+ */
+export function passageTarget(arg: string): PassageTarget {
+  const expression = arg.trim();
+  const name = stringLiteralValue(expression);
+  return name === null
+    ? { kind: 'expression', expression }
+    : { kind: 'name', name };
+}
+
 /**
  * The passage a `passage` argument names (`{goto "Hall"}`, `{goto $room}`):
  * the value of its expression. Throws what evaluating it throws, and if no

package/src/components/macros/option-utils.ts

@@ -16,13 +16,19 @@ export function parseVarArgs(rawArgs: string): {
 
 /** The parameters of an `{option}` sub-macro: its value. */
 const OPTION_PARAMETERS = [
-  { name: 'value', type: 'string', required: true },
+  { name: 'value', type: 'string', holds: 'text', required: true },
 ] as const;
 
 /** The parameters of the built-in sub-macros that take arguments. */
-export const SUB_MACRO_PARAMETERS: Record<string, readonly ParameterDef[]> = {
-  option: OPTION_PARAMETERS,
-};
+const SUB_MACRO_PARAMETERS: ReadonlyMap<string, readonly ParameterDef[]> =
+  new Map([['option', OPTION_PARAMETERS]]);
+
+/** The parameters of the built-in sub-macro `name`, if it takes any. */
+export function subMacroParameters(
+  name: string,
+): readonly ParameterDef[] | undefined {
+  return SUB_MACRO_PARAMETERS.get(name.toLowerCase());
+}
 
 /**
  * Walk AST children to find {option} macro nodes, returning their values.

package/src/expression.ts

@@ -2,13 +2,12 @@ import type { StoryState } from './store';
 import { useStoryStore } from './store';
 import type { Passage } from './parser';
 import { random, randomInt } from './prng';
-import { parseCode, type JsGoal, type Sigil } from './js-lexer';
+import { clearTransformCache, transform } from './transform';
 import {
   EMPTY_NAMESPACE,
   asNamespace,
   countOf,
   isNamespace,
-  variableNameError,
   type Counts,
   type Namespace,
 } from './utils/namespace';
@@ -36,60 +35,11 @@ type CompiledExpression = (
   transient: Record<string, unknown>,
 ) => unknown;
 
+export { transform };
+
 const FN_CACHE_MAX = 500;
 const fnCache = new Map<string, CompiledExpression>();
 
-const NAMESPACES: Record<Sigil, string> = {
-  $: 'variables',
-  _: 'temporary',
-  '@': 'locals',
-  '%': 'transient',
-};
-
-/** Ends with an identifier character. */
-const IDENT_END_RE = /[\p{ID_Continue}$\u200c\u200d]$/u;
-
-/**
- * Transform expression: $var → variables["var"], _var → temporary["var"],
- * @var → locals["var"], %var → transient["var"].
- * The lexer (`lexJs`) finds the references: only in code (string, template
- * and regex literal text and comments are left untouched), only where an
- * identifier starts (not in `a$b`, nor as the property name in `obj._x`),
- * and `%var` only where an operand is expected, since `%` is also the
- * modulo operator. `goal` tells whether `expr` is an expression or a list of
- * statements, as that decides whether a leading `{` opens an object literal
- * or a block. A reference to a variable named `__proto__` throws a
- * SyntaxError: no namespace can hold one (see utils/namespace.ts).
- * Exported for tests.
- */
-export function transform(expr: string, goal: JsGoal = 'expression'): string {
-  const key = goal === 'statements' ? 's' + expr : 'e' + expr;
-  const cached = transformCache.get(key);
-  if (cached !== undefined) return cached;
-  const { refs } = parseCode(expr, goal);
-  let result = '';
-  let at = 0;
-  for (const ref of refs) {
-    const error = variableNameError(ref.name, ref.sigil + ref.name);
-    if (error) throw new SyntaxError(`spindle: ${error}`);
-    result += expr.slice(at, ref.start);
-    if (ref.shorthand) result += `${ref.name}: `;
-    // `typeof%x` needs a space once `%x` turns into an identifier.
-    else if (IDENT_END_RE.test(result.slice(-2))) result += ' ';
-    result += `${NAMESPACES[ref.sigil]}["${ref.name}"]`;
-    at = ref.end;
-  }
-  result += expr.slice(at);
-  if (transformCache.size >= FN_CACHE_MAX) {
-    transformCache.delete(transformCache.keys().next().value!);
-  }
-  transformCache.set(key, result);
-  return result;
-}
-
-/** Transformed code by goal and source. */
-const transformCache = new Map<string, string>();
-
 const preamble =
   'const {currentPassage,previousPassage,visited,hasVisited,hasVisitedAny,hasVisitedAll,rendered,hasRendered,hasRenderedAny,hasRenderedAll,random,randomInt}=__fns;';
 
@@ -269,7 +219,7 @@ export function execute(code: string, ...namespaces: Namespaces): void {
 /** Clear the compiled expression cache. Useful for testing and HMR. */
 export function clearExpressionCache(): void {
   fnCache.clear();
-  transformCache.clear();
+  clearTransformCache();
   cachedFns = null;
   cachedVisitCounts = null;
   cachedRenderCounts = null;

package/src/index.tsx

@@ -17,6 +17,7 @@ import {
   extractDefaults,
 } from './story-variables';
 import { getMacro, getMacroRegistry, isSubMacro } from './registry';
+import { parameterLookup } from './code-check';
 import { getWidget } from './widgets/widget-registry';
 import {
   formatDiagnostic,
@@ -71,14 +72,11 @@ function markupErrors(
   only: (passage: MarkupPassage) => boolean,
 ): string[] {
   const macros = getMacroRegistry();
-  const parameters = new Map(
-    macros.map((m) => [m.name.toLowerCase(), m.parameters]),
-  );
   return validateMarkup(storyData.passages.values(), {
     isKnownMacro: (name) =>
       !!getMacro(name) || isSubMacro(name) || !!getWidget(name),
     macroNames: macros.map((m) => m.name),
-    parametersOf: (name) => parameters.get(name),
+    parametersOf: parameterLookup(macros),
     only,
   }).map(formatDiagnostic);
 }

package/src/js-lexer.ts

@@ -1122,6 +1122,25 @@ function regexEnd(src: string, start: number): number {
   return -1;
 }
 
+/**
+ * The value of `src` when it is exactly one `"…"` or `'…'` string literal,
+ * as JavaScript reads it (`"\u0048all"` is `Hall`, `"\1"` is U+0001 in
+ * the non-strict code passages run), otherwise `null`: other code, or a
+ * literal that is not well-formed (`"\u{110000}"`).
+ */
+export function stringLiteralValue(src: string): string | null {
+  if (src[0] !== '"' && src[0] !== "'") return null;
+  try {
+    const p = tokenizerAt(src, 0, 'expression');
+    p.nextToken();
+    return p.type === tt.string && p.end === src.length
+      ? (p.value as string)
+      : null;
+  } catch {
+    return null;
+  }
+}
+
 /**
  * Scan the `"…"` or `'…'` string literal opening at `start`. `end` is the
  * index just past its closing quote, or `src.length` when it is unterminated

package/src/markup/grammar-shared.ts

@@ -0,0 +1,8 @@
+/**
+ * What the grammar (spindle.peggy) imports from the rest of Spindle, so that
+ * it holds no second copy: the generated parser imports this module's
+ * default export as `shared` (see scripts/peggy.ts).
+ */
+import { SIGIL_SCOPES, withSelectors } from './tokens';
+
+export default { SIGIL_SCOPES, withSelectors };

package/src/markup/parse.ts

@@ -5,7 +5,7 @@
  */
 import { parse as pegParse } from './spindle.peggy';
 import { isBlockMacro, type ASTNode } from './ast';
-import type { Token } from './tokens';
+import type { Selectors, Token } from './tokens';
 import { isCodeAttribute } from './code-attributes';
 import { defaultCodeEnd, type CodeEnd } from './code-end';
 
@@ -74,7 +74,7 @@ export function lineColumn(
 
 function run(
   source: string,
-  startRule: 'Markup' | 'Tokens',
+  startRule: 'Markup' | 'Tokens' | 'SelectorsPrefix',
   options: ParseMarkupOptions,
 ): unknown {
   const hooks = options.hooks
@@ -116,3 +116,108 @@ export function tokenizeMarkup(
 ): Token[] {
   return run(source, 'Tokens', options) as Token[];
 }
+
+/**
+ * The `.class#id` selectors that start `source` at `at`, with the index just
+ * past them and the one space that may follow (`end`; `at` if there are
+ * none). A selector name may hold `{$name}` interpolations, as in markup.
+ */
+export function parseSelectors(
+  source: string,
+  at = 0,
+): Selectors & { end: number } {
+  const found = run(source.slice(at), 'SelectorsPrefix', {}) as Selectors & {
+    end: number;
+  };
+  return { ...found, end: at + found.end };
+}
+
+/** The tokens of markup that may be malformed, and its errors. */
+export interface TolerantTokens {
+  tokens: Token[];
+  /** One for each malformed tag, in source order, with offsets in the source. */
+  errors: MarkupError[];
+}
+
+/**
+ * `tokenizeMarkup` for half-typed markup: the tokens it can read, and every
+ * error, instead of throwing at the first. At a malformed tag, the tokens
+ * before it are kept, its first character is read as text and tokenizing
+ * resumes after it, so what follows is read as if the tag had not been
+ * started. Tokens and errors have offsets in `source`; for well-formed
+ * markup the tokens are those of `tokenizeMarkup` and there are no errors.
+ */
+export function tokenizeMarkupTolerant(
+  source: string,
+  options: ParseMarkupOptions = {},
+): TolerantTokens {
+  const tokens: Token[] = [];
+  const errors: MarkupError[] = [];
+  // The code of a tag ends at a }: with none after it, the tag is unclosed
+  // without reading its code to the end of the source, which every
+  // unclosed tag would do again (#265).
+  const lastBrace = source.lastIndexOf('}');
+  const { closeBrace } = { ...defaultHooks, ...options.hooks };
+  /** `text` shifted by `base`, tokenized: the tokens or the error. */
+  const attempt = (text: string, base: number): Token[] | MarkupError => {
+    const hooks: Partial<MarkupHooks> = {
+      ...options.hooks,
+      closeBrace: (input, codeStart, lenientStart) =>
+        base + Math.min(codeStart, lenientStart) > lastBrace
+          ? -1
+          : closeBrace(input, codeStart, lenientStart),
+    };
+    try {
+      return tokenizeMarkup(text, { ...options, hooks }).map((token) =>
+        shift(token, base),
+      );
+    } catch (error) {
+      if (!(error instanceof MarkupError)) throw error;
+      return error;
+    }
+  };
+  let base = 0;
+  while (base <= source.length) {
+    const rest = source.slice(base);
+    const result = attempt(rest, base);
+    if (Array.isArray(result)) {
+      tokens.push(...result);
+      break;
+    }
+    const at = base + result.offset;
+    const { line, column } = lineColumn(source, at);
+    errors.push(new MarkupError(result.reason, at, line, column));
+    // The longest prefix that tokenizes: the error may be inside a tag
+    // (an attribute value) of which the prefix is itself malformed
+    let cut = result.offset;
+    let prefix = attempt(rest.slice(0, cut), base);
+    while (!Array.isArray(prefix)) {
+      cut = prefix.offset;
+      prefix = attempt(rest.slice(0, cut), base);
+    }
+    tokens.push(...prefix);
+    // The character that started the damage is text
+    if (cut < rest.length) {
+      const start = base + cut;
+      tokens.push({
+        type: 'text',
+        value: rest[cut]!,
+        start,
+        end: start + 1,
+      });
+    }
+    base += cut + 1;
+  }
+  return { tokens, errors };
+}
+
+/** `token` with its offsets moved `by` on. */
+function shift(token: Token, by: number): Token {
+  if (by === 0) return token;
+  const moved = { ...token, start: token.start + by, end: token.end + by };
+  if (moved.type === 'link') {
+    moved.targetStart += by;
+    moved.targetEnd += by;
+  }
+  return moved;
+}

package/src/markup/spindle.d.peggy.ts

@@ -13,12 +13,15 @@ export declare class SyntaxError extends globalThis.SyntaxError {
 }
 
 export interface ParseOptions {
-  startRule: 'Markup' | 'Tokens';
+  startRule: 'Markup' | 'Tokens' | 'SelectorsPrefix';
   /** Text mode (attribute values, labels). */
   text: boolean;
   /** What the grammar leaves to code (see parse.ts MarkupHooks). */
   hooks: unknown;
 }
 
-/** Parse markup: the AST for `Markup`, the tokens for `Tokens`. */
+/**
+ * Parse markup: the AST for `Markup`, the tokens for `Tokens`, the selectors
+ * and where they end for `SelectorsPrefix`.
+ */
 export declare function parse(input: string, options: ParseOptions): unknown;