@rohal12/spindle 0.57.0 → 0.58.0

package/src/markup/spindle.d.peggy.ts

@@ -13,12 +13,15 @@ export declare class SyntaxError extends globalThis.SyntaxError {
 }
 
 export interface ParseOptions {
-  startRule: 'Markup' | 'Tokens';
+  startRule: 'Markup' | 'Tokens' | 'SelectorsPrefix';
   /** Text mode (attribute values, labels). */
   text: boolean;
   /** What the grammar leaves to code (see parse.ts MarkupHooks). */
   hooks: unknown;
 }
 
-/** Parse markup: the AST for `Markup`, the tokens for `Tokens`. */
+/**
+ * Parse markup: the AST for `Markup`, the tokens for `Tokens`, the selectors
+ * and where they end for `SelectorsPrefix`.
+ */
 export declare function parse(input: string, options: ParseOptions): unknown;

package/src/markup/spindle.peggy

@@ -18,13 +18,14 @@
 //   isBlock(name), isRaw(name)  whether a macro takes a body / a raw body
 //   isCodeAttribute(name)  whether an attribute value is code (onclick)
 //
+// SIGIL_SCOPES and withSelectors come from tokens.ts (see grammar-shared.ts)
 // Note: Peggy's code blocks count braces without reading JavaScript strings,
 // so the code here writes literal braces as LB and RB.
 
 {{
 const LB = String.fromCharCode(123);
 const RB = String.fromCharCode(125);
-const SIGIL_SCOPES = { $: 'variable', _: 'temporary', '@': 'local', '%': 'transient' };
+const { SIGIL_SCOPES, withSelectors } = shared;
 const VOID_TAGS = new Set(['area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link', 'meta', 'param', 'source', 'track', 'wbr']);
 const BRANCH_PARENT = { elseif: 'if', else: 'if', case: 'switch', default: 'switch', next: 'timed' };
 const BRANCHING = new Set(['if', 'switch', 'timed']);
@@ -47,12 +48,6 @@ function quoteArg(value) {
   return '"' + value.replace(/[\\"]/g, '\\$&') + '"';
 }
 
-function withSelectors(target, from) {
-  if (from.className) target.className = from.className;
-  if (from.id) target.id = from.id;
-  return target;
-}
-
 function selectorsOf(parts) {
   const classes = [];
   let id = '';
@@ -359,6 +354,10 @@ Selectors
   = parts:SelectorPart+ " "? { return selectorsOf(parts); }
   / "" { return NO_SELECTORS; }
 
+/** The selectors at the start of the input, and where they end. */
+SelectorsPrefix
+  = s:Selectors end:Here .* { return { ...s, end }; }
+
 SelectorPart
   = prefix:[.#] name:$SelectorChar* { return [prefix, name]; }
 

package/src/markup/validate.ts

@@ -11,7 +11,7 @@ import { parseWidgetDef } from '../widgets/widget-def';
 import { codeAndText, parseOrError, withParseCache } from '../code-check';
 import { CodeSyntaxError } from '../js-lexer';
 import type { ParameterDef } from '../registry';
-import { SUB_MACRO_PARAMETERS } from '../components/macros/option-utils';
+import { subMacroParameters } from '../components/macros/option-utils';
 
 /** A passage to validate. */
 export interface MarkupPassage {
@@ -67,8 +67,15 @@ export interface MarkupValidationOptions {
   parametersOf?(name: string): readonly ParameterDef[] | undefined;
 }
 
-/** Passages that hold no markup. */
-const NOT_MARKUP = new Set(['StoryVariables', 'StoryTransients']);
+/**
+ * Passages that hold no markup: declarations, and the JavaScript function
+ * body of SaveTitle.
+ */
+export const NOT_MARKUP = new Set([
+  'StoryVariables',
+  'StoryTransients',
+  'SaveTitle',
+]);
 const NOT_MARKUP_TAGS = ['script', 'stylesheet'];
 
 /** Branch macros, which their parent macro renders. */
@@ -222,7 +229,7 @@ export function validateMarkup(
   };
 
   const parametersOf = (name: string) =>
-    options.parametersOf?.(name) ?? SUB_MACRO_PARAMETERS[name.toLowerCase()];
+    options.parametersOf?.(name) ?? subMacroParameters(name);
 
   /**
    * Check `tokens`, the tokens of `src`, which starts at `base` in the

package/src/registry.ts

@@ -178,6 +178,13 @@ export function registerMacroMetadata(
   metadataRegistry.set(name, metadata);
 }
 
+/** The declared parameters of the macro `name`, if it declares any. */
+export function getMacroParameters(
+  name: string,
+): readonly ParameterDef[] | undefined {
+  return metadataRegistry.get(name)?.parameters;
+}
+
 export function getMacroRegistry(): MacroMetadata[] {
   return Array.from(metadataRegistry.values());
 }

package/src/story-variables.ts

@@ -9,7 +9,9 @@ import {
   scanStringLiteral,
   type JsGoal,
 } from './js-lexer';
-import { parseOrError, withParseCache } from './code-check';
+import { argPieces, parseOrError, withParseCache } from './code-check';
+import { getMacroParameters, type ParameterDef } from './registry';
+import { NOT_MARKUP } from './markup/validate';
 import { createNamespace, variableNameError } from './utils/namespace';
 
 /**
@@ -319,18 +321,47 @@ function collectPassageRefs(
   content: string,
   storeVarMacros: ReadonlySet<string>,
   onRef: RefCallback,
+  parametersOf: ParametersOf,
 ): void {
   withParseCache(() =>
-    collectTokenRefs(content, tokensOf(content, false), storeVarMacros, onRef),
+    collectTokenRefs(
+      content,
+      tokensOf(content, false),
+      storeVarMacros,
+      onRef,
+      parametersOf,
+    ),
   );
 }
 
+/** The declared parameters of a macro, if it has any. */
+type ParametersOf = (macro: string) => readonly ParameterDef[] | undefined;
+
+/**
+ * Report the references in the code that a macro's quoted arguments hold,
+ * the condition and `run` action of `{watch}`: the argument scan sees only
+ * the strings.
+ */
+function scanStringCode(
+  token: Extract<Token, { type: 'macro' }>,
+  params: readonly ParameterDef[] | undefined,
+  onRef: RefCallback,
+): void {
+  if (!params || !token.rawArgs) return;
+  for (const piece of argPieces(token.rawArgs, 0, params, token.name, '')) {
+    if (piece.kind === 'code' && piece.inString) {
+      scanCode(piece.code, onRef, piece.goal);
+    }
+  }
+}
+
 /** Report the `$var` references in the tokens of `content`. */
 function collectTokenRefs(
   content: string,
   tokens: Token[],
   storeVarMacros: ReadonlySet<string>,
   onRef: RefCallback,
+  parametersOf: ParametersOf = () => undefined,
 ): void {
   for (let t = 0; t < tokens.length; t++) {
     const token = tokens[t]!;
@@ -345,6 +376,7 @@ function collectTokenRefs(
       }
     } else if (token.type === 'macro' && !token.isClose) {
       scanCode(token.rawArgs, onRef);
+      scanStringCode(token, parametersOf(token.name), onRef);
 
       if (storeVarMacros.has(token.name.toLowerCase())) {
         const first = token.rawArgs.trim().split(/\s+/)[0] ?? '';
@@ -383,6 +415,7 @@ export function validatePassages(
   passages: Map<string, Passage>,
   schema: Map<string, VariableSchema>,
   storeVarMacros: Iterable<string> = BUILTIN_STORE_VAR_MACROS,
+  parametersOf: ParametersOf = getMacroParameters,
 ): string[] {
   const errors: string[] = [];
   const storeVarSet = new Set(
@@ -390,16 +423,21 @@ export function validatePassages(
   );
 
   for (const [name, passage] of passages) {
-    // Don't validate the StoryVariables/StoryTransients passages themselves
-    if (name === 'StoryVariables' || name === 'StoryTransients') continue;
+    // Don't validate the declarations or the SaveTitle code themselves
+    if (NOT_MARKUP.has(name)) continue;
 
     const forLocals = extractForLocals(passage.content);
-    collectPassageRefs(passage.content, storeVarSet, (ref) => {
-      const error = validateRef(ref, schema, forLocals);
-      if (error) {
-        errors.push(`Passage "${name}": ${error}`);
-      }
-    });
+    collectPassageRefs(
+      passage.content,
+      storeVarSet,
+      (ref) => {
+        const error = validateRef(ref, schema, forLocals);
+        if (error) {
+          errors.push(`Passage "${name}": ${error}`);
+        }
+      },
+      parametersOf,
+    );
   }
 
   return errors;

package/src/tooling.ts

@@ -10,12 +10,50 @@ import {
   type MarkupPassage,
 } from './markup/validate';
 import { isBlockMacro } from './markup/ast';
+import { tokenizeMarkupTolerant } from './markup/parse';
+import { collectPassageReferences, type PassageReference } from './code-check';
+import { subMacroParameters } from './components/macros/option-utils';
 import { blockWidgetNames } from './widgets/widget-def';
 import type { ParameterDef } from './registry';
 
 export { parseStoryVariables } from './story-variables';
 export { checkParameterTypes } from './registry';
 export { formatDiagnostic } from './markup/validate';
+
+// The parsing rules (see "Tooling API" in docs/tooling.md): the leaf rules
+// the runtime parses with, so editor tooling needn't mirror them.
+export { findCodeEnd, lexJs, lexTemplate, scanStringLiteral } from './js-lexer';
+export type {
+  FindCodeEndOptions,
+  JsGoal,
+  JsLexHandlers,
+  Sigil,
+} from './js-lexer';
+export {
+  MarkupError,
+  parseSelectors,
+  tokenizeMarkup,
+  tokenizeMarkupTolerant,
+} from './markup/parse';
+export type { ParseMarkupOptions, TolerantTokens } from './markup/parse';
+export { SIGIL_SCOPES, isSigil } from './markup/tokens';
+export type { Selectors, Token, VariableScope } from './markup/tokens';
+export {
+  endsWithOperator,
+  readQuoted,
+  splitArgs,
+  splitTopLevel,
+  stripLooseQuotes,
+  unescapeQuoted,
+} from './components/macros/arg-utils';
+export { splitIncludeFlag } from './components/macros/include-args';
+export {
+  evaluatePassageName,
+  passageTarget,
+} from './components/macros/macro-args';
+export type { PassageTarget } from './components/macros/macro-args';
+export { transform } from './transform';
+export type { PassageReference } from './code-check';
 export type { MarkupDiagnostic, MarkupPassage } from './markup/validate';
 
 /** What tooling knows about a macro (see MacroMetadata). */
@@ -68,3 +106,24 @@ export function validateStoryMarkup(
     checkPassageNames: options.checkPassageNames,
   });
 }
+
+/**
+ * The passages the markup of `source` names (see PassageReference), against
+ * the given macros: what the story-start check looks up. Malformed tags are
+ * skipped, so it reads half-typed markup.
+ */
+export function collectStoryPassageReferences(
+  source: string,
+  macros: Iterable<ToolingMacro>,
+): PassageReference[] {
+  const parameters = new Map<string, readonly ParameterDef[] | undefined>();
+  for (const macro of macros) {
+    parameters.set(macro.name.toLowerCase(), macro.parameters);
+  }
+  const { tokens } = tokenizeMarkupTolerant(source);
+  return collectPassageReferences(
+    source,
+    tokens,
+    (name) => parameters.get(name.toLowerCase()) ?? subMacroParameters(name),
+  );
+}

package/src/transform.ts

@@ -0,0 +1,65 @@
+/**
+ * The sigil transform: the story's code with its variable references
+ * (`$var`, `_var`, `@var`, `%var`) turned into the namespace lookups the
+ * expression engine runs. Pure string-to-string, so tooling can use it
+ * (see src/tooling.ts).
+ */
+import { parseCode, type JsGoal, type Sigil } from './js-lexer';
+import { variableNameError } from './utils/namespace';
+
+const TRANSFORM_CACHE_MAX = 500;
+
+const NAMESPACES: Record<Sigil, string> = {
+  $: 'variables',
+  _: 'temporary',
+  '@': 'locals',
+  '%': 'transient',
+};
+
+/** Ends with an identifier character. */
+const IDENT_END_RE = /[\p{ID_Continue}$\u200c\u200d]$/u;
+
+/**
+ * Transform expression: $var → variables["var"], _var → temporary["var"],
+ * @var → locals["var"], %var → transient["var"].
+ * The lexer (`lexJs`) finds the references: only in code (string, template
+ * and regex literal text and comments are left untouched), only where an
+ * identifier starts (not in `a$b`, nor as the property name in `obj._x`),
+ * and `%var` only where an operand is expected, since `%` is also the
+ * modulo operator. `goal` tells whether `expr` is an expression or a list of
+ * statements, as that decides whether a leading `{` opens an object literal
+ * or a block. A reference to a variable named `__proto__` throws a
+ * SyntaxError: no namespace can hold one (see utils/namespace.ts).
+ *  */
+export function transform(expr: string, goal: JsGoal = 'expression'): string {
+  const key = goal === 'statements' ? 's' + expr : 'e' + expr;
+  const cached = transformCache.get(key);
+  if (cached !== undefined) return cached;
+  const { refs } = parseCode(expr, goal);
+  let result = '';
+  let at = 0;
+  for (const ref of refs) {
+    const error = variableNameError(ref.name, ref.sigil + ref.name);
+    if (error) throw new SyntaxError(`spindle: ${error}`);
+    result += expr.slice(at, ref.start);
+    if (ref.shorthand) result += `${ref.name}: `;
+    // `typeof%x` needs a space once `%x` turns into an identifier.
+    else if (IDENT_END_RE.test(result.slice(-2))) result += ' ';
+    result += `${NAMESPACES[ref.sigil]}["${ref.name}"]`;
+    at = ref.end;
+  }
+  result += expr.slice(at);
+  if (transformCache.size >= TRANSFORM_CACHE_MAX) {
+    transformCache.delete(transformCache.keys().next().value!);
+  }
+  transformCache.set(key, result);
+  return result;
+}
+
+/** Transformed code by goal and source. */
+const transformCache = new Map<string, string>();
+
+/** Clear the transformed code. Useful for testing and HMR. */
+export function clearTransformCache(): void {
+  transformCache.clear();
+}

package/types/tooling.d.ts

@@ -160,3 +160,314 @@ export interface ValidateMarkupOptions {
  * `Passage "Start", line 3, column 5 (story.twee:12): Unclosed {if}: …`.
  */
 export declare function formatDiagnostic(diagnostic: MarkupDiagnostic): string;
+
+// ---------------------------------------------------------------------------
+// Parsing rules
+//
+// The leaf rules Spindle parses passages with, so that editor tooling can use
+// them instead of mirroring them (see docs/tooling.md). They are pure
+// string-to-data functions that do not load the runtime.
+// ---------------------------------------------------------------------------
+
+/** The sigil of a variable reference: story, temporary, local, transient. */
+export type Sigil = '$' | '_' | '@' | '%';
+
+/**
+ * What a piece of code is: one expression (a macro argument) or a list of
+ * statements (a `{do}` body).
+ */
+export type JsGoal = 'expression' | 'statements';
+
+/** What `lexJs` reports, in source order, every character exactly once. */
+export interface JsLexHandlers {
+  /**
+   * One character of code, outside literals and comments. `nesting` is the
+   * number of template-literal `${…}` interpolations around it.
+   */
+  code?(ch: string, index: number, nesting: number): void;
+  /**
+   * Literal text passed through verbatim: a string or regex literal, a
+   * comment, or a piece of a template literal.
+   */
+  literal?(text: string, index: number, nesting: number): void;
+  /** A sigil variable reference (`$name`), covering the sigil and name. */
+  variable?(sigil: Sigil, name: string, index: number, nesting: number): void;
+}
+
+/**
+ * Walk `src` as JavaScript, reporting code characters, literal text and
+ * variable references to `handlers`. Text that is no JavaScript is read
+ * leniently: an unterminated literal runs to the end. Returns `src.length`.
+ */
+export declare function lexJs(
+  src: string,
+  handlers: JsLexHandlers,
+  goal?: JsGoal,
+): number;
+
+/**
+ * Lex the template literal opening at `start` (a backtick) as `lexJs` does,
+ * its interpolations one nesting level deeper. Returns the index just past
+ * its closing backtick, or `src.length` if it is unterminated.
+ */
+export declare function lexTemplate(
+  src: string,
+  start: number,
+  handlers?: JsLexHandlers,
+  nesting?: number,
+): number;
+
+export interface FindCodeEndOptions {
+  /** What the code is (default `expression`). */
+  goal?: JsGoal;
+  /** End the code at a `{` in code, at any depth, for which this holds. */
+  stop?: (index: number) => boolean;
+}
+
+/**
+ * Where the code starting at `start` ends: the index of the `}` that closes
+ * the `{…}` around it (or, with `stop`, of the first `{` for which it holds).
+ * Braces and quotes in literals and comments don't count. `-1` when there is
+ * no such end, or the code before it can't be JavaScript.
+ */
+export declare function findCodeEnd(
+  src: string,
+  start: number,
+  options?: FindCodeEndOptions,
+): number;
+
+/**
+ * Scan the `"…"` or `'…'` string literal opening at `start`: `end` is just
+ * past its closing quote, or `src.length` when unterminated (`closed` false).
+ */
+export declare function scanStringLiteral(
+  src: string,
+  start: number,
+): { end: number; closed: boolean };
+
+/** The namespace a variable reference reads. */
+export type VariableScope = 'variable' | 'temporary' | 'local' | 'transient';
+
+/** The scope each variable sigil names. */
+export declare const SIGIL_SCOPES: Readonly<Record<Sigil, VariableScope>>;
+
+/** Whether `c` is a variable sigil. */
+export declare function isSigil(c: string | undefined): c is Sigil;
+
+/** The `.class#id` selectors written before a link, variable or macro. */
+export interface Selectors {
+  className?: string;
+  id?: string;
+}
+
+/**
+ * The selectors that start `source` at `at`, and `end`, the index just past
+ * them and the one space that may follow (`at` if there are none). A name
+ * may hold `{$name}` interpolations.
+ */
+export declare function parseSelectors(
+  source: string,
+  at?: number,
+): Selectors & { end: number };
+
+/** Where a token is in the input: from `start` up to `end`. */
+interface TokenSpan {
+  start: number;
+  end: number;
+}
+
+export interface TextToken extends TokenSpan {
+  type: 'text';
+  value: string;
+}
+
+export interface LinkToken extends TokenSpan, Selectors {
+  type: 'link';
+  display: string;
+  target: string;
+}
+
+export interface MacroToken extends TokenSpan, Selectors {
+  type: 'macro';
+  name: string;
+  rawArgs: string;
+  isClose: boolean;
+}
+
+export interface VariableToken extends TokenSpan, Selectors {
+  type: 'variable';
+  name: string;
+  scope: VariableScope;
+}
+
+export interface ExpressionToken extends TokenSpan, Selectors {
+  type: 'expression';
+  expression: string;
+}
+
+export interface HtmlToken extends TokenSpan {
+  type: 'html';
+  tag: string;
+  attributes: Record<string, string>;
+  isClose: boolean;
+  isSelfClose: boolean;
+}
+
+/** A flat markup token, with its offsets (UTF-16 code units) in the input. */
+export type Token =
+  | TextToken
+  | LinkToken
+  | MacroToken
+  | VariableToken
+  | ExpressionToken
+  | HtmlToken;
+
+/** Malformed markup, with where it starts (0-based offset, 1-based line and column). */
+export declare class MarkupError extends Error {
+  reason: string;
+  offset: number;
+  line: number;
+  column: number;
+}
+
+export interface ParseMarkupOptions {
+  /**
+   * Text mode, for markup that becomes a string (HTML attribute values,
+   * macro labels): only `{…}` markup and brace escapes are recognized.
+   */
+  text?: boolean;
+}
+
+/**
+ * The flat tokens of markup, without nesting, so unclosed or mismatched
+ * macros and elements are no error here. Throws a `MarkupError` for a
+ * malformed tag, such as an unclosed `{`, `[[` or attribute value.
+ */
+export declare function tokenizeMarkup(
+  source: string,
+  options?: ParseMarkupOptions,
+): Token[];
+
+/**
+ * Split macro arguments at top-level commas (outside strings, templates and
+ * brackets); with no comma, adjacent standalone values separated by
+ * whitespace (`"Label" "target"`).
+ */
+export declare function splitArgs(raw: string): string[];
+
+/**
+ * Split `src` at every top-level character for which `isSeparator` holds
+ * (code outside literals, comments and brackets). Segments are untrimmed and
+ * empty ones are kept.
+ */
+export declare function splitTopLevel(
+  src: string,
+  isSeparator: (ch: string) => boolean,
+): string[];
+
+/**
+ * Read the `"…"` or `'…'` string whose opening quote is at `start`: its
+ * unescaped value (`\"`, `\'` and `\\` only) and the index just past it, or
+ * `null` if there is none or it is unterminated.
+ */
+export declare function readQuoted(
+  src: string,
+  start: number,
+): { value: string; end: number } | null;
+
+/** Undo `\"`, `\'` and `\\` in the body of a quoted macro argument. */
+export declare function unescapeQuoted(body: string): string;
+
+/** Strip an optional quote from each end of a loosely quoted argument. */
+export declare function stripLooseQuotes(src: string): string;
+
+/** Whether code ends with an operator that still needs an operand. */
+export declare function endsWithOperator(src: string): boolean;
+
+/**
+ * The arguments of `{include}`: whether it has the standalone `inline` flag
+ * (first or last word, outside quotes and brackets), and the passage
+ * expression that is left.
+ */
+export declare function splitIncludeFlag(rawArgs: string): {
+  inline: boolean;
+  passage: string | undefined;
+};
+
+/** The tokens of markup that may be malformed, and its errors. */
+export interface TolerantTokens {
+  tokens: Token[];
+  /** One for each malformed tag, in source order, with offsets in the source. */
+  errors: MarkupError[];
+}
+
+/**
+ * `tokenizeMarkup` for half-typed markup: the tokens it can read, and every
+ * error, instead of throwing at the first. At a malformed tag the tokens
+ * before it are kept, its first character is read as text, and tokenizing
+ * resumes after it. Offsets are in `source`. For well-formed markup the
+ * tokens are those of `tokenizeMarkup` and there are no errors.
+ */
+export declare function tokenizeMarkupTolerant(
+  source: string,
+  options?: ParseMarkupOptions,
+): TolerantTokens;
+
+/**
+ * The code Spindle runs for `expr`: its variable references (`$var`, `_var`,
+ * `@var`, `%var`) turned into namespace lookups (`variables["var"]`).
+ * String, template and regex literals and comments are untouched. `goal` is
+ * `'statements'` for a `{do}` body. Throws a `SyntaxError` for code that is
+ * not well-formed, and for a reference to a variable named `__proto__`.
+ */
+export declare function transform(expr: string, goal?: JsGoal): string;
+
+/** What a `passage` argument is: a quoted name, or an expression. */
+export type PassageTarget =
+  | { kind: 'name'; name: string }
+  | { kind: 'expression'; expression: string };
+
+/**
+ * Read a `passage` argument (`{goto "Hall"}`, `{include $room}`) as written:
+ * a quoted string is the name its JavaScript literal has (`"Hall"` is
+ * `Hall`), anything else an expression whose value, as a string, is the name
+ * when it runs.
+ */
+export declare function passageTarget(arg: string): PassageTarget;
+
+/**
+ * The passage a `passage` argument names, as `{goto}`, `{include}` and
+ * `{link}` find it: `String(evaluate(expr))`. Throws what `evaluate` throws,
+ * and an error naming the current passage if the story has no passage of
+ * that name.
+ */
+export declare function evaluatePassageName(
+  expr: string | undefined,
+  evaluate: (expr: string) => unknown,
+  state: {
+    storyData: { passages: { has(name: string): boolean } } | null;
+    currentPassage: string;
+  },
+): string;
+
+/** A passage that markup names, and where. */
+export interface PassageReference {
+  /** The macro it is the argument of; `link` for `[[…]]` links. */
+  macro: string;
+  /** The name written out, or the expression whose value is the name. */
+  target: PassageTarget;
+  /** Where it is written, as written (quotes included): offsets in `source`. */
+  start: number;
+  end: number;
+}
+
+/**
+ * The passages the markup of `source` names, in source order: `[[…]]` links,
+ * the passage of `{goto}`, `{include}` and `{link}` (and of macros that
+ * declare a `passage` argument), the `goto` and `dialog` actions of
+ * `{watch}` and the body of `{dialog}`. Malformed tags are skipped, so
+ * half-typed markup reads.
+ */
+export declare function collectPassageReferences(
+  source: string,
+): PassageReference[];