@rohal12/spindle 0.56.0 → 0.58.0

package/src/expression.ts

@@ -2,13 +2,12 @@ import type { StoryState } from './store';
 import { useStoryStore } from './store';
 import type { Passage } from './parser';
 import { random, randomInt } from './prng';
-import { parseCode, type JsGoal, type Sigil } from './js-lexer';
+import { clearTransformCache, transform } from './transform';
 import {
   EMPTY_NAMESPACE,
   asNamespace,
   countOf,
   isNamespace,
-  variableNameError,
   type Counts,
   type Namespace,
 } from './utils/namespace';
@@ -36,60 +35,11 @@ type CompiledExpression = (
   transient: Record<string, unknown>,
 ) => unknown;
 
+export { transform };
+
 const FN_CACHE_MAX = 500;
 const fnCache = new Map<string, CompiledExpression>();
 
-const NAMESPACES: Record<Sigil, string> = {
-  $: 'variables',
-  _: 'temporary',
-  '@': 'locals',
-  '%': 'transient',
-};
-
-/** Ends with an identifier character. */
-const IDENT_END_RE = /[\p{ID_Continue}$\u200c\u200d]$/u;
-
-/**
- * Transform expression: $var → variables["var"], _var → temporary["var"],
- * @var → locals["var"], %var → transient["var"].
- * The lexer (`lexJs`) finds the references: only in code (string, template
- * and regex literal text and comments are left untouched), only where an
- * identifier starts (not in `a$b`, nor as the property name in `obj._x`),
- * and `%var` only where an operand is expected, since `%` is also the
- * modulo operator. `goal` tells whether `expr` is an expression or a list of
- * statements, as that decides whether a leading `{` opens an object literal
- * or a block. A reference to a variable named `__proto__` throws a
- * SyntaxError: no namespace can hold one (see utils/namespace.ts).
- * Exported for tests.
- */
-export function transform(expr: string, goal: JsGoal = 'expression'): string {
-  const key = goal === 'statements' ? 's' + expr : 'e' + expr;
-  const cached = transformCache.get(key);
-  if (cached !== undefined) return cached;
-  const { refs } = parseCode(expr, goal);
-  let result = '';
-  let at = 0;
-  for (const ref of refs) {
-    const error = variableNameError(ref.name, ref.sigil + ref.name);
-    if (error) throw new SyntaxError(`spindle: ${error}`);
-    result += expr.slice(at, ref.start);
-    if (ref.shorthand) result += `${ref.name}: `;
-    // `typeof%x` needs a space once `%x` turns into an identifier.
-    else if (IDENT_END_RE.test(result.slice(-2))) result += ' ';
-    result += `${NAMESPACES[ref.sigil]}["${ref.name}"]`;
-    at = ref.end;
-  }
-  result += expr.slice(at);
-  if (transformCache.size >= FN_CACHE_MAX) {
-    transformCache.delete(transformCache.keys().next().value!);
-  }
-  transformCache.set(key, result);
-  return result;
-}
-
-/** Transformed code by goal and source. */
-const transformCache = new Map<string, string>();
-
 const preamble =
   'const {currentPassage,previousPassage,visited,hasVisited,hasVisitedAny,hasVisitedAll,rendered,hasRendered,hasRenderedAny,hasRenderedAll,random,randomInt}=__fns;';
 
@@ -269,7 +219,7 @@ export function execute(code: string, ...namespaces: Namespaces): void {
 /** Clear the compiled expression cache. Useful for testing and HMR. */
 export function clearExpressionCache(): void {
   fnCache.clear();
-  transformCache.clear();
+  clearTransformCache();
   cachedFns = null;
   cachedVisitCounts = null;
   cachedRenderCounts = null;

package/src/markup/grammar-shared.ts

@@ -0,0 +1,8 @@
+/**
+ * What the grammar (spindle.peggy) imports from the rest of Spindle, so that
+ * it holds no second copy: the generated parser imports this module's
+ * default export as `shared` (see scripts/peggy.ts).
+ */
+import { SIGIL_SCOPES, withSelectors } from './tokens';
+
+export default { SIGIL_SCOPES, withSelectors };

package/src/markup/parse.ts

@@ -5,7 +5,7 @@
  */
 import { parse as pegParse } from './spindle.peggy';
 import { isBlockMacro, type ASTNode } from './ast';
-import type { Token } from './tokens';
+import type { Selectors, Token } from './tokens';
 import { isCodeAttribute } from './code-attributes';
 import { defaultCodeEnd, type CodeEnd } from './code-end';
 
@@ -74,7 +74,7 @@ export function lineColumn(
 
 function run(
   source: string,
-  startRule: 'Markup' | 'Tokens',
+  startRule: 'Markup' | 'Tokens' | 'SelectorsPrefix',
   options: ParseMarkupOptions,
 ): unknown {
   const hooks = options.hooks
@@ -116,3 +116,90 @@ export function tokenizeMarkup(
 ): Token[] {
   return run(source, 'Tokens', options) as Token[];
 }
+
+/**
+ * The `.class#id` selectors that start `source` at `at`, with the index just
+ * past them and the one space that may follow (`end`; `at` if there are
+ * none). A selector name may hold `{$name}` interpolations, as in markup.
+ */
+export function parseSelectors(
+  source: string,
+  at = 0,
+): Selectors & { end: number } {
+  const found = run(source.slice(at), 'SelectorsPrefix', {}) as Selectors & {
+    end: number;
+  };
+  return { ...found, end: at + found.end };
+}
+
+/** The tokens of markup that may be malformed, and its errors. */
+export interface TolerantTokens {
+  tokens: Token[];
+  /** One for each malformed tag, in source order, with offsets in the source. */
+  errors: MarkupError[];
+}
+
+/**
+ * `tokenizeMarkup` for half-typed markup: the tokens it can read, and every
+ * error, instead of throwing at the first. At a malformed tag, the tokens
+ * before it are kept, its first character is read as text and tokenizing
+ * resumes after it, so what follows is read as if the tag had not been
+ * started. Tokens and errors have offsets in `source`; for well-formed
+ * markup the tokens are those of `tokenizeMarkup` and there are no errors.
+ */
+export function tokenizeMarkupTolerant(
+  source: string,
+  options: ParseMarkupOptions = {},
+): TolerantTokens {
+  const tokens: Token[] = [];
+  const errors: MarkupError[] = [];
+  /** `text` shifted by `base`, tokenized: the tokens or the error. */
+  const attempt = (text: string, base: number): Token[] | MarkupError => {
+    try {
+      return tokenizeMarkup(text, options).map((token) => shift(token, base));
+    } catch (error) {
+      if (!(error instanceof MarkupError)) throw error;
+      return error;
+    }
+  };
+  let base = 0;
+  while (base <= source.length) {
+    const rest = source.slice(base);
+    const result = attempt(rest, base);
+    if (Array.isArray(result)) {
+      tokens.push(...result);
+      break;
+    }
+    const at = base + result.offset;
+    const { line, column } = lineColumn(source, at);
+    errors.push(new MarkupError(result.reason, at, line, column));
+    // The longest prefix that tokenizes: the error may be inside a tag
+    // (an attribute value) of which the prefix is itself malformed
+    let cut = result.offset;
+    let prefix = attempt(rest.slice(0, cut), base);
+    while (!Array.isArray(prefix)) {
+      cut = prefix.offset;
+      prefix = attempt(rest.slice(0, cut), base);
+    }
+    tokens.push(...prefix);
+    // The character that started the damage is text
+    if (cut < rest.length) {
+      const start = base + cut;
+      tokens.push({
+        type: 'text',
+        value: rest[cut]!,
+        start,
+        end: start + 1,
+      });
+    }
+    base += cut + 1;
+  }
+  return { tokens, errors };
+}
+
+/** `token` with its offsets moved `by` on. */
+function shift(token: Token, by: number): Token {
+  return by === 0
+    ? token
+    : { ...token, start: token.start + by, end: token.end + by };
+}

package/src/markup/spindle.d.peggy.ts

@@ -13,12 +13,15 @@ export declare class SyntaxError extends globalThis.SyntaxError {
 }
 
 export interface ParseOptions {
-  startRule: 'Markup' | 'Tokens';
+  startRule: 'Markup' | 'Tokens' | 'SelectorsPrefix';
   /** Text mode (attribute values, labels). */
   text: boolean;
   /** What the grammar leaves to code (see parse.ts MarkupHooks). */
   hooks: unknown;
 }
 
-/** Parse markup: the AST for `Markup`, the tokens for `Tokens`. */
+/**
+ * Parse markup: the AST for `Markup`, the tokens for `Tokens`, the selectors
+ * and where they end for `SelectorsPrefix`.
+ */
 export declare function parse(input: string, options: ParseOptions): unknown;

package/src/markup/spindle.peggy

@@ -18,13 +18,14 @@
 //   isBlock(name), isRaw(name)  whether a macro takes a body / a raw body
 //   isCodeAttribute(name)  whether an attribute value is code (onclick)
 //
+// SIGIL_SCOPES and withSelectors come from tokens.ts (see grammar-shared.ts)
 // Note: Peggy's code blocks count braces without reading JavaScript strings,
 // so the code here writes literal braces as LB and RB.
 
 {{
 const LB = String.fromCharCode(123);
 const RB = String.fromCharCode(125);
-const SIGIL_SCOPES = { $: 'variable', _: 'temporary', '@': 'local', '%': 'transient' };
+const { SIGIL_SCOPES, withSelectors } = shared;
 const VOID_TAGS = new Set(['area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link', 'meta', 'param', 'source', 'track', 'wbr']);
 const BRANCH_PARENT = { elseif: 'if', else: 'if', case: 'switch', default: 'switch', next: 'timed' };
 const BRANCHING = new Set(['if', 'switch', 'timed']);
@@ -47,12 +48,6 @@ function quoteArg(value) {
   return '"' + value.replace(/[\\"]/g, '\\$&') + '"';
 }
 
-function withSelectors(target, from) {
-  if (from.className) target.className = from.className;
-  if (from.id) target.id = from.id;
-  return target;
-}
-
 function selectorsOf(parts) {
   const classes = [];
   let id = '';
@@ -359,6 +354,10 @@ Selectors
   = parts:SelectorPart+ " "? { return selectorsOf(parts); }
   / "" { return NO_SELECTORS; }
 
+/** The selectors at the start of the input, and where they end. */
+SelectorsPrefix
+  = s:Selectors end:Here .* { return { ...s, end }; }
+
 SelectorPart
   = prefix:[.#] name:$SelectorChar* { return [prefix, name]; }
 

package/src/markup/validate.ts

@@ -11,6 +11,7 @@ import { parseWidgetDef } from '../widgets/widget-def';
 import { codeAndText, parseOrError, withParseCache } from '../code-check';
 import { CodeSyntaxError } from '../js-lexer';
 import type { ParameterDef } from '../registry';
+import { subMacroParameters } from '../components/macros/option-utils';
 
 /** A passage to validate. */
 export interface MarkupPassage {
@@ -66,8 +67,15 @@ export interface MarkupValidationOptions {
   parametersOf?(name: string): readonly ParameterDef[] | undefined;
 }
 
-/** Passages that hold no markup. */
-const NOT_MARKUP = new Set(['StoryVariables', 'StoryTransients']);
+/**
+ * Passages that hold no markup: declarations, and the JavaScript function
+ * body of SaveTitle.
+ */
+export const NOT_MARKUP = new Set([
+  'StoryVariables',
+  'StoryTransients',
+  'SaveTitle',
+]);
 const NOT_MARKUP_TAGS = ['script', 'stylesheet'];
 
 /** Branch macros, which their parent macro renders. */
@@ -220,7 +228,8 @@ export function validateMarkup(
     }
   };
 
-  const parametersOf = options.parametersOf ?? (() => undefined);
+  const parametersOf = (name: string) =>
+    options.parametersOf?.(name) ?? subMacroParameters(name);
 
   /**
    * Check `tokens`, the tokens of `src`, which starts at `base` in the
@@ -236,6 +245,10 @@ export function validateMarkup(
     checkMacros(passage, tokens, base, where);
     for (const piece of codeAndText(src, tokens, parametersOf)) {
       const at = base + piece.offset;
+      if (piece.kind === 'argument-error') {
+        report(passage, at, `${where}${piece.message} in ${piece.label}`);
+        continue;
+      }
       if (piece.kind === 'passage') {
         if (
           options.checkPassageNames !== false &&

package/src/registry.ts

@@ -178,6 +178,13 @@ export function registerMacroMetadata(
   metadataRegistry.set(name, metadata);
 }
 
+/** The declared parameters of the macro `name`, if it declares any. */
+export function getMacroParameters(
+  name: string,
+): readonly ParameterDef[] | undefined {
+  return metadataRegistry.get(name)?.parameters;
+}
+
 export function getMacroRegistry(): MacroMetadata[] {
   return Array.from(metadataRegistry.values());
 }

package/src/story-variables.ts

@@ -9,7 +9,9 @@ import {
   scanStringLiteral,
   type JsGoal,
 } from './js-lexer';
-import { parseOrError, withParseCache } from './code-check';
+import { argPieces, parseOrError, withParseCache } from './code-check';
+import { getMacroParameters, type ParameterDef } from './registry';
+import { NOT_MARKUP } from './markup/validate';
 import { createNamespace, variableNameError } from './utils/namespace';
 
 /**
@@ -319,18 +321,47 @@ function collectPassageRefs(
   content: string,
   storeVarMacros: ReadonlySet<string>,
   onRef: RefCallback,
+  parametersOf: ParametersOf,
 ): void {
   withParseCache(() =>
-    collectTokenRefs(content, tokensOf(content, false), storeVarMacros, onRef),
+    collectTokenRefs(
+      content,
+      tokensOf(content, false),
+      storeVarMacros,
+      onRef,
+      parametersOf,
+    ),
   );
 }
 
+/** The declared parameters of a macro, if it has any. */
+type ParametersOf = (macro: string) => readonly ParameterDef[] | undefined;
+
+/**
+ * Report the references in the code that a macro's quoted arguments hold,
+ * the condition and `run` action of `{watch}`: the argument scan sees only
+ * the strings.
+ */
+function scanStringCode(
+  token: Extract<Token, { type: 'macro' }>,
+  params: readonly ParameterDef[] | undefined,
+  onRef: RefCallback,
+): void {
+  if (!params || !token.rawArgs) return;
+  for (const piece of argPieces(token.rawArgs, 0, params, token.name, '')) {
+    if (piece.kind === 'code' && piece.inString) {
+      scanCode(piece.code, onRef, piece.goal);
+    }
+  }
+}
+
 /** Report the `$var` references in the tokens of `content`. */
 function collectTokenRefs(
   content: string,
   tokens: Token[],
   storeVarMacros: ReadonlySet<string>,
   onRef: RefCallback,
+  parametersOf: ParametersOf = () => undefined,
 ): void {
   for (let t = 0; t < tokens.length; t++) {
     const token = tokens[t]!;
@@ -345,6 +376,7 @@ function collectTokenRefs(
       }
     } else if (token.type === 'macro' && !token.isClose) {
       scanCode(token.rawArgs, onRef);
+      scanStringCode(token, parametersOf(token.name), onRef);
 
       if (storeVarMacros.has(token.name.toLowerCase())) {
         const first = token.rawArgs.trim().split(/\s+/)[0] ?? '';
@@ -383,6 +415,7 @@ export function validatePassages(
   passages: Map<string, Passage>,
   schema: Map<string, VariableSchema>,
   storeVarMacros: Iterable<string> = BUILTIN_STORE_VAR_MACROS,
+  parametersOf: ParametersOf = getMacroParameters,
 ): string[] {
   const errors: string[] = [];
   const storeVarSet = new Set(
@@ -390,16 +423,21 @@ export function validatePassages(
   );
 
   for (const [name, passage] of passages) {
-    // Don't validate the StoryVariables/StoryTransients passages themselves
-    if (name === 'StoryVariables' || name === 'StoryTransients') continue;
+    // Don't validate the declarations or the SaveTitle code themselves
+    if (NOT_MARKUP.has(name)) continue;
 
     const forLocals = extractForLocals(passage.content);
-    collectPassageRefs(passage.content, storeVarSet, (ref) => {
-      const error = validateRef(ref, schema, forLocals);
-      if (error) {
-        errors.push(`Passage "${name}": ${error}`);
-      }
-    });
+    collectPassageRefs(
+      passage.content,
+      storeVarSet,
+      (ref) => {
+        const error = validateRef(ref, schema, forLocals);
+        if (error) {
+          errors.push(`Passage "${name}": ${error}`);
+        }
+      },
+      parametersOf,
+    );
   }
 
   return errors;

package/src/tooling.ts

@@ -10,12 +10,50 @@ import {
   type MarkupPassage,
 } from './markup/validate';
 import { isBlockMacro } from './markup/ast';
+import { tokenizeMarkupTolerant } from './markup/parse';
+import { collectPassageReferences, type PassageReference } from './code-check';
+import { subMacroParameters } from './components/macros/option-utils';
 import { blockWidgetNames } from './widgets/widget-def';
 import type { ParameterDef } from './registry';
 
 export { parseStoryVariables } from './story-variables';
 export { checkParameterTypes } from './registry';
 export { formatDiagnostic } from './markup/validate';
+
+// The parsing rules (see "Tooling API" in docs/tooling.md): the leaf rules
+// the runtime parses with, so editor tooling needn't mirror them.
+export { findCodeEnd, lexJs, lexTemplate, scanStringLiteral } from './js-lexer';
+export type {
+  FindCodeEndOptions,
+  JsGoal,
+  JsLexHandlers,
+  Sigil,
+} from './js-lexer';
+export {
+  MarkupError,
+  parseSelectors,
+  tokenizeMarkup,
+  tokenizeMarkupTolerant,
+} from './markup/parse';
+export type { ParseMarkupOptions, TolerantTokens } from './markup/parse';
+export { SIGIL_SCOPES, isSigil } from './markup/tokens';
+export type { Selectors, Token, VariableScope } from './markup/tokens';
+export {
+  endsWithOperator,
+  readQuoted,
+  splitArgs,
+  splitTopLevel,
+  stripLooseQuotes,
+  unescapeQuoted,
+} from './components/macros/arg-utils';
+export { splitIncludeFlag } from './components/macros/include-args';
+export {
+  evaluatePassageName,
+  passageTarget,
+} from './components/macros/macro-args';
+export type { PassageTarget } from './components/macros/macro-args';
+export { transform } from './transform';
+export type { PassageReference } from './code-check';
 export type { MarkupDiagnostic, MarkupPassage } from './markup/validate';
 
 /** What tooling knows about a macro (see MacroMetadata). */
@@ -68,3 +106,24 @@ export function validateStoryMarkup(
     checkPassageNames: options.checkPassageNames,
   });
 }
+
+/**
+ * The passages the markup of `source` names (see PassageReference), against
+ * the given macros: what the story-start check looks up. Malformed tags are
+ * skipped, so it reads half-typed markup.
+ */
+export function collectStoryPassageReferences(
+  source: string,
+  macros: Iterable<ToolingMacro>,
+): PassageReference[] {
+  const parameters = new Map<string, readonly ParameterDef[] | undefined>();
+  for (const macro of macros) {
+    parameters.set(macro.name.toLowerCase(), macro.parameters);
+  }
+  const { tokens } = tokenizeMarkupTolerant(source);
+  return collectPassageReferences(
+    source,
+    tokens,
+    (name) => parameters.get(name.toLowerCase()) ?? subMacroParameters(name),
+  );
+}

package/src/transform.ts

@@ -0,0 +1,65 @@
+/**
+ * The sigil transform: the story's code with its variable references
+ * (`$var`, `_var`, `@var`, `%var`) turned into the namespace lookups the
+ * expression engine runs. Pure string-to-string, so tooling can use it
+ * (see src/tooling.ts).
+ */
+import { parseCode, type JsGoal, type Sigil } from './js-lexer';
+import { variableNameError } from './utils/namespace';
+
+const TRANSFORM_CACHE_MAX = 500;
+
+const NAMESPACES: Record<Sigil, string> = {
+  $: 'variables',
+  _: 'temporary',
+  '@': 'locals',
+  '%': 'transient',
+};
+
+/** Ends with an identifier character. */
+const IDENT_END_RE = /[\p{ID_Continue}$\u200c\u200d]$/u;
+
+/**
+ * Transform expression: $var → variables["var"], _var → temporary["var"],
+ * @var → locals["var"], %var → transient["var"].
+ * The lexer (`lexJs`) finds the references: only in code (string, template
+ * and regex literal text and comments are left untouched), only where an
+ * identifier starts (not in `a$b`, nor as the property name in `obj._x`),
+ * and `%var` only where an operand is expected, since `%` is also the
+ * modulo operator. `goal` tells whether `expr` is an expression or a list of
+ * statements, as that decides whether a leading `{` opens an object literal
+ * or a block. A reference to a variable named `__proto__` throws a
+ * SyntaxError: no namespace can hold one (see utils/namespace.ts).
+ *  */
+export function transform(expr: string, goal: JsGoal = 'expression'): string {
+  const key = goal === 'statements' ? 's' + expr : 'e' + expr;
+  const cached = transformCache.get(key);
+  if (cached !== undefined) return cached;
+  const { refs } = parseCode(expr, goal);
+  let result = '';
+  let at = 0;
+  for (const ref of refs) {
+    const error = variableNameError(ref.name, ref.sigil + ref.name);
+    if (error) throw new SyntaxError(`spindle: ${error}`);
+    result += expr.slice(at, ref.start);
+    if (ref.shorthand) result += `${ref.name}: `;
+    // `typeof%x` needs a space once `%x` turns into an identifier.
+    else if (IDENT_END_RE.test(result.slice(-2))) result += ' ';
+    result += `${NAMESPACES[ref.sigil]}["${ref.name}"]`;
+    at = ref.end;
+  }
+  result += expr.slice(at);
+  if (transformCache.size >= TRANSFORM_CACHE_MAX) {
+    transformCache.delete(transformCache.keys().next().value!);
+  }
+  transformCache.set(key, result);
+  return result;
+}
+
+/** Transformed code by goal and source. */
+const transformCache = new Map<string, string>();
+
+/** Clear the transformed code. Useful for testing and HMR. */
+export function clearTransformCache(): void {
+  transformCache.clear();
+}