@rohal12/spindle 0.57.0 → 0.59.0

package/src/markup/spindle.peggy

@@ -18,41 +18,48 @@
 //   isBlock(name), isRaw(name)  whether a macro takes a body / a raw body
 //   isCodeAttribute(name)  whether an attribute value is code (onclick)
 //
+// SIGIL_SCOPES and withSelectors come from tokens.ts (see grammar-shared.ts)
 // Note: Peggy's code blocks count braces without reading JavaScript strings,
 // so the code here writes literal braces as LB and RB.
 
 {{
 const LB = String.fromCharCode(123);
 const RB = String.fromCharCode(125);
-const SIGIL_SCOPES = { $: 'variable', _: 'temporary', '@': 'local', '%': 'transient' };
+const { SIGIL_SCOPES, withSelectors } = shared;
 const VOID_TAGS = new Set(['area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link', 'meta', 'param', 'source', 'track', 'wbr']);
 const BRANCH_PARENT = { elseif: 'if', else: 'if', case: 'switch', default: 'switch', next: 'timed' };
 const BRANCHING = new Set(['if', 'switch', 'timed']);
 const LINK_SEPARATORS = [['|', false], ['->', false], ['<-', true]];
 const NO_SELECTORS = {};
 
-function parseLink(inner) {
+/**
+ * A link's display text and target, from its text `inner` (at `at` in the
+ * input), with where the target is written.
+ */
+function parseLink(inner, at) {
+  /** The trimmed text from `start` to `end` of inner, and where it is. */
+  const part = (start, end) => {
+    const raw = inner.slice(start, end);
+    const value = raw.trim();
+    const from = at + start + (value ? raw.indexOf(value) : 0);
+    return { value, start: from, end: from + value.length };
+  };
   for (const [separator, targetFirst] of LINK_SEPARATORS) {
     const idx = inner.indexOf(separator);
     if (idx === -1) continue;
-    const before = inner.slice(0, idx).trim();
-    const after = inner.slice(idx + separator.length).trim();
-    return targetFirst ? { display: after, target: before } : { display: before, target: after };
+    const before = part(0, idx);
+    const after = part(idx + separator.length, inner.length);
+    const [display, target] = targetFirst ? [after, before] : [before, after];
+    return { display: display.value, target: target.value, targetStart: target.start, targetEnd: target.end };
   }
-  const trimmed = inner.trim();
-  return { display: trimmed, target: trimmed };
+  const whole = part(0, inner.length);
+  return { display: whole.value, target: whole.value, targetStart: whole.start, targetEnd: whole.end };
 }
 
 function quoteArg(value) {
   return '"' + value.replace(/[\\"]/g, '\\$&') + '"';
 }
 
-function withSelectors(target, from) {
-  if (from.className) target.className = from.className;
-  if (from.id) target.id = from.id;
-  return target;
-}
-
 function selectorsOf(parts) {
   const classes = [];
   let id = '';
@@ -111,7 +118,7 @@ function tokenOf(x) {
     case 'text':
       return { type: 'text', value: x.value, ...span };
     case 'link':
-      return withSelectors({ type: 'link', display: x.display, target: x.target, ...span }, x);
+      return withSelectors({ type: 'link', display: x.display, target: x.target, targetStart: x.targetStart, targetEnd: x.targetEnd, ...span }, x);
     case 'variable':
       return withSelectors({ type: 'variable', name: x.name, scope: x.scope, ...span }, x);
     case 'expression':
@@ -341,8 +348,8 @@ Literal
   / "[" { return lexeme('text', location(), { value: '[' }); }
 
 LinkLexeme
-  = &{ return !textMode; } "[[" s:Selectors inner:$LinkText "]]" {
-      return withSelectors(lexeme('link', location(), parseLink(inner)), s);
+  = &{ return !textMode; } "[[" s:Selectors at:Here inner:$LinkText "]]" {
+      return withSelectors(lexeme('link', location(), parseLink(inner, at)), s);
     }
   / &{ return !textMode; } "[[" { fail('Unclosed link: [[ without ]]', location()); }
 
@@ -359,6 +366,10 @@ Selectors
   = parts:SelectorPart+ " "? { return selectorsOf(parts); }
   / "" { return NO_SELECTORS; }
 
+/** The selectors at the start of the input, and where they end. */
+SelectorsPrefix
+  = s:Selectors end:Here .* { return { ...s, end }; }
+
 SelectorPart
   = prefix:[.#] name:$SelectorChar* { return [prefix, name]; }
 

package/src/markup/tokens.ts

@@ -58,6 +58,9 @@ export interface LinkToken extends Span, Selectors {
   type: 'link';
   display: string;
   target: string;
+  /** Where the target is written, from `targetStart` to `targetEnd`. */
+  targetStart: number;
+  targetEnd: number;
 }
 
 export interface MacroToken extends Span, Selectors {

package/src/markup/validate.ts

@@ -8,10 +8,13 @@
 import { lineColumn, MarkupError, parseMarkup, tokenizeMarkup } from './parse';
 import type { Token } from './tokens';
 import { parseWidgetDef } from '../widgets/widget-def';
-import { codeAndText, parseOrError, withParseCache } from '../code-check';
+import {
+  codeAndText,
+  parseOrError,
+  withParseCache,
+  type ParametersOf,
+} from '../code-check';
 import { CodeSyntaxError } from '../js-lexer';
-import type { ParameterDef } from '../registry';
-import { SUB_MACRO_PARAMETERS } from '../components/macros/option-utils';
 
 /** A passage to validate. */
 export interface MarkupPassage {
@@ -59,16 +62,24 @@ export interface MarkupValidationOptions {
    */
   checkPassageNames?: boolean;
   /**
-   * The declared parameters of a macro, whose `expression` and `statements`
-   * arguments are checked as code and whose `text` and `string` arguments
-   * as markup (see code-check.ts). Without them only the code of `{$…}`,
-   * `{do}`, branch conditions and attributes is checked.
+   * The declared parameters of a macro (see parameterLookup in
+   * code-check.ts), whose `expression` and `statements` arguments are
+   * checked as code and whose `text` and `string` arguments as what they
+   * hold. Without them only the code of `{$…}`, `{do}`, branch conditions
+   * and attributes is checked.
    */
-  parametersOf?(name: string): readonly ParameterDef[] | undefined;
+  parametersOf?: ParametersOf;
 }
 
-/** Passages that hold no markup. */
-const NOT_MARKUP = new Set(['StoryVariables', 'StoryTransients']);
+/**
+ * Passages that hold no markup: declarations, and the JavaScript function
+ * body of SaveTitle.
+ */
+export const NOT_MARKUP = new Set([
+  'StoryVariables',
+  'StoryTransients',
+  'SaveTitle',
+]);
 const NOT_MARKUP_TAGS = ['script', 'stylesheet'];
 
 /** Branch macros, which their parent macro renders. */
@@ -221,8 +232,7 @@ export function validateMarkup(
     }
   };
 
-  const parametersOf = (name: string) =>
-    options.parametersOf?.(name) ?? SUB_MACRO_PARAMETERS[name.toLowerCase()];
+  const parametersOf: ParametersOf = options.parametersOf ?? (() => undefined);
 
   /**
    * Check `tokens`, the tokens of `src`, which starts at `base` in the

package/src/registry.ts

@@ -104,11 +104,36 @@ export const PARAMETER_TYPES = [
 
 export type ParameterType = (typeof PARAMETER_TYPES)[number];
 
+/**
+ * What the value of a `string` or `text` argument holds, for the check at
+ * story start and for tooling (see code-check.ts):
+ * - `markup`: markup the macro renders (`{button}`'s label): its markup
+ *   is checked.
+ * - `text`: plain text the macro uses as written (`{checkbox}`'s label).
+ * - `passage`: a passage name (`{watch}`'s `goto`): the passage must exist.
+ * - `expression`, `statements`: code (`{watch}`'s condition and `run`): it
+ *   is checked as code, and its variable references against the schema.
+ *
+ * Without it, the argument of a macro with `interpolate` holds `markup`,
+ * any other `text`.
+ */
+export const STRING_HOLDS = [
+  'markup',
+  'text',
+  'passage',
+  'expression',
+  'statements',
+] as const;
+
+export type StringHolds = (typeof STRING_HOLDS)[number];
+
 export interface ParameterDef {
   name: string;
   required?: boolean;
   description?: string;
   type: ParameterType;
+  /** What a `string` or `text` argument holds (see StringHolds). */
+  holds?: StringHolds;
   /** The options of an `options` parameter. */
   parameters?: readonly ParameterDef[];
 }
@@ -116,30 +141,62 @@ export interface ParameterDef {
 /**
  * Throw if a parameter `macro` declares (or an option of one) has no type,
  * or one that isn't a ParameterType: arguments are read by their type, and
- * there is no default.
+ * there is no default. Throw too if it declares what it `holds` wrongly.
  */
 export function checkParameterTypes(
   macro: string,
   parameters: readonly ParameterDef[],
 ): void {
+  const holdsFix =
+    `Give a \`string\` or \`text\` parameter one of ${STRING_HOLDS.join(', ')} ` +
+    'to hold (see docs/custom-macros.md#what-a-string-holds).';
   for (const param of parameters) {
     const type: unknown = param.type;
     if (!PARAMETER_TYPES.includes(type as ParameterType)) {
-      const problem =
+      throw parameterError(
+        macro,
+        param,
         type === undefined
           ? 'has no type'
-          : `has the unknown type ${JSON.stringify(type)}`;
-      throw new Error(
-        `spindle: The parameter "${param.name}" of the macro {${macro}} ${problem}. ` +
-          `Give it one of the types ${PARAMETER_TYPES.join(', ')} ` +
+          : `has the unknown type ${JSON.stringify(type)}`,
+        `Give it one of the types ${PARAMETER_TYPES.join(', ')} ` +
           '(see docs/custom-macros.md#parameter-types), ' +
           'or declare no parameters and read props.rawArgs.',
       );
     }
+    const holds: unknown = param.holds;
+    if (holds !== undefined && type !== 'string' && type !== 'text') {
+      throw parameterError(
+        macro,
+        param,
+        `is of the type ${type}, which holds what it is`,
+        holdsFix,
+      );
+    }
+    if (holds !== undefined && !STRING_HOLDS.includes(holds as StringHolds)) {
+      throw parameterError(
+        macro,
+        param,
+        `holds the unknown ${JSON.stringify(holds)}`,
+        holdsFix,
+      );
+    }
     if (param.parameters) checkParameterTypes(macro, param.parameters);
   }
 }
 
+/** The error for the parameter `param` of `macro`: what is wrong, and the fix. */
+function parameterError(
+  macro: string,
+  param: ParameterDef,
+  problem: string,
+  fix: string,
+): Error {
+  return new Error(
+    `spindle: The parameter "${param.name}" of the macro {${macro}} ${problem}. ${fix}`,
+  );
+}
+
 type ArgValue<T, D> = T extends 'flag' | 'separator'
   ? boolean
   : T extends 'names'
@@ -178,6 +235,13 @@ export function registerMacroMetadata(
   metadataRegistry.set(name, metadata);
 }
 
+/** The declared parameters of the macro `name`, if it declares any. */
+export function getMacroParameters(
+  name: string,
+): readonly ParameterDef[] | undefined {
+  return metadataRegistry.get(name)?.parameters;
+}
+
 export function getMacroRegistry(): MacroMetadata[] {
   return Array.from(metadataRegistry.values());
 }

package/src/story-variables.ts

@@ -9,7 +9,15 @@ import {
   scanStringLiteral,
   type JsGoal,
 } from './js-lexer';
-import { parseOrError, withParseCache } from './code-check';
+import {
+  argPieces,
+  holdsCode,
+  parseOrError,
+  withParseCache,
+  type ParametersOf,
+} from './code-check';
+import { getMacroParameters, type ParameterDef } from './registry';
+import { NOT_MARKUP } from './markup/validate';
 import { createNamespace, variableNameError } from './utils/namespace';
 
 /**
@@ -319,18 +327,45 @@ function collectPassageRefs(
   content: string,
   storeVarMacros: ReadonlySet<string>,
   onRef: RefCallback,
+  parametersOf: ParametersOf,
 ): void {
   withParseCache(() =>
-    collectTokenRefs(content, tokensOf(content, false), storeVarMacros, onRef),
+    collectTokenRefs(
+      content,
+      tokensOf(content, false),
+      storeVarMacros,
+      onRef,
+      parametersOf,
+    ),
   );
 }
 
+/**
+ * Report the references in the code that a macro's quoted arguments hold
+ * (see ParameterDef.holds), the condition and `run` action of `{watch}`:
+ * the argument scan sees only the strings. Only the macros that declare
+ * code in a string have their arguments read.
+ */
+function scanStringCode(
+  token: Extract<Token, { type: 'macro' }>,
+  params: readonly ParameterDef[] | undefined,
+  onRef: RefCallback,
+): void {
+  if (!params || !token.rawArgs || !holdsCode(params)) return;
+  for (const piece of argPieces(token.rawArgs, 0, params, token.name, '')) {
+    if (piece.kind === 'code' && piece.inString) {
+      scanCode(piece.code, onRef, piece.goal);
+    }
+  }
+}
+
 /** Report the `$var` references in the tokens of `content`. */
 function collectTokenRefs(
   content: string,
   tokens: Token[],
   storeVarMacros: ReadonlySet<string>,
   onRef: RefCallback,
+  parametersOf: ParametersOf = () => undefined,
 ): void {
   for (let t = 0; t < tokens.length; t++) {
     const token = tokens[t]!;
@@ -345,6 +380,7 @@ function collectTokenRefs(
       }
     } else if (token.type === 'macro' && !token.isClose) {
       scanCode(token.rawArgs, onRef);
+      scanStringCode(token, parametersOf(token.name), onRef);
 
       if (storeVarMacros.has(token.name.toLowerCase())) {
         const first = token.rawArgs.trim().split(/\s+/)[0] ?? '';
@@ -383,6 +419,7 @@ export function validatePassages(
   passages: Map<string, Passage>,
   schema: Map<string, VariableSchema>,
   storeVarMacros: Iterable<string> = BUILTIN_STORE_VAR_MACROS,
+  parametersOf: ParametersOf = getMacroParameters,
 ): string[] {
   const errors: string[] = [];
   const storeVarSet = new Set(
@@ -390,16 +427,21 @@ export function validatePassages(
   );
 
   for (const [name, passage] of passages) {
-    // Don't validate the StoryVariables/StoryTransients passages themselves
-    if (name === 'StoryVariables' || name === 'StoryTransients') continue;
+    // Don't validate the declarations or the SaveTitle code themselves
+    if (NOT_MARKUP.has(name)) continue;
 
     const forLocals = extractForLocals(passage.content);
-    collectPassageRefs(passage.content, storeVarSet, (ref) => {
-      const error = validateRef(ref, schema, forLocals);
-      if (error) {
-        errors.push(`Passage "${name}": ${error}`);
-      }
-    });
+    collectPassageRefs(
+      passage.content,
+      storeVarSet,
+      (ref) => {
+        const error = validateRef(ref, schema, forLocals);
+        if (error) {
+          errors.push(`Passage "${name}": ${error}`);
+        }
+      },
+      parametersOf,
+    );
   }
 
   return errors;

package/src/tooling.ts

@@ -10,12 +10,53 @@ import {
   type MarkupPassage,
 } from './markup/validate';
 import { isBlockMacro } from './markup/ast';
+import { tokenizeMarkupTolerant } from './markup/parse';
+import {
+  collectPassageReferences,
+  parameterLookup,
+  type PassageReference,
+} from './code-check';
 import { blockWidgetNames } from './widgets/widget-def';
 import type { ParameterDef } from './registry';
 
 export { parseStoryVariables } from './story-variables';
 export { checkParameterTypes } from './registry';
 export { formatDiagnostic } from './markup/validate';
+
+// The parsing rules (see "Tooling API" in docs/tooling.md): the leaf rules
+// the runtime parses with, so editor tooling needn't mirror them.
+export { findCodeEnd, lexJs, lexTemplate, scanStringLiteral } from './js-lexer';
+export type {
+  FindCodeEndOptions,
+  JsGoal,
+  JsLexHandlers,
+  Sigil,
+} from './js-lexer';
+export {
+  MarkupError,
+  parseSelectors,
+  tokenizeMarkup,
+  tokenizeMarkupTolerant,
+} from './markup/parse';
+export type { ParseMarkupOptions, TolerantTokens } from './markup/parse';
+export { SIGIL_SCOPES, isSigil } from './markup/tokens';
+export type { Selectors, Token, VariableScope } from './markup/tokens';
+export {
+  endsWithOperator,
+  readQuoted,
+  splitArgs,
+  splitTopLevel,
+  stripLooseQuotes,
+  unescapeQuoted,
+} from './components/macros/arg-utils';
+export { splitIncludeFlag } from './components/macros/include-args';
+export {
+  evaluatePassageName,
+  passageTarget,
+} from './components/macros/macro-args';
+export type { PassageTarget } from './components/macros/macro-args';
+export { transform } from './transform';
+export type { PassageReference } from './code-check';
 export type { MarkupDiagnostic, MarkupPassage } from './markup/validate';
 
 /** What tooling knows about a macro (see MacroMetadata). */
@@ -33,8 +74,13 @@ export interface ToolingMacro {
   name: string;
   block: boolean;
   subMacros: string[];
-  /** Its declared parameters: their types tell which arguments are code. */
+  /**
+   * Its declared parameters: their types tell which arguments are code, and
+   * what a `string` or `text` argument holds (see ParameterDef.holds).
+   */
   parameters?: readonly ParameterDef[];
+  /** Whether it resolves markup: its strings hold markup by default. */
+  interpolate?: boolean;
 }
 
 /**
@@ -49,22 +95,34 @@ export function validateStoryMarkup(
   options: ValidateMarkupOptions = {},
 ): MarkupDiagnostic[] {
   const list = [...passages];
+  const all = [...macros];
   const known = new Set<string>();
-  const parameters = new Map<string, readonly ParameterDef[] | undefined>();
   const blocks = new Set(blockWidgetNames(list).map((n) => n.toLowerCase()));
-  for (const macro of macros) {
+  for (const macro of all) {
     const name = macro.name.toLowerCase();
     known.add(name);
-    parameters.set(name, macro.parameters);
     for (const sub of macro.subMacros) known.add(sub.toLowerCase());
     if (macro.block) blocks.add(name);
   }
   return validateMarkup(list, {
     isKnownMacro: (name) => known.has(name),
     macroNames: known,
-    parametersOf: (name) => parameters.get(name),
+    parametersOf: parameterLookup(all),
     isBlockMacro: (name) =>
       blocks.has(name.toLowerCase()) || isBlockMacro(name),
     checkPassageNames: options.checkPassageNames,
   });
 }
+
+/**
+ * The passages the markup of `source` names (see PassageReference), against
+ * the given macros: what the story-start check looks up. Malformed tags are
+ * skipped, so it reads half-typed markup.
+ */
+export function collectStoryPassageReferences(
+  source: string,
+  macros: Iterable<ToolingMacro>,
+): PassageReference[] {
+  const { tokens } = tokenizeMarkupTolerant(source);
+  return collectPassageReferences(source, tokens, parameterLookup(macros));
+}

package/src/transform.ts

@@ -0,0 +1,65 @@
+/**
+ * The sigil transform: the story's code with its variable references
+ * (`$var`, `_var`, `@var`, `%var`) turned into the namespace lookups the
+ * expression engine runs. Pure string-to-string, so tooling can use it
+ * (see src/tooling.ts).
+ */
+import { parseCode, type JsGoal, type Sigil } from './js-lexer';
+import { variableNameError } from './utils/namespace';
+
+const TRANSFORM_CACHE_MAX = 500;
+
+const NAMESPACES: Record<Sigil, string> = {
+  $: 'variables',
+  _: 'temporary',
+  '@': 'locals',
+  '%': 'transient',
+};
+
+/** Ends with an identifier character. */
+const IDENT_END_RE = /[\p{ID_Continue}$\u200c\u200d]$/u;
+
+/**
+ * Transform expression: $var → variables["var"], _var → temporary["var"],
+ * @var → locals["var"], %var → transient["var"].
+ * The lexer (`lexJs`) finds the references: only in code (string, template
+ * and regex literal text and comments are left untouched), only where an
+ * identifier starts (not in `a$b`, nor as the property name in `obj._x`),
+ * and `%var` only where an operand is expected, since `%` is also the
+ * modulo operator. `goal` tells whether `expr` is an expression or a list of
+ * statements, as that decides whether a leading `{` opens an object literal
+ * or a block. A reference to a variable named `__proto__` throws a
+ * SyntaxError: no namespace can hold one (see utils/namespace.ts).
+ *  */
+export function transform(expr: string, goal: JsGoal = 'expression'): string {
+  const key = goal === 'statements' ? 's' + expr : 'e' + expr;
+  const cached = transformCache.get(key);
+  if (cached !== undefined) return cached;
+  const { refs } = parseCode(expr, goal);
+  let result = '';
+  let at = 0;
+  for (const ref of refs) {
+    const error = variableNameError(ref.name, ref.sigil + ref.name);
+    if (error) throw new SyntaxError(`spindle: ${error}`);
+    result += expr.slice(at, ref.start);
+    if (ref.shorthand) result += `${ref.name}: `;
+    // `typeof%x` needs a space once `%x` turns into an identifier.
+    else if (IDENT_END_RE.test(result.slice(-2))) result += ' ';
+    result += `${NAMESPACES[ref.sigil]}["${ref.name}"]`;
+    at = ref.end;
+  }
+  result += expr.slice(at);
+  if (transformCache.size >= TRANSFORM_CACHE_MAX) {
+    transformCache.delete(transformCache.keys().next().value!);
+  }
+  transformCache.set(key, result);
+  return result;
+}
+
+/** Transformed code by goal and source. */
+const transformCache = new Map<string, string>();
+
+/** Clear the transformed code. Useful for testing and HMR. */
+export function clearTransformCache(): void {
+  transformCache.clear();
+}

package/src/widgets/widget-def.ts

@@ -7,8 +7,8 @@ import type { MacroArgs } from '../registry';
 
 /** A {widget} definition's name, then its `@` parameters. */
 export const WIDGET_PARAMETERS = [
-  { name: 'name', type: 'text', required: true },
-  { name: 'parameters', type: 'text' },
+  { name: 'name', type: 'text', holds: 'text', required: true },
+  { name: 'parameters', type: 'text', holds: 'text' },
 ] as const;
 
 export interface WidgetDef {

package/types/index.d.ts

@@ -272,6 +272,27 @@ export type ParameterType =
   | 'separator'
   | 'options';
 
+/**
+ * What the value of a `string` or `text` argument holds, for the check at
+ * story start and for tooling:
+ * - `markup`: markup the macro renders (`{button}`'s label): its markup
+ *   is checked.
+ * - `text`: plain text the macro uses as written (`{checkbox}`'s label).
+ * - `passage`: a passage name (`{watch}`'s `goto`): the passage must exist.
+ * - `expression`, `statements`: code (`{watch}`'s condition and `run`): it
+ *   is checked as code, and its variable references against the schema.
+ *
+ * Without it, the argument of a macro with `interpolate` holds `markup`,
+ * any other `text`.
+ * @see {@link ../../src/registry.ts} for the implementation.
+ */
+export type StringHolds =
+  | 'markup'
+  | 'text'
+  | 'passage'
+  | 'expression'
+  | 'statements';
+
 /**
  * Parameter metadata for a macro definition.
  * @see {@link ../../src/registry.ts} for the implementation.
@@ -282,6 +303,8 @@ export interface ParameterDef {
   description?: string;
   /** How the argument is read: required, there is no default. */
   type: ParameterType;
+  /** What a `string` or `text` argument holds (see StringHolds). */
+  holds?: StringHolds;
   /** The options of an `options` parameter. */
   parameters?: readonly ParameterDef[];
 }