@wdprlib/parser 3.1.2 → 4.0.0

package/src/parser/rules/block/module/resolution/dynamic-modules.ts

@@ -0,0 +1,93 @@
+import type { Element } from "@wdprlib/ast";
+import { isListPagesModule, resolveListPages } from "../listpages/resolve";
+import { isListUsersModule, resolveListUsers } from "../listusers/resolve";
+import { walkElements } from "../walk";
+import type { ListPagesContext, ListUsersContext } from "./contexts";
+
+export interface DynamicModuleContext {
+  listPages: ListPagesContext | null;
+  listUsers: ListUsersContext | null;
+  fetchListPagesProvided: boolean;
+  fetchListUsersProvided: boolean;
+}
+
+export interface DynamicModuleIds {
+  listPagesId: number;
+  listUsersId: number;
+}
+
+export interface DynamicModuleResolution {
+  handled: boolean;
+  elements: Element[];
+  ids: DynamicModuleIds;
+}
+
+export function resolveDynamicModuleElement(
+  element: Element,
+  ctx: DynamicModuleContext,
+  ids: DynamicModuleIds,
+): DynamicModuleResolution {
+  if (element.element !== "module") {
+    return { handled: false, elements: [element], ids };
+  }
+
+  if (isListPagesModule(element.data)) {
+    const elements: Element[] = [];
+    const listPagesId = ids.listPagesId;
+
+    if (ctx.listPages) {
+      const moduleData = ctx.listPages.dataMap.get(listPagesId);
+      const template = ctx.listPages.compiledTemplates.get(listPagesId);
+
+      if (moduleData && template) {
+        elements.push(...resolveListPages(element.data, moduleData, template, ctx.listPages.parse));
+      }
+    } else if (!ctx.fetchListPagesProvided) {
+      elements.push(element);
+    }
+
+    return {
+      handled: true,
+      elements,
+      ids: { ...ids, listPagesId: listPagesId + 1 },
+    };
+  }
+
+  if (isListUsersModule(element.data)) {
+    const elements: Element[] = [];
+    const listUsersId = ids.listUsersId;
+
+    if (ctx.listUsers) {
+      const moduleData = ctx.listUsers.dataMap.get(listUsersId);
+      const template = ctx.listUsers.compiledTemplates.get(listUsersId);
+
+      if (moduleData && template) {
+        elements.push(...resolveListUsers(element.data, moduleData, template, ctx.listUsers.parse));
+      }
+    } else if (!ctx.fetchListUsersProvided) {
+      elements.push(element);
+    }
+
+    return {
+      handled: true,
+      elements,
+      ids: { ...ids, listUsersId: listUsersId + 1 },
+    };
+  }
+
+  return { handled: false, elements: [element], ids };
+}
+
+export function countDynamicModules(elements: Element[]): DynamicModuleIds {
+  const counts: DynamicModuleIds = { listPagesId: 0, listUsersId: 0 };
+  walkElements(elements, (element) => {
+    if (element.element !== "module") return;
+
+    if (isListPagesModule(element.data)) {
+      counts.listPagesId++;
+    } else if (isListUsersModule(element.data)) {
+      counts.listUsersId++;
+    }
+  });
+  return counts;
+}

package/src/parser/rules/block/module/resolution/styles.ts

@@ -0,0 +1,53 @@
+import type { Element } from "@wdprlib/ast";
+import { STYLE_SLOT_PREFIX } from "@wdprlib/ast";
+import { mapElementChildren } from "../walk";
+import type { IfTagsData } from "../iftags/resolve";
+
+type IfTagsDataWithStyleSlot = IfTagsData & {
+  _styleSlot: number;
+};
+
+/**
+ * Collect and remove style elements from the AST.
+ *
+ * Unresolved `if-tags` elements receive style-slot placeholders so render-time
+ * iftags evaluation can preserve style order relative to already collected CSS.
+ */
+export function collectStyles(elements: Element[]): { elements: Element[]; styles: string[] } {
+  const styles: string[] = [];
+  const ctx = { nextSlotId: 0 };
+  const filtered = collectStylesFromElements(elements, styles, ctx);
+  return { elements: filtered, styles };
+}
+
+function collectStylesFromElements(
+  elements: Element[],
+  styles: string[],
+  ctx: { nextSlotId: number },
+): Element[] {
+  const result: Element[] = [];
+
+  for (const element of elements) {
+    if (element.element === "style") {
+      styles.push(element.data as string);
+      continue;
+    }
+
+    if (element.element === "if-tags") {
+      const slotId = ctx.nextSlotId++;
+      styles.push(`${STYLE_SLOT_PREFIX}${slotId}`);
+      const data: IfTagsDataWithStyleSlot = { ...(element.data as IfTagsData), _styleSlot: slotId };
+      result.push({
+        element: "if-tags",
+        data,
+      });
+      continue;
+    }
+
+    result.push(
+      mapElementChildren(element, (children) => collectStylesFromElements(children, styles, ctx)),
+    );
+  }
+
+  return result;
+}

package/src/parser/rules/block/module/resolution/walk-resolve.ts

@@ -0,0 +1,107 @@
+import type { Element } from "@wdprlib/ast";
+import { mapElementChildrenWithState } from "../walk";
+import { isIfTagsElement, resolveIfTags, type IfTagsData } from "../iftags/resolve";
+import type { ListPagesContext, ListUsersContext } from "./contexts";
+import { countDynamicModules, resolveDynamicModuleElement } from "./dynamic-modules";
+
+/**
+ * Resolution context passed through AST traversal.
+ */
+export interface WalkContext {
+  listPages: ListPagesContext | null;
+  listUsers: ListUsersContext | null;
+  /** Whether fetchListPages callback was provided, even if no data returned. */
+  fetchListPagesProvided: boolean;
+  /** Whether fetchListUsers callback was provided, even if no data returned. */
+  fetchListUsersProvided: boolean;
+  pageTags: string[] | null;
+  listPagesIdCounter: number;
+  listUsersIdCounter: number;
+}
+
+export interface WalkResult {
+  elements: Element[];
+  nextListPagesId: number;
+  nextListUsersId: number;
+}
+
+/**
+ * Walk AST and resolve modules/iftags.
+ */
+export function walkAndResolve(elements: Element[], ctx: WalkContext): WalkResult {
+  const result: Element[] = [];
+  let listPagesId = ctx.listPagesIdCounter;
+  let listUsersId = ctx.listUsersIdCounter;
+
+  for (const element of elements) {
+    const dynamicModule = resolveDynamicModuleElement(element, ctx, { listPagesId, listUsersId });
+    if (dynamicModule.handled) {
+      result.push(...dynamicModule.elements);
+      listPagesId = dynamicModule.ids.listPagesId;
+      listUsersId = dynamicModule.ids.listUsersId;
+      continue;
+    }
+
+    if (isIfTagsElement(element)) {
+      const ifTagsData = element.data as IfTagsData;
+      const resolveResult = resolveIfTags(ifTagsData, ctx.pageTags);
+
+      if (resolveResult.evaluated) {
+        if (resolveResult.matched) {
+          const childResult = walkAndResolve(ifTagsData.elements, {
+            ...ctx,
+            listPagesIdCounter: listPagesId,
+            listUsersIdCounter: listUsersId,
+          });
+          result.push(...childResult.elements);
+          listPagesId = childResult.nextListPagesId;
+          listUsersId = childResult.nextListUsersId;
+        } else {
+          const counts = countDynamicModules(ifTagsData.elements);
+          listPagesId += counts.listPagesId;
+          listUsersId += counts.listUsersId;
+        }
+      } else {
+        const childResult = walkAndResolve(ifTagsData.elements, {
+          ...ctx,
+          listPagesIdCounter: listPagesId,
+          listUsersIdCounter: listUsersId,
+        });
+        result.push({
+          element: "if-tags",
+          data: {
+            ...ifTagsData,
+            elements: childResult.elements,
+          },
+        });
+        listPagesId = childResult.nextListPagesId;
+        listUsersId = childResult.nextListUsersId;
+      }
+      continue;
+    }
+
+    const mapped = mapElementChildrenWithState(
+      element,
+      { listPagesId, listUsersId },
+      (children, state) => {
+        const childResult = walkAndResolve(children, {
+          ...ctx,
+          listPagesIdCounter: state.listPagesId,
+          listUsersIdCounter: state.listUsersId,
+        });
+        return {
+          elements: childResult.elements,
+          state: {
+            listPagesId: childResult.nextListPagesId,
+            listUsersId: childResult.nextListUsersId,
+          },
+        };
+      },
+    );
+    result.push(mapped.element);
+    listPagesId = mapped.state.listPagesId;
+    listUsersId = mapped.state.listUsersId;
+  }
+
+  return { elements: result, nextListPagesId: listPagesId, nextListUsersId: listUsersId };
+}

package/src/parser/rules/block/module/resolve.ts

@@ -0,0 +1,198 @@
+/**
+ * Unified resolver that walks a parsed AST and expands dynamic modules.
+ *
+ * Handles three module families in a single traversal:
+ *
+ * - **ListPages** — fetches page data via {@link DataProvider.fetchListPages},
+ *   resolves `@URL` parameters from the page path (HPC support), and
+ *   expands `%%variable%%` templates.
+ * - **ListUsers** — fetches user data via {@link DataProvider.fetchListUsers}
+ *   and expands `%%variable%%` templates.
+ * - **IfTags** — evaluates tag conditions against the current page's tags
+ *   (from {@link DataProvider.getPageTags}) and keeps or discards content.
+ *
+ * The main entry point is {@link resolveModules}.
+ *
+ * @module
+ */
+
+import type { SyntaxTree } from "@wdprlib/ast";
+import type { DataProvider } from "./types-common";
+import { resolveIncludes } from "./include";
+import type { ListPagesDataRequirement, CompiledTemplate } from "./listpages/types";
+import type { ListUsersDataRequirement, ListUsersCompiledTemplate } from "./listusers/types";
+import type { ParseFunction } from "./listpages/resolve";
+import { buildListPagesContext, buildListUsersContext } from "./resolution/contexts";
+import { walkAndResolve } from "./resolution/walk-resolve";
+import { collectStyles } from "./resolution/styles";
+
+const MODULE_SECONDARY_INCLUDE_MAX_ITERATIONS = 5;
+
+// Re-export from listpages/resolve for external use
+export type { ParseFunction } from "./listpages/resolve";
+
+/**
+ * Transform module-generated wikitext before it is parsed back into AST nodes.
+ *
+ * Applications can use this hook to inject include resolution, parse caches, or
+ * diagnostics for ListPages/ListUsers secondary transformations.
+ *
+ * This hook receives the string after module variables such as `%%title%%` have
+ * already been substituted. Returning a different string changes what is parsed
+ * for that module item. When this hook is supplied it replaces the built-in
+ * secondary include pass, so callers that still need Wikidot-style secondary
+ * includes must call `resolveIncludes()` or equivalent logic from the hook.
+ */
+export type ModuleSourceTransform = (source: string) => string;
+
+/**
+ * Configuration for {@link resolveModules}.
+ *
+ * Callers must supply pre-extracted requirements and pre-compiled
+ * templates (obtained from `extractDataRequirements()` and
+ * `compileTemplate()` / `compileListUsersTemplate()`).
+ *
+ * @group Module Resolution
+ */
+export interface ResolveOptions {
+  /** Parser function used to re-parse expanded template markup into AST nodes */
+  parse: ParseFunction;
+
+  /** Pre-compiled ListPages body templates, keyed by requirement ID */
+  compiledListPagesTemplates: Map<number, CompiledTemplate>;
+
+  /** Pre-compiled ListUsers body templates, keyed by requirement ID */
+  compiledListUsersTemplates?: Map<number, ListUsersCompiledTemplate>;
+
+  /**
+   * Data requirements grouped by module type.
+   * Obtained from `extractDataRequirements()`.
+   */
+  requirements: {
+    listPages?: ListPagesDataRequirement[];
+    listUsers?: ListUsersDataRequirement[];
+  };
+
+  /**
+   * URL path for `@URL` parameter resolution (HPC / pagination support).
+   *
+   * Wikidot encodes pagination state in the URL path as key/value pairs
+   * after the page name, e.g. `"/scp-001/offset/10/page2_limit/5"`.
+   * When provided, `@URL` references in ListPages queries are replaced
+   * with the corresponding values from this path.
+   */
+  urlPath?: string;
+
+  /**
+   * Maximum include expansion iterations for secondary transformations inside
+   * modules such as ListPages/ListUsers. Defaults to Wikidot's observed
+   * secondary transformation limit.
+   */
+  includeMaxIterations?: number;
+
+  /**
+   * Transform module-generated wikitext before re-parsing it.
+   *
+   * When omitted, `resolveModules()` preserves the existing default: if
+   * `dataProvider.fetchInclude` is present, secondary `[[include]]` expansion
+   * is performed with `includeMaxIterations`; otherwise the source is parsed
+   * unchanged. When supplied, this hook owns the whole transformation.
+   *
+   * This is the extension point for application-level parse/include caches. The
+   * parser package does not persist cache entries itself because cache keys and
+   * invalidation depend on site/page revisions, tags, URL parameters, and user
+   * state outside wdpr.
+   */
+  transformModuleSource?: ModuleSourceTransform;
+}
+
+/**
+ * Resolve all modules in the AST
+ *
+ * Fetches data for each module using the provided callback,
+ * then expands the modules with the fetched data.
+ *
+ * Handles:
+ * - ListPages: fetches page data and expands templates
+ * - IfTags: evaluates tag conditions and includes/excludes content
+ *
+ * @param ast - Parsed AST
+ * @param dataProvider - Callback provider to fetch data for each module
+ * @param options - Resolution options including requirements
+ */
+export async function resolveModules(
+  ast: SyntaxTree,
+  dataProvider: DataProvider,
+  options: ResolveOptions,
+): Promise<SyntaxTree> {
+  const parse = createModuleParseFunction(options, dataProvider);
+  const listPagesCtx = await buildListPagesContext(
+    dataProvider,
+    options.requirements.listPages ?? [],
+    options.compiledListPagesTemplates,
+    parse,
+    options.urlPath,
+  );
+  const listUsersCtx = await buildListUsersContext(
+    dataProvider,
+    options.requirements.listUsers ?? [],
+    options.compiledListUsersTemplates,
+    parse,
+  );
+  const pageTags = dataProvider.getPageTags?.() ?? null;
+
+  // Resolve AST
+  const resolvedElements = walkAndResolve(ast.elements, {
+    listPages: listPagesCtx,
+    listUsers: listUsersCtx,
+    fetchListPagesProvided: dataProvider.fetchListPages !== undefined,
+    fetchListUsersProvided: dataProvider.fetchListUsers !== undefined,
+    pageTags,
+    listPagesIdCounter: 0,
+    listUsersIdCounter: 0,
+  });
+
+  // Collect style elements from resolved AST
+  const { elements: finalElements, styles } = collectStyles(resolvedElements.elements);
+
+  const result: SyntaxTree = {
+    ...ast,
+    elements: finalElements,
+  };
+
+  if (styles.length > 0) {
+    result.styles = styles;
+  }
+
+  return result;
+}
+
+function createModuleParseFunction(
+  options: ResolveOptions,
+  dataProvider: DataProvider,
+): ParseFunction {
+  const transform = createModuleSourceTransform(options, dataProvider);
+  if (!transform) {
+    return options.parse;
+  }
+
+  return (source: string) => options.parse(transform(source));
+}
+
+function createModuleSourceTransform(
+  options: ResolveOptions,
+  dataProvider: DataProvider,
+): ModuleSourceTransform | null {
+  if (options.transformModuleSource) {
+    return options.transformModuleSource;
+  }
+
+  if (!dataProvider.fetchInclude) {
+    return null;
+  }
+
+  return (source: string) =>
+    resolveIncludes(source, dataProvider.fetchInclude!, {
+      maxIterations: options.includeMaxIterations ?? MODULE_SECONDARY_INCLUDE_MAX_ITERATIONS,
+    });
+}

package/src/parser/rules/block/module/rule.ts

@@ -0,0 +1,56 @@
+import type { Element } from "@wdprlib/ast";
+import type { BlockRule, ParseContext, RuleResult } from "../../types";
+import { currentToken } from "../../types";
+import { parseModuleBody } from "./body";
+import { createUnknownModuleElement, moduleParseResultToElement } from "./element";
+import { getModuleRuleByName } from "./mapping";
+import { parseModuleOpen } from "./open";
+
+export const moduleRule: BlockRule = {
+  name: "module",
+  startTokens: ["BLOCK_OPEN"],
+  requiresLineStart: false,
+
+  parse(ctx: ParseContext): RuleResult<Element> {
+    const openToken = currentToken(ctx);
+    if (openToken.type !== "BLOCK_OPEN") {
+      return { success: false };
+    }
+
+    const openResult = parseModuleOpen(ctx);
+    if (!openResult) {
+      return { success: false };
+    }
+
+    // Page syntax disabled (e.g., forum-post mode)
+    if (!ctx.settings.enablePageSyntax) {
+      return { success: false };
+    }
+
+    // Dispatch to specific module parser
+    const moduleParseRule = getModuleRuleByName(openResult.moduleName);
+
+    // Check for body based on module rule's hasBody flag
+    // For unknown modules, default to no body (they will be parsed as bodyless)
+    const bodyResult = parseModuleBody(ctx, openResult.pos, moduleParseRule?.hasBody ?? false);
+    const consumed = openResult.consumed + bodyResult.consumed;
+
+    if (moduleParseRule) {
+      const result = moduleParseRule.parse(ctx, bodyResult.pos, openResult.attrs, bodyResult.body);
+
+      return {
+        success: true,
+        elements: [moduleParseResultToElement(result)],
+        consumed,
+      };
+    }
+
+    return {
+      success: true,
+      elements: [
+        createUnknownModuleElement(openResult.moduleName, openResult.attrs, bodyResult.body),
+      ],
+      consumed,
+    };
+  },
+};

package/src/parser/rules/block/module/types-common.ts

@@ -0,0 +1,70 @@
+/**
+ * Shared callback interface for module resolution.
+ *
+ * `DataProvider` is the single object that callers pass to
+ * `resolveModules()` to supply external data. Each property is an
+ * optional async callback; if omitted the corresponding module type is
+ * left unresolved in the AST.
+ *
+ * Include resolution uses a separate API (`resolveIncludes()`)
+ * because it operates on raw wikitext before parsing, not on AST nodes.
+ *
+ * @module
+ */
+
+import type { ListPagesDataFetcher } from "./listpages/types";
+import type { ListUsersDataFetcher } from "./listusers/types";
+import type { IfTagsResolver } from "./iftags/types";
+import type { IncludeFetcher } from "./include/resolve/types";
+
+/**
+ * Callback bag for supplying external data during module resolution.
+ *
+ * Pass an instance to `resolveModules()`. Every callback is optional:
+ * when a callback is missing the corresponding module node is kept as-is
+ * in the output AST — useful when you only need to resolve a subset of
+ * modules (e.g. only `[[iftags]]` on the client side).
+ *
+ * @group Module Resolution
+ */
+export interface DataProvider {
+  /**
+   * Fetch page data for `[[module ListPages]]` expansion.
+   *
+   * Called once per ListPages instance in the AST with the normalised
+   * query parameters extracted from the module's wikitext attributes.
+   *
+   * @security The query fields originate from **untrusted user input**.
+   * When building database queries from the returned requirement:
+   * - **Never** interpolate `req.query` / `req.rawAttributes` into SQL
+   * - **Always** use parameterised queries or prepared statements
+   * - For `order` (ORDER BY), validate against a whitelist of column names
+   */
+  fetchListPages?: ListPagesDataFetcher;
+
+  /**
+   * Fetch user data for `[[module ListUsers]]` expansion.
+   *
+   * Called once per ListUsers instance with the parsed query parameters.
+   */
+  fetchListUsers?: ListUsersDataFetcher;
+
+  /**
+   * Fetch raw wikitext for `[[include]]` directives that are created during
+   * module secondary transformations such as ListPages/ListUsers item rendering.
+   *
+   * Top-level include expansion still uses `resolveIncludes()` directly before
+   * parsing. This callback is only for module-rendered wikitext that needs its
+   * own include pass before being parsed again.
+   */
+  fetchInclude?: IncludeFetcher;
+
+  /**
+   * Return the current page's tags for `[[iftags]]` evaluation.
+   *
+   * If provided, `[[iftags]]` blocks are evaluated and either kept or
+   * discarded based on whether the page's tags satisfy the condition.
+   * If omitted, `[[iftags]]` blocks pass through unresolved.
+   */
+  getPageTags?: IfTagsResolver;
+}

package/src/parser/rules/block/module/types.ts

@@ -0,0 +1,61 @@
+/**
+ *
+ * Core type definitions for the Wikidot module system.
+ *
+ * Wikidot modules are block-level constructs invoked with `[[module Name ...]]`
+ * syntax. Each module type (ListPages, CSS, Rate, etc.) is implemented as a
+ * `ModuleRule` that defines how to parse the module's attributes and body into
+ * an AST node.
+ *
+ * @module
+ */
+
+import type { Element, Module } from "@wdprlib/ast";
+import type { ParseContext } from "../../types";
+
+/**
+ * Parser function type for re-parsing substituted template output as wikitext.
+ *
+ * Used by ListPages and ListUsers modules during the resolution phase. After
+ * template variables are substituted with actual data, the resulting string
+ * needs to be parsed as wikitext to produce AST elements.
+ *
+ * @param input - Wikitext string to parse
+ * @returns Object containing the parsed elements
+ */
+export type ParseFunction = (input: string) => { elements: Element[] };
+
+/**
+ * Definition of a module rule that handles a specific Wikidot module type.
+ *
+ * Each module rule declares which module names it handles (e.g., "listpages",
+ * "css"), whether the module accepts a body (content between `[[module Name]]`
+ * and `[[/module]]`), and a parse function that produces the AST representation.
+ *
+ * The parse function's return type determines how the result is handled:
+ * - `Module`: Wrapped as `{ element: "module", data: Module }` in the AST
+ * - `Element`: Used directly as an AST element (e.g., CSS module returns a `style` element)
+ */
+export interface ModuleRule {
+  /** Internal identifier for the rule (e.g., "module-listpages") */
+  name: string;
+  /** Module names this rule handles, matched case-insensitively (e.g., ["listpages"]) */
+  acceptsNames: string[];
+  /** Whether this module accepts a body (content between `[[module]]` and `[[/module]]`) */
+  hasBody: boolean;
+  /**
+   * Parse the module's attributes and optional body into an AST node.
+   *
+   * @param ctx - Current parse context (token stream, settings, etc.)
+   * @param pos - Current token position
+   * @param args - Key-value attributes from the module's opening tag
+   * @param body - Raw text content between opening and closing tags (only if `hasBody` is true)
+   * @returns A Module data object or a direct Element
+   */
+  parse: (
+    ctx: ParseContext,
+    pos: number,
+    args: Record<string, string>,
+    body?: string,
+  ) => Module | Element;
+}

package/src/parser/rules/block/module/utils.ts

@@ -0,0 +1,43 @@
+/**
+ *
+ * Utility functions for parsing Wikidot module attribute values.
+ *
+ * Wikidot modules accept attributes as string key-value pairs. These utilities
+ * convert common attribute types (booleans, integers) from their string
+ * representation to proper TypeScript types, following Wikidot's conventions
+ * for truthy/falsy values.
+ *
+ * @module
+ */
+
+/**
+ * Parse a boolean value from a Wikidot attribute string.
+ *
+ * Wikidot accepts both "yes"/"no" and "true"/"false" as boolean attribute values.
+ * If the value does not match any recognized boolean string, the default is returned.
+ *
+ * @param value - The attribute string value, or undefined if the attribute was not specified
+ * @param defaultValue - Value to return when the attribute is undefined or unrecognized
+ * @returns The parsed boolean value
+ */
+export function parseBool(value: string | undefined, defaultValue: boolean): boolean {
+  if (value === undefined) return defaultValue;
+  if (value === "yes" || value === "true") return true;
+  if (value === "no" || value === "false") return false;
+  return defaultValue;
+}
+
+/**
+ * Parse a 32-bit integer value from a Wikidot attribute string.
+ *
+ * Uses base-10 parsing. Returns undefined for non-numeric strings or
+ * when the attribute is not specified.
+ *
+ * @param value - The attribute string value, or undefined if not specified
+ * @returns The parsed integer, or undefined if parsing fails
+ */
+export function parseInt32(value: string | undefined): number | undefined {
+  if (value === undefined) return undefined;
+  const num = Number.parseInt(value, 10);
+  return Number.isNaN(num) ? undefined : num;
+}