@wdprlib/parser 3.1.2 → 3.2.0

package/src/parser/rules/block/module/resolve.ts

@@ -0,0 +1,411 @@
+/**
+ * Unified resolver that walks a parsed AST and expands dynamic modules.
+ *
+ * Handles three module families in a single traversal:
+ *
+ * - **ListPages** — fetches page data via {@link DataProvider.fetchListPages},
+ *   resolves `@URL` parameters from the page path (HPC support), and
+ *   expands `%%variable%%` templates.
+ * - **ListUsers** — fetches user data via {@link DataProvider.fetchListUsers}
+ *   and expands `%%variable%%` templates.
+ * - **IfTags** — evaluates tag conditions against the current page's tags
+ *   (from {@link DataProvider.getPageTags}) and keeps or discards content.
+ *
+ * The main entry point is {@link resolveModules}.
+ *
+ * @module
+ */
+
+import type { Element, SyntaxTree } from "@wdprlib/ast";
+import { STYLE_SLOT_PREFIX } from "@wdprlib/ast";
+import type { DataProvider } from "./types-common";
+import { walkElements, mapElementChildren, mapElementChildrenWithState } from "./walk";
+import type {
+  ListPagesDataRequirement,
+  ListPagesExternalData,
+  CompiledTemplate,
+} from "./listpages/types";
+import type {
+  ListUsersDataRequirement,
+  ListUsersExternalData,
+  ListUsersCompiledTemplate,
+} from "./listusers/types";
+import { isListPagesModule, resolveListPages, type ParseFunction } from "./listpages/resolve";
+import { isListUsersModule, resolveListUsers } from "./listusers/resolve";
+import { isIfTagsElement, resolveIfTags, type IfTagsData } from "./iftags/resolve";
+import { parseUrlParams, resolveAndNormalizeQuery } from "./listpages/url-resolver";
+
+// Re-export from listpages/resolve for external use
+export type { ParseFunction } from "./listpages/resolve";
+
+/**
+ * Configuration for {@link resolveModules}.
+ *
+ * Callers must supply pre-extracted requirements and pre-compiled
+ * templates (obtained from `extractDataRequirements()` and
+ * `compileTemplate()` / `compileListUsersTemplate()`).
+ *
+ * @group Module Resolution
+ */
+export interface ResolveOptions {
+  /** Parser function used to re-parse expanded template markup into AST nodes */
+  parse: ParseFunction;
+
+  /** Pre-compiled ListPages body templates, keyed by requirement ID */
+  compiledListPagesTemplates: Map<number, CompiledTemplate>;
+
+  /** Pre-compiled ListUsers body templates, keyed by requirement ID */
+  compiledListUsersTemplates?: Map<number, ListUsersCompiledTemplate>;
+
+  /**
+   * Data requirements grouped by module type.
+   * Obtained from `extractDataRequirements()`.
+   */
+  requirements: {
+    listPages?: ListPagesDataRequirement[];
+    listUsers?: ListUsersDataRequirement[];
+  };
+
+  /**
+   * URL path for `@URL` parameter resolution (HPC / pagination support).
+   *
+   * Wikidot encodes pagination state in the URL path as key/value pairs
+   * after the page name, e.g. `"/scp-001/offset/10/page2_limit/5"`.
+   * When provided, `@URL` references in ListPages queries are replaced
+   * with the corresponding values from this path.
+   */
+  urlPath?: string;
+}
+
+/**
+ * Context for ListPages resolution (internal)
+ */
+interface ListPagesContext {
+  dataMap: Map<number, ListPagesExternalData>;
+  compiledTemplates: Map<number, CompiledTemplate>;
+  parse: ParseFunction;
+}
+
+/**
+ * Context for ListUsers resolution (internal)
+ */
+interface ListUsersContext {
+  dataMap: Map<number, ListUsersExternalData>;
+  compiledTemplates: Map<number, ListUsersCompiledTemplate>;
+  parse: ParseFunction;
+}
+
+/**
+ * Resolve all modules in the AST
+ *
+ * Fetches data for each module using the provided callback,
+ * then expands the modules with the fetched data.
+ *
+ * Handles:
+ * - ListPages: fetches page data and expands templates
+ * - IfTags: evaluates tag conditions and includes/excludes content
+ *
+ * @param ast - Parsed AST
+ * @param dataProvider - Callback provider to fetch data for each module
+ * @param options - Resolution options including requirements
+ */
+export async function resolveModules(
+  ast: SyntaxTree,
+  dataProvider: DataProvider,
+  options: ResolveOptions,
+): Promise<SyntaxTree> {
+  // Build ListPages context if requirements provided
+  let listPagesCtx: ListPagesContext | null = null;
+  const listPagesReqs = options.requirements.listPages ?? [];
+
+  if (listPagesReqs.length > 0 && dataProvider.fetchListPages) {
+    const dataMap = new Map<number, ListPagesExternalData>();
+
+    // Parse URL parameters once for all modules
+    const urlParams = parseUrlParams(options.urlPath ?? "");
+
+    for (const req of listPagesReqs) {
+      // Resolve @URL parameters and normalize query
+      const normalizedQuery = resolveAndNormalizeQuery(req, urlParams);
+      const data = await dataProvider.fetchListPages(normalizedQuery, req);
+      if (data) {
+        dataMap.set(req.id, data);
+      }
+    }
+
+    if (dataMap.size > 0) {
+      listPagesCtx = {
+        dataMap,
+        compiledTemplates: options.compiledListPagesTemplates,
+        parse: options.parse,
+      };
+    }
+  }
+
+  // Build ListUsers context if requirements provided
+  let listUsersCtx: ListUsersContext | null = null;
+  const listUsersReqs = options.requirements.listUsers ?? [];
+
+  if (listUsersReqs.length > 0 && dataProvider.fetchListUsers) {
+    const dataMap = new Map<number, ListUsersExternalData>();
+
+    for (const req of listUsersReqs) {
+      const data = await dataProvider.fetchListUsers(req);
+      if (data) {
+        dataMap.set(req.id, data);
+      }
+    }
+
+    if (dataMap.size > 0) {
+      listUsersCtx = {
+        dataMap,
+        compiledTemplates: options.compiledListUsersTemplates ?? new Map(),
+        parse: options.parse,
+      };
+    }
+  }
+
+  // Get page tags if callback provided
+  const pageTags = dataProvider.getPageTags?.() ?? null;
+
+  // Resolve AST
+  const resolvedElements = walkAndResolve(ast.elements, {
+    listPages: listPagesCtx,
+    listUsers: listUsersCtx,
+    fetchListPagesProvided: dataProvider.fetchListPages !== undefined,
+    fetchListUsersProvided: dataProvider.fetchListUsers !== undefined,
+    pageTags,
+    listPagesIdCounter: 0,
+    listUsersIdCounter: 0,
+  });
+
+  // Collect style elements from resolved AST
+  const { elements: finalElements, styles } = collectStyles(resolvedElements.elements);
+
+  const result: SyntaxTree = {
+    ...ast,
+    elements: finalElements,
+  };
+
+  if (styles.length > 0) {
+    result.styles = styles;
+  }
+
+  return result;
+}
+
+/**
+ * Resolution context passed through AST traversal
+ */
+interface WalkContext {
+  listPages: ListPagesContext | null;
+  listUsers: ListUsersContext | null;
+  /** Whether fetchListPages callback was provided (even if no data returned) */
+  fetchListPagesProvided: boolean;
+  /** Whether fetchListUsers callback was provided (even if no data returned) */
+  fetchListUsersProvided: boolean;
+  pageTags: string[] | null;
+  listPagesIdCounter: number;
+  listUsersIdCounter: number;
+}
+
+interface WalkResult {
+  elements: Element[];
+  nextListPagesId: number;
+  nextListUsersId: number;
+}
+
+/**
+ * Walk AST and resolve modules/iftags
+ */
+function walkAndResolve(elements: Element[], ctx: WalkContext): WalkResult {
+  const result: Element[] = [];
+  let listPagesId = ctx.listPagesIdCounter;
+  let listUsersId = ctx.listUsersIdCounter;
+
+  for (const element of elements) {
+    // ListPages module
+    if (element.element === "module" && isListPagesModule(element.data)) {
+      if (ctx.listPages) {
+        const moduleData = ctx.listPages.dataMap.get(listPagesId);
+        const template = ctx.listPages.compiledTemplates.get(listPagesId);
+
+        if (moduleData && template) {
+          const resolved = resolveListPages(
+            element.data,
+            moduleData,
+            template,
+            ctx.listPages.parse,
+          );
+          result.push(...resolved);
+        }
+      } else if (!ctx.fetchListPagesProvided) {
+        result.push(element);
+      }
+      listPagesId++;
+      continue;
+    }
+
+    // ListUsers module
+    if (element.element === "module" && isListUsersModule(element.data)) {
+      if (ctx.listUsers) {
+        const moduleData = ctx.listUsers.dataMap.get(listUsersId);
+        const template = ctx.listUsers.compiledTemplates.get(listUsersId);
+
+        if (moduleData && template) {
+          const resolved = resolveListUsers(
+            element.data,
+            moduleData,
+            template,
+            ctx.listUsers.parse,
+          );
+          result.push(...resolved);
+        }
+      } else if (!ctx.fetchListUsersProvided) {
+        result.push(element);
+      }
+      listUsersId++;
+      continue;
+    }
+
+    // IfTags
+    if (isIfTagsElement(element)) {
+      const ifTagsData = element.data as IfTagsData;
+      const resolveResult = resolveIfTags(ifTagsData, ctx.pageTags);
+
+      if (resolveResult.evaluated) {
+        if (resolveResult.matched) {
+          const childResult = walkAndResolve(ifTagsData.elements, {
+            ...ctx,
+            listPagesIdCounter: listPagesId,
+            listUsersIdCounter: listUsersId,
+          });
+          result.push(...childResult.elements);
+          listPagesId = childResult.nextListPagesId;
+          listUsersId = childResult.nextListUsersId;
+        } else {
+          const counts = countModulesInElements(ifTagsData.elements);
+          listPagesId += counts.listPages;
+          listUsersId += counts.listUsers;
+        }
+      } else {
+        const childResult = walkAndResolve(ifTagsData.elements, {
+          ...ctx,
+          listPagesIdCounter: listPagesId,
+          listUsersIdCounter: listUsersId,
+        });
+        result.push({
+          element: "if-tags",
+          data: {
+            ...ifTagsData,
+            elements: childResult.elements,
+          },
+        });
+        listPagesId = childResult.nextListPagesId;
+        listUsersId = childResult.nextListUsersId;
+      }
+      continue;
+    }
+
+    // Recurse into child elements (list, table, definition-list, tab-view, generic)
+    const mapped = mapElementChildrenWithState(
+      element,
+      { listPagesId, listUsersId },
+      (children, state) => {
+        const childResult = walkAndResolve(children, {
+          ...ctx,
+          listPagesIdCounter: state.listPagesId,
+          listUsersIdCounter: state.listUsersId,
+        });
+        return {
+          elements: childResult.elements,
+          state: {
+            listPagesId: childResult.nextListPagesId,
+            listUsersId: childResult.nextListUsersId,
+          },
+        };
+      },
+    );
+    result.push(mapped.element);
+    listPagesId = mapped.state.listPagesId;
+    listUsersId = mapped.state.listUsersId;
+  }
+
+  return { elements: result, nextListPagesId: listPagesId, nextListUsersId: listUsersId };
+}
+
+/**
+ * Count resolved modules in elements without resolving them
+ *
+ * Used to advance ID counters without expensive resolution when
+ * IfTags condition doesn't match (elements are skipped but IDs must sync)
+ */
+function countModulesInElements(elements: Element[]): { listPages: number; listUsers: number } {
+  let listPages = 0;
+  let listUsers = 0;
+  walkElements(elements, (element) => {
+    if (element.element === "module") {
+      if (isListPagesModule(element.data)) {
+        listPages++;
+      } else if (isListUsersModule(element.data)) {
+        listUsers++;
+      }
+    }
+  });
+  return { listPages, listUsers };
+}
+
+/**
+ * Collect and remove style elements from the AST.
+ *
+ * Walks the element tree recursively, extracting style elements
+ * from any depth and returning them separately. Unresolved `if-tags`
+ * elements are skipped — their internal styles remain in the AST and
+ * are rendered inline at render time when the condition is evaluated.
+ *
+ * The order of collected styles reflects their appearance order in the AST.
+ */
+
+function collectStyles(elements: Element[]): { elements: Element[]; styles: string[] } {
+  const styles: string[] = [];
+  const ctx = { nextSlotId: 0 };
+  const filtered = collectStylesFromElements(elements, styles, ctx);
+  return { elements: filtered, styles };
+}
+
+function collectStylesFromElements(
+  elements: Element[],
+  styles: string[],
+  ctx: { nextSlotId: number },
+): Element[] {
+  const result: Element[] = [];
+
+  for (const element of elements) {
+    if (element.element === "style") {
+      styles.push(element.data as string);
+      continue;
+    }
+
+    // Unresolved iftags: insert a style-slot placeholder to preserve
+    // source-order of styles relative to other collected styles.
+    // The slot ID is attached to the element data so the renderer can
+    // collect styles into the correct slot at render time.
+    if (element.element === "if-tags") {
+      const slotId = ctx.nextSlotId++;
+      styles.push(`${STYLE_SLOT_PREFIX}${slotId}`);
+      result.push({
+        element: "if-tags",
+        data: { ...(element.data as IfTagsData), _styleSlot: slotId },
+      } as unknown as Element);
+      continue;
+    }
+
+    // Recurse into children using mapElementChildren
+    const mapped = mapElementChildren(element, (children) =>
+      collectStylesFromElements(children, styles, ctx),
+    );
+    result.push(mapped);
+  }
+
+  return result;
+}

package/src/parser/rules/block/module/types-common.ts

@@ -0,0 +1,59 @@
+/**
+ * Shared callback interface for module resolution.
+ *
+ * `DataProvider` is the single object that callers pass to
+ * `resolveModules()` to supply external data. Each property is an
+ * optional async callback; if omitted the corresponding module type is
+ * left unresolved in the AST.
+ *
+ * Include resolution uses a separate API (`resolveIncludes()`)
+ * because it operates on raw wikitext before parsing, not on AST nodes.
+ *
+ * @module
+ */
+
+import type { ListPagesDataFetcher } from "./listpages/types";
+import type { ListUsersDataFetcher } from "./listusers/types";
+import type { IfTagsResolver } from "./iftags/types";
+
+/**
+ * Callback bag for supplying external data during module resolution.
+ *
+ * Pass an instance to `resolveModules()`. Every callback is optional:
+ * when a callback is missing the corresponding module node is kept as-is
+ * in the output AST — useful when you only need to resolve a subset of
+ * modules (e.g. only `[[iftags]]` on the client side).
+ *
+ * @group Module Resolution
+ */
+export interface DataProvider {
+  /**
+   * Fetch page data for `[[module ListPages]]` expansion.
+   *
+   * Called once per ListPages instance in the AST with the normalised
+   * query parameters extracted from the module's wikitext attributes.
+   *
+   * @security The query fields originate from **untrusted user input**.
+   * When building database queries from the returned requirement:
+   * - **Never** interpolate `req.query` / `req.rawAttributes` into SQL
+   * - **Always** use parameterised queries or prepared statements
+   * - For `order` (ORDER BY), validate against a whitelist of column names
+   */
+  fetchListPages?: ListPagesDataFetcher;
+
+  /**
+   * Fetch user data for `[[module ListUsers]]` expansion.
+   *
+   * Called once per ListUsers instance with the parsed query parameters.
+   */
+  fetchListUsers?: ListUsersDataFetcher;
+
+  /**
+   * Return the current page's tags for `[[iftags]]` evaluation.
+   *
+   * If provided, `[[iftags]]` blocks are evaluated and either kept or
+   * discarded based on whether the page's tags satisfy the condition.
+   * If omitted, `[[iftags]]` blocks pass through unresolved.
+   */
+  getPageTags?: IfTagsResolver;
+}

package/src/parser/rules/block/module/types.ts

@@ -0,0 +1,61 @@
+/**
+ *
+ * Core type definitions for the Wikidot module system.
+ *
+ * Wikidot modules are block-level constructs invoked with `[[module Name ...]]`
+ * syntax. Each module type (ListPages, CSS, Rate, etc.) is implemented as a
+ * `ModuleRule` that defines how to parse the module's attributes and body into
+ * an AST node.
+ *
+ * @module
+ */
+
+import type { Element, Module } from "@wdprlib/ast";
+import type { ParseContext } from "../../types";
+
+/**
+ * Parser function type for re-parsing substituted template output as wikitext.
+ *
+ * Used by ListPages and ListUsers modules during the resolution phase. After
+ * template variables are substituted with actual data, the resulting string
+ * needs to be parsed as wikitext to produce AST elements.
+ *
+ * @param input - Wikitext string to parse
+ * @returns Object containing the parsed elements
+ */
+export type ParseFunction = (input: string) => { elements: Element[] };
+
+/**
+ * Definition of a module rule that handles a specific Wikidot module type.
+ *
+ * Each module rule declares which module names it handles (e.g., "listpages",
+ * "css"), whether the module accepts a body (content between `[[module Name]]`
+ * and `[[/module]]`), and a parse function that produces the AST representation.
+ *
+ * The parse function's return type determines how the result is handled:
+ * - `Module`: Wrapped as `{ element: "module", data: Module }` in the AST
+ * - `Element`: Used directly as an AST element (e.g., CSS module returns a `style` element)
+ */
+export interface ModuleRule {
+  /** Internal identifier for the rule (e.g., "module-listpages") */
+  name: string;
+  /** Module names this rule handles, matched case-insensitively (e.g., ["listpages"]) */
+  acceptsNames: string[];
+  /** Whether this module accepts a body (content between `[[module]]` and `[[/module]]`) */
+  hasBody: boolean;
+  /**
+   * Parse the module's attributes and optional body into an AST node.
+   *
+   * @param ctx - Current parse context (token stream, settings, etc.)
+   * @param pos - Current token position
+   * @param args - Key-value attributes from the module's opening tag
+   * @param body - Raw text content between opening and closing tags (only if `hasBody` is true)
+   * @returns A Module data object or a direct Element
+   */
+  parse: (
+    ctx: ParseContext,
+    pos: number,
+    args: Record<string, string>,
+    body?: string,
+  ) => Module | Element;
+}

package/src/parser/rules/block/module/utils.ts

@@ -0,0 +1,43 @@
+/**
+ *
+ * Utility functions for parsing Wikidot module attribute values.
+ *
+ * Wikidot modules accept attributes as string key-value pairs. These utilities
+ * convert common attribute types (booleans, integers) from their string
+ * representation to proper TypeScript types, following Wikidot's conventions
+ * for truthy/falsy values.
+ *
+ * @module
+ */
+
+/**
+ * Parse a boolean value from a Wikidot attribute string.
+ *
+ * Wikidot accepts both "yes"/"no" and "true"/"false" as boolean attribute values.
+ * If the value does not match any recognized boolean string, the default is returned.
+ *
+ * @param value - The attribute string value, or undefined if the attribute was not specified
+ * @param defaultValue - Value to return when the attribute is undefined or unrecognized
+ * @returns The parsed boolean value
+ */
+export function parseBool(value: string | undefined, defaultValue: boolean): boolean {
+  if (value === undefined) return defaultValue;
+  if (value === "yes" || value === "true") return true;
+  if (value === "no" || value === "false") return false;
+  return defaultValue;
+}
+
+/**
+ * Parse a 32-bit integer value from a Wikidot attribute string.
+ *
+ * Uses base-10 parsing. Returns undefined for non-numeric strings or
+ * when the attribute is not specified.
+ *
+ * @param value - The attribute string value, or undefined if not specified
+ * @returns The parsed integer, or undefined if parsing fails
+ */
+export function parseInt32(value: string | undefined): number | undefined {
+  if (value === undefined) return undefined;
+  const num = Number.parseInt(value, 10);
+  return Number.isNaN(num) ? undefined : num;
+}