@wdprlib/parser 3.2.0 → 4.0.0

package/src/parser/rules/block/module/resolution/walk-resolve.ts

@@ -0,0 +1,107 @@
+import type { Element } from "@wdprlib/ast";
+import { mapElementChildrenWithState } from "../walk";
+import { isIfTagsElement, resolveIfTags, type IfTagsData } from "../iftags/resolve";
+import type { ListPagesContext, ListUsersContext } from "./contexts";
+import { countDynamicModules, resolveDynamicModuleElement } from "./dynamic-modules";
+
+/**
+ * Resolution context passed through AST traversal.
+ */
+export interface WalkContext {
+  listPages: ListPagesContext | null;
+  listUsers: ListUsersContext | null;
+  /** Whether fetchListPages callback was provided, even if no data returned. */
+  fetchListPagesProvided: boolean;
+  /** Whether fetchListUsers callback was provided, even if no data returned. */
+  fetchListUsersProvided: boolean;
+  pageTags: string[] | null;
+  listPagesIdCounter: number;
+  listUsersIdCounter: number;
+}
+
+export interface WalkResult {
+  elements: Element[];
+  nextListPagesId: number;
+  nextListUsersId: number;
+}
+
+/**
+ * Walk AST and resolve modules/iftags.
+ */
+export function walkAndResolve(elements: Element[], ctx: WalkContext): WalkResult {
+  const result: Element[] = [];
+  let listPagesId = ctx.listPagesIdCounter;
+  let listUsersId = ctx.listUsersIdCounter;
+
+  for (const element of elements) {
+    const dynamicModule = resolveDynamicModuleElement(element, ctx, { listPagesId, listUsersId });
+    if (dynamicModule.handled) {
+      result.push(...dynamicModule.elements);
+      listPagesId = dynamicModule.ids.listPagesId;
+      listUsersId = dynamicModule.ids.listUsersId;
+      continue;
+    }
+
+    if (isIfTagsElement(element)) {
+      const ifTagsData = element.data as IfTagsData;
+      const resolveResult = resolveIfTags(ifTagsData, ctx.pageTags);
+
+      if (resolveResult.evaluated) {
+        if (resolveResult.matched) {
+          const childResult = walkAndResolve(ifTagsData.elements, {
+            ...ctx,
+            listPagesIdCounter: listPagesId,
+            listUsersIdCounter: listUsersId,
+          });
+          result.push(...childResult.elements);
+          listPagesId = childResult.nextListPagesId;
+          listUsersId = childResult.nextListUsersId;
+        } else {
+          const counts = countDynamicModules(ifTagsData.elements);
+          listPagesId += counts.listPagesId;
+          listUsersId += counts.listUsersId;
+        }
+      } else {
+        const childResult = walkAndResolve(ifTagsData.elements, {
+          ...ctx,
+          listPagesIdCounter: listPagesId,
+          listUsersIdCounter: listUsersId,
+        });
+        result.push({
+          element: "if-tags",
+          data: {
+            ...ifTagsData,
+            elements: childResult.elements,
+          },
+        });
+        listPagesId = childResult.nextListPagesId;
+        listUsersId = childResult.nextListUsersId;
+      }
+      continue;
+    }
+
+    const mapped = mapElementChildrenWithState(
+      element,
+      { listPagesId, listUsersId },
+      (children, state) => {
+        const childResult = walkAndResolve(children, {
+          ...ctx,
+          listPagesIdCounter: state.listPagesId,
+          listUsersIdCounter: state.listUsersId,
+        });
+        return {
+          elements: childResult.elements,
+          state: {
+            listPagesId: childResult.nextListPagesId,
+            listUsersId: childResult.nextListUsersId,
+          },
+        };
+      },
+    );
+    result.push(mapped.element);
+    listPagesId = mapped.state.listPagesId;
+    listUsersId = mapped.state.listUsersId;
+  }
+
+  return { elements: result, nextListPagesId: listPagesId, nextListUsersId: listUsersId };
+}

package/src/parser/rules/block/module/resolve.ts

@@ -16,28 +16,35 @@
  * @module
  */
 
-import type { Element, SyntaxTree } from "@wdprlib/ast";
-import { STYLE_SLOT_PREFIX } from "@wdprlib/ast";
+import type { SyntaxTree } from "@wdprlib/ast";
 import type { DataProvider } from "./types-common";
-import { walkElements, mapElementChildren, mapElementChildrenWithState } from "./walk";
-import type {
-  ListPagesDataRequirement,
-  ListPagesExternalData,
-  CompiledTemplate,
-} from "./listpages/types";
-import type {
-  ListUsersDataRequirement,
-  ListUsersExternalData,
-  ListUsersCompiledTemplate,
-} from "./listusers/types";
-import { isListPagesModule, resolveListPages, type ParseFunction } from "./listpages/resolve";
-import { isListUsersModule, resolveListUsers } from "./listusers/resolve";
-import { isIfTagsElement, resolveIfTags, type IfTagsData } from "./iftags/resolve";
-import { parseUrlParams, resolveAndNormalizeQuery } from "./listpages/url-resolver";
+import { resolveIncludes } from "./include";
+import type { ListPagesDataRequirement, CompiledTemplate } from "./listpages/types";
+import type { ListUsersDataRequirement, ListUsersCompiledTemplate } from "./listusers/types";
+import type { ParseFunction } from "./listpages/resolve";
+import { buildListPagesContext, buildListUsersContext } from "./resolution/contexts";
+import { walkAndResolve } from "./resolution/walk-resolve";
+import { collectStyles } from "./resolution/styles";
+
+const MODULE_SECONDARY_INCLUDE_MAX_ITERATIONS = 5;
 
 // Re-export from listpages/resolve for external use
 export type { ParseFunction } from "./listpages/resolve";
 
+/**
+ * Transform module-generated wikitext before it is parsed back into AST nodes.
+ *
+ * Applications can use this hook to inject include resolution, parse caches, or
+ * diagnostics for ListPages/ListUsers secondary transformations.
+ *
+ * This hook receives the string after module variables such as `%%title%%` have
+ * already been substituted. Returning a different string changes what is parsed
+ * for that module item. When this hook is supplied it replaces the built-in
+ * secondary include pass, so callers that still need Wikidot-style secondary
+ * includes must call `resolveIncludes()` or equivalent logic from the hook.
+ */
+export type ModuleSourceTransform = (source: string) => string;
+
 /**
  * Configuration for {@link resolveModules}.
  *
@@ -75,24 +82,28 @@ export interface ResolveOptions {
    * with the corresponding values from this path.
    */
   urlPath?: string;
-}
 
-/**
- * Context for ListPages resolution (internal)
- */
-interface ListPagesContext {
-  dataMap: Map<number, ListPagesExternalData>;
-  compiledTemplates: Map<number, CompiledTemplate>;
-  parse: ParseFunction;
-}
+  /**
+   * Maximum include expansion iterations for secondary transformations inside
+   * modules such as ListPages/ListUsers. Defaults to Wikidot's observed
+   * secondary transformation limit.
+   */
+  includeMaxIterations?: number;
 
-/**
- * Context for ListUsers resolution (internal)
- */
-interface ListUsersContext {
-  dataMap: Map<number, ListUsersExternalData>;
-  compiledTemplates: Map<number, ListUsersCompiledTemplate>;
-  parse: ParseFunction;
+  /**
+   * Transform module-generated wikitext before re-parsing it.
+   *
+   * When omitted, `resolveModules()` preserves the existing default: if
+   * `dataProvider.fetchInclude` is present, secondary `[[include]]` expansion
+   * is performed with `includeMaxIterations`; otherwise the source is parsed
+   * unchanged. When supplied, this hook owns the whole transformation.
+   *
+   * This is the extension point for application-level parse/include caches. The
+   * parser package does not persist cache entries itself because cache keys and
+   * invalidation depend on site/page revisions, tags, URL parameters, and user
+   * state outside wdpr.
+   */
+  transformModuleSource?: ModuleSourceTransform;
 }
 
 /**
@@ -114,58 +125,20 @@ export async function resolveModules(
   dataProvider: DataProvider,
   options: ResolveOptions,
 ): Promise<SyntaxTree> {
-  // Build ListPages context if requirements provided
-  let listPagesCtx: ListPagesContext | null = null;
-  const listPagesReqs = options.requirements.listPages ?? [];
-
-  if (listPagesReqs.length > 0 && dataProvider.fetchListPages) {
-    const dataMap = new Map<number, ListPagesExternalData>();
-
-    // Parse URL parameters once for all modules
-    const urlParams = parseUrlParams(options.urlPath ?? "");
-
-    for (const req of listPagesReqs) {
-      // Resolve @URL parameters and normalize query
-      const normalizedQuery = resolveAndNormalizeQuery(req, urlParams);
-      const data = await dataProvider.fetchListPages(normalizedQuery, req);
-      if (data) {
-        dataMap.set(req.id, data);
-      }
-    }
-
-    if (dataMap.size > 0) {
-      listPagesCtx = {
-        dataMap,
-        compiledTemplates: options.compiledListPagesTemplates,
-        parse: options.parse,
-      };
-    }
-  }
-
-  // Build ListUsers context if requirements provided
-  let listUsersCtx: ListUsersContext | null = null;
-  const listUsersReqs = options.requirements.listUsers ?? [];
-
-  if (listUsersReqs.length > 0 && dataProvider.fetchListUsers) {
-    const dataMap = new Map<number, ListUsersExternalData>();
-
-    for (const req of listUsersReqs) {
-      const data = await dataProvider.fetchListUsers(req);
-      if (data) {
-        dataMap.set(req.id, data);
-      }
-    }
-
-    if (dataMap.size > 0) {
-      listUsersCtx = {
-        dataMap,
-        compiledTemplates: options.compiledListUsersTemplates ?? new Map(),
-        parse: options.parse,
-      };
-    }
-  }
-
-  // Get page tags if callback provided
+  const parse = createModuleParseFunction(options, dataProvider);
+  const listPagesCtx = await buildListPagesContext(
+    dataProvider,
+    options.requirements.listPages ?? [],
+    options.compiledListPagesTemplates,
+    parse,
+    options.urlPath,
+  );
+  const listUsersCtx = await buildListUsersContext(
+    dataProvider,
+    options.requirements.listUsers ?? [],
+    options.compiledListUsersTemplates,
+    parse,
+  );
   const pageTags = dataProvider.getPageTags?.() ?? null;
 
   // Resolve AST
@@ -194,218 +167,32 @@ export async function resolveModules(
   return result;
 }
 
-/**
- * Resolution context passed through AST traversal
- */
-interface WalkContext {
-  listPages: ListPagesContext | null;
-  listUsers: ListUsersContext | null;
-  /** Whether fetchListPages callback was provided (even if no data returned) */
-  fetchListPagesProvided: boolean;
-  /** Whether fetchListUsers callback was provided (even if no data returned) */
-  fetchListUsersProvided: boolean;
-  pageTags: string[] | null;
-  listPagesIdCounter: number;
-  listUsersIdCounter: number;
-}
-
-interface WalkResult {
-  elements: Element[];
-  nextListPagesId: number;
-  nextListUsersId: number;
-}
-
-/**
- * Walk AST and resolve modules/iftags
- */
-function walkAndResolve(elements: Element[], ctx: WalkContext): WalkResult {
-  const result: Element[] = [];
-  let listPagesId = ctx.listPagesIdCounter;
-  let listUsersId = ctx.listUsersIdCounter;
-
-  for (const element of elements) {
-    // ListPages module
-    if (element.element === "module" && isListPagesModule(element.data)) {
-      if (ctx.listPages) {
-        const moduleData = ctx.listPages.dataMap.get(listPagesId);
-        const template = ctx.listPages.compiledTemplates.get(listPagesId);
-
-        if (moduleData && template) {
-          const resolved = resolveListPages(
-            element.data,
-            moduleData,
-            template,
-            ctx.listPages.parse,
-          );
-          result.push(...resolved);
-        }
-      } else if (!ctx.fetchListPagesProvided) {
-        result.push(element);
-      }
-      listPagesId++;
-      continue;
-    }
-
-    // ListUsers module
-    if (element.element === "module" && isListUsersModule(element.data)) {
-      if (ctx.listUsers) {
-        const moduleData = ctx.listUsers.dataMap.get(listUsersId);
-        const template = ctx.listUsers.compiledTemplates.get(listUsersId);
-
-        if (moduleData && template) {
-          const resolved = resolveListUsers(
-            element.data,
-            moduleData,
-            template,
-            ctx.listUsers.parse,
-          );
-          result.push(...resolved);
-        }
-      } else if (!ctx.fetchListUsersProvided) {
-        result.push(element);
-      }
-      listUsersId++;
-      continue;
-    }
-
-    // IfTags
-    if (isIfTagsElement(element)) {
-      const ifTagsData = element.data as IfTagsData;
-      const resolveResult = resolveIfTags(ifTagsData, ctx.pageTags);
-
-      if (resolveResult.evaluated) {
-        if (resolveResult.matched) {
-          const childResult = walkAndResolve(ifTagsData.elements, {
-            ...ctx,
-            listPagesIdCounter: listPagesId,
-            listUsersIdCounter: listUsersId,
-          });
-          result.push(...childResult.elements);
-          listPagesId = childResult.nextListPagesId;
-          listUsersId = childResult.nextListUsersId;
-        } else {
-          const counts = countModulesInElements(ifTagsData.elements);
-          listPagesId += counts.listPages;
-          listUsersId += counts.listUsers;
-        }
-      } else {
-        const childResult = walkAndResolve(ifTagsData.elements, {
-          ...ctx,
-          listPagesIdCounter: listPagesId,
-          listUsersIdCounter: listUsersId,
-        });
-        result.push({
-          element: "if-tags",
-          data: {
-            ...ifTagsData,
-            elements: childResult.elements,
-          },
-        });
-        listPagesId = childResult.nextListPagesId;
-        listUsersId = childResult.nextListUsersId;
-      }
-      continue;
-    }
-
-    // Recurse into child elements (list, table, definition-list, tab-view, generic)
-    const mapped = mapElementChildrenWithState(
-      element,
-      { listPagesId, listUsersId },
-      (children, state) => {
-        const childResult = walkAndResolve(children, {
-          ...ctx,
-          listPagesIdCounter: state.listPagesId,
-          listUsersIdCounter: state.listUsersId,
-        });
-        return {
-          elements: childResult.elements,
-          state: {
-            listPagesId: childResult.nextListPagesId,
-            listUsersId: childResult.nextListUsersId,
-          },
-        };
-      },
-    );
-    result.push(mapped.element);
-    listPagesId = mapped.state.listPagesId;
-    listUsersId = mapped.state.listUsersId;
+function createModuleParseFunction(
+  options: ResolveOptions,
+  dataProvider: DataProvider,
+): ParseFunction {
+  const transform = createModuleSourceTransform(options, dataProvider);
+  if (!transform) {
+    return options.parse;
   }
 
-  return { elements: result, nextListPagesId: listPagesId, nextListUsersId: listUsersId };
+  return (source: string) => options.parse(transform(source));
 }
 
-/**
- * Count resolved modules in elements without resolving them
- *
- * Used to advance ID counters without expensive resolution when
- * IfTags condition doesn't match (elements are skipped but IDs must sync)
- */
-function countModulesInElements(elements: Element[]): { listPages: number; listUsers: number } {
-  let listPages = 0;
-  let listUsers = 0;
-  walkElements(elements, (element) => {
-    if (element.element === "module") {
-      if (isListPagesModule(element.data)) {
-        listPages++;
-      } else if (isListUsersModule(element.data)) {
-        listUsers++;
-      }
-    }
-  });
-  return { listPages, listUsers };
-}
-
-/**
- * Collect and remove style elements from the AST.
- *
- * Walks the element tree recursively, extracting style elements
- * from any depth and returning them separately. Unresolved `if-tags`
- * elements are skipped — their internal styles remain in the AST and
- * are rendered inline at render time when the condition is evaluated.
- *
- * The order of collected styles reflects their appearance order in the AST.
- */
-
-function collectStyles(elements: Element[]): { elements: Element[]; styles: string[] } {
-  const styles: string[] = [];
-  const ctx = { nextSlotId: 0 };
-  const filtered = collectStylesFromElements(elements, styles, ctx);
-  return { elements: filtered, styles };
-}
-
-function collectStylesFromElements(
-  elements: Element[],
-  styles: string[],
-  ctx: { nextSlotId: number },
-): Element[] {
-  const result: Element[] = [];
-
-  for (const element of elements) {
-    if (element.element === "style") {
-      styles.push(element.data as string);
-      continue;
-    }
-
-    // Unresolved iftags: insert a style-slot placeholder to preserve
-    // source-order of styles relative to other collected styles.
-    // The slot ID is attached to the element data so the renderer can
-    // collect styles into the correct slot at render time.
-    if (element.element === "if-tags") {
-      const slotId = ctx.nextSlotId++;
-      styles.push(`${STYLE_SLOT_PREFIX}${slotId}`);
-      result.push({
-        element: "if-tags",
-        data: { ...(element.data as IfTagsData), _styleSlot: slotId },
-      } as unknown as Element);
-      continue;
-    }
+function createModuleSourceTransform(
+  options: ResolveOptions,
+  dataProvider: DataProvider,
+): ModuleSourceTransform | null {
+  if (options.transformModuleSource) {
+    return options.transformModuleSource;
+  }
 
-    // Recurse into children using mapElementChildren
-    const mapped = mapElementChildren(element, (children) =>
-      collectStylesFromElements(children, styles, ctx),
-    );
-    result.push(mapped);
+  if (!dataProvider.fetchInclude) {
+    return null;
   }
 
-  return result;
+  return (source: string) =>
+    resolveIncludes(source, dataProvider.fetchInclude!, {
+      maxIterations: options.includeMaxIterations ?? MODULE_SECONDARY_INCLUDE_MAX_ITERATIONS,
+    });
 }

package/src/parser/rules/block/module/rule.ts

@@ -0,0 +1,56 @@
+import type { Element } from "@wdprlib/ast";
+import type { BlockRule, ParseContext, RuleResult } from "../../types";
+import { currentToken } from "../../types";
+import { parseModuleBody } from "./body";
+import { createUnknownModuleElement, moduleParseResultToElement } from "./element";
+import { getModuleRuleByName } from "./mapping";
+import { parseModuleOpen } from "./open";
+
+export const moduleRule: BlockRule = {
+  name: "module",
+  startTokens: ["BLOCK_OPEN"],
+  requiresLineStart: false,
+
+  parse(ctx: ParseContext): RuleResult<Element> {
+    const openToken = currentToken(ctx);
+    if (openToken.type !== "BLOCK_OPEN") {
+      return { success: false };
+    }
+
+    const openResult = parseModuleOpen(ctx);
+    if (!openResult) {
+      return { success: false };
+    }
+
+    // Page syntax disabled (e.g., forum-post mode)
+    if (!ctx.settings.enablePageSyntax) {
+      return { success: false };
+    }
+
+    // Dispatch to specific module parser
+    const moduleParseRule = getModuleRuleByName(openResult.moduleName);
+
+    // Check for body based on module rule's hasBody flag
+    // For unknown modules, default to no body (they will be parsed as bodyless)
+    const bodyResult = parseModuleBody(ctx, openResult.pos, moduleParseRule?.hasBody ?? false);
+    const consumed = openResult.consumed + bodyResult.consumed;
+
+    if (moduleParseRule) {
+      const result = moduleParseRule.parse(ctx, bodyResult.pos, openResult.attrs, bodyResult.body);
+
+      return {
+        success: true,
+        elements: [moduleParseResultToElement(result)],
+        consumed,
+      };
+    }
+
+    return {
+      success: true,
+      elements: [
+        createUnknownModuleElement(openResult.moduleName, openResult.attrs, bodyResult.body),
+      ],
+      consumed,
+    };
+  },
+};

package/src/parser/rules/block/module/types-common.ts

@@ -15,6 +15,7 @@
 import type { ListPagesDataFetcher } from "./listpages/types";
 import type { ListUsersDataFetcher } from "./listusers/types";
 import type { IfTagsResolver } from "./iftags/types";
+import type { IncludeFetcher } from "./include/resolve/types";
 
 /**
  * Callback bag for supplying external data during module resolution.
@@ -48,6 +49,16 @@ export interface DataProvider {
    */
   fetchListUsers?: ListUsersDataFetcher;
 
+  /**
+   * Fetch raw wikitext for `[[include]]` directives that are created during
+   * module secondary transformations such as ListPages/ListUsers item rendering.
+   *
+   * Top-level include expansion still uses `resolveIncludes()` directly before
+   * parsing. This callback is only for module-rendered wikitext that needs its
+   * own include pass before being parsed again.
+   */
+  fetchInclude?: IncludeFetcher;
+
   /**
    * Return the current page's tags for `[[iftags]]` evaluation.
    *

package/src/parser/rules/block/module/walk/children.ts

@@ -0,0 +1,35 @@
+import type { Element, ElementOf, ListData } from "@wdprlib/ast";
+
+export type ListElement = ElementOf<"list">;
+
+export function listElement(data: ListData): ListElement {
+  return { element: "list", data };
+}
+
+export function getGenericElementChildren(element: Element): Element[] | null {
+  if (!("data" in element) || !element.data || typeof element.data !== "object") {
+    return null;
+  }
+
+  const data = element.data as Record<string, unknown>;
+  return Array.isArray(data.elements) ? (data.elements as Element[]) : null;
+}
+
+export function withGenericElementChildren(element: Element, children: Element[]): Element {
+  if (!("data" in element) || !element.data || typeof element.data !== "object") {
+    return element;
+  }
+
+  const data = element.data as Record<string, unknown>;
+  if (!Array.isArray(data.elements)) {
+    return element;
+  }
+
+  return {
+    ...element,
+    data: {
+      ...data,
+      elements: children,
+    },
+  } as Element;
+}

package/src/parser/rules/block/module/walk/index.ts

@@ -0,0 +1,9 @@
+/**
+ *
+ * AST element traversal and transformation utilities.
+ *
+ * @module
+ */
+
+export { walkElements } from "./traverse";
+export { mapElementChildren, mapElementChildrenWithState } from "./map";

package/src/parser/rules/block/module/walk/map/index.ts

@@ -0,0 +1,2 @@
+export { mapElementChildren } from "./stateless";
+export { mapElementChildrenWithState } from "./stateful";

package/src/parser/rules/block/module/walk/map/stateful-definition-list.ts

@@ -0,0 +1,25 @@
+import type { DefinitionListItem, Element } from "@wdprlib/ast";
+import type { StatefulTransformResult } from "./types";
+
+export function mapDefinitionListItemsWithState<S>(
+  items: DefinitionListItem[],
+  state: S,
+  transform: (elements: Element[], state: S) => StatefulTransformResult<S>,
+): { items: DefinitionListItem[]; state: S } {
+  const newItems: DefinitionListItem[] = [];
+  let currentState = state;
+
+  for (const item of items) {
+    const keyResult = transform(item.key, currentState);
+    currentState = keyResult.state;
+    const valueResult = transform(item.value, currentState);
+    currentState = valueResult.state;
+    newItems.push({
+      key_string: item.key_string,
+      key: keyResult.elements,
+      value: valueResult.elements,
+    });
+  }
+
+  return { items: newItems, state: currentState };
+}