@wdprlib/parser 4.3.0 → 4.4.0

package/src/parser/rules/block/module/resolution/resolve-async.ts

@@ -0,0 +1,269 @@
+import type { Diagnostic, Element, ParseResult, SyntaxTree } from "@wdprlib/ast";
+import type { DataProvider } from "../types-common";
+import type { ModuleParseResult } from "../types";
+import { isListPagesModule, type ListPagesModuleData } from "../listpages/resolve";
+import type {
+  CompiledTemplate,
+  ListPagesDataRequirement,
+  ListPagesExternalData,
+  VariableContext,
+} from "../listpages/types";
+import { isListUsersModule } from "../listusers/resolve";
+import type {
+  ListUsersCompiledTemplate,
+  ListUsersDataRequirement,
+  ListUsersExternalData,
+  ListUsersVariableContext,
+} from "../listusers/types";
+import { isTagCloudModule, resolveTagCloud } from "../tagcloud/resolve";
+import type { TagCloudDataRequirement } from "../tagcloud/types";
+import { resolveIfTags } from "../iftags/resolve";
+import {
+  getGenericElementChildren,
+  listElement,
+  withGenericElementChildren,
+} from "../walk/children";
+import { buildListPagesDataMap, buildListUsersDataMap, buildTagCloudDataMap } from "./data-maps";
+import { ModuleDocumentRegistry } from "./document";
+import { collectStyles, mergeCollectedStyles } from "./styles";
+
+export type AsyncModuleParseFunction = (source: string) => Promise<ModuleParseResult>;
+
+export interface ResolveModulesWithAsyncParseOptions {
+  parse: AsyncModuleParseFunction;
+  compiledListPagesTemplates: Map<number, CompiledTemplate>;
+  compiledListUsersTemplates?: Map<number, ListUsersCompiledTemplate>;
+  requirements: {
+    listPages?: ListPagesDataRequirement[];
+    listUsers?: ListUsersDataRequirement[];
+    tagCloud?: TagCloudDataRequirement[];
+  };
+  urlPath?: string;
+  pageTags: string[];
+}
+
+export interface AsyncModuleResolutionResult {
+  ast: SyntaxTree;
+  diagnostics: Diagnostic[];
+}
+
+interface ResolutionState {
+  listPagesId: number;
+  listUsersId: number;
+  tagCloudId: number;
+}
+
+interface ResolutionContext {
+  dataProvider: DataProvider;
+  listPagesData: Map<number, ListPagesExternalData>;
+  listUsersData: Map<number, ListUsersExternalData>;
+  tagCloudData: Awaited<ReturnType<typeof buildTagCloudDataMap>>;
+  options: ResolveModulesWithAsyncParseOptions;
+  parse: (source: string) => Promise<SyntaxTree>;
+}
+
+export async function resolveModulesWithAsyncParse(
+  ast: SyntaxTree,
+  dataProvider: DataProvider,
+  options: ResolveModulesWithAsyncParseOptions,
+): Promise<AsyncModuleResolutionResult> {
+  const registry = new ModuleDocumentRegistry();
+  registry.register(ast);
+  const parse = async (source: string): Promise<SyntaxTree> =>
+    registry.register(await options.parse(source));
+
+  const [listPagesData, listUsersData, tagCloudData] = await Promise.all([
+    buildListPagesDataMap(dataProvider, options.requirements.listPages ?? [], options.urlPath),
+    buildListUsersDataMap(dataProvider, options.requirements.listUsers ?? []),
+    buildTagCloudDataMap(dataProvider, options.requirements.tagCloud ?? []),
+  ]);
+  const context: ResolutionContext = {
+    dataProvider,
+    listPagesData,
+    listUsersData,
+    tagCloudData,
+    options,
+    parse,
+  };
+  const state: ResolutionState = { listPagesId: 0, listUsersId: 0, tagCloudId: 0 };
+  const resolvedElements = await resolveElements(ast.elements, context, state);
+  const { elements, styles, anchoredStyles } = collectStyles(resolvedElements);
+  const intermediate: SyntaxTree = { ...ast, elements };
+  const mergedStyles = mergeCollectedStyles(ast.styles, styles, new Map(), anchoredStyles);
+  if (mergedStyles.length > 0) intermediate.styles = mergedStyles;
+  else delete intermediate.styles;
+
+  return {
+    ast: registry.finalize(intermediate, elements, options.pageTags),
+    diagnostics: registry.diagnostics,
+  };
+}
+
+async function resolveElements(
+  elements: Element[],
+  context: ResolutionContext,
+  state: ResolutionState,
+): Promise<Element[]> {
+  const result: Element[] = [];
+  for (const element of elements) {
+    if (element.element === "module") {
+      const resolved = await resolveModuleElement(element, context, state);
+      if (resolved !== null) {
+        result.push(...resolved);
+        continue;
+      }
+    }
+
+    if (element.element === "if-tags") {
+      const resolution = resolveIfTags(element.data, context.options.pageTags);
+      if (resolution.matched) {
+        result.push(...(await resolveElements(element.data.elements, context, state)));
+      }
+      continue;
+    }
+
+    result.push(await resolveElementChildren(element, context, state));
+  }
+  return result;
+}
+
+async function resolveModuleElement(
+  element: Extract<Element, { element: "module" }>,
+  context: ResolutionContext,
+  state: ResolutionState,
+): Promise<Element[] | null> {
+  if (isListPagesModule(element.data)) {
+    const id = state.listPagesId++;
+    const data = context.listPagesData.get(id);
+    const template = context.options.compiledListPagesTemplates.get(id);
+    if (data && template) return resolveListPagesAsync(element.data, data, template, context.parse);
+    return context.dataProvider.fetchListPages ? [] : [element];
+  }
+  if (isListUsersModule(element.data)) {
+    const id = state.listUsersId++;
+    const data = context.listUsersData.get(id);
+    const template = context.options.compiledListUsersTemplates?.get(id);
+    if (data && template) {
+      const variableContext: ListUsersVariableContext = { user: data.user };
+      return (await context.parse(template(variableContext))).elements;
+    }
+    return context.dataProvider.fetchListUsers ? [] : [element];
+  }
+  if (isTagCloudModule(element.data)) {
+    const id = state.tagCloudId++;
+    const data = context.tagCloudData.get(id);
+    if (data) return resolveTagCloud(element.data, data);
+    return context.dataProvider.fetchTagCloud ? [] : [element];
+  }
+  return null;
+}
+
+async function resolveListPagesAsync(
+  module: ListPagesModuleData,
+  data: ListPagesExternalData,
+  template: CompiledTemplate,
+  parse: (source: string) => Promise<SyntaxTree>,
+): Promise<Element[]> {
+  if (data.pages.length === 0) return [];
+  const result: Element[] = [];
+
+  if (module["prepend-line"] && !module.separate) {
+    result.push(...(await parse(module["prepend-line"])).elements);
+  }
+  for (let i = 0; i < data.pages.length; i++) {
+    const page = data.pages[i];
+    if (!page) continue;
+    const variableContext: VariableContext = {
+      page,
+      index: i + 1,
+      total: data.totalCount,
+      limit: module.limit,
+      site: data.site,
+    };
+    const parsed = await parse(template(variableContext));
+    if (module.separate) {
+      result.push({
+        element: "container",
+        data: {
+          type: "div",
+          attributes: { class: "list-pages-item" },
+          elements: parsed.elements,
+        },
+      });
+    } else {
+      result.push(...parsed.elements);
+    }
+  }
+  if (module["append-line"] && !module.separate) {
+    result.push(...(await parse(module["append-line"])).elements);
+  }
+
+  return module.wrapper
+    ? [
+        {
+          element: "container",
+          data: {
+            type: "div",
+            attributes: { class: "list-pages-box" },
+            elements: result,
+          },
+        },
+      ]
+    : result;
+}
+
+async function resolveElementChildren(
+  element: Element,
+  context: ResolutionContext,
+  state: ResolutionState,
+): Promise<Element> {
+  if (element.element === "list") {
+    const items = [];
+    for (const item of element.data.items) {
+      if (item["item-type"] === "elements") {
+        items.push({ ...item, elements: await resolveElements(item.elements, context, state) });
+      } else {
+        const resolved = (await resolveElements([listElement(item.data)], context, state))[0];
+        items.push(resolved?.element === "list" ? { ...item, data: resolved.data } : item);
+      }
+    }
+    return { ...element, data: { ...element.data, items } };
+  }
+  if (element.element === "table") {
+    const rows = [];
+    for (const row of element.data.rows) {
+      const cells = [];
+      for (const cell of row.cells) {
+        cells.push({ ...cell, elements: await resolveElements(cell.elements, context, state) });
+      }
+      rows.push({ ...row, cells });
+    }
+    return { ...element, data: { ...element.data, rows } };
+  }
+  if (element.element === "definition-list") {
+    const entries = [];
+    for (const entry of element.data) {
+      entries.push({
+        ...entry,
+        key: await resolveElements(entry.key, context, state),
+        value: await resolveElements(entry.value, context, state),
+      });
+    }
+    return { ...element, data: entries };
+  }
+  if (element.element === "tab-view") {
+    const tabs = [];
+    for (const tab of element.data) {
+      tabs.push({ ...tab, elements: await resolveElements(tab.elements, context, state) });
+    }
+    return { ...element, data: tabs };
+  }
+  const children = getGenericElementChildren(element);
+  return children === null
+    ? element
+    : withGenericElementChildren(element, await resolveElements(children, context, state));
+}
+
+export function asParseResult(ast: SyntaxTree, diagnostics: Diagnostic[]): ParseResult {
+  return { ast, diagnostics };
+}

package/src/parser/rules/block/module/resolution/styles.ts

@@ -1,41 +1,161 @@
 import type { Element } from "@wdprlib/ast";
-import { STYLE_SLOT_PREFIX } from "@wdprlib/ast";
-import { mapElementChildren } from "../walk";
+import { STYLE_ANCHOR_PREFIX, STYLE_SLOT_PREFIX } from "@wdprlib/ast";
+import { mapElementChildren, walkElements } from "../walk";
 import type { IfTagsData } from "../iftags/resolve";
 
 type IfTagsDataWithStyleSlot = IfTagsData & {
-  _styleSlot: number;
+  _styleSlot?: number;
 };
 
+export type ResolvedStyleSlots = ReadonlyMap<number, readonly string[]>;
+
+export interface StyleCollectionResult {
+  elements: Element[];
+  styles: string[];
+  anchoredStyles: string[];
+  anchors: Element[];
+}
+
 /**
  * Collect and remove style elements from the AST.
  *
  * Unresolved `if-tags` elements receive style-slot placeholders so render-time
  * iftags evaluation can preserve style order relative to already collected CSS.
  */
-export function collectStyles(elements: Element[]): { elements: Element[]; styles: string[] } {
+export function collectStyles(
+  elements: Element[],
+  ignoredAnchors: WeakSet<Element> = new WeakSet(),
+): StyleCollectionResult {
   const styles: string[] = [];
-  const ctx = { nextSlotId: 0 };
-  const filtered = collectStylesFromElements(elements, styles, ctx);
-  return { elements: filtered, styles };
+  const anchoredStyles: string[] = [];
+  const anchors: Element[] = [];
+  const usedSlots = collectExistingStyleSlots(elements);
+  const ctx = { nextSlotId: 0, usedSlots };
+  const filtered = collectStylesFromElements(
+    elements,
+    styles,
+    anchoredStyles,
+    anchors,
+    ignoredAnchors,
+    ctx,
+  );
+  return { elements: filtered, styles, anchoredStyles, anchors };
+}
+
+/**
+ * Preserve styles collected by earlier resolution passes while keeping
+ * unresolved IfTags style-slot markers idempotent across repeated passes.
+ */
+export function mergeCollectedStyles(
+  existing: readonly string[] | undefined,
+  collected: readonly string[],
+  resolvedSlots: ResolvedStyleSlots = new Map(),
+  anchoredStyles: readonly string[] = [],
+): string[] {
+  const unanchoredPrevious = [...(existing ?? [])];
+  for (const style of anchoredStyles) removeFirst(unanchoredPrevious, style);
+  for (const style of collected) {
+    if (isStyleSlotMarker(style)) removeFirst(unanchoredPrevious, style);
+  }
+
+  const merged = unanchoredPrevious.filter((style) => !isStyleSlotMarker(style));
+  for (const style of collected) {
+    if (isStyleSlotMarker(style)) appendStyleSlot(merged, style, resolvedSlots);
+    else merged.push(style);
+  }
+  return merged;
+}
+
+function isStyleSlotMarker(style: string): boolean {
+  return style.startsWith(STYLE_SLOT_PREFIX);
+}
+
+export function createStyleSlotMarker(slotId: number): string {
+  return `${STYLE_SLOT_PREFIX}${slotId}`;
+}
+
+export function getStyleSlotId(data: IfTagsData): number | undefined {
+  const slotId = (data as IfTagsDataWithStyleSlot)._styleSlot;
+  return Number.isSafeInteger(slotId) && slotId! >= 0 ? slotId : undefined;
+}
+
+function removeFirst(values: string[], value: string): void {
+  const index = values.indexOf(value);
+  if (index >= 0) values.splice(index, 1);
+}
+
+function appendStyleSlot(
+  target: string[],
+  marker: string,
+  resolvedSlots: ResolvedStyleSlots,
+): void {
+  const slotId = Number(marker.slice(STYLE_SLOT_PREFIX.length));
+  const replacement = resolvedSlots.get(slotId);
+  if (replacement) target.push(...replacement);
+  else target.push(marker);
+}
+
+function collectExistingStyleSlots(elements: Element[]): Set<number> {
+  const slots = new Set<number>();
+  walkElements(elements, (element) => {
+    if (element.element !== "if-tags") return;
+    const slotId = getStyleSlotId(element.data);
+    if (slotId !== undefined) slots.add(slotId);
+  });
+  return slots;
+}
+
+function allocateStyleSlot(ctx: StyleCollectionContext): number {
+  while (ctx.usedSlots.has(ctx.nextSlotId)) ctx.nextSlotId++;
+  const slotId = ctx.nextSlotId++;
+  ctx.usedSlots.add(slotId);
+  return slotId;
+}
+
+interface StyleCollectionContext {
+  nextSlotId: number;
+  usedSlots: Set<number>;
 }
 
 function collectStylesFromElements(
   elements: Element[],
   styles: string[],
-  ctx: { nextSlotId: number },
+  anchoredStyles: string[],
+  anchors: Element[],
+  ignoredAnchors: WeakSet<Element>,
+  ctx: StyleCollectionContext,
 ): Element[] {
   const result: Element[] = [];
 
   for (const element of elements) {
     if (element.element === "style") {
-      styles.push(element.data as string);
+      const css = element.data as string;
+      if (css.startsWith(STYLE_SLOT_PREFIX)) {
+        styles.push(css);
+        continue;
+      }
+      if (css.startsWith(STYLE_ANCHOR_PREFIX)) {
+        anchors.push(element);
+        if (ignoredAnchors.has(element)) {
+          result.push(element);
+          continue;
+        }
+        const anchoredCss = css.slice(STYLE_ANCHOR_PREFIX.length);
+        styles.push(anchoredCss);
+        anchoredStyles.push(anchoredCss);
+        result.push(element);
+        continue;
+      }
+      styles.push(css);
+      const anchor: Element = { element: "style", data: `${STYLE_ANCHOR_PREFIX}${css}` };
+      anchors.push(anchor);
+      result.push(anchor);
       continue;
     }
 
     if (element.element === "if-tags") {
-      const slotId = ctx.nextSlotId++;
-      styles.push(`${STYLE_SLOT_PREFIX}${slotId}`);
+      const slotId = getStyleSlotId(element.data) ?? allocateStyleSlot(ctx);
+      styles.push(createStyleSlotMarker(slotId));
       const data: IfTagsDataWithStyleSlot = { ...(element.data as IfTagsData), _styleSlot: slotId };
       result.push({
         element: "if-tags",
@@ -45,7 +165,9 @@ function collectStylesFromElements(
     }
 
     result.push(
-      mapElementChildren(element, (children) => collectStylesFromElements(children, styles, ctx)),
+      mapElementChildren(element, (children) =>
+        collectStylesFromElements(children, styles, anchoredStyles, anchors, ignoredAnchors, ctx),
+      ),
     );
   }
 

package/src/parser/rules/block/module/resolution/walk-resolve.ts

@@ -3,6 +3,7 @@ import { mapElementChildrenWithState } from "../walk";
 import { isIfTagsElement, resolveIfTags, type IfTagsData } from "../iftags/resolve";
 import type { ListPagesContext, ListUsersContext, TagCloudContext } from "./contexts";
 import { countDynamicModules, resolveDynamicModuleElement } from "./dynamic-modules";
+import { collectStyles, createStyleSlotMarker, getStyleSlotId } from "./styles";
 
 /**
  * Resolution context passed through AST traversal.
@@ -21,6 +22,8 @@ export interface WalkContext {
   listPagesIdCounter: number;
   listUsersIdCounter: number;
   tagCloudIdCounter: number;
+  resolvedStyleSlots: Map<number, string[]>;
+  routedStyleAnchors: WeakSet<Element>;
 }
 
 export interface WalkResult {
@@ -56,6 +59,7 @@ export function walkAndResolve(elements: Element[], ctx: WalkContext): WalkResul
     if (isIfTagsElement(element)) {
       const ifTagsData = element.data as IfTagsData;
       const resolveResult = resolveIfTags(ifTagsData, ctx.pageTags);
+      const styleSlotId = getStyleSlotId(ifTagsData);
 
       if (resolveResult.evaluated) {
         if (resolveResult.matched) {
@@ -65,11 +69,25 @@ export function walkAndResolve(elements: Element[], ctx: WalkContext): WalkResul
             listUsersIdCounter: listUsersId,
             tagCloudIdCounter: tagCloudId,
           });
-          result.push(...childResult.elements);
+          if (styleSlotId === undefined) {
+            result.push(...childResult.elements);
+          } else {
+            const collected = collectStyles(childResult.elements);
+            ctx.resolvedStyleSlots.set(styleSlotId, collected.styles);
+            for (const anchor of collected.anchors) ctx.routedStyleAnchors.add(anchor);
+            result.push(
+              { element: "style", data: createStyleSlotMarker(styleSlotId) },
+              ...collected.elements,
+            );
+          }
           listPagesId = childResult.nextListPagesId;
           listUsersId = childResult.nextListUsersId;
           tagCloudId = childResult.nextTagCloudId;
         } else {
+          if (styleSlotId !== undefined) {
+            ctx.resolvedStyleSlots.set(styleSlotId, []);
+            result.push({ element: "style", data: createStyleSlotMarker(styleSlotId) });
+          }
           const counts = countDynamicModules(ifTagsData.elements);
           listPagesId += counts.listPagesId;
           listUsersId += counts.listUsersId;

package/src/parser/rules/block/module/resolve.ts

@@ -16,7 +16,7 @@
  * @module
  */
 
-import type { SyntaxTree } from "@wdprlib/ast";
+import type { Diagnostic, Element, SyntaxTree } from "@wdprlib/ast";
 import type { DataProvider } from "./types-common";
 import { resolveIncludes } from "./include";
 import type { ListPagesDataRequirement, CompiledTemplate } from "./listpages/types";
@@ -29,7 +29,8 @@ import {
   buildTagCloudContext,
 } from "./resolution/contexts";
 import { walkAndResolve } from "./resolution/walk-resolve";
-import { collectStyles } from "./resolution/styles";
+import { collectStyles, mergeCollectedStyles } from "./resolution/styles";
+import { containsSyntaxFootnoteBlock, ModuleDocumentRegistry } from "./resolution/document";
 
 const MODULE_SECONDARY_INCLUDE_MAX_ITERATIONS = 5;
 
@@ -110,6 +111,9 @@ export interface ResolveOptions {
    * state outside wdpr.
    */
   transformModuleSource?: ModuleSourceTransform;
+
+  /** Receives diagnostics emitted by ListPages/ListUsers secondary parses. */
+  onDiagnostics?: (diagnostics: Diagnostic[]) => void;
 }
 
 /**
@@ -131,7 +135,9 @@ export async function resolveModules(
   dataProvider: DataProvider,
   options: ResolveOptions,
 ): Promise<SyntaxTree> {
-  const parse = createModuleParseFunction(options, dataProvider);
+  const registry = new ModuleDocumentRegistry();
+  registry.register(ast);
+  const parse = createModuleParseFunction(options, dataProvider, registry);
   const listPagesCtx = await buildListPagesContext(
     dataProvider,
     options.requirements.listPages ?? [],
@@ -147,6 +153,8 @@ export async function resolveModules(
   );
   const tagCloudCtx = await buildTagCloudContext(dataProvider, options.requirements.tagCloud ?? []);
   const pageTags = dataProvider.getPageTags?.() ?? null;
+  const resolvedStyleSlots = new Map<number, string[]>();
+  const routedStyleAnchors = new WeakSet<Element>();
 
   // Resolve AST
   const resolvedElements = walkAndResolve(ast.elements, {
@@ -160,33 +168,44 @@ export async function resolveModules(
     listPagesIdCounter: 0,
     listUsersIdCounter: 0,
     tagCloudIdCounter: 0,
+    resolvedStyleSlots,
+    routedStyleAnchors,
   });
 
   // Collect style elements from resolved AST
-  const { elements: finalElements, styles } = collectStyles(resolvedElements.elements);
+  const {
+    elements: finalElements,
+    styles,
+    anchoredStyles,
+  } = collectStyles(resolvedElements.elements, routedStyleAnchors);
 
   const result: SyntaxTree = {
     ...ast,
     elements: finalElements,
   };
 
-  if (styles.length > 0) {
-    result.styles = styles;
-  }
+  const mergedStyles = mergeCollectedStyles(ast.styles, styles, resolvedStyleSlots, anchoredStyles);
+  if (mergedStyles.length > 0) result.styles = mergedStyles;
 
-  return result;
+  options.onDiagnostics?.(registry.diagnostics);
+  return registry.finalize(
+    result,
+    finalElements,
+    pageTags,
+    containsSyntaxFootnoteBlock(ast.elements),
+  );
 }
 
 function createModuleParseFunction(
   options: ResolveOptions,
   dataProvider: DataProvider,
+  registry: ModuleDocumentRegistry,
 ): ParseFunction {
   const transform = createModuleSourceTransform(options, dataProvider);
-  if (!transform) {
-    return options.parse;
-  }
-
-  return (source: string) => options.parse(transform(source));
+  return (source: string) =>
+    registry.register(options.parse(transform ? transform(source) : source), {
+      stripLegacyImplicitFootnoteBlock: true,
+    });
 }
 
 function createModuleSourceTransform(

package/src/parser/rules/block/module/types.ts

@@ -10,7 +10,7 @@
  * @module
  */
 
-import type { Element, Module } from "@wdprlib/ast";
+import type { Element, Module, ParseResult, SyntaxTree } from "@wdprlib/ast";
 import type { ParseContext } from "../../types";
 
 /**
@@ -23,7 +23,12 @@ import type { ParseContext } from "../../types";
  * @param input - Wikitext string to parse
  * @returns Object containing the parsed elements
  */
-export type ParseFunction = (input: string) => { elements: Element[] };
+export type ModuleParseResult = SyntaxTree | ParseResult;
+export type ParseFunction = (input: string) => ModuleParseResult;
+
+export function getModuleParseAst(result: ModuleParseResult): SyntaxTree {
+  return "ast" in result ? result.ast : result;
+}
 
 /**
  * Definition of a module rule that handles a specific Wikidot module type.

package/src/parser/rules/contracts/parse-context.ts

@@ -25,6 +25,7 @@ export interface ParseContext {
   version: Version;
   trackPositions: boolean;
   settings: WikitextSettings;
+  appendImplicitFootnoteBlock: boolean;
   footnotes: Element[][];
   tocEntries: TocEntry[];
   codeBlocks: CodeBlockData[];

package/src/pipeline/index.ts

@@ -0,0 +1,7 @@
+export { processWikitext } from "./process";
+export type {
+  ProcessedWikitextDocument,
+  ProcessWikitextCallbackContext,
+  ProcessWikitextDataProvider,
+  ProcessWikitextOptions,
+} from "./types";