@wdprlib/parser 4.3.0 → 5.0.0

package/src/parser/rules/block/module/resolution/styles.ts

@@ -1,41 +1,161 @@
 import type { Element } from "@wdprlib/ast";
-import { STYLE_SLOT_PREFIX } from "@wdprlib/ast";
-import { mapElementChildren } from "../walk";
+import { STYLE_ANCHOR_PREFIX, STYLE_SLOT_PREFIX } from "@wdprlib/ast";
+import { mapElementChildren, walkElements } from "../walk";
 import type { IfTagsData } from "../iftags/resolve";
 
 type IfTagsDataWithStyleSlot = IfTagsData & {
-  _styleSlot: number;
+  _styleSlot?: number;
 };
 
+export type ResolvedStyleSlots = ReadonlyMap<number, readonly string[]>;
+
+export interface StyleCollectionResult {
+  elements: Element[];
+  styles: string[];
+  anchoredStyles: string[];
+  anchors: Element[];
+}
+
 /**
  * Collect and remove style elements from the AST.
  *
  * Unresolved `if-tags` elements receive style-slot placeholders so render-time
  * iftags evaluation can preserve style order relative to already collected CSS.
  */
-export function collectStyles(elements: Element[]): { elements: Element[]; styles: string[] } {
+export function collectStyles(
+  elements: Element[],
+  ignoredAnchors: WeakSet<Element> = new WeakSet(),
+): StyleCollectionResult {
   const styles: string[] = [];
-  const ctx = { nextSlotId: 0 };
-  const filtered = collectStylesFromElements(elements, styles, ctx);
-  return { elements: filtered, styles };
+  const anchoredStyles: string[] = [];
+  const anchors: Element[] = [];
+  const usedSlots = collectExistingStyleSlots(elements);
+  const ctx = { nextSlotId: 0, usedSlots };
+  const filtered = collectStylesFromElements(
+    elements,
+    styles,
+    anchoredStyles,
+    anchors,
+    ignoredAnchors,
+    ctx,
+  );
+  return { elements: filtered, styles, anchoredStyles, anchors };
+}
+
+/**
+ * Preserve styles collected by earlier resolution passes while keeping
+ * unresolved IfTags style-slot markers idempotent across repeated passes.
+ */
+export function mergeCollectedStyles(
+  existing: readonly string[] | undefined,
+  collected: readonly string[],
+  resolvedSlots: ResolvedStyleSlots = new Map(),
+  anchoredStyles: readonly string[] = [],
+): string[] {
+  const unanchoredPrevious = [...(existing ?? [])];
+  for (const style of anchoredStyles) removeFirst(unanchoredPrevious, style);
+  for (const style of collected) {
+    if (isStyleSlotMarker(style)) removeFirst(unanchoredPrevious, style);
+  }
+
+  const merged = unanchoredPrevious.filter((style) => !isStyleSlotMarker(style));
+  for (const style of collected) {
+    if (isStyleSlotMarker(style)) appendStyleSlot(merged, style, resolvedSlots);
+    else merged.push(style);
+  }
+  return merged;
+}
+
+function isStyleSlotMarker(style: string): boolean {
+  return style.startsWith(STYLE_SLOT_PREFIX);
+}
+
+export function createStyleSlotMarker(slotId: number): string {
+  return `${STYLE_SLOT_PREFIX}${slotId}`;
+}
+
+export function getStyleSlotId(data: IfTagsData): number | undefined {
+  const slotId = (data as IfTagsDataWithStyleSlot)._styleSlot;
+  return Number.isSafeInteger(slotId) && slotId! >= 0 ? slotId : undefined;
+}
+
+function removeFirst(values: string[], value: string): void {
+  const index = values.indexOf(value);
+  if (index >= 0) values.splice(index, 1);
+}
+
+function appendStyleSlot(
+  target: string[],
+  marker: string,
+  resolvedSlots: ResolvedStyleSlots,
+): void {
+  const slotId = Number(marker.slice(STYLE_SLOT_PREFIX.length));
+  const replacement = resolvedSlots.get(slotId);
+  if (replacement) target.push(...replacement);
+  else target.push(marker);
+}
+
+function collectExistingStyleSlots(elements: Element[]): Set<number> {
+  const slots = new Set<number>();
+  walkElements(elements, (element) => {
+    if (element.element !== "if-tags") return;
+    const slotId = getStyleSlotId(element.data);
+    if (slotId !== undefined) slots.add(slotId);
+  });
+  return slots;
+}
+
+function allocateStyleSlot(ctx: StyleCollectionContext): number {
+  while (ctx.usedSlots.has(ctx.nextSlotId)) ctx.nextSlotId++;
+  const slotId = ctx.nextSlotId++;
+  ctx.usedSlots.add(slotId);
+  return slotId;
+}
+
+interface StyleCollectionContext {
+  nextSlotId: number;
+  usedSlots: Set<number>;
 }
 
 function collectStylesFromElements(
   elements: Element[],
   styles: string[],
-  ctx: { nextSlotId: number },
+  anchoredStyles: string[],
+  anchors: Element[],
+  ignoredAnchors: WeakSet<Element>,
+  ctx: StyleCollectionContext,
 ): Element[] {
   const result: Element[] = [];
 
   for (const element of elements) {
     if (element.element === "style") {
-      styles.push(element.data as string);
+      const css = element.data as string;
+      if (css.startsWith(STYLE_SLOT_PREFIX)) {
+        styles.push(css);
+        continue;
+      }
+      if (css.startsWith(STYLE_ANCHOR_PREFIX)) {
+        anchors.push(element);
+        if (ignoredAnchors.has(element)) {
+          result.push(element);
+          continue;
+        }
+        const anchoredCss = css.slice(STYLE_ANCHOR_PREFIX.length);
+        styles.push(anchoredCss);
+        anchoredStyles.push(anchoredCss);
+        result.push(element);
+        continue;
+      }
+      styles.push(css);
+      const anchor: Element = { element: "style", data: `${STYLE_ANCHOR_PREFIX}${css}` };
+      anchors.push(anchor);
+      result.push(anchor);
       continue;
     }
 
     if (element.element === "if-tags") {
-      const slotId = ctx.nextSlotId++;
-      styles.push(`${STYLE_SLOT_PREFIX}${slotId}`);
+      const slotId = getStyleSlotId(element.data) ?? allocateStyleSlot(ctx);
+      styles.push(createStyleSlotMarker(slotId));
       const data: IfTagsDataWithStyleSlot = { ...(element.data as IfTagsData), _styleSlot: slotId };
       result.push({
         element: "if-tags",
@@ -45,7 +165,9 @@ function collectStylesFromElements(
     }
 
     result.push(
-      mapElementChildren(element, (children) => collectStylesFromElements(children, styles, ctx)),
+      mapElementChildren(element, (children) =>
+        collectStylesFromElements(children, styles, anchoredStyles, anchors, ignoredAnchors, ctx),
+      ),
     );
   }
 

package/src/parser/rules/block/module/resolution/walk-resolve.ts

@@ -3,6 +3,7 @@ import { mapElementChildrenWithState } from "../walk";
 import { isIfTagsElement, resolveIfTags, type IfTagsData } from "../iftags/resolve";
 import type { ListPagesContext, ListUsersContext, TagCloudContext } from "./contexts";
 import { countDynamicModules, resolveDynamicModuleElement } from "./dynamic-modules";
+import { collectStyles, createStyleSlotMarker, getStyleSlotId } from "./styles";
 
 /**
  * Resolution context passed through AST traversal.
@@ -21,6 +22,8 @@ export interface WalkContext {
   listPagesIdCounter: number;
   listUsersIdCounter: number;
   tagCloudIdCounter: number;
+  resolvedStyleSlots: Map<number, string[]>;
+  routedStyleAnchors: WeakSet<Element>;
 }
 
 export interface WalkResult {
@@ -56,6 +59,7 @@ export function walkAndResolve(elements: Element[], ctx: WalkContext): WalkResul
     if (isIfTagsElement(element)) {
       const ifTagsData = element.data as IfTagsData;
       const resolveResult = resolveIfTags(ifTagsData, ctx.pageTags);
+      const styleSlotId = getStyleSlotId(ifTagsData);
 
       if (resolveResult.evaluated) {
         if (resolveResult.matched) {
@@ -65,11 +69,25 @@ export function walkAndResolve(elements: Element[], ctx: WalkContext): WalkResul
             listUsersIdCounter: listUsersId,
             tagCloudIdCounter: tagCloudId,
           });
-          result.push(...childResult.elements);
+          if (styleSlotId === undefined) {
+            result.push(...childResult.elements);
+          } else {
+            const collected = collectStyles(childResult.elements);
+            ctx.resolvedStyleSlots.set(styleSlotId, collected.styles);
+            for (const anchor of collected.anchors) ctx.routedStyleAnchors.add(anchor);
+            result.push(
+              { element: "style", data: createStyleSlotMarker(styleSlotId) },
+              ...collected.elements,
+            );
+          }
           listPagesId = childResult.nextListPagesId;
           listUsersId = childResult.nextListUsersId;
           tagCloudId = childResult.nextTagCloudId;
         } else {
+          if (styleSlotId !== undefined) {
+            ctx.resolvedStyleSlots.set(styleSlotId, []);
+            result.push({ element: "style", data: createStyleSlotMarker(styleSlotId) });
+          }
           const counts = countDynamicModules(ifTagsData.elements);
           listPagesId += counts.listPagesId;
           listUsersId += counts.listUsersId;

package/src/parser/rules/block/module/resolve.ts

@@ -16,7 +16,7 @@
  * @module
  */
 
-import type { SyntaxTree } from "@wdprlib/ast";
+import type { Diagnostic, Element, SyntaxTree } from "@wdprlib/ast";
 import type { DataProvider } from "./types-common";
 import { resolveIncludes } from "./include";
 import type { ListPagesDataRequirement, CompiledTemplate } from "./listpages/types";
@@ -29,7 +29,8 @@ import {
   buildTagCloudContext,
 } from "./resolution/contexts";
 import { walkAndResolve } from "./resolution/walk-resolve";
-import { collectStyles } from "./resolution/styles";
+import { collectStyles, mergeCollectedStyles } from "./resolution/styles";
+import { containsSyntaxFootnoteBlock, ModuleDocumentRegistry } from "./resolution/document";
 
 const MODULE_SECONDARY_INCLUDE_MAX_ITERATIONS = 5;
 
@@ -110,6 +111,9 @@ export interface ResolveOptions {
    * state outside wdpr.
    */
   transformModuleSource?: ModuleSourceTransform;
+
+  /** Receives diagnostics emitted by ListPages/ListUsers secondary parses. */
+  onDiagnostics?: (diagnostics: Diagnostic[]) => void;
 }
 
 /**
@@ -131,7 +135,9 @@ export async function resolveModules(
   dataProvider: DataProvider,
   options: ResolveOptions,
 ): Promise<SyntaxTree> {
-  const parse = createModuleParseFunction(options, dataProvider);
+  const registry = new ModuleDocumentRegistry();
+  registry.register(ast);
+  const parse = createModuleParseFunction(options, dataProvider, registry);
   const listPagesCtx = await buildListPagesContext(
     dataProvider,
     options.requirements.listPages ?? [],
@@ -147,6 +153,8 @@ export async function resolveModules(
   );
   const tagCloudCtx = await buildTagCloudContext(dataProvider, options.requirements.tagCloud ?? []);
   const pageTags = dataProvider.getPageTags?.() ?? null;
+  const resolvedStyleSlots = new Map<number, string[]>();
+  const routedStyleAnchors = new WeakSet<Element>();
 
   // Resolve AST
   const resolvedElements = walkAndResolve(ast.elements, {
@@ -160,33 +168,44 @@ export async function resolveModules(
     listPagesIdCounter: 0,
     listUsersIdCounter: 0,
     tagCloudIdCounter: 0,
+    resolvedStyleSlots,
+    routedStyleAnchors,
   });
 
   // Collect style elements from resolved AST
-  const { elements: finalElements, styles } = collectStyles(resolvedElements.elements);
+  const {
+    elements: finalElements,
+    styles,
+    anchoredStyles,
+  } = collectStyles(resolvedElements.elements, routedStyleAnchors);
 
   const result: SyntaxTree = {
     ...ast,
     elements: finalElements,
   };
 
-  if (styles.length > 0) {
-    result.styles = styles;
-  }
+  const mergedStyles = mergeCollectedStyles(ast.styles, styles, resolvedStyleSlots, anchoredStyles);
+  if (mergedStyles.length > 0) result.styles = mergedStyles;
 
-  return result;
+  options.onDiagnostics?.(registry.diagnostics);
+  return registry.finalize(
+    result,
+    finalElements,
+    pageTags,
+    containsSyntaxFootnoteBlock(ast.elements),
+  );
 }
 
 function createModuleParseFunction(
   options: ResolveOptions,
   dataProvider: DataProvider,
+  registry: ModuleDocumentRegistry,
 ): ParseFunction {
   const transform = createModuleSourceTransform(options, dataProvider);
-  if (!transform) {
-    return options.parse;
-  }
-
-  return (source: string) => options.parse(transform(source));
+  return (source: string) =>
+    registry.register(options.parse(transform ? transform(source) : source), {
+      stripLegacyImplicitFootnoteBlock: true,
+    });
 }
 
 function createModuleSourceTransform(

package/src/parser/rules/block/module/types.ts

@@ -10,7 +10,7 @@
  * @module
  */
 
-import type { Element, Module } from "@wdprlib/ast";
+import type { Element, Module, ParseResult, SyntaxTree } from "@wdprlib/ast";
 import type { ParseContext } from "../../types";
 
 /**
@@ -23,7 +23,12 @@ import type { ParseContext } from "../../types";
  * @param input - Wikitext string to parse
  * @returns Object containing the parsed elements
  */
-export type ParseFunction = (input: string) => { elements: Element[] };
+export type ModuleParseResult = SyntaxTree | ParseResult;
+export type ParseFunction = (input: string) => ModuleParseResult;
+
+export function getModuleParseAst(result: ModuleParseResult): SyntaxTree {
+  return "ast" in result ? result.ast : result;
+}
 
 /**
  * Definition of a module rule that handles a specific Wikidot module type.

package/src/parser/rules/contracts/parse-context.ts

@@ -25,6 +25,7 @@ export interface ParseContext {
   version: Version;
   trackPositions: boolean;
   settings: WikitextSettings;
+  appendImplicitFootnoteBlock: boolean;
   footnotes: Element[][];
   tocEntries: TocEntry[];
   codeBlocks: CodeBlockData[];

package/src/pipeline/index.ts

@@ -0,0 +1,7 @@
+export { processWikitext } from "./process";
+export type {
+  ProcessedWikitextDocument,
+  ProcessWikitextCallbackContext,
+  ProcessWikitextDataProvider,
+  ProcessWikitextOptions,
+} from "./types";

package/src/pipeline/process.ts

@@ -0,0 +1,172 @@
+import {
+  DEFAULT_SETTINGS,
+  type Diagnostic,
+  type PageRef,
+  type WikitextPageContext,
+} from "@wdprlib/ast";
+import { parse } from "../parser";
+import { extractDataRequirements } from "../parser/rules/block/module/listpages/extract";
+import {
+  resolveIncludesAsyncWithTrace,
+  type AsyncIncludeFetcher,
+  type IncludeDependency,
+} from "../parser/rules/block/module/include";
+import type { DataProvider } from "../parser/rules/block/module/types-common";
+import { resolveModulesWithAsyncParse } from "../parser/rules/block/module/resolution/resolve-async";
+import type {
+  ProcessedWikitextDocument,
+  ProcessWikitextCallbackContext,
+  ProcessWikitextOptions,
+} from "./types";
+
+const DEFAULT_MODULE_MAX_PASSES = 5;
+const PIPELINE_DIAGNOSTIC_POSITION = {
+  start: { line: 1, column: 1, offset: 0 },
+  end: { line: 1, column: 1, offset: 0 },
+};
+
+export async function processWikitext<TPage extends WikitextPageContext>(
+  source: string,
+  options: ProcessWikitextOptions<TPage>,
+): Promise<ProcessedWikitextDocument<TPage>> {
+  const settings = options.settings ?? DEFAULT_SETTINGS;
+  const callbackContext: ProcessWikitextCallbackContext<TPage> = {
+    page: options.page,
+    settings,
+  };
+  const dependencies: IncludeDependency[] = [];
+  const diagnostics: Diagnostic[] = [];
+  const fetchInclude = createRequestIncludeFetcher(
+    options.dataProvider?.fetchInclude
+      ? (pageRef) => options.dataProvider!.fetchInclude!(pageRef, callbackContext)
+      : undefined,
+  );
+  const resolveSource = async (input: string): Promise<string> => {
+    if (!fetchInclude) return input;
+    const resolution = await resolveIncludesAsyncWithTrace(input, fetchInclude, {
+      maxIterations: options.includeMaxIterations,
+      settings,
+    });
+    dependencies.push(...resolution.dependencies);
+    if (resolution.reachedMaxIterations) {
+      diagnostics.push(
+        createLimitDiagnostic(
+          "include-resolution-limit",
+          `Include expansion stopped after ${options.includeMaxIterations ?? 10} iterations.`,
+        ),
+      );
+    }
+    return resolution.source;
+  };
+
+  const expandedSource = await resolveSource(source);
+  const initial = parse(expandedSource, {
+    settings,
+    pageTags: options.page.tags,
+    appendImplicitFootnoteBlock: false,
+  });
+  diagnostics.push(...initial.diagnostics);
+  const dataProvider = createModuleDataProvider(options, callbackContext);
+  const parseFragment = async (fragmentSource: string) =>
+    parse(await resolveSource(fragmentSource), {
+      settings,
+      pageTags: options.page.tags,
+      appendImplicitFootnoteBlock: false,
+    });
+  let ast = initial.ast;
+
+  for (let pass = 0; pass < DEFAULT_MODULE_MAX_PASSES; pass++) {
+    const extraction = extractDataRequirements(ast);
+    if (pass > 0 && !hasResolvableRequirements(extraction.requirements, options.dataProvider)) {
+      break;
+    }
+
+    const resolved = await resolveModulesWithAsyncParse(ast, dataProvider, {
+      parse: parseFragment,
+      compiledListPagesTemplates: extraction.compiledListPagesTemplates,
+      compiledListUsersTemplates: extraction.compiledListUsersTemplates,
+      requirements: extraction.requirements,
+      urlPath: options.page.urlPath,
+      pageTags: options.page.tags,
+    });
+    ast = resolved.ast;
+    diagnostics.push(...resolved.diagnostics);
+  }
+
+  if (hasResolvableRequirements(extractDataRequirements(ast).requirements, options.dataProvider)) {
+    diagnostics.push(
+      createLimitDiagnostic(
+        "module-resolution-limit",
+        `Module resolution stopped after ${DEFAULT_MODULE_MAX_PASSES} passes.`,
+      ),
+    );
+  }
+
+  return {
+    ast,
+    page: options.page,
+    settings,
+    diagnostics,
+    dependencies,
+  };
+}
+
+function hasResolvableRequirements(
+  requirements: ReturnType<typeof extractDataRequirements>["requirements"],
+  provider:
+    | {
+        fetchListPages?: unknown;
+        fetchListUsers?: unknown;
+        fetchTagCloud?: unknown;
+      }
+    | undefined,
+): boolean {
+  return Boolean(
+    (provider?.fetchListPages && requirements.listPages.length > 0) ||
+    (provider?.fetchListUsers && requirements.listUsers.length > 0) ||
+    (provider?.fetchTagCloud && requirements.tagCloud.length > 0),
+  );
+}
+
+function createLimitDiagnostic(code: string, message: string): Diagnostic {
+  return {
+    severity: "warning",
+    code,
+    message,
+    position: PIPELINE_DIAGNOSTIC_POSITION,
+  };
+}
+
+function createModuleDataProvider<TPage extends WikitextPageContext>(
+  options: ProcessWikitextOptions<TPage>,
+  context: ProcessWikitextCallbackContext<TPage>,
+): DataProvider {
+  const provider = options.dataProvider;
+  return {
+    fetchListPages: provider?.fetchListPages
+      ? (query, requirement) => provider.fetchListPages!(query, requirement, context)
+      : undefined,
+    fetchListUsers: provider?.fetchListUsers
+      ? (requirement) => provider.fetchListUsers!(requirement, context)
+      : undefined,
+    fetchTagCloud: provider?.fetchTagCloud
+      ? (requirement) => provider.fetchTagCloud!(requirement, context)
+      : undefined,
+    getPageTags: () => options.page.tags,
+  };
+}
+
+function createRequestIncludeFetcher(
+  fetcher: AsyncIncludeFetcher | undefined,
+): AsyncIncludeFetcher | undefined {
+  if (!fetcher) return undefined;
+  const cache = new Map<string, Promise<string | null>>();
+  return (pageRef: PageRef): Promise<string | null> => {
+    const key = `${pageRef.site ?? ""}:${pageRef.page.toLowerCase()}`;
+    const cached = cache.get(key);
+    if (cached) return cached;
+    const result = fetcher(pageRef).catch(() => null);
+    cache.set(key, result);
+    return result;
+  };
+}

package/src/pipeline/types.ts

@@ -0,0 +1,63 @@
+import type {
+  Diagnostic,
+  PageRef,
+  SyntaxTree,
+  WikitextPageContext,
+  WikitextSettings,
+} from "@wdprlib/ast";
+import type { IncludeDependency } from "../parser/rules/block/module/include";
+import type {
+  ListPagesDataRequirement,
+  ListPagesExternalData,
+  NormalizedListPagesQuery,
+} from "../parser/rules/block/module/listpages/types";
+import type {
+  ListUsersDataRequirement,
+  ListUsersExternalData,
+} from "../parser/rules/block/module/listusers/types";
+import type {
+  TagCloudDataRequirement,
+  TagCloudExternalData,
+} from "../parser/rules/block/module/tagcloud/types";
+
+export interface ProcessWikitextCallbackContext<TPage extends WikitextPageContext> {
+  page: TPage;
+  settings: WikitextSettings;
+}
+
+export interface ProcessWikitextDataProvider<TPage extends WikitextPageContext> {
+  fetchInclude?: (
+    pageRef: PageRef,
+    context: ProcessWikitextCallbackContext<TPage>,
+  ) => Promise<string | null>;
+  fetchListPages?: (
+    query: NormalizedListPagesQuery,
+    requirement: ListPagesDataRequirement,
+    context: ProcessWikitextCallbackContext<TPage>,
+  ) => Promise<ListPagesExternalData | null | undefined>;
+  fetchListUsers?: (
+    requirement: ListUsersDataRequirement,
+    context: ProcessWikitextCallbackContext<TPage>,
+  ) => Promise<ListUsersExternalData | null | undefined>;
+  fetchTagCloud?: (
+    requirement: TagCloudDataRequirement,
+    context: ProcessWikitextCallbackContext<TPage>,
+  ) => Promise<TagCloudExternalData | null | undefined>;
+}
+
+export interface ProcessWikitextOptions<TPage extends WikitextPageContext> {
+  page: TPage;
+  settings?: WikitextSettings;
+  dataProvider?: ProcessWikitextDataProvider<TPage>;
+  includeMaxIterations?: number;
+}
+
+export interface ProcessedWikitextDocument<
+  TPage extends WikitextPageContext = WikitextPageContext,
+> {
+  ast: SyntaxTree;
+  page: TPage;
+  settings: WikitextSettings;
+  diagnostics: Diagnostic[];
+  dependencies: IncludeDependency[];
+}