@wdprlib/parser 4.3.0 → 4.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wdprlib/parser",
-  "version": "4.3.0",
+  "version": "4.4.0",
   "description": "Parser for Wikidot markup",
   "keywords": [
     "ast",
@@ -41,6 +41,6 @@
   },
   "dependencies": {
     "@braintree/sanitize-url": "^7.1.1",
-    "@wdprlib/ast": "2.3.0"
+    "@wdprlib/ast": "2.4.0"
   }
 }

package/src/index.ts

@@ -98,6 +98,15 @@ export { Lexer, tokenize, createToken } from "./lexer";
 export type { ParserOptions } from "./parser";
 export { Parser, parse } from "./parser";
 
+// High-level parser pipeline
+export { processWikitext } from "./pipeline";
+export type {
+  ProcessedWikitextDocument,
+  ProcessWikitextCallbackContext,
+  ProcessWikitextDataProvider,
+  ProcessWikitextOptions,
+} from "./pipeline";
+
 // Modules (ListPages, ListUsers, IfTags, Include, etc.)
 export type {
   // ListPages query types
@@ -121,6 +130,7 @@ export type {
   ExtractionResult,
   // Resolution types
   ParseFunction,
+  ModuleParseResult,
   ModuleSourceTransform,
   ResolveOptions,
   // Include resolution
@@ -163,6 +173,7 @@ export {
   extractIncludeReferences,
   resolveIncludes,
   resolveIncludesAsync,
+  resolveIncludesAsyncWithTrace,
   resolveIncludesWithTrace,
   // IfTags source-level preprocessing (run between include expansion and parse)
   preprocessIftags,

package/src/parser/parse/context.ts

@@ -10,6 +10,7 @@ export function createParseContext(tokens: Token[], options: ParserOptions = {})
     version: options.version ?? "wikidot",
     trackPositions: options.trackPositions ?? true,
     settings: options.settings ?? DEFAULT_SETTINGS,
+    appendImplicitFootnoteBlock: options.appendImplicitFootnoteBlock ?? true,
     footnotes: [],
     tocEntries: [],
     codeBlocks: [],

package/src/parser/parse/options.ts

@@ -31,4 +31,10 @@ export interface ParserOptions {
    * - `string[]`: every iftags block is evaluated against the given tags eagerly.
    */
   pageTags?: string[] | null;
+  /**
+   * Append the implicit document-level footnote block when no explicit block
+   * exists. Defaults to true. Fragment pipelines can disable this and add one
+   * block after all fragments have been merged.
+   */
+  appendImplicitFootnoteBlock?: boolean;
 }

package/src/parser/parse/result.ts

@@ -10,7 +10,9 @@ import { containsFootnoteBlock } from "./footnotes";
 
 export function finalizeParseResult(ctx: ParseContext, children: Element[]): ParseResult {
   const cleanedChildren = postprocessChildren(children);
-  appendImplicitFootnoteBlock(cleanedChildren);
+  if (ctx.appendImplicitFootnoteBlock) {
+    appendImplicitFootnoteBlock(cleanedChildren);
+  }
 
   return {
     ast: buildSyntaxTree(ctx, cleanedChildren),

package/src/parser/rules/block/module/include/index.ts

@@ -16,7 +16,12 @@
  * @module
  */
 
-export { resolveIncludes, resolveIncludesAsync, resolveIncludesWithTrace } from "./resolve";
+export {
+  resolveIncludes,
+  resolveIncludesAsync,
+  resolveIncludesAsyncWithTrace,
+  resolveIncludesWithTrace,
+} from "./resolve";
 export { extractIncludeReferences } from "./references";
 export type {
   IncludeFetcher,

package/src/parser/rules/block/module/include/resolve/index.ts

@@ -7,7 +7,11 @@
 
 import type { PageRef } from "@wdprlib/ast";
 import { createCachedAsyncIncludeFetcher, createCachedIncludeFetcher } from "./cache";
-import { expandIterative, expandIterativeAsync, expandIterativeWithTrace } from "./iterate";
+import {
+  expandIterative,
+  expandIterativeAsyncWithTrace,
+  expandIterativeWithTrace,
+} from "./iterate";
 import type {
   AsyncIncludeFetcher,
   IncludeFetcher,
@@ -84,13 +88,29 @@ export async function resolveIncludesAsync(
   fetcher: AsyncIncludeFetcher,
   options?: ResolveIncludesOptions,
 ): Promise<string> {
+  return (await resolveIncludesAsyncWithTrace(source, fetcher, options)).source;
+}
+
+/**
+ * Async include expansion with dependency and iteration trace data.
+ */
+export async function resolveIncludesAsyncWithTrace(
+  source: string,
+  fetcher: AsyncIncludeFetcher,
+  options?: ResolveIncludesOptions,
+): Promise<ResolveIncludesTraceResult> {
   if (options?.settings && !options.settings.enablePageSyntax) {
-    return source;
+    return {
+      source,
+      dependencies: [],
+      iterations: [],
+      reachedMaxIterations: false,
+    };
   }
 
   const maxIterations = options?.maxIterations ?? 10;
   const cachedFetcher = createCachedAsyncIncludeFetcher(fetcher, normalizePageKey);
-  return expandIterativeAsync(source, cachedFetcher, maxIterations);
+  return expandIterativeAsyncWithTrace(source, cachedFetcher, maxIterations);
 }
 
 /**

package/src/parser/rules/block/module/include/resolve/iterate.ts

@@ -102,6 +102,41 @@ export async function expandIterativeAsync(
   return current;
 }
 
+/** Async counterpart of {@link expandIterativeWithTrace}. */
+export async function expandIterativeAsyncWithTrace(
+  source: string,
+  fetcher: AsyncIncludeFetcher,
+  maxIterations: number,
+): Promise<ResolveIncludesTraceResult> {
+  let current = source;
+  const replacementCache = new Map<string, Promise<string>>();
+  const dependencies: IncludeDependency[] = [];
+  const iterations: ResolveIncludesTraceResult["iterations"] = [];
+
+  for (let i = 0; i < maxIterations; i++) {
+    const expanded = await expandOneIterationAsyncWithTrace(current, fetcher, replacementCache, i);
+    if (expanded === null) break;
+
+    dependencies.push(...expanded.dependencies);
+    iterations.push({
+      iteration: i,
+      directives: expanded.references,
+      changed: expanded.source !== current,
+    });
+
+    if (expanded.source === current) break;
+    current = expanded.source;
+  }
+
+  return {
+    source: current,
+    dependencies,
+    iterations,
+    reachedMaxIterations:
+      iterations.length === maxIterations && scanIncludeDirectives(current).length > 0,
+  };
+}
+
 function expandOneIteration(
   source: string,
   fetcher: IncludeFetcher,
@@ -179,6 +214,42 @@ async function expandOneIterationAsync(
   return parts.join("");
 }
 
+async function expandOneIterationAsyncWithTrace(
+  source: string,
+  fetcher: AsyncIncludeFetcher,
+  replacementCache: Map<string, Promise<string>>,
+  iteration: number,
+): Promise<{
+  source: string;
+  references: IncludeReference[];
+  dependencies: IncludeDependency[];
+} | null> {
+  if (!MAYBE_INCLUDE_PATTERN.test(source)) return null;
+
+  const directives = scanIncludeDirectives(source);
+  if (directives.length === 0) return null;
+
+  const references = directives.map(createIncludeReference);
+  const replacements = await Promise.all(
+    directives.map(({ inner }) => replaceCachedAsync(inner, fetcher, replacementCache)),
+  );
+  const parts: string[] = [];
+  let lastPos = 0;
+
+  for (let i = 0; i < directives.length; i++) {
+    const { start, end } = directives[i]!;
+    parts.push(source.slice(lastPos, start), replacements[i]!);
+    lastPos = end;
+  }
+  parts.push(source.slice(lastPos));
+
+  return {
+    source: parts.join(""),
+    references,
+    dependencies: references.map((reference) => ({ ...reference, iteration })),
+  };
+}
+
 function replaceCached(inner: string, fetcher: IncludeFetcher, cache: Map<string, string>): string {
   const cached = cache.get(inner);
   if (cached !== undefined) return cached;

package/src/parser/rules/block/module/index.ts

@@ -20,7 +20,7 @@
  */
 
 // Module rule types and registry
-export type { ModuleRule } from "./types";
+export type { ModuleParseResult, ModuleRule } from "./types";
 export { MODULE_RULES, getModuleRuleByName } from "./mapping";
 export { moduleRule } from "./rule";
 
@@ -106,6 +106,7 @@ export {
   extractIncludeReferences,
   resolveIncludes,
   resolveIncludesAsync,
+  resolveIncludesAsyncWithTrace,
   resolveIncludesWithTrace,
 } from "./include";
 

package/src/parser/rules/block/module/listpages/resolution/items.ts

@@ -1,6 +1,6 @@
 import type { Element } from "@wdprlib/ast";
 import type { CompiledTemplate, ListPagesExternalData, VariableContext } from "../types";
-import type { ParseFunction } from "../../types";
+import { getModuleParseAst, type ParseFunction } from "../../types";
 import type { ListPagesModuleData } from "../resolve";
 
 export function renderListPagesItems(
@@ -23,7 +23,7 @@ export function renderListPagesItems(
       site: data.site,
     };
 
-    const itemAst = parse(compiledTemplate(ctx));
+    const itemAst = getModuleParseAst(parse(compiledTemplate(ctx)));
 
     if (module.separate) {
       items.push({

package/src/parser/rules/block/module/listpages/resolution/wrapper.ts

@@ -1,5 +1,5 @@
 import type { Element } from "@wdprlib/ast";
-import type { ParseFunction } from "../../types";
+import { getModuleParseAst, type ParseFunction } from "../../types";
 import type { ListPagesModuleData } from "../resolve";
 
 export function wrapListPagesResult(
@@ -14,14 +14,14 @@ export function wrapListPagesResult(
   const result: Element[] = [];
 
   if (module["prepend-line"] && !module.separate) {
-    const prependAst = parse(module["prepend-line"]);
+    const prependAst = getModuleParseAst(parse(module["prepend-line"]));
     result.push(...prependAst.elements);
   }
 
   result.push(...items);
 
   if (module["append-line"] && !module.separate) {
-    const appendAst = parse(module["append-line"]);
+    const appendAst = getModuleParseAst(parse(module["append-line"]));
     result.push(...appendAst.elements);
   }
 

package/src/parser/rules/block/module/listusers/resolve.ts

@@ -15,7 +15,7 @@ import type {
   ListUsersCompiledTemplate,
   ListUsersVariableContext,
 } from "./types";
-import type { ParseFunction } from "../types";
+import { getModuleParseAst, type ParseFunction } from "../types";
 
 /**
  * Narrowed type for the list-users variant of the Module discriminated union.
@@ -53,6 +53,6 @@ export function resolveListUsers(
 ): Element[] {
   const ctx: ListUsersVariableContext = { user: data.user };
   const substituted = compiledTemplate(ctx);
-  const itemAst = parse(substituted);
+  const itemAst = getModuleParseAst(parse(substituted));
   return itemAst.elements;
 }

package/src/parser/rules/block/module/resolution/document.ts

@@ -0,0 +1,357 @@
+import {
+  evaluateExpression,
+  isTruthy,
+  type Diagnostic,
+  type Element,
+  type ParseResult,
+  type SyntaxTree,
+  type TocEntry,
+} from "@wdprlib/ast";
+import { extractHeadingText } from "../../heading/toc-text";
+import { buildTableOfContents } from "../../../../toc";
+import { resolveIfTags } from "../iftags/resolve";
+import {
+  getGenericElementChildren,
+  listElement,
+  withGenericElementChildren,
+} from "../walk/children";
+import type { ModuleParseResult } from "../types";
+
+interface RegisterOptions {
+  stripLegacyImplicitFootnoteBlock?: boolean;
+}
+
+/** Tracks parse-local side channels while module fragments are merged. */
+export class ModuleDocumentRegistry {
+  private readonly footnotesByElement = new WeakMap<Element, Element[]>();
+  readonly diagnostics: Diagnostic[] = [];
+
+  register(result: ModuleParseResult, options: RegisterOptions = {}): SyntaxTree {
+    const parseResult = isParseResult(result) ? result : null;
+    const ast: SyntaxTree = parseResult ? parseResult.ast : (result as SyntaxTree);
+    const elements = options.stripLegacyImplicitFootnoteBlock
+      ? stripTrailingDefaultFootnoteBlock(ast.elements)
+      : ast.elements;
+    const registeredAst = elements === ast.elements ? ast : { ...ast, elements };
+
+    if (parseResult) {
+      this.diagnostics.push(...parseResult.diagnostics);
+    }
+    this.registerFootnotes(registeredAst);
+    return registeredAst;
+  }
+
+  finalize(
+    ast: SyntaxTree,
+    elements: Element[],
+    pageTags: string[] | null,
+    ensureFootnoteBlock = true,
+  ): SyntaxTree {
+    const normalizedElements = normalizeFootnoteBlocks(elements, ensureFootnoteBlock);
+    const footnotes: Element[][] = [];
+    const htmlBlocks: string[] = [];
+    const codeBlocks: NonNullable<SyntaxTree["code-blocks"]> = [];
+    const tocEntries: TocEntry[] = [];
+
+    const collectElement = (element: Element): void => {
+      if (element.element === "footnote") {
+        const content = this.footnotesByElement.get(element);
+        if (content) footnotes.push(content);
+      } else if (element.element === "html") {
+        htmlBlocks.push(element.data.contents);
+      } else if (element.element === "code") {
+        codeBlocks.push(element.data);
+      } else if (isTocHeading(element)) {
+        tocEntries.push({
+          level: element.data.type.header.level,
+          text: extractHeadingText(element.data.elements),
+        });
+      }
+    };
+    walkRenderOrder(normalizedElements, pageTags, collectElement);
+
+    const result: SyntaxTree = { ...ast, elements: normalizedElements };
+    setOptionalArray(result, "footnotes", footnotes);
+    setOptionalArray(result, "html-blocks", htmlBlocks);
+    setOptionalArray(result, "code-blocks", codeBlocks);
+    setOptionalArray(result, "table-of-contents", buildTableOfContents(tocEntries));
+    return result;
+  }
+
+  private registerFootnotes(ast: SyntaxTree): void {
+    const footnoteElements: Element[] = [];
+    walkSyntaxOrder(ast.elements, (element) => {
+      if (element.element === "footnote") footnoteElements.push(element);
+    });
+
+    for (let i = 0; i < footnoteElements.length; i++) {
+      const content = ast.footnotes?.[i];
+      if (content) this.footnotesByElement.set(footnoteElements[i]!, content);
+    }
+  }
+}
+
+function isParseResult(result: ModuleParseResult): result is ParseResult {
+  return "ast" in result && "diagnostics" in result;
+}
+
+function setOptionalArray<K extends keyof SyntaxTree>(
+  tree: SyntaxTree,
+  key: K,
+  value: NonNullable<SyntaxTree[K]>,
+): void {
+  if ((value as unknown[]).length > 0) {
+    tree[key] = value;
+  } else {
+    delete tree[key];
+  }
+}
+
+function stripTrailingDefaultFootnoteBlock(elements: Element[]): Element[] {
+  const last = elements.at(-1);
+  if (last?.element !== "footnote-block" || last.data.title !== null || last.data.hide === true) {
+    return elements;
+  }
+  return elements.slice(0, -1);
+}
+
+function normalizeFootnoteBlocks(elements: Element[], ensureFootnoteBlock: boolean): Element[] {
+  const state = { found: false };
+  const normalized = mapSyntaxElements(elements, (element) => {
+    if (element.element !== "footnote-block") return element;
+    if (state.found) return null;
+    state.found = true;
+    return element;
+  });
+
+  if (!state.found && ensureFootnoteBlock) {
+    normalized.push({
+      element: "footnote-block",
+      data: { title: null, hide: false },
+    });
+  }
+  return normalized;
+}
+
+export function containsSyntaxFootnoteBlock(elements: Element[]): boolean {
+  let found = false;
+  walkSyntaxOrder(elements, (element) => {
+    if (element.element === "footnote-block") found = true;
+  });
+  return found;
+}
+
+function mapSyntaxElements(
+  elements: Element[],
+  transform: (element: Element) => Element | null,
+): Element[] {
+  const result: Element[] = [];
+  for (const original of elements) {
+    const element = transform(original);
+    if (!element) continue;
+    result.push(mapSyntaxChildren(element, (children) => mapSyntaxElements(children, transform)));
+  }
+  return result;
+}
+
+function mapSyntaxChildren(
+  element: Element,
+  transform: (elements: Element[]) => Element[],
+): Element {
+  if (element.element === "if") {
+    return {
+      ...element,
+      data: {
+        ...element.data,
+        // oxlint-disable-next-line unicorn/no-thenable -- `then` is part of the public AST schema
+        then: transform(element.data.then),
+        else: transform(element.data.else),
+      },
+    };
+  }
+  if (element.element === "ifexpr") {
+    return {
+      ...element,
+      data: {
+        ...element.data,
+        // oxlint-disable-next-line unicorn/no-thenable -- `then` is part of the public AST schema
+        then: transform(element.data.then),
+        else: transform(element.data.else),
+      },
+    };
+  }
+  if (element.element === "bibliography-block") {
+    return {
+      ...element,
+      data: {
+        ...element.data,
+        entries: element.data.entries.map((entry) => ({
+          ...entry,
+          key: transform(entry.key),
+          value: transform(entry.value),
+        })),
+      },
+    };
+  }
+  if (element.element === "list") {
+    return {
+      ...element,
+      data: {
+        ...element.data,
+        items: element.data.items.map((item) =>
+          item["item-type"] === "elements"
+            ? { ...item, elements: transform(item.elements) }
+            : mapSubList(item, transform),
+        ),
+      },
+    };
+  }
+  if (element.element === "table") {
+    return {
+      ...element,
+      data: {
+        ...element.data,
+        rows: element.data.rows.map((row) => ({
+          ...row,
+          cells: row.cells.map((cell) => ({ ...cell, elements: transform(cell.elements) })),
+        })),
+      },
+    };
+  }
+  if (element.element === "definition-list") {
+    return {
+      ...element,
+      data: element.data.map((entry) => ({
+        ...entry,
+        key: transform(entry.key),
+        value: transform(entry.value),
+      })),
+    };
+  }
+  if (element.element === "tab-view") {
+    return {
+      ...element,
+      data: element.data.map((tab) => ({ ...tab, elements: transform(tab.elements) })),
+    };
+  }
+
+  const children = getGenericElementChildren(element);
+  return children === null ? element : withGenericElementChildren(element, transform(children));
+}
+
+function mapSubList(
+  item: Extract<ElementOfListItem, { "item-type": "sub-list" }>,
+  transform: (elements: Element[]) => Element[],
+): ElementOfListItem {
+  const transformed = transform([listElement(item.data)])[0];
+  return transformed?.element === "list" ? { ...item, data: transformed.data } : item;
+}
+
+type ElementOfListItem = Extract<Element, { element: "list" }>["data"]["items"][number];
+
+function walkSyntaxOrder(elements: Element[], callback: (element: Element) => void): void {
+  for (const element of elements) {
+    callback(element);
+    walkAllSyntaxChildren(element, (children) => walkSyntaxOrder(children, callback));
+  }
+}
+
+function walkRenderOrder(
+  elements: Element[],
+  pageTags: string[] | null,
+  callback: (element: Element) => void,
+): void {
+  for (const element of elements) {
+    callback(element);
+    if (element.element === "if") {
+      walkRenderOrder(
+        isTruthy(element.data.condition) ? element.data.then : element.data.else,
+        pageTags,
+        callback,
+      );
+      continue;
+    }
+    if (element.element === "ifexpr") {
+      const evaluated = evaluateExpression(element.data.expression);
+      if (evaluated.success) {
+        walkRenderOrder(
+          evaluated.value !== 0 ? element.data.then : element.data.else,
+          pageTags,
+          callback,
+        );
+      }
+      continue;
+    }
+    if (element.element === "if-tags") {
+      const resolved = resolveIfTags(element.data, pageTags);
+      if (!resolved.evaluated || resolved.matched) {
+        walkRenderOrder(element.data.elements, pageTags, callback);
+      }
+      continue;
+    }
+    if (element.element === "bibliography-block") {
+      if (!element.data.hide) {
+        for (const entry of element.data.entries) {
+          walkRenderOrder(entry.value, pageTags, callback);
+        }
+      }
+      continue;
+    }
+    walkBasicChildren(element, (children) => walkRenderOrder(children, pageTags, callback));
+  }
+}
+
+function walkAllSyntaxChildren(element: Element, callback: (elements: Element[]) => void): void {
+  if (element.element === "if" || element.element === "ifexpr") {
+    callback(element.data.then);
+    callback(element.data.else);
+    return;
+  }
+  if (element.element === "bibliography-block") {
+    for (const entry of element.data.entries) {
+      callback(entry.key);
+      callback(entry.value);
+    }
+    return;
+  }
+  walkBasicChildren(element, callback);
+}
+
+function walkBasicChildren(element: Element, callback: (elements: Element[]) => void): void {
+  if (element.element === "list") {
+    for (const item of element.data.items) {
+      callback(item["item-type"] === "elements" ? item.elements : [listElement(item.data)]);
+    }
+    return;
+  }
+  if (element.element === "table") {
+    for (const row of element.data.rows) {
+      for (const cell of row.cells) callback(cell.elements);
+    }
+    return;
+  }
+  if (element.element === "definition-list") {
+    for (const entry of element.data) {
+      callback(entry.key);
+      callback(entry.value);
+    }
+    return;
+  }
+  if (element.element === "tab-view") {
+    for (const tab of element.data) callback(tab.elements);
+    return;
+  }
+  const children = getGenericElementChildren(element);
+  if (children !== null) callback(children);
+}
+
+function isTocHeading(element: Element): element is Extract<Element, { element: "container" }> & {
+  data: { type: { header: { level: number; "has-toc": true } } };
+} {
+  return (
+    element.element === "container" &&
+    typeof element.data.type === "object" &&
+    "header" in element.data.type &&
+    element.data.type.header["has-toc"]
+  );
+}