@wdprlib/parser 3.2.0 → 4.0.0

package/src/parser/rules/block/module/include/resolve/iterate.ts

@@ -0,0 +1,202 @@
+import { scanIncludeDirectives } from "../scanner";
+import { createIncludeReference } from "../references";
+import { replaceOneInclude, replaceOneIncludeAsync } from "./replace";
+import type {
+  AsyncIncludeFetcher,
+  IncludeDependency,
+  IncludeFetcher,
+  IncludeReference,
+  ResolveIncludesTraceResult,
+} from "./types";
+
+const MAYBE_INCLUDE_PATTERN = /\[\[include\s/i;
+
+/**
+ * Iteratively expand all `[[include]]` directives in source text.
+ *
+ * Each iteration replaces every include directive in the current source
+ * with its fetched content after variable substitution. No recursion
+ * into individual includes: the next iteration handles nested includes.
+ */
+export function expandIterative(
+  source: string,
+  fetcher: IncludeFetcher,
+  maxIterations: number,
+): string {
+  let current = source;
+  const replacementCache = new Map<string, string>();
+  for (let i = 0; i < maxIterations; i++) {
+    const expanded = expandOneIteration(current, fetcher, replacementCache);
+    if (expanded === null || expanded === current) break;
+    current = expanded;
+  }
+  return current;
+}
+
+/**
+ * Trace-enabled version of {@link expandIterative}.
+ *
+ * The expansion behavior is identical to the normal sync path; this function
+ * additionally records directives seen at each iteration so callers can build
+ * dependency graphs and debug max-iteration leftovers.
+ */
+export function expandIterativeWithTrace(
+  source: string,
+  fetcher: IncludeFetcher,
+  maxIterations: number,
+): ResolveIncludesTraceResult {
+  let current = source;
+  const replacementCache = new Map<string, string>();
+  const dependencies: IncludeDependency[] = [];
+  const iterations: ResolveIncludesTraceResult["iterations"] = [];
+
+  for (let i = 0; i < maxIterations; i++) {
+    const expanded = expandOneIterationWithTrace(current, fetcher, replacementCache, i);
+    if (expanded === null) break;
+
+    dependencies.push(...expanded.dependencies);
+    iterations.push({
+      iteration: i,
+      directives: expanded.references,
+      changed: expanded.source !== current,
+    });
+
+    // Stable self-includes can keep scanning as directives forever; Wikidot-style
+    // expansion stops when a whole iteration no longer changes the source.
+    if (expanded.source === current) break;
+    current = expanded.source;
+  }
+
+  return {
+    source: current,
+    dependencies,
+    iterations,
+    // Hitting the cap only matters when the final source still contains a
+    // recognizable include directive. A fully expanded document can use exactly
+    // maxIterations passes without leaving work behind.
+    reachedMaxIterations:
+      iterations.length === maxIterations && scanIncludeDirectives(current).length > 0,
+  };
+}
+
+/**
+ * Async iterative expansion of `[[include]]` directives.
+ *
+ * Each iteration scans the current source for include directives, fetches
+ * sibling includes concurrently, and builds the replacement string in source
+ * order. Nested includes introduced by replacements are still handled by the
+ * next iteration.
+ */
+export async function expandIterativeAsync(
+  source: string,
+  fetcher: AsyncIncludeFetcher,
+  maxIterations: number,
+): Promise<string> {
+  let current = source;
+  const replacementCache = new Map<string, Promise<string>>();
+  for (let i = 0; i < maxIterations; i++) {
+    const expanded = await expandOneIterationAsync(current, fetcher, replacementCache);
+    if (expanded === null || expanded === current) break;
+    current = expanded;
+  }
+  return current;
+}
+
+function expandOneIteration(
+  source: string,
+  fetcher: IncludeFetcher,
+  replacementCache: Map<string, string>,
+): string | null {
+  if (!MAYBE_INCLUDE_PATTERN.test(source)) return null;
+
+  const directives = scanIncludeDirectives(source);
+  if (directives.length === 0) return null;
+
+  const parts: string[] = [];
+  let lastPos = 0;
+  for (const { start, end, inner } of directives) {
+    parts.push(source.slice(lastPos, start), replaceCached(inner, fetcher, replacementCache));
+    lastPos = end;
+  }
+  parts.push(source.slice(lastPos));
+  return parts.join("");
+}
+
+function expandOneIterationWithTrace(
+  source: string,
+  fetcher: IncludeFetcher,
+  replacementCache: Map<string, string>,
+  iteration: number,
+): {
+  source: string;
+  references: IncludeReference[];
+  dependencies: IncludeDependency[];
+} | null {
+  if (!MAYBE_INCLUDE_PATTERN.test(source)) return null;
+
+  const directives = scanIncludeDirectives(source);
+  if (directives.length === 0) return null;
+
+  const references = directives.map(createIncludeReference);
+  const dependencies = references.map((reference) => ({ ...reference, iteration }));
+  const parts: string[] = [];
+  let lastPos = 0;
+  for (const { start, end, inner } of directives) {
+    parts.push(source.slice(lastPos, start), replaceCached(inner, fetcher, replacementCache));
+    lastPos = end;
+  }
+  parts.push(source.slice(lastPos));
+
+  return {
+    source: parts.join(""),
+    references,
+    dependencies,
+  };
+}
+
+async function expandOneIterationAsync(
+  source: string,
+  fetcher: AsyncIncludeFetcher,
+  replacementCache: Map<string, Promise<string>>,
+): Promise<string | null> {
+  if (!MAYBE_INCLUDE_PATTERN.test(source)) return null;
+
+  const directives = scanIncludeDirectives(source);
+  if (directives.length === 0) return null;
+
+  const replacements = await Promise.all(
+    directives.map(({ inner }) => replaceCachedAsync(inner, fetcher, replacementCache)),
+  );
+
+  const parts: string[] = [];
+  let lastPos = 0;
+  for (let i = 0; i < directives.length; i++) {
+    const { start, end } = directives[i]!;
+    parts.push(source.slice(lastPos, start), replacements[i]!);
+    lastPos = end;
+  }
+  parts.push(source.slice(lastPos));
+  return parts.join("");
+}
+
+function replaceCached(inner: string, fetcher: IncludeFetcher, cache: Map<string, string>): string {
+  const cached = cache.get(inner);
+  if (cached !== undefined) return cached;
+
+  const replacement = replaceOneInclude(inner, fetcher);
+  cache.set(inner, replacement);
+  return replacement;
+}
+
+function replaceCachedAsync(
+  inner: string,
+  fetcher: AsyncIncludeFetcher,
+  cache: Map<string, Promise<string>>,
+): Promise<string> {
+  const cached = cache.get(inner);
+  if (cached) return cached;
+
+  const replacement = replaceOneIncludeAsync(inner, fetcher);
+  cache.set(inner, replacement);
+  return replacement;
+}

package/src/parser/rules/block/module/include/resolve/replace.ts

@@ -0,0 +1,31 @@
+import { parseIncludeDirective, substituteVariables } from "../directive";
+import type { AsyncIncludeFetcher, IncludeFetcher } from "./types";
+
+/**
+ * Replace a single include match with its fetched + variable-substituted content.
+ * Returns the replacement text for a single directive's `inner` content.
+ */
+export function replaceOneInclude(inner: string, fetcher: IncludeFetcher): string {
+  const { location, assignments } = parseIncludeDirective(inner);
+  const content = fetcher(location);
+  if (content === null) {
+    return missingIncludeError(location.page);
+  }
+  return substituteVariables(content, assignments);
+}
+
+export async function replaceOneIncludeAsync(
+  inner: string,
+  fetcher: AsyncIncludeFetcher,
+): Promise<string> {
+  const { location, assignments } = parseIncludeDirective(inner);
+  const content = await fetcher(location);
+  if (content === null) {
+    return missingIncludeError(location.page);
+  }
+  return substituteVariables(content, assignments);
+}
+
+function missingIncludeError(page: string): string {
+  return `[[div class="error-block"]]\nPage to be included "${page}" cannot be found!\n[[/div]]`;
+}

package/src/parser/rules/block/module/include/resolve/types.ts

@@ -0,0 +1,105 @@
+import type { PageRef, WikitextSettings } from "@wdprlib/ast";
+import type { IncludeAssignment } from "../directive";
+
+/**
+ * Callback to fetch page content for include resolution.
+ * Returns the wikitext source of the page, or null if the page does not exist.
+ *
+ * @security The fetcher is called with user-provided page references.
+ * Implementations should validate and sanitize page references before
+ * using them in database queries or file system access.
+ */
+export type IncludeFetcher = (pageRef: PageRef) => string | null;
+
+/**
+ * Async callback to fetch page content for include resolution.
+ * Returns a promise of the wikitext source, or null if the page does not exist.
+ *
+ * @security The fetcher is called with user-provided page references.
+ * Implementations should validate and sanitize page references before
+ * using them in database queries or file system access.
+ */
+export type AsyncIncludeFetcher = (pageRef: PageRef) => Promise<string | null>;
+
+/**
+ * Options for resolveIncludes / resolveIncludesAsync.
+ */
+export interface ResolveIncludesOptions {
+  /**
+   * Maximum number of expansion iterations (default: 10).
+   *
+   * Each iteration replaces all `[[include]]` directives in the current
+   * source with fetched content. Iteration stops when the source is
+   * unchanged or this limit is reached.
+   */
+  maxIterations?: number;
+  /** Wikitext settings. If enablePageSyntax is false, includes are not expanded. */
+  settings?: WikitextSettings;
+}
+
+/**
+ * A single include directive found in raw source text.
+ *
+ * This represents the directive as it existed during the include expansion
+ * phase. It does not imply that the target exists, and it does not include
+ * nested dependencies introduced by the target source.
+ */
+export interface IncludeReference {
+  /** Target page reference parsed from the directive. */
+  location: PageRef;
+  /** Variable assignments parsed from the directive, preserving source order. */
+  assignments: IncludeAssignment[];
+  /** Index of the opening `[[`. */
+  start: number;
+  /** Index just past the closing `]]`. */
+  end: number;
+  /** Text between `[[include ` and the closing `]]`. */
+  inner: string;
+}
+
+/**
+ * Include dependency observed during iterative expansion.
+ *
+ * Unlike `IncludeReference`, this is collected after a fetchable include layer
+ * has actually been expanded. If an included page introduces another include,
+ * that nested directive appears as a later `IncludeDependency` with a higher
+ * `iteration`.
+ */
+export interface IncludeDependency extends IncludeReference {
+  /** Zero-based expansion iteration where this directive was observed. */
+  iteration: number;
+}
+
+/**
+ * Trace data for one include expansion iteration.
+ *
+ * `directives` is the scan result before replacements for that iteration.
+ * `changed` is false for stable self-includes where replacing the directive
+ * produces the same source again; the resolver stops at that point.
+ */
+export interface IncludeIterationTrace {
+  /** Zero-based expansion iteration. */
+  iteration: number;
+  /** Include directives found in the source for this iteration. */
+  directives: IncludeReference[];
+  /** Whether replacing the directives changed the source. */
+  changed: boolean;
+}
+
+/**
+ * Result returned by trace-enabled include expansion.
+ *
+ * The `source` field is intended to be byte-for-byte equivalent to
+ * `resolveIncludes(source, fetcher, options)` for the same inputs. The rest of
+ * the object is diagnostic data for dependency collection and cache planning.
+ */
+export interface ResolveIncludesTraceResult {
+  /** Final expanded source. */
+  source: string;
+  /** Include dependencies observed across all expansion iterations. */
+  dependencies: IncludeDependency[];
+  /** Per-iteration include scan and change information. */
+  iterations: IncludeIterationTrace[];
+  /** True when expansion stopped with directives still present after hitting the iteration cap. */
+  reachedMaxIterations: boolean;
+}

package/src/parser/rules/block/module/include/scanner.ts

@@ -0,0 +1,121 @@
+/**
+ * Matches the opening `[[include` token at the start of a line.
+ *
+ * The `m` flag makes `^` match at line boundaries, enforcing the Wikidot
+ * rule that `[[include]]` must appear at the start of a line. The trailing
+ * `\s` separates the directive name from its arguments. The actual extent
+ * of each directive is found by scanIncludeDirectives, which balances nested
+ * `[[ ... ]]` so that block markup inside a parameter value does not terminate
+ * the directive at the first `]]`.
+ */
+const INCLUDE_OPEN_PATTERN = /^\[\[include\s/gim;
+const OPEN_BRACKET = 91;
+const CLOSE_BRACKET = 93;
+
+/** A located `[[include ...]]` directive with bracket-balanced extent. */
+export interface IncludeDirectiveMatch {
+  /** Index of the opening `[[`. */
+  start: number;
+  /** Index just past the closing `]]`. */
+  end: number;
+  /** Text between `[[include ` and the closing `]]`. */
+  inner: string;
+}
+
+/**
+ * Find all `[[include ...]]` directives in `source`, choosing each
+ * closing `]]` so that block markup inside a parameter value does not
+ * end the directive prematurely.
+ */
+export function scanIncludeDirectives(source: string): IncludeDirectiveMatch[] {
+  const matches: IncludeDirectiveMatch[] = [];
+  const opener = INCLUDE_OPEN_PATTERN;
+  opener.lastIndex = 0;
+  let match: RegExpExecArray | null;
+
+  while ((match = opener.exec(source)) !== null) {
+    const start = match.index;
+    const contentStart = start + match[0].length;
+    const firstNewline = source.indexOf("\n", start);
+
+    let depth = 0;
+    let linkDepth = 0;
+    let i = start;
+    let closeEnd = -1;
+
+    while (i < source.length) {
+      const ch = source.charCodeAt(i);
+      const next = source.charCodeAt(i + 1);
+      const nextNext = source.charCodeAt(i + 2);
+
+      if (ch === OPEN_BRACKET && next === OPEN_BRACKET && nextNext === OPEN_BRACKET) {
+        linkDepth++;
+        i += 3;
+      } else if (
+        linkDepth > 0 &&
+        ch === CLOSE_BRACKET &&
+        next === CLOSE_BRACKET &&
+        nextNext === CLOSE_BRACKET
+      ) {
+        linkDepth--;
+        i += 3;
+      } else if (linkDepth > 0) {
+        i++;
+      } else if (ch === OPEN_BRACKET && next === OPEN_BRACKET) {
+        depth++;
+        i += 2;
+      } else if (ch === CLOSE_BRACKET && next === CLOSE_BRACKET) {
+        const closeStart = i;
+        depth--;
+        i += 2;
+
+        if (depth <= 0) {
+          const innerSoFar = source.slice(contentStart, closeStart);
+          if (hasAttributes(innerSoFar)) {
+            while (i < source.length && source[i] === "]") {
+              i++;
+            }
+          }
+          const onOpenerLine = firstNewline === -1 || closeStart < firstNewline;
+          if (onOpenerLine || isRestOfLineBlank(source, i)) {
+            closeEnd = i;
+            break;
+          }
+        }
+      } else {
+        i++;
+      }
+    }
+
+    if (closeEnd === -1) {
+      opener.lastIndex = start + 2;
+      continue;
+    }
+
+    matches.push({ start, end: closeEnd, inner: source.slice(contentStart, closeEnd - 2) });
+    opener.lastIndex = closeEnd;
+  }
+
+  return matches;
+}
+
+/**
+ * Returns true when the directive's inner content carries an attribute section,
+ * either pipe-delimited (`|key=value`) or space-separated after the page name.
+ */
+function hasAttributes(innerSoFar: string): boolean {
+  if (innerSoFar.includes("|")) return true;
+  const trimmed = innerSoFar.trimStart();
+  const whitespaceIndex = trimmed.search(/\s/);
+  if (whitespaceIndex === -1) return false;
+  return trimmed.slice(whitespaceIndex).trim().length > 0;
+}
+
+function isRestOfLineBlank(source: string, pos: number): boolean {
+  for (let i = pos; i < source.length; i++) {
+    const ch = source[i];
+    if (ch === "\n") return true;
+    if (ch !== " " && ch !== "\t" && ch !== "\r") return false;
+  }
+  return true;
+}

package/src/parser/rules/block/module/index.ts

@@ -22,6 +22,7 @@
 // Module rule types and registry
 export type { ModuleRule } from "./types";
 export { MODULE_RULES, getModuleRuleByName } from "./mapping";
+export { moduleRule } from "./rule";
 
 // Module parsers
 export { rateModuleRule } from "./rate/index";
@@ -95,7 +96,18 @@ export {
 
 // Include module
 export type { IncludeFetcher, AsyncIncludeFetcher, ResolveIncludesOptions } from "./include";
-export { resolveIncludes, resolveIncludesAsync } from "./include";
+export type {
+  IncludeReference,
+  IncludeDependency,
+  IncludeIterationTrace,
+  ResolveIncludesTraceResult,
+} from "./include";
+export {
+  extractIncludeReferences,
+  resolveIncludes,
+  resolveIncludesAsync,
+  resolveIncludesWithTrace,
+} from "./include";
 
 // ListUsers module
 export type {
@@ -117,6 +129,6 @@ export {
 } from "./listusers";
 
 // Module resolver
-export type { ResolveOptions } from "./resolve";
+export type { ModuleSourceTransform, ResolveOptions } from "./resolve";
 export { resolveModules } from "./resolve";
 export { STYLE_SLOT_PREFIX } from "@wdprlib/ast";