@wdprlib/parser 3.1.2 → 4.0.0

package/src/parser/rules/block/module/include/directive.ts

@@ -0,0 +1,91 @@
+import type { PageRef } from "@wdprlib/ast";
+
+export interface IncludeAssignment {
+  key: string;
+  value: string;
+}
+
+/**
+ * Parse the inner content of an `[[include ...]]` directive into a page reference
+ * and variable assignments.
+ */
+export function parseIncludeDirective(inner: string): {
+  location: PageRef;
+  assignments: IncludeAssignment[];
+} {
+  const normalized = inner.includes("\n") ? inner.replaceAll("\n", " ") : inner;
+  const parts = normalized.split("|");
+  const firstSegment = parts[0]!.trim();
+
+  const spaceIndex = firstSegment.indexOf(" ");
+  let target: string;
+  const varSegments: string[] = [];
+
+  if (spaceIndex !== -1) {
+    target = firstSegment.slice(0, spaceIndex);
+    const rest = firstSegment.slice(spaceIndex + 1).trim();
+    if (rest) {
+      varSegments.push(rest);
+    }
+  } else {
+    target = firstSegment;
+  }
+
+  for (let i = 1; i < parts.length; i++) {
+    const segment = parts[i]!.trim();
+    if (segment) {
+      varSegments.push(segment);
+    }
+  }
+
+  const assignments = parseIncludeAssignments(varSegments);
+  return { location: parseIncludePageRef(target), assignments };
+}
+
+export function substituteVariables(
+  content: string,
+  assignments: readonly IncludeAssignment[],
+): string {
+  if (assignments.length === 0) return content;
+  if (!content.includes("{$")) return content;
+
+  let substituted = content;
+  for (const { key, value } of assignments) {
+    substituted = substituted.replaceAll(`{$${key}}`, value);
+  }
+  return substituted;
+}
+
+function parseIncludePageRef(target: string): PageRef {
+  if (target.startsWith(":")) {
+    const rest = target.slice(1);
+    const colonIndex = rest.indexOf(":");
+    if (colonIndex !== -1) {
+      return { site: rest.slice(0, colonIndex), page: rest.slice(colonIndex + 1) };
+    }
+  }
+  return { site: null, page: target };
+}
+
+function parseIncludeAssignments(varSegments: string[]): IncludeAssignment[] {
+  const assignments: IncludeAssignment[] = [];
+
+  for (const segment of varSegments) {
+    const assignment = parseVariableSegment(segment);
+    if (assignment) {
+      assignments.push(assignment);
+    }
+  }
+
+  return assignments;
+}
+
+function parseVariableSegment(segment: string): IncludeAssignment | null {
+  const eqIndex = segment.indexOf("=");
+  if (eqIndex === -1) return null;
+  const key = segment.slice(0, eqIndex).trim();
+  if (!key) return null;
+  const value = segment.slice(eqIndex + 1).trim();
+  if (value === "") return null;
+  return { key, value };
+}

package/src/parser/rules/block/module/include/index.ts

@@ -0,0 +1,29 @@
+/**
+ *
+ * Include directive resolution for Wikidot's `[[include page]]` syntax.
+ *
+ * The include system performs text-level macro expansion before parsing. Each
+ * `[[include page | var=value]]` directive is replaced with the fetched page's
+ * content (after variable substitution), allowing block structures like `[[div]]`
+ * to span across include boundaries.
+ *
+ * Supports:
+ * - Same-site includes: `[[include page-name]]`
+ * - Cross-site includes: `[[include :site-name:page-name]]`
+ * - Variable substitution: `[[include page | key=value]]` replaces `{$key}` in the included content
+ * - Iterative expansion with configurable iteration limit (Wikidot-compatible)
+ *
+ * @module
+ */
+
+export { resolveIncludes, resolveIncludesAsync, resolveIncludesWithTrace } from "./resolve";
+export { extractIncludeReferences } from "./references";
+export type {
+  IncludeFetcher,
+  AsyncIncludeFetcher,
+  ResolveIncludesOptions,
+  IncludeReference,
+  IncludeDependency,
+  IncludeIterationTrace,
+  ResolveIncludesTraceResult,
+} from "./resolve";

package/src/parser/rules/block/module/include/references.ts

@@ -0,0 +1,42 @@
+import { parseIncludeDirective } from "./directive";
+import { scanIncludeDirectives, type IncludeDirectiveMatch } from "./scanner";
+import type { IncludeReference, ResolveIncludesOptions } from "./resolve/types";
+
+/**
+ * Extract include directives from raw source without fetching or expanding them.
+ *
+ * This uses the same scanner and directive parser as `resolveIncludes`, so it
+ * reports only directives that the include expansion pass would recognize.
+ *
+ * This is a source-phase API. It intentionally does not evaluate later syntax
+ * such as comments, expressions, or iftags. Wikidot expands includes before
+ * those phases, so an include inside a later comment block is still reported,
+ * while a token that only becomes `[[include ...]]` after comment/expression/
+ * iftags processing is not reported.
+ *
+ * The result is not a complete transitive dependency graph. Nested includes
+ * only become visible after fetching and expanding the current layer; use
+ * `resolveIncludesWithTrace` when the caller needs observed dependency edges
+ * across iterative expansion.
+ */
+export function extractIncludeReferences(
+  source: string,
+  options?: ResolveIncludesOptions,
+): IncludeReference[] {
+  if (options?.settings && !options.settings.enablePageSyntax) {
+    return [];
+  }
+
+  return scanIncludeDirectives(source).map(createIncludeReference);
+}
+
+export function createIncludeReference(match: IncludeDirectiveMatch): IncludeReference {
+  const { location, assignments } = parseIncludeDirective(match.inner);
+  return {
+    location,
+    assignments,
+    start: match.start,
+    end: match.end,
+    inner: match.inner,
+  };
+}

package/src/parser/rules/block/module/include/resolve/cache.ts

@@ -0,0 +1,44 @@
+import type { PageRef } from "@wdprlib/ast";
+import type { AsyncIncludeFetcher, IncludeFetcher } from "./types";
+
+export function createCachedIncludeFetcher(
+  fetcher: IncludeFetcher,
+  normalizeKey: (pageRef: PageRef) => string,
+): IncludeFetcher {
+  const cache = new Map<string, string | null>();
+
+  return (pageRef: PageRef): string | null => {
+    const key = normalizeKey(pageRef);
+    if (cache.has(key)) {
+      return cache.get(key)!;
+    }
+
+    let result: string | null;
+    try {
+      result = fetcher(pageRef);
+    } catch {
+      result = null;
+    }
+    cache.set(key, result);
+    return result;
+  };
+}
+
+export function createCachedAsyncIncludeFetcher(
+  fetcher: AsyncIncludeFetcher,
+  normalizeKey: (pageRef: PageRef) => string,
+): AsyncIncludeFetcher {
+  const cache = new Map<string, Promise<string | null>>();
+
+  return async (pageRef: PageRef): Promise<string | null> => {
+    const key = normalizeKey(pageRef);
+    const cached = cache.get(key);
+    if (cached) {
+      return cached;
+    }
+
+    const result = fetcher(pageRef).catch(() => null);
+    cache.set(key, result);
+    return result;
+  };
+}

package/src/parser/rules/block/module/include/resolve/index.ts

@@ -0,0 +1,106 @@
+/**
+ *
+ * Text-level expansion of `[[include]]` directives.
+ *
+ * @module
+ */
+
+import type { PageRef } from "@wdprlib/ast";
+import { createCachedAsyncIncludeFetcher, createCachedIncludeFetcher } from "./cache";
+import { expandIterative, expandIterativeAsync, expandIterativeWithTrace } from "./iterate";
+import type {
+  AsyncIncludeFetcher,
+  IncludeFetcher,
+  ResolveIncludesOptions,
+  ResolveIncludesTraceResult,
+} from "./types";
+
+export type {
+  IncludeFetcher,
+  AsyncIncludeFetcher,
+  ResolveIncludesOptions,
+  IncludeReference,
+  IncludeDependency,
+  IncludeIterationTrace,
+  ResolveIncludesTraceResult,
+} from "./types";
+
+/**
+ * Expand all [[include]] directives in the source text.
+ *
+ * Uses Wikidot-compatible iterative expansion: each iteration replaces
+ * all include directives in the current source with fetched and
+ * variable-substituted content. Iteration continues until no further
+ * changes occur or `maxIterations` is reached.
+ */
+export function resolveIncludes(
+  source: string,
+  fetcher: IncludeFetcher,
+  options?: ResolveIncludesOptions,
+): string {
+  if (options?.settings && !options.settings.enablePageSyntax) {
+    return source;
+  }
+
+  const maxIterations = options?.maxIterations ?? 10;
+  const cachedFetcher = createCachedIncludeFetcher(fetcher, normalizePageKey);
+  return expandIterative(source, cachedFetcher, maxIterations);
+}
+
+/**
+ * Expand all [[include]] directives and return dependency/iteration trace data.
+ *
+ * This follows the same sync expansion behavior as {@link resolveIncludes};
+ * the additional trace data is intended for application-level dependency
+ * graphs, cache invalidation, and diagnostics.
+ */
+export function resolveIncludesWithTrace(
+  source: string,
+  fetcher: IncludeFetcher,
+  options?: ResolveIncludesOptions,
+): ResolveIncludesTraceResult {
+  if (options?.settings && !options.settings.enablePageSyntax) {
+    return {
+      source,
+      dependencies: [],
+      iterations: [],
+      reachedMaxIterations: false,
+    };
+  }
+
+  const maxIterations = options?.maxIterations ?? 10;
+  const cachedFetcher = createCachedIncludeFetcher(fetcher, normalizePageKey);
+  return expandIterativeWithTrace(source, cachedFetcher, maxIterations);
+}
+
+/**
+ * Async version of {@link resolveIncludes}.
+ *
+ * Expand all [[include]] directives using an async fetcher, allowing
+ * page content to be loaded from async sources such as databases.
+ */
+export async function resolveIncludesAsync(
+  source: string,
+  fetcher: AsyncIncludeFetcher,
+  options?: ResolveIncludesOptions,
+): Promise<string> {
+  if (options?.settings && !options.settings.enablePageSyntax) {
+    return source;
+  }
+
+  const maxIterations = options?.maxIterations ?? 10;
+  const cachedFetcher = createCachedAsyncIncludeFetcher(fetcher, normalizePageKey);
+  return expandIterativeAsync(source, cachedFetcher, maxIterations);
+}
+
+/**
+ * Normalize a PageRef into a consistent string key for cache lookups.
+ *
+ * Page names are lowercased for case-insensitive matching. Cross-site
+ * references include the site name as a prefix.
+ */
+function normalizePageKey(location: PageRef): string {
+  const site = location.site ?? "";
+  const page = location.page.toLowerCase();
+  return site ? `${site}:${page}` : page;
+}

package/src/parser/rules/block/module/include/resolve/iterate.ts

@@ -0,0 +1,202 @@
+import { scanIncludeDirectives } from "../scanner";
+import { createIncludeReference } from "../references";
+import { replaceOneInclude, replaceOneIncludeAsync } from "./replace";
+import type {
+  AsyncIncludeFetcher,
+  IncludeDependency,
+  IncludeFetcher,
+  IncludeReference,
+  ResolveIncludesTraceResult,
+} from "./types";
+
+const MAYBE_INCLUDE_PATTERN = /\[\[include\s/i;
+
+/**
+ * Iteratively expand all `[[include]]` directives in source text.
+ *
+ * Each iteration replaces every include directive in the current source
+ * with its fetched content after variable substitution. No recursion
+ * into individual includes: the next iteration handles nested includes.
+ */
+export function expandIterative(
+  source: string,
+  fetcher: IncludeFetcher,
+  maxIterations: number,
+): string {
+  let current = source;
+  const replacementCache = new Map<string, string>();
+  for (let i = 0; i < maxIterations; i++) {
+    const expanded = expandOneIteration(current, fetcher, replacementCache);
+    if (expanded === null || expanded === current) break;
+    current = expanded;
+  }
+  return current;
+}
+
+/**
+ * Trace-enabled version of {@link expandIterative}.
+ *
+ * The expansion behavior is identical to the normal sync path; this function
+ * additionally records directives seen at each iteration so callers can build
+ * dependency graphs and debug max-iteration leftovers.
+ */
+export function expandIterativeWithTrace(
+  source: string,
+  fetcher: IncludeFetcher,
+  maxIterations: number,
+): ResolveIncludesTraceResult {
+  let current = source;
+  const replacementCache = new Map<string, string>();
+  const dependencies: IncludeDependency[] = [];
+  const iterations: ResolveIncludesTraceResult["iterations"] = [];
+
+  for (let i = 0; i < maxIterations; i++) {
+    const expanded = expandOneIterationWithTrace(current, fetcher, replacementCache, i);
+    if (expanded === null) break;
+
+    dependencies.push(...expanded.dependencies);
+    iterations.push({
+      iteration: i,
+      directives: expanded.references,
+      changed: expanded.source !== current,
+    });
+
+    // Stable self-includes can keep scanning as directives forever; Wikidot-style
+    // expansion stops when a whole iteration no longer changes the source.
+    if (expanded.source === current) break;
+    current = expanded.source;
+  }
+
+  return {
+    source: current,
+    dependencies,
+    iterations,
+    // Hitting the cap only matters when the final source still contains a
+    // recognizable include directive. A fully expanded document can use exactly
+    // maxIterations passes without leaving work behind.
+    reachedMaxIterations:
+      iterations.length === maxIterations && scanIncludeDirectives(current).length > 0,
+  };
+}
+
+/**
+ * Async iterative expansion of `[[include]]` directives.
+ *
+ * Each iteration scans the current source for include directives, fetches
+ * sibling includes concurrently, and builds the replacement string in source
+ * order. Nested includes introduced by replacements are still handled by the
+ * next iteration.
+ */
+export async function expandIterativeAsync(
+  source: string,
+  fetcher: AsyncIncludeFetcher,
+  maxIterations: number,
+): Promise<string> {
+  let current = source;
+  const replacementCache = new Map<string, Promise<string>>();
+  for (let i = 0; i < maxIterations; i++) {
+    const expanded = await expandOneIterationAsync(current, fetcher, replacementCache);
+    if (expanded === null || expanded === current) break;
+    current = expanded;
+  }
+  return current;
+}
+
+function expandOneIteration(
+  source: string,
+  fetcher: IncludeFetcher,
+  replacementCache: Map<string, string>,
+): string | null {
+  if (!MAYBE_INCLUDE_PATTERN.test(source)) return null;
+
+  const directives = scanIncludeDirectives(source);
+  if (directives.length === 0) return null;
+
+  const parts: string[] = [];
+  let lastPos = 0;
+  for (const { start, end, inner } of directives) {
+    parts.push(source.slice(lastPos, start), replaceCached(inner, fetcher, replacementCache));
+    lastPos = end;
+  }
+  parts.push(source.slice(lastPos));
+  return parts.join("");
+}
+
+function expandOneIterationWithTrace(
+  source: string,
+  fetcher: IncludeFetcher,
+  replacementCache: Map<string, string>,
+  iteration: number,
+): {
+  source: string;
+  references: IncludeReference[];
+  dependencies: IncludeDependency[];
+} | null {
+  if (!MAYBE_INCLUDE_PATTERN.test(source)) return null;
+
+  const directives = scanIncludeDirectives(source);
+  if (directives.length === 0) return null;
+
+  const references = directives.map(createIncludeReference);
+  const dependencies = references.map((reference) => ({ ...reference, iteration }));
+  const parts: string[] = [];
+  let lastPos = 0;
+  for (const { start, end, inner } of directives) {
+    parts.push(source.slice(lastPos, start), replaceCached(inner, fetcher, replacementCache));
+    lastPos = end;
+  }
+  parts.push(source.slice(lastPos));
+
+  return {
+    source: parts.join(""),
+    references,
+    dependencies,
+  };
+}
+
+async function expandOneIterationAsync(
+  source: string,
+  fetcher: AsyncIncludeFetcher,
+  replacementCache: Map<string, Promise<string>>,
+): Promise<string | null> {
+  if (!MAYBE_INCLUDE_PATTERN.test(source)) return null;
+
+  const directives = scanIncludeDirectives(source);
+  if (directives.length === 0) return null;
+
+  const replacements = await Promise.all(
+    directives.map(({ inner }) => replaceCachedAsync(inner, fetcher, replacementCache)),
+  );
+
+  const parts: string[] = [];
+  let lastPos = 0;
+  for (let i = 0; i < directives.length; i++) {
+    const { start, end } = directives[i]!;
+    parts.push(source.slice(lastPos, start), replacements[i]!);
+    lastPos = end;
+  }
+  parts.push(source.slice(lastPos));
+  return parts.join("");
+}
+
+function replaceCached(inner: string, fetcher: IncludeFetcher, cache: Map<string, string>): string {
+  const cached = cache.get(inner);
+  if (cached !== undefined) return cached;
+
+  const replacement = replaceOneInclude(inner, fetcher);
+  cache.set(inner, replacement);
+  return replacement;
+}
+
+function replaceCachedAsync(
+  inner: string,
+  fetcher: AsyncIncludeFetcher,
+  cache: Map<string, Promise<string>>,
+): Promise<string> {
+  const cached = cache.get(inner);
+  if (cached) return cached;
+
+  const replacement = replaceOneIncludeAsync(inner, fetcher);
+  cache.set(inner, replacement);
+  return replacement;
+}

package/src/parser/rules/block/module/include/resolve/replace.ts

@@ -0,0 +1,31 @@
+import { parseIncludeDirective, substituteVariables } from "../directive";
+import type { AsyncIncludeFetcher, IncludeFetcher } from "./types";
+
+/**
+ * Replace a single include match with its fetched + variable-substituted content.
+ * Returns the replacement text for a single directive's `inner` content.
+ */
+export function replaceOneInclude(inner: string, fetcher: IncludeFetcher): string {
+  const { location, assignments } = parseIncludeDirective(inner);
+  const content = fetcher(location);
+  if (content === null) {
+    return missingIncludeError(location.page);
+  }
+  return substituteVariables(content, assignments);
+}
+
+export async function replaceOneIncludeAsync(
+  inner: string,
+  fetcher: AsyncIncludeFetcher,
+): Promise<string> {
+  const { location, assignments } = parseIncludeDirective(inner);
+  const content = await fetcher(location);
+  if (content === null) {
+    return missingIncludeError(location.page);
+  }
+  return substituteVariables(content, assignments);
+}
+
+function missingIncludeError(page: string): string {
+  return `[[div class="error-block"]]\nPage to be included "${page}" cannot be found!\n[[/div]]`;
+}

package/src/parser/rules/block/module/include/resolve/types.ts

@@ -0,0 +1,105 @@
+import type { PageRef, WikitextSettings } from "@wdprlib/ast";
+import type { IncludeAssignment } from "../directive";
+
+/**
+ * Callback to fetch page content for include resolution.
+ * Returns the wikitext source of the page, or null if the page does not exist.
+ *
+ * @security The fetcher is called with user-provided page references.
+ * Implementations should validate and sanitize page references before
+ * using them in database queries or file system access.
+ */
+export type IncludeFetcher = (pageRef: PageRef) => string | null;
+
+/**
+ * Async callback to fetch page content for include resolution.
+ * Returns a promise of the wikitext source, or null if the page does not exist.
+ *
+ * @security The fetcher is called with user-provided page references.
+ * Implementations should validate and sanitize page references before
+ * using them in database queries or file system access.
+ */
+export type AsyncIncludeFetcher = (pageRef: PageRef) => Promise<string | null>;
+
+/**
+ * Options for resolveIncludes / resolveIncludesAsync.
+ */
+export interface ResolveIncludesOptions {
+  /**
+   * Maximum number of expansion iterations (default: 10).
+   *
+   * Each iteration replaces all `[[include]]` directives in the current
+   * source with fetched content. Iteration stops when the source is
+   * unchanged or this limit is reached.
+   */
+  maxIterations?: number;
+  /** Wikitext settings. If enablePageSyntax is false, includes are not expanded. */
+  settings?: WikitextSettings;
+}
+
+/**
+ * A single include directive found in raw source text.
+ *
+ * This represents the directive as it existed during the include expansion
+ * phase. It does not imply that the target exists, and it does not include
+ * nested dependencies introduced by the target source.
+ */
+export interface IncludeReference {
+  /** Target page reference parsed from the directive. */
+  location: PageRef;
+  /** Variable assignments parsed from the directive, preserving source order. */
+  assignments: IncludeAssignment[];
+  /** Index of the opening `[[`. */
+  start: number;
+  /** Index just past the closing `]]`. */
+  end: number;
+  /** Text between `[[include ` and the closing `]]`. */
+  inner: string;
+}
+
+/**
+ * Include dependency observed during iterative expansion.
+ *
+ * Unlike `IncludeReference`, this is collected after a fetchable include layer
+ * has actually been expanded. If an included page introduces another include,
+ * that nested directive appears as a later `IncludeDependency` with a higher
+ * `iteration`.
+ */
+export interface IncludeDependency extends IncludeReference {
+  /** Zero-based expansion iteration where this directive was observed. */
+  iteration: number;
+}
+
+/**
+ * Trace data for one include expansion iteration.
+ *
+ * `directives` is the scan result before replacements for that iteration.
+ * `changed` is false for stable self-includes where replacing the directive
+ * produces the same source again; the resolver stops at that point.
+ */
+export interface IncludeIterationTrace {
+  /** Zero-based expansion iteration. */
+  iteration: number;
+  /** Include directives found in the source for this iteration. */
+  directives: IncludeReference[];
+  /** Whether replacing the directives changed the source. */
+  changed: boolean;
+}
+
+/**
+ * Result returned by trace-enabled include expansion.
+ *
+ * The `source` field is intended to be byte-for-byte equivalent to
+ * `resolveIncludes(source, fetcher, options)` for the same inputs. The rest of
+ * the object is diagnostic data for dependency collection and cache planning.
+ */
+export interface ResolveIncludesTraceResult {
+  /** Final expanded source. */
+  source: string;
+  /** Include dependencies observed across all expansion iterations. */
+  dependencies: IncludeDependency[];
+  /** Per-iteration include scan and change information. */
+  iterations: IncludeIterationTrace[];
+  /** True when expansion stopped with directives still present after hitting the iteration cap. */
+  reachedMaxIterations: boolean;
+}