@wdprlib/parser 3.2.0 → 4.1.0

package/src/parser/rules/block/math/index.ts

@@ -0,0 +1,106 @@
+/**
+ *
+ * Block rule for the Wikidot math block: `[[math name]]...[[/math]]`.
+ *
+ * A math block captures LaTeX source code between the tags and stores it
+ * as a `math` element in the AST. The content is not parsed for inline
+ * markup.
+ *
+ * @module
+ */
+import type { Element } from "@wdprlib/ast";
+import type { BlockRule, ParseContext, RuleResult } from "../../types";
+import { currentToken } from "../../types";
+import { parseBlockName } from "../utils";
+import { collectMathContent, consumeMathClose } from "./content";
+import { parseMathName } from "./name";
+
+/**
+ * Block rule for `[[math name]]...[[/math]]`.
+ *
+ * Content is captured as raw LaTeX source. BACKSLASH_BREAK tokens are
+ * restored to their original `\\\n` form for correct LaTeX rendering.
+ */
+export const mathBlockRule: BlockRule = {
+  name: "math",
+  startTokens: ["BLOCK_OPEN"],
+  requiresLineStart: false,
+
+  parse(ctx: ParseContext): RuleResult<Element> {
+    const openToken = currentToken(ctx);
+    if (openToken.type !== "BLOCK_OPEN") {
+      return { success: false };
+    }
+
+    let pos = ctx.pos + 1;
+    let consumed = 1;
+
+    const nameResult = parseBlockName(ctx, pos);
+    if (!nameResult || nameResult.name !== "math") {
+      return { success: false };
+    }
+    pos += nameResult.consumed;
+    consumed += nameResult.consumed;
+
+    while (ctx.tokens[pos]?.type === "WHITESPACE") {
+      pos++;
+      consumed++;
+    }
+
+    const mathName = parseMathName(ctx, pos);
+    pos += mathName.consumed;
+    consumed += mathName.consumed;
+
+    while (ctx.tokens[pos]?.type === "WHITESPACE") {
+      pos++;
+      consumed++;
+    }
+
+    if (ctx.tokens[pos]?.type !== "BLOCK_CLOSE") {
+      return { success: false };
+    }
+    pos++;
+    consumed++;
+
+    if (ctx.tokens[pos]?.type === "NEWLINE") {
+      pos++;
+      consumed++;
+    }
+
+    const contentResult = collectMathContent(ctx, pos);
+    const latexSource = contentResult.latexSource.trim();
+    consumed += contentResult.consumed;
+    pos += contentResult.consumed;
+
+    if (!contentResult.foundClose) {
+      ctx.diagnostics.push({
+        severity: "warning",
+        code: "unclosed-block",
+        message: "Missing closing tag [[/math]] for [[math]]",
+        position: openToken.position,
+      });
+    } else {
+      const closeConsumed = consumeMathClose(ctx, pos);
+      pos += closeConsumed;
+      consumed += closeConsumed;
+    }
+
+    if (!latexSource) {
+      return { success: false };
+    }
+
+    return {
+      success: true,
+      elements: [
+        {
+          element: "math",
+          data: {
+            name: mathName.name,
+            "latex-source": latexSource,
+          },
+        },
+      ],
+      consumed,
+    };
+  },
+};

package/src/parser/rules/block/math/name.ts

@@ -0,0 +1,35 @@
+import type { ParseContext } from "../../types";
+
+export interface MathNameResult {
+  name: string | null;
+  consumed: number;
+}
+
+export function parseMathName(ctx: ParseContext, startPos: number): MathNameResult {
+  let pos = startPos;
+  let consumed = 0;
+  let name = "";
+
+  const first = ctx.tokens[pos];
+  if (first?.type !== "IDENTIFIER" && first?.type !== "TEXT") {
+    return { name: null, consumed: 0 };
+  }
+
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (
+      !token ||
+      token.type === "BLOCK_CLOSE" ||
+      token.type === "WHITESPACE" ||
+      token.type === "NEWLINE"
+    ) {
+      break;
+    }
+
+    name += token.value;
+    pos++;
+    consumed++;
+  }
+
+  return { name: name === "" ? null : name, consumed };
+}

package/src/parser/rules/block/module/body.ts

@@ -0,0 +1,92 @@
+import type { ParseContext } from "../../types";
+import { currentToken } from "../../types";
+import { parseBlockName } from "../utils";
+
+export interface ModuleBodyResult {
+  body?: string;
+  pos: number;
+  consumed: number;
+}
+
+export function parseModuleBody(
+  ctx: ParseContext,
+  startPos: number,
+  moduleHasBody: boolean,
+): ModuleBodyResult {
+  if (!moduleHasBody || ctx.tokens[startPos]?.type !== "NEWLINE") {
+    return { pos: startPos, consumed: 0 };
+  }
+
+  let pos = startPos + 1;
+  let consumed = 1;
+  const bodyParts: string[] = [];
+  let foundClose = false;
+
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (!token || token.type === "EOF") {
+      break;
+    }
+
+    if (token.type === "BLOCK_END_OPEN") {
+      const closeResult = parseModuleClose(ctx, pos);
+      if (closeResult) {
+        foundClose = true;
+        pos = closeResult.pos;
+        consumed += closeResult.consumed;
+        break;
+      }
+    }
+
+    bodyParts.push(token.value);
+    pos++;
+    consumed++;
+  }
+
+  if (!foundClose) {
+    ctx.diagnostics.push({
+      severity: "warning",
+      code: "unclosed-block",
+      message: "Missing closing tag [[/module]] for [[module]]",
+      position: currentToken(ctx).position,
+    });
+  }
+
+  const trimmed = bodyParts.join("").trim();
+  return {
+    body: trimmed ? trimmed : undefined,
+    pos,
+    consumed,
+  };
+}
+
+function parseModuleClose(
+  ctx: ParseContext,
+  startPos: number,
+): { pos: number; consumed: number } | null {
+  if (ctx.tokens[startPos]?.type !== "BLOCK_END_OPEN") {
+    return null;
+  }
+
+  const closeNameResult = parseBlockName(ctx, startPos + 1);
+  if (
+    !closeNameResult ||
+    (closeNameResult.name !== "module" && closeNameResult.name !== "module654")
+  ) {
+    return null;
+  }
+
+  let pos = startPos + 1 + closeNameResult.consumed;
+  let consumed = 1 + closeNameResult.consumed;
+
+  if (ctx.tokens[pos]?.type === "BLOCK_CLOSE") {
+    pos++;
+    consumed++;
+  }
+  if (ctx.tokens[pos]?.type === "NEWLINE") {
+    pos++;
+    consumed++;
+  }
+
+  return { pos, consumed };
+}

package/src/parser/rules/block/module/element.ts

@@ -0,0 +1,33 @@
+import type { Element, Module } from "@wdprlib/ast";
+
+export function moduleParseResultToElement(result: Module | Element): Element {
+  if (isElement(result)) {
+    return result;
+  }
+
+  return createModuleElement(result);
+}
+
+export function createUnknownModuleElement(
+  name: string,
+  args: Record<string, string>,
+  body: string | undefined,
+): Element {
+  return createModuleElement({
+    module: "unknown",
+    name,
+    arguments: args,
+    body,
+  });
+}
+
+function createModuleElement(data: Module): Element {
+  return {
+    element: "module",
+    data,
+  };
+}
+
+function isElement(value: Module | Element): value is Element {
+  return "element" in value;
+}

package/src/parser/rules/block/module/include/directive.ts

@@ -0,0 +1,91 @@
+import type { PageRef } from "@wdprlib/ast";
+
+export interface IncludeAssignment {
+  key: string;
+  value: string;
+}
+
+/**
+ * Parse the inner content of an `[[include ...]]` directive into a page reference
+ * and variable assignments.
+ */
+export function parseIncludeDirective(inner: string): {
+  location: PageRef;
+  assignments: IncludeAssignment[];
+} {
+  const normalized = inner.includes("\n") ? inner.replaceAll("\n", " ") : inner;
+  const parts = normalized.split("|");
+  const firstSegment = parts[0]!.trim();
+
+  const spaceIndex = firstSegment.indexOf(" ");
+  let target: string;
+  const varSegments: string[] = [];
+
+  if (spaceIndex !== -1) {
+    target = firstSegment.slice(0, spaceIndex);
+    const rest = firstSegment.slice(spaceIndex + 1).trim();
+    if (rest) {
+      varSegments.push(rest);
+    }
+  } else {
+    target = firstSegment;
+  }
+
+  for (let i = 1; i < parts.length; i++) {
+    const segment = parts[i]!.trim();
+    if (segment) {
+      varSegments.push(segment);
+    }
+  }
+
+  const assignments = parseIncludeAssignments(varSegments);
+  return { location: parseIncludePageRef(target), assignments };
+}
+
+export function substituteVariables(
+  content: string,
+  assignments: readonly IncludeAssignment[],
+): string {
+  if (assignments.length === 0) return content;
+  if (!content.includes("{$")) return content;
+
+  let substituted = content;
+  for (const { key, value } of assignments) {
+    substituted = substituted.replaceAll(`{$${key}}`, value);
+  }
+  return substituted;
+}
+
+function parseIncludePageRef(target: string): PageRef {
+  if (target.startsWith(":")) {
+    const rest = target.slice(1);
+    const colonIndex = rest.indexOf(":");
+    if (colonIndex !== -1) {
+      return { site: rest.slice(0, colonIndex), page: rest.slice(colonIndex + 1) };
+    }
+  }
+  return { site: null, page: target };
+}
+
+function parseIncludeAssignments(varSegments: string[]): IncludeAssignment[] {
+  const assignments: IncludeAssignment[] = [];
+
+  for (const segment of varSegments) {
+    const assignment = parseVariableSegment(segment);
+    if (assignment) {
+      assignments.push(assignment);
+    }
+  }
+
+  return assignments;
+}
+
+function parseVariableSegment(segment: string): IncludeAssignment | null {
+  const eqIndex = segment.indexOf("=");
+  if (eqIndex === -1) return null;
+  const key = segment.slice(0, eqIndex).trim();
+  if (!key) return null;
+  const value = segment.slice(eqIndex + 1).trim();
+  if (value === "") return null;
+  return { key, value };
+}

package/src/parser/rules/block/module/include/index.ts

@@ -16,5 +16,14 @@
  * @module
  */
 
-export { resolveIncludes, resolveIncludesAsync } from "./resolve";
-export type { IncludeFetcher, AsyncIncludeFetcher, ResolveIncludesOptions } from "./resolve";
+export { resolveIncludes, resolveIncludesAsync, resolveIncludesWithTrace } from "./resolve";
+export { extractIncludeReferences } from "./references";
+export type {
+  IncludeFetcher,
+  AsyncIncludeFetcher,
+  ResolveIncludesOptions,
+  IncludeReference,
+  IncludeDependency,
+  IncludeIterationTrace,
+  ResolveIncludesTraceResult,
+} from "./resolve";

package/src/parser/rules/block/module/include/references.ts

@@ -0,0 +1,42 @@
+import { parseIncludeDirective } from "./directive";
+import { scanIncludeDirectives, type IncludeDirectiveMatch } from "./scanner";
+import type { IncludeReference, ResolveIncludesOptions } from "./resolve/types";
+
+/**
+ * Extract include directives from raw source without fetching or expanding them.
+ *
+ * This uses the same scanner and directive parser as `resolveIncludes`, so it
+ * reports only directives that the include expansion pass would recognize.
+ *
+ * This is a source-phase API. It intentionally does not evaluate later syntax
+ * such as comments, expressions, or iftags. Wikidot expands includes before
+ * those phases, so an include inside a later comment block is still reported,
+ * while a token that only becomes `[[include ...]]` after comment/expression/
+ * iftags processing is not reported.
+ *
+ * The result is not a complete transitive dependency graph. Nested includes
+ * only become visible after fetching and expanding the current layer; use
+ * `resolveIncludesWithTrace` when the caller needs observed dependency edges
+ * across iterative expansion.
+ */
+export function extractIncludeReferences(
+  source: string,
+  options?: ResolveIncludesOptions,
+): IncludeReference[] {
+  if (options?.settings && !options.settings.enablePageSyntax) {
+    return [];
+  }
+
+  return scanIncludeDirectives(source).map(createIncludeReference);
+}
+
+export function createIncludeReference(match: IncludeDirectiveMatch): IncludeReference {
+  const { location, assignments } = parseIncludeDirective(match.inner);
+  return {
+    location,
+    assignments,
+    start: match.start,
+    end: match.end,
+    inner: match.inner,
+  };
+}

package/src/parser/rules/block/module/include/resolve/cache.ts

@@ -0,0 +1,44 @@
+import type { PageRef } from "@wdprlib/ast";
+import type { AsyncIncludeFetcher, IncludeFetcher } from "./types";
+
+export function createCachedIncludeFetcher(
+  fetcher: IncludeFetcher,
+  normalizeKey: (pageRef: PageRef) => string,
+): IncludeFetcher {
+  const cache = new Map<string, string | null>();
+
+  return (pageRef: PageRef): string | null => {
+    const key = normalizeKey(pageRef);
+    if (cache.has(key)) {
+      return cache.get(key)!;
+    }
+
+    let result: string | null;
+    try {
+      result = fetcher(pageRef);
+    } catch {
+      result = null;
+    }
+    cache.set(key, result);
+    return result;
+  };
+}
+
+export function createCachedAsyncIncludeFetcher(
+  fetcher: AsyncIncludeFetcher,
+  normalizeKey: (pageRef: PageRef) => string,
+): AsyncIncludeFetcher {
+  const cache = new Map<string, Promise<string | null>>();
+
+  return async (pageRef: PageRef): Promise<string | null> => {
+    const key = normalizeKey(pageRef);
+    const cached = cache.get(key);
+    if (cached) {
+      return cached;
+    }
+
+    const result = fetcher(pageRef).catch(() => null);
+    cache.set(key, result);
+    return result;
+  };
+}

package/src/parser/rules/block/module/include/resolve/index.ts

@@ -0,0 +1,106 @@
+/**
+ *
+ * Text-level expansion of `[[include]]` directives.
+ *
+ * @module
+ */
+
+import type { PageRef } from "@wdprlib/ast";
+import { createCachedAsyncIncludeFetcher, createCachedIncludeFetcher } from "./cache";
+import { expandIterative, expandIterativeAsync, expandIterativeWithTrace } from "./iterate";
+import type {
+  AsyncIncludeFetcher,
+  IncludeFetcher,
+  ResolveIncludesOptions,
+  ResolveIncludesTraceResult,
+} from "./types";
+
+export type {
+  IncludeFetcher,
+  AsyncIncludeFetcher,
+  ResolveIncludesOptions,
+  IncludeReference,
+  IncludeDependency,
+  IncludeIterationTrace,
+  ResolveIncludesTraceResult,
+} from "./types";
+
+/**
+ * Expand all [[include]] directives in the source text.
+ *
+ * Uses Wikidot-compatible iterative expansion: each iteration replaces
+ * all include directives in the current source with fetched and
+ * variable-substituted content. Iteration continues until no further
+ * changes occur or `maxIterations` is reached.
+ */
+export function resolveIncludes(
+  source: string,
+  fetcher: IncludeFetcher,
+  options?: ResolveIncludesOptions,
+): string {
+  if (options?.settings && !options.settings.enablePageSyntax) {
+    return source;
+  }
+
+  const maxIterations = options?.maxIterations ?? 10;
+  const cachedFetcher = createCachedIncludeFetcher(fetcher, normalizePageKey);
+  return expandIterative(source, cachedFetcher, maxIterations);
+}
+
+/**
+ * Expand all [[include]] directives and return dependency/iteration trace data.
+ *
+ * This follows the same sync expansion behavior as {@link resolveIncludes};
+ * the additional trace data is intended for application-level dependency
+ * graphs, cache invalidation, and diagnostics.
+ */
+export function resolveIncludesWithTrace(
+  source: string,
+  fetcher: IncludeFetcher,
+  options?: ResolveIncludesOptions,
+): ResolveIncludesTraceResult {
+  if (options?.settings && !options.settings.enablePageSyntax) {
+    return {
+      source,
+      dependencies: [],
+      iterations: [],
+      reachedMaxIterations: false,
+    };
+  }
+
+  const maxIterations = options?.maxIterations ?? 10;
+  const cachedFetcher = createCachedIncludeFetcher(fetcher, normalizePageKey);
+  return expandIterativeWithTrace(source, cachedFetcher, maxIterations);
+}
+
+/**
+ * Async version of {@link resolveIncludes}.
+ *
+ * Expand all [[include]] directives using an async fetcher, allowing
+ * page content to be loaded from async sources such as databases.
+ */
+export async function resolveIncludesAsync(
+  source: string,
+  fetcher: AsyncIncludeFetcher,
+  options?: ResolveIncludesOptions,
+): Promise<string> {
+  if (options?.settings && !options.settings.enablePageSyntax) {
+    return source;
+  }
+
+  const maxIterations = options?.maxIterations ?? 10;
+  const cachedFetcher = createCachedAsyncIncludeFetcher(fetcher, normalizePageKey);
+  return expandIterativeAsync(source, cachedFetcher, maxIterations);
+}
+
+/**
+ * Normalize a PageRef into a consistent string key for cache lookups.
+ *
+ * Page names are lowercased for case-insensitive matching. Cross-site
+ * references include the site name as a prefix.
+ */
+function normalizePageKey(location: PageRef): string {
+  const site = location.site ?? "";
+  const page = location.page.toLowerCase();
+  return site ? `${site}:${page}` : page;
+}