@wdprlib/parser 5.0.0 → 5.1.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wdprlib/parser",
-  "version": "5.0.0",
+  "version": "5.1.2",
   "description": "Parser for Wikidot markup",
   "keywords": [
     "ast",
@@ -41,6 +41,6 @@
   },
   "dependencies": {
     "@braintree/sanitize-url": "^7.1.1",
-    "@wdprlib/ast": "3.0.0"
+    "@wdprlib/ast": "3.0.1"
   }
 }

package/src/parser/parse/context.ts

@@ -1,9 +1,13 @@
-import { DEFAULT_SETTINGS } from "@wdprlib/ast";
+import { DEFAULT_SETTINGS, type PageRef } from "@wdprlib/ast";
 import type { Token } from "../../lexer";
 import { blockFallbackRule, blockRules, inlineRules, type ParseContext } from "../rules";
 import type { ParserOptions } from "./options";
 
-export function createParseContext(tokens: Token[], options: ParserOptions = {}): ParseContext {
+export function createParseContext(
+  tokens: Token[],
+  options: ParserOptions = {},
+  deferInclude?: (location: PageRef) => boolean,
+): ParseContext {
   return {
     tokens,
     pos: 0,
@@ -11,6 +15,7 @@ export function createParseContext(tokens: Token[], options: ParserOptions = {})
     trackPositions: options.trackPositions ?? true,
     settings: options.settings ?? DEFAULT_SETTINGS,
     appendImplicitFootnoteBlock: options.appendImplicitFootnoteBlock ?? true,
+    deferInclude,
     footnotes: [],
     tocEntries: [],
     codeBlocks: [],

package/src/parser/parse/index.ts

@@ -1,6 +1,6 @@
-import type { ParseResult } from "@wdprlib/ast";
+import type { PageRef, ParseResult } from "@wdprlib/ast";
 import { tokenize } from "../../lexer";
-import { Parser } from "./parser";
+import { Parser, parseTokensWithIncludeDeferral } from "./parser";
 import { parseLargePlainTextDocument, parsePlainNonAsciiDocument } from "./plain-non-ascii";
 import { prepareSourceForParse } from "./source";
 import type { ParserOptions } from "./options";
@@ -23,6 +23,22 @@ export { Parser } from "./parser";
  * @since 2.0.0
  */
 export function parse(source: string, options?: ParserOptions): ParseResult {
+  return parseSource(source, options);
+}
+
+export function parseWithIncludeDeferral(
+  source: string,
+  options: ParserOptions,
+  deferInclude: (location: PageRef) => boolean,
+): ParseResult {
+  return parseSource(source, options, deferInclude);
+}
+
+function parseSource(
+  source: string,
+  options?: ParserOptions,
+  deferInclude?: (location: PageRef) => boolean,
+): ParseResult {
   const plainResult = parsePlainNonAsciiDocument(source);
   if (plainResult) {
     return plainResult;
@@ -38,5 +54,7 @@ export function parse(source: string, options?: ParserOptions): ParseResult {
     trackPositions: options?.trackPositions,
     compactTextRuns: preprocessed.length >= COMPACT_TEXT_RUN_SOURCE_LENGTH,
   });
-  return new Parser(tokens, options).parse();
+  return deferInclude
+    ? parseTokensWithIncludeDeferral(tokens, options ?? {}, deferInclude)
+    : new Parser(tokens, options).parse();
 }

package/src/parser/parse/parser.ts

@@ -1,4 +1,4 @@
-import type { Element, ParseResult } from "@wdprlib/ast";
+import type { Element, PageRef, ParseResult } from "@wdprlib/ast";
 import type { Token } from "../../lexer";
 import type { ParseContext } from "../rules";
 import type { ParserOptions } from "./options";
@@ -33,26 +33,35 @@ export class Parser {
    * @since 2.0.0
    */
   parse(): ParseResult {
-    const children: Element[] = [];
+    return parseContext(this.ctx);
+  }
+}
 
-    while (!this.isAtEnd()) {
-      const blocks = this.parseBlock();
-      children.push(...blocks);
-    }
+export function parseTokensWithIncludeDeferral(
+  tokens: Token[],
+  options: ParserOptions,
+  deferInclude: (location: PageRef) => boolean,
+): ParseResult {
+  return parseContext(createParseContext(tokens, options, deferInclude));
+}
 
-    return finalizeParseResult(this.ctx, children);
-  }
+function parseContext(ctx: ParseContext): ParseResult {
+  const children: Element[] = [];
 
-  private isAtEnd(): boolean {
-    return this.ctx.pos >= this.ctx.tokens.length || this.currentToken().type === "EOF";
+  while (!isAtEnd(ctx)) {
+    children.push(...parseBlock(ctx));
   }
 
-  private currentToken(): Token {
-    return this.ctx.tokens[this.ctx.pos] ?? this.eofToken();
-  }
+  return finalizeParseResult(ctx, children);
+}
 
-  private eofToken(): Token {
-    return {
+function isAtEnd(ctx: ParseContext): boolean {
+  return ctx.pos >= ctx.tokens.length || currentToken(ctx).type === "EOF";
+}
+
+function currentToken(ctx: ParseContext): Token {
+  return (
+    ctx.tokens[ctx.pos] ?? {
       type: "EOF",
       value: "",
       position: {
@@ -60,20 +69,20 @@ export class Parser {
         end: { line: 0, column: 0, offset: 0 },
       },
       lineStart: false,
-    };
-  }
-
-  private skipWhitespace(): void {
-    while (this.currentToken().type === "WHITESPACE") {
-      this.ctx.pos++;
     }
-  }
+  );
+}
 
-  private parseBlock(): Element[] {
-    return parseNextBlock(
-      this.ctx,
-      () => this.skipWhitespace(),
-      () => this.isAtEnd(),
-    );
+function skipWhitespace(ctx: ParseContext): void {
+  while (currentToken(ctx).type === "WHITESPACE") {
+    ctx.pos++;
   }
 }
+
+function parseBlock(ctx: ParseContext): Element[] {
+  return parseNextBlock(
+    ctx,
+    () => skipWhitespace(ctx),
+    () => isAtEnd(ctx),
+  );
+}

package/src/parser/rules/block/include/index.ts

@@ -1,4 +1,4 @@
-import type { Element } from "@wdprlib/ast";
+import { lineBreak, paragraph, text, type Element } from "@wdprlib/ast";
 import type { BlockRule, ParseContext, RuleResult } from "../../types";
 import { currentToken } from "../../types";
 import { parseBlockName } from "../utils";
@@ -46,6 +46,7 @@ export const includeRule: BlockRule = {
     }
     pos++;
     consumed++;
+    const directiveEnd = pos;
 
     if (ctx.tokens[pos]?.type === "NEWLINE") {
       pos++;
@@ -56,6 +57,23 @@ export const includeRule: BlockRule = {
       return { success: false };
     }
 
+    const location = parsePageRef(args.target);
+    if (ctx.deferInclude?.(location)) {
+      // Keep Wikidot's parser extent here. Nested block markup in an include
+      // value must delay its closing brackets through variable expansion.
+      const source = ctx.tokens
+        .slice(ctx.pos, directiveEnd)
+        .map((token) => token.value)
+        .join("");
+      const elements: Element[] = [];
+      const lines = source.split("\n");
+      for (let index = 0; index < lines.length; index++) {
+        if (index > 0) elements.push(lineBreak());
+        if (lines[index] !== "") elements.push(text(lines[index]!));
+      }
+      return { success: true, elements: [paragraph(elements)], consumed };
+    }
+
     return {
       success: true,
       elements: [
@@ -64,7 +82,7 @@ export const includeRule: BlockRule = {
           data: {
             "paragraph-safe": false,
             variables: parseVariables(args.argumentTokens),
-            location: parsePageRef(args.target),
+            location,
             elements: [],
           },
         },

package/src/parser/rules/block/module/include/resolve/index.ts

@@ -15,6 +15,7 @@ import {
 import type {
   AsyncIncludeFetcher,
   IncludeFetcher,
+  IncludeReference,
   ResolveIncludesOptions,
   ResolveIncludesTraceResult,
 } from "./types";
@@ -113,6 +114,33 @@ export async function resolveIncludesAsyncWithTrace(
   return expandIterativeAsyncWithTrace(source, cachedFetcher, maxIterations);
 }
 
+/**
+ * Resolve includes while leaving selected directives untouched for the
+ * high-level parser pipeline to render literally.
+ *
+ * This helper is intentionally not exported from the package barrel so the
+ * public low-level resolver contract remains unchanged.
+ */
+export async function resolveIncludesAsyncWithTraceSelective(
+  source: string,
+  fetcher: AsyncIncludeFetcher,
+  shouldDefer: (reference: IncludeReference) => boolean,
+  options?: ResolveIncludesOptions,
+): Promise<ResolveIncludesTraceResult> {
+  if (options?.settings && !options.settings.enablePageSyntax) {
+    return {
+      source,
+      dependencies: [],
+      iterations: [],
+      reachedMaxIterations: false,
+    };
+  }
+
+  const maxIterations = options?.maxIterations ?? 10;
+  const cachedFetcher = createCachedAsyncIncludeFetcher(fetcher, normalizePageKey);
+  return expandIterativeAsyncWithTrace(source, cachedFetcher, maxIterations, shouldDefer);
+}
+
 /**
  * Normalize a PageRef into a consistent string key for cache lookups.
  *

package/src/parser/rules/block/module/include/resolve/iterate.ts

@@ -107,6 +107,7 @@ export async function expandIterativeAsyncWithTrace(
   source: string,
   fetcher: AsyncIncludeFetcher,
   maxIterations: number,
+  shouldDefer?: (reference: IncludeReference) => boolean,
 ): Promise<ResolveIncludesTraceResult> {
   let current = source;
   const replacementCache = new Map<string, Promise<string>>();
@@ -114,7 +115,13 @@ export async function expandIterativeAsyncWithTrace(
   const iterations: ResolveIncludesTraceResult["iterations"] = [];
 
   for (let i = 0; i < maxIterations; i++) {
-    const expanded = await expandOneIterationAsyncWithTrace(current, fetcher, replacementCache, i);
+    const expanded = await expandOneIterationAsyncWithTrace(
+      current,
+      fetcher,
+      replacementCache,
+      i,
+      shouldDefer,
+    );
     if (expanded === null) break;
 
     dependencies.push(...expanded.dependencies);
@@ -133,7 +140,7 @@ export async function expandIterativeAsyncWithTrace(
     dependencies,
     iterations,
     reachedMaxIterations:
-      iterations.length === maxIterations && scanIncludeDirectives(current).length > 0,
+      iterations.length === maxIterations && hasResolvableDirectives(current, shouldDefer),
   };
 }
 
@@ -219,6 +226,7 @@ async function expandOneIterationAsyncWithTrace(
   fetcher: AsyncIncludeFetcher,
   replacementCache: Map<string, Promise<string>>,
   iteration: number,
+  shouldDefer?: (reference: IncludeReference) => boolean,
 ): Promise<{
   source: string;
   references: IncludeReference[];
@@ -230,8 +238,13 @@ async function expandOneIterationAsyncWithTrace(
   if (directives.length === 0) return null;
 
   const references = directives.map(createIncludeReference);
+  const deferred = references.map((reference) => shouldDefer?.(reference) ?? false);
   const replacements = await Promise.all(
-    directives.map(({ inner }) => replaceCachedAsync(inner, fetcher, replacementCache)),
+    directives.map((directive, index) =>
+      deferred[index]
+        ? Promise.resolve(source.slice(directive.start, directive.end))
+        : replaceCachedAsync(directive.inner, fetcher, replacementCache),
+    ),
   );
   const parts: string[] = [];
   let lastPos = 0;
@@ -246,10 +259,21 @@ async function expandOneIterationAsyncWithTrace(
   return {
     source: parts.join(""),
     references,
-    dependencies: references.map((reference) => ({ ...reference, iteration })),
+    dependencies: references
+      .filter((_, index) => !deferred[index])
+      .map((reference) => ({ ...reference, iteration })),
   };
 }
 
+function hasResolvableDirectives(
+  source: string,
+  shouldDefer: ((reference: IncludeReference) => boolean) | undefined,
+): boolean {
+  const directives = scanIncludeDirectives(source);
+  if (!shouldDefer) return directives.length > 0;
+  return directives.some((directive) => !shouldDefer(createIncludeReference(directive)));
+}
+
 function replaceCached(inner: string, fetcher: IncludeFetcher, cache: Map<string, string>): string {
   const cached = cache.get(inner);
   if (cached !== undefined) return cached;

package/src/parser/rules/block/module/include/scanner.ts

@@ -40,6 +40,8 @@ export function scanIncludeDirectives(source: string): IncludeDirectiveMatch[] {
 
     let depth = 0;
     let linkDepth = 0;
+    const nestedQuotes: Array<number | null> = [];
+    const nestedQuoteAllowed: boolean[] = [];
     let i = start;
     let closeEnd = -1;
 
@@ -48,7 +50,37 @@ export function scanIncludeDirectives(source: string): IncludeDirectiveMatch[] {
       const next = source.charCodeAt(i + 1);
       const nextNext = source.charCodeAt(i + 2);
 
-      if (ch === OPEN_BRACKET && next === OPEN_BRACKET && nextNext === OPEN_BRACKET) {
+      if (ch === 10 || ch === 13) {
+        linkDepth = 0;
+        if (depth > 1) {
+          depth = 1;
+          nestedQuotes.length = 0;
+          nestedQuoteAllowed.length = 0;
+        }
+        i++;
+        continue;
+      }
+
+      const nestedQuote = nestedQuotes.at(-1);
+      if (nestedQuote !== undefined && nestedQuote !== null) {
+        if (ch === 92) {
+          i += 2;
+        } else {
+          if (ch === nestedQuote) {
+            nestedQuotes[nestedQuotes.length - 1] = null;
+            nestedQuoteAllowed[nestedQuoteAllowed.length - 1] = false;
+          }
+          i++;
+        }
+      } else if (
+        nestedQuote === null &&
+        nestedQuoteAllowed.at(-1) === true &&
+        (ch === 34 || ch === 39)
+      ) {
+        nestedQuotes[nestedQuotes.length - 1] = ch;
+        nestedQuoteAllowed[nestedQuoteAllowed.length - 1] = false;
+        i++;
+      } else if (ch === OPEN_BRACKET && next === OPEN_BRACKET && nextNext === OPEN_BRACKET) {
         linkDepth++;
         i += 3;
       } else if (
@@ -62,10 +94,28 @@ export function scanIncludeDirectives(source: string): IncludeDirectiveMatch[] {
       } else if (linkDepth > 0) {
         i++;
       } else if (ch === OPEN_BRACKET && next === OPEN_BRACKET) {
+        if (nestedQuoteAllowed.length > 0) {
+          nestedQuoteAllowed[nestedQuoteAllowed.length - 1] = false;
+        }
+        if (depth > 0) {
+          nestedQuotes.push(null);
+          nestedQuoteAllowed.push(false);
+        }
         depth++;
         i += 2;
+      } else if (ch === CLOSE_BRACKET && next !== CLOSE_BRACKET && depth > 1) {
+        // A lone `]` is the first close of an intentionally incomplete nested
+        // token (for example `[[module ...]` passed through an include value).
+        depth--;
+        nestedQuotes.pop();
+        nestedQuoteAllowed.pop();
+        i++;
       } else if (ch === CLOSE_BRACKET && next === CLOSE_BRACKET) {
         const closeStart = i;
+        if (depth > 1) {
+          nestedQuotes.pop();
+          nestedQuoteAllowed.pop();
+        }
         depth--;
         i += 2;
 
@@ -83,6 +133,13 @@ export function scanIncludeDirectives(source: string): IncludeDirectiveMatch[] {
           }
         }
       } else {
+        if (nestedQuoteAllowed.length > 0) {
+          if (ch === 61) {
+            nestedQuoteAllowed[nestedQuoteAllowed.length - 1] = true;
+          } else if (ch !== 32 && ch !== 9) {
+            nestedQuoteAllowed[nestedQuoteAllowed.length - 1] = false;
+          }
+        }
         i++;
       }
     }

package/src/parser/rules/contracts/parse-context.ts

@@ -3,6 +3,7 @@ import type {
   CodeBlockData,
   Diagnostic,
   Element,
+  PageRef,
   TocEntry,
   Version,
   WikitextSettings,
@@ -26,6 +27,7 @@ export interface ParseContext {
   trackPositions: boolean;
   settings: WikitextSettings;
   appendImplicitFootnoteBlock: boolean;
+  deferInclude?: (location: PageRef) => boolean;
   footnotes: Element[][];
   tocEntries: TocEntry[];
   codeBlocks: CodeBlockData[];

package/src/pipeline/process.ts

@@ -4,13 +4,10 @@ import {
   type PageRef,
   type WikitextPageContext,
 } from "@wdprlib/ast";
-import { parse } from "../parser";
+import { parseWithIncludeDeferral } from "../parser/parse";
 import { extractDataRequirements } from "../parser/rules/block/module/listpages/extract";
-import {
-  resolveIncludesAsyncWithTrace,
-  type AsyncIncludeFetcher,
-  type IncludeDependency,
-} from "../parser/rules/block/module/include";
+import type { AsyncIncludeFetcher, IncludeDependency } from "../parser/rules/block/module/include";
+import { resolveIncludesAsyncWithTraceSelective } from "../parser/rules/block/module/include/resolve";
 import type { DataProvider } from "../parser/rules/block/module/types-common";
 import { resolveModulesWithAsyncParse } from "../parser/rules/block/module/resolution/resolve-async";
 import type {
@@ -42,37 +39,48 @@ export async function processWikitext<TPage extends WikitextPageContext>(
       : undefined,
   );
   const resolveSource = async (input: string): Promise<string> => {
-    if (!fetchInclude) return input;
-    const resolution = await resolveIncludesAsyncWithTrace(input, fetchInclude, {
-      maxIterations: options.includeMaxIterations,
-      settings,
-    });
-    dependencies.push(...resolution.dependencies);
-    if (resolution.reachedMaxIterations) {
-      diagnostics.push(
-        createLimitDiagnostic(
-          "include-resolution-limit",
-          `Include expansion stopped after ${options.includeMaxIterations ?? 10} iterations.`,
-        ),
+    let expanded = input;
+    if (fetchInclude) {
+      const resolution = await resolveIncludesAsyncWithTraceSelective(
+        input,
+        fetchInclude,
+        (reference) => isCurrentPageInclude(reference.location, options.page),
+        {
+          maxIterations: options.includeMaxIterations,
+          settings,
+        },
       );
+      dependencies.push(...resolution.dependencies);
+      if (resolution.reachedMaxIterations) {
+        diagnostics.push(
+          createLimitDiagnostic(
+            "include-resolution-limit",
+            `Include expansion stopped after ${options.includeMaxIterations ?? 10} iterations.`,
+          ),
+        );
+      }
+      expanded = resolution.source;
     }
-    return resolution.source;
+    return expanded;
   };
 
-  const expandedSource = await resolveSource(source);
-  const initial = parse(expandedSource, {
-    settings,
-    pageTags: options.page.tags,
-    appendImplicitFootnoteBlock: false,
-  });
+  const parseSource = async (input: string) => {
+    const expanded = await resolveSource(input);
+    return parseWithIncludeDeferral(
+      expanded,
+      {
+        settings,
+        pageTags: options.page.tags,
+        appendImplicitFootnoteBlock: false,
+      },
+      (location) => isCurrentPageInclude(location, options.page),
+    );
+  };
+
+  const initial = await parseSource(source);
   diagnostics.push(...initial.diagnostics);
   const dataProvider = createModuleDataProvider(options, callbackContext);
-  const parseFragment = async (fragmentSource: string) =>
-    parse(await resolveSource(fragmentSource), {
-      settings,
-      pageTags: options.page.tags,
-      appendImplicitFootnoteBlock: false,
-    });
+  const parseFragment = parseSource;
   let ast = initial.ast;
 
   for (let pass = 0; pass < DEFAULT_MODULE_MAX_PASSES; pass++) {
@@ -111,6 +119,15 @@ export async function processWikitext<TPage extends WikitextPageContext>(
   };
 }
 
+function isCurrentPageInclude(location: PageRef, page: WikitextPageContext): boolean {
+  if (location.site !== null) {
+    if (!page.site || location.site.toLowerCase() !== page.site.toLowerCase()) return false;
+  }
+
+  const target = location.page.toLowerCase();
+  return target === page.fullName.toLowerCase() || target === page.unixName?.toLowerCase();
+}
+
 function hasResolvableRequirements(
   requirements: ReturnType<typeof extractDataRequirements>["requirements"],
   provider: