@wdprlib/parser 4.4.0 → 5.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wdprlib/parser",
-  "version": "4.4.0",
+  "version": "5.1.0",
   "description": "Parser for Wikidot markup",
   "keywords": [
     "ast",
@@ -41,6 +41,6 @@
   },
   "dependencies": {
     "@braintree/sanitize-url": "^7.1.1",
-    "@wdprlib/ast": "2.4.0"
+    "@wdprlib/ast": "3.0.0"
   }
 }

package/src/index.ts

@@ -185,6 +185,8 @@ export {
   parseParent,
   parseDateSelector,
   parseNumericSelector,
+  definePageData,
+  matchesListPagesSelectors,
   // ListUsers
   extractListUsersVariables,
   compileListUsersTemplate,

package/src/parser/parse/context.ts

@@ -1,9 +1,13 @@
-import { DEFAULT_SETTINGS } from "@wdprlib/ast";
+import { DEFAULT_SETTINGS, type PageRef } from "@wdprlib/ast";
 import type { Token } from "../../lexer";
 import { blockFallbackRule, blockRules, inlineRules, type ParseContext } from "../rules";
 import type { ParserOptions } from "./options";
 
-export function createParseContext(tokens: Token[], options: ParserOptions = {}): ParseContext {
+export function createParseContext(
+  tokens: Token[],
+  options: ParserOptions = {},
+  deferInclude?: (location: PageRef) => boolean,
+): ParseContext {
   return {
     tokens,
     pos: 0,
@@ -11,6 +15,7 @@ export function createParseContext(tokens: Token[], options: ParserOptions = {})
     trackPositions: options.trackPositions ?? true,
     settings: options.settings ?? DEFAULT_SETTINGS,
     appendImplicitFootnoteBlock: options.appendImplicitFootnoteBlock ?? true,
+    deferInclude,
     footnotes: [],
     tocEntries: [],
     codeBlocks: [],

package/src/parser/parse/index.ts

@@ -1,6 +1,6 @@
-import type { ParseResult } from "@wdprlib/ast";
+import type { PageRef, ParseResult } from "@wdprlib/ast";
 import { tokenize } from "../../lexer";
-import { Parser } from "./parser";
+import { Parser, parseTokensWithIncludeDeferral } from "./parser";
 import { parseLargePlainTextDocument, parsePlainNonAsciiDocument } from "./plain-non-ascii";
 import { prepareSourceForParse } from "./source";
 import type { ParserOptions } from "./options";
@@ -23,6 +23,22 @@ export { Parser } from "./parser";
  * @since 2.0.0
  */
 export function parse(source: string, options?: ParserOptions): ParseResult {
+  return parseSource(source, options);
+}
+
+export function parseWithIncludeDeferral(
+  source: string,
+  options: ParserOptions,
+  deferInclude: (location: PageRef) => boolean,
+): ParseResult {
+  return parseSource(source, options, deferInclude);
+}
+
+function parseSource(
+  source: string,
+  options?: ParserOptions,
+  deferInclude?: (location: PageRef) => boolean,
+): ParseResult {
   const plainResult = parsePlainNonAsciiDocument(source);
   if (plainResult) {
     return plainResult;
@@ -38,5 +54,7 @@ export function parse(source: string, options?: ParserOptions): ParseResult {
     trackPositions: options?.trackPositions,
     compactTextRuns: preprocessed.length >= COMPACT_TEXT_RUN_SOURCE_LENGTH,
   });
-  return new Parser(tokens, options).parse();
+  return deferInclude
+    ? parseTokensWithIncludeDeferral(tokens, options ?? {}, deferInclude)
+    : new Parser(tokens, options).parse();
 }

package/src/parser/parse/parser.ts

@@ -1,4 +1,4 @@
-import type { Element, ParseResult } from "@wdprlib/ast";
+import type { Element, PageRef, ParseResult } from "@wdprlib/ast";
 import type { Token } from "../../lexer";
 import type { ParseContext } from "../rules";
 import type { ParserOptions } from "./options";
@@ -33,26 +33,35 @@ export class Parser {
    * @since 2.0.0
    */
   parse(): ParseResult {
-    const children: Element[] = [];
+    return parseContext(this.ctx);
+  }
+}
 
-    while (!this.isAtEnd()) {
-      const blocks = this.parseBlock();
-      children.push(...blocks);
-    }
+export function parseTokensWithIncludeDeferral(
+  tokens: Token[],
+  options: ParserOptions,
+  deferInclude: (location: PageRef) => boolean,
+): ParseResult {
+  return parseContext(createParseContext(tokens, options, deferInclude));
+}
 
-    return finalizeParseResult(this.ctx, children);
-  }
+function parseContext(ctx: ParseContext): ParseResult {
+  const children: Element[] = [];
 
-  private isAtEnd(): boolean {
-    return this.ctx.pos >= this.ctx.tokens.length || this.currentToken().type === "EOF";
+  while (!isAtEnd(ctx)) {
+    children.push(...parseBlock(ctx));
   }
 
-  private currentToken(): Token {
-    return this.ctx.tokens[this.ctx.pos] ?? this.eofToken();
-  }
+  return finalizeParseResult(ctx, children);
+}
 
-  private eofToken(): Token {
-    return {
+function isAtEnd(ctx: ParseContext): boolean {
+  return ctx.pos >= ctx.tokens.length || currentToken(ctx).type === "EOF";
+}
+
+function currentToken(ctx: ParseContext): Token {
+  return (
+    ctx.tokens[ctx.pos] ?? {
       type: "EOF",
       value: "",
       position: {
@@ -60,20 +69,20 @@ export class Parser {
         end: { line: 0, column: 0, offset: 0 },
       },
       lineStart: false,
-    };
-  }
-
-  private skipWhitespace(): void {
-    while (this.currentToken().type === "WHITESPACE") {
-      this.ctx.pos++;
     }
-  }
+  );
+}
 
-  private parseBlock(): Element[] {
-    return parseNextBlock(
-      this.ctx,
-      () => this.skipWhitespace(),
-      () => this.isAtEnd(),
-    );
+function skipWhitespace(ctx: ParseContext): void {
+  while (currentToken(ctx).type === "WHITESPACE") {
+    ctx.pos++;
   }
 }
+
+function parseBlock(ctx: ParseContext): Element[] {
+  return parseNextBlock(
+    ctx,
+    () => skipWhitespace(ctx),
+    () => isAtEnd(ctx),
+  );
+}

package/src/parser/preprocess/expr/index.ts

@@ -7,8 +7,11 @@
  */
 
 import { makeUniqueSentinels, maskRawRegions, restorePlaceholders } from "../utils";
+import { matchDirectiveKind } from "./kind";
 import { expandInnermost } from "./scan";
 
+const MAX_EXPR_NESTING = 64;
+
 /**
  * Resolve every `[[#if]]` / `[[#ifexpr]]` / `[[#expr]]` that sits inside
  * another block's opener (depth > 0). Top-level directives are left for
@@ -21,6 +24,7 @@ export function preprocessExpr(source: string): string {
 
   const sentinels = makeUniqueSentinels(source);
   const { masked, placeholders } = maskRawRegions(source, sentinels);
+  if (exceedsExprNestingLimit(masked)) return source;
   const reduced = reduceExpr(masked);
   return restorePlaceholders(reduced, placeholders, sentinels);
 }
@@ -43,3 +47,25 @@ function reduceExpr(source: string): string {
   }
   return current;
 }
+
+function exceedsExprNestingLimit(source: string): boolean {
+  const expressionStack: boolean[] = [];
+  let expressionDepth = 0;
+
+  for (let i = 0; i < source.length; i++) {
+    if (source.startsWith("[[", i)) {
+      const isExpression = matchDirectiveKind(source, i) !== null;
+      expressionStack.push(isExpression);
+      if (isExpression && ++expressionDepth > MAX_EXPR_NESTING) return true;
+      i++;
+      continue;
+    }
+
+    if (source.startsWith("]]", i)) {
+      if (expressionStack.pop()) expressionDepth--;
+      i++;
+    }
+  }
+
+  return false;
+}

package/src/parser/preprocess/utils/raw-regions.ts

@@ -13,16 +13,25 @@ export interface Sentinels {
  * Choose sentinel strings that are guaranteed not to appear in `source`.
  * The placeholders we splice into the masked source have the form
  * `<open><digits><close>`, so the restore pass must not confuse them
- * with content. Extends both sentinel characters until neither appears.
+ * with content.
  */
 export function makeUniqueSentinels(source: string): Sentinels {
-  let open = BASE_PLACEHOLDER_OPEN;
-  let close = BASE_PLACEHOLDER_CLOSE;
-  while (source.includes(open) || source.includes(close)) {
-    open += BASE_PLACEHOLDER_OPEN;
-    close += BASE_PLACEHOLDER_CLOSE;
+  let openRun = 0;
+  let closeRun = 0;
+  let longestOpenRun = 0;
+  let longestCloseRun = 0;
+
+  for (const char of source) {
+    openRun = char === BASE_PLACEHOLDER_OPEN ? openRun + 1 : 0;
+    closeRun = char === BASE_PLACEHOLDER_CLOSE ? closeRun + 1 : 0;
+    longestOpenRun = Math.max(longestOpenRun, openRun);
+    longestCloseRun = Math.max(longestCloseRun, closeRun);
   }
-  return { open, close };
+
+  return {
+    open: BASE_PLACEHOLDER_OPEN.repeat(longestOpenRun + 1),
+    close: BASE_PLACEHOLDER_CLOSE.repeat(longestCloseRun + 1),
+  };
 }
 
 /**

package/src/parser/rules/block/include/index.ts

@@ -1,4 +1,4 @@
-import type { Element } from "@wdprlib/ast";
+import { lineBreak, paragraph, text, type Element } from "@wdprlib/ast";
 import type { BlockRule, ParseContext, RuleResult } from "../../types";
 import { currentToken } from "../../types";
 import { parseBlockName } from "../utils";
@@ -46,6 +46,7 @@ export const includeRule: BlockRule = {
     }
     pos++;
     consumed++;
+    const directiveEnd = pos;
 
     if (ctx.tokens[pos]?.type === "NEWLINE") {
       pos++;
@@ -56,6 +57,23 @@ export const includeRule: BlockRule = {
       return { success: false };
     }
 
+    const location = parsePageRef(args.target);
+    if (ctx.deferInclude?.(location)) {
+      // Keep Wikidot's parser extent here. Nested block markup in an include
+      // value must delay its closing brackets through variable expansion.
+      const source = ctx.tokens
+        .slice(ctx.pos, directiveEnd)
+        .map((token) => token.value)
+        .join("");
+      const elements: Element[] = [];
+      const lines = source.split("\n");
+      for (let index = 0; index < lines.length; index++) {
+        if (index > 0) elements.push(lineBreak());
+        if (lines[index] !== "") elements.push(text(lines[index]!));
+      }
+      return { success: true, elements: [paragraph(elements)], consumed };
+    }
+
     return {
       success: true,
       elements: [
@@ -64,7 +82,7 @@ export const includeRule: BlockRule = {
           data: {
             "paragraph-safe": false,
             variables: parseVariables(args.argumentTokens),
-            location: parsePageRef(args.target),
+            location,
             elements: [],
           },
         },

package/src/parser/rules/block/module/include/resolve/index.ts

@@ -15,6 +15,7 @@ import {
 import type {
   AsyncIncludeFetcher,
   IncludeFetcher,
+  IncludeReference,
   ResolveIncludesOptions,
   ResolveIncludesTraceResult,
 } from "./types";
@@ -113,6 +114,33 @@ export async function resolveIncludesAsyncWithTrace(
   return expandIterativeAsyncWithTrace(source, cachedFetcher, maxIterations);
 }
 
+/**
+ * Resolve includes while leaving selected directives untouched for the
+ * high-level parser pipeline to render literally.
+ *
+ * This helper is intentionally not exported from the package barrel so the
+ * public low-level resolver contract remains unchanged.
+ */
+export async function resolveIncludesAsyncWithTraceSelective(
+  source: string,
+  fetcher: AsyncIncludeFetcher,
+  shouldDefer: (reference: IncludeReference) => boolean,
+  options?: ResolveIncludesOptions,
+): Promise<ResolveIncludesTraceResult> {
+  if (options?.settings && !options.settings.enablePageSyntax) {
+    return {
+      source,
+      dependencies: [],
+      iterations: [],
+      reachedMaxIterations: false,
+    };
+  }
+
+  const maxIterations = options?.maxIterations ?? 10;
+  const cachedFetcher = createCachedAsyncIncludeFetcher(fetcher, normalizePageKey);
+  return expandIterativeAsyncWithTrace(source, cachedFetcher, maxIterations, shouldDefer);
+}
+
 /**
  * Normalize a PageRef into a consistent string key for cache lookups.
  *

package/src/parser/rules/block/module/include/resolve/iterate.ts

@@ -107,6 +107,7 @@ export async function expandIterativeAsyncWithTrace(
   source: string,
   fetcher: AsyncIncludeFetcher,
   maxIterations: number,
+  shouldDefer?: (reference: IncludeReference) => boolean,
 ): Promise<ResolveIncludesTraceResult> {
   let current = source;
   const replacementCache = new Map<string, Promise<string>>();
@@ -114,7 +115,13 @@ export async function expandIterativeAsyncWithTrace(
   const iterations: ResolveIncludesTraceResult["iterations"] = [];
 
   for (let i = 0; i < maxIterations; i++) {
-    const expanded = await expandOneIterationAsyncWithTrace(current, fetcher, replacementCache, i);
+    const expanded = await expandOneIterationAsyncWithTrace(
+      current,
+      fetcher,
+      replacementCache,
+      i,
+      shouldDefer,
+    );
     if (expanded === null) break;
 
     dependencies.push(...expanded.dependencies);
@@ -133,7 +140,7 @@ export async function expandIterativeAsyncWithTrace(
     dependencies,
     iterations,
     reachedMaxIterations:
-      iterations.length === maxIterations && scanIncludeDirectives(current).length > 0,
+      iterations.length === maxIterations && hasResolvableDirectives(current, shouldDefer),
   };
 }
 
@@ -219,6 +226,7 @@ async function expandOneIterationAsyncWithTrace(
   fetcher: AsyncIncludeFetcher,
   replacementCache: Map<string, Promise<string>>,
   iteration: number,
+  shouldDefer?: (reference: IncludeReference) => boolean,
 ): Promise<{
   source: string;
   references: IncludeReference[];
@@ -230,8 +238,13 @@ async function expandOneIterationAsyncWithTrace(
   if (directives.length === 0) return null;
 
   const references = directives.map(createIncludeReference);
+  const deferred = references.map((reference) => shouldDefer?.(reference) ?? false);
   const replacements = await Promise.all(
-    directives.map(({ inner }) => replaceCachedAsync(inner, fetcher, replacementCache)),
+    directives.map((directive, index) =>
+      deferred[index]
+        ? Promise.resolve(source.slice(directive.start, directive.end))
+        : replaceCachedAsync(directive.inner, fetcher, replacementCache),
+    ),
   );
   const parts: string[] = [];
   let lastPos = 0;
@@ -246,10 +259,21 @@ async function expandOneIterationAsyncWithTrace(
   return {
     source: parts.join(""),
     references,
-    dependencies: references.map((reference) => ({ ...reference, iteration })),
+    dependencies: references
+      .filter((_, index) => !deferred[index])
+      .map((reference) => ({ ...reference, iteration })),
   };
 }
 
+function hasResolvableDirectives(
+  source: string,
+  shouldDefer: ((reference: IncludeReference) => boolean) | undefined,
+): boolean {
+  const directives = scanIncludeDirectives(source);
+  if (!shouldDefer) return directives.length > 0;
+  return directives.some((directive) => !shouldDefer(createIncludeReference(directive)));
+}
+
 function replaceCached(inner: string, fetcher: IncludeFetcher, cache: Map<string, string>): string {
   const cached = cache.get(inner);
   if (cached !== undefined) return cached;

package/src/parser/rules/block/module/include/scanner.ts

@@ -40,6 +40,8 @@ export function scanIncludeDirectives(source: string): IncludeDirectiveMatch[] {
 
     let depth = 0;
     let linkDepth = 0;
+    const nestedQuotes: Array<number | null> = [];
+    const nestedQuoteAllowed: boolean[] = [];
     let i = start;
     let closeEnd = -1;
 
@@ -48,7 +50,37 @@ export function scanIncludeDirectives(source: string): IncludeDirectiveMatch[] {
       const next = source.charCodeAt(i + 1);
       const nextNext = source.charCodeAt(i + 2);
 
-      if (ch === OPEN_BRACKET && next === OPEN_BRACKET && nextNext === OPEN_BRACKET) {
+      if (ch === 10 || ch === 13) {
+        linkDepth = 0;
+        if (depth > 1) {
+          depth = 1;
+          nestedQuotes.length = 0;
+          nestedQuoteAllowed.length = 0;
+        }
+        i++;
+        continue;
+      }
+
+      const nestedQuote = nestedQuotes.at(-1);
+      if (nestedQuote !== undefined && nestedQuote !== null) {
+        if (ch === 92) {
+          i += 2;
+        } else {
+          if (ch === nestedQuote) {
+            nestedQuotes[nestedQuotes.length - 1] = null;
+            nestedQuoteAllowed[nestedQuoteAllowed.length - 1] = false;
+          }
+          i++;
+        }
+      } else if (
+        nestedQuote === null &&
+        nestedQuoteAllowed.at(-1) === true &&
+        (ch === 34 || ch === 39)
+      ) {
+        nestedQuotes[nestedQuotes.length - 1] = ch;
+        nestedQuoteAllowed[nestedQuoteAllowed.length - 1] = false;
+        i++;
+      } else if (ch === OPEN_BRACKET && next === OPEN_BRACKET && nextNext === OPEN_BRACKET) {
         linkDepth++;
         i += 3;
       } else if (
@@ -62,10 +94,28 @@ export function scanIncludeDirectives(source: string): IncludeDirectiveMatch[] {
       } else if (linkDepth > 0) {
         i++;
       } else if (ch === OPEN_BRACKET && next === OPEN_BRACKET) {
+        if (nestedQuoteAllowed.length > 0) {
+          nestedQuoteAllowed[nestedQuoteAllowed.length - 1] = false;
+        }
+        if (depth > 0) {
+          nestedQuotes.push(null);
+          nestedQuoteAllowed.push(false);
+        }
         depth++;
         i += 2;
+      } else if (ch === CLOSE_BRACKET && next !== CLOSE_BRACKET && depth > 1) {
+        // A lone `]` is the first close of an intentionally incomplete nested
+        // token (for example `[[module ...]` passed through an include value).
+        depth--;
+        nestedQuotes.pop();
+        nestedQuoteAllowed.pop();
+        i++;
       } else if (ch === CLOSE_BRACKET && next === CLOSE_BRACKET) {
         const closeStart = i;
+        if (depth > 1) {
+          nestedQuotes.pop();
+          nestedQuoteAllowed.pop();
+        }
         depth--;
         i += 2;
 
@@ -83,6 +133,13 @@ export function scanIncludeDirectives(source: string): IncludeDirectiveMatch[] {
           }
         }
       } else {
+        if (nestedQuoteAllowed.length > 0) {
+          if (ch === 61) {
+            nestedQuoteAllowed[nestedQuoteAllowed.length - 1] = true;
+          } else if (ch !== 32 && ch !== 9) {
+            nestedQuoteAllowed[nestedQuoteAllowed.length - 1] = false;
+          }
+        }
         i++;
       }
     }

package/src/parser/rules/block/module/index.ts

@@ -82,6 +82,8 @@ export {
   parseParent,
   parseDateSelector,
   parseNumericSelector,
+  definePageData,
+  matchesListPagesSelectors,
 } from "./listpages";
 
 // IfTags module

package/src/parser/rules/block/module/listpages/index.ts

@@ -59,6 +59,8 @@ export { extractDataRequirements } from "./extract";
 // Resolution
 export type { ParseFunction, ListPagesModuleData } from "./resolve";
 export { isListPagesModule, resolveListPages } from "./resolve";
+export { definePageData } from "./types/external-data";
+export { matchesListPagesSelectors } from "./selectors";
 
 // Compiler
 export { compileTemplate } from "./compiler";

package/src/parser/rules/block/module/listpages/selectors.ts

@@ -0,0 +1,64 @@
+import type { PageData } from "./types/external-data";
+import type {
+  NormalizedCategory,
+  NormalizedListPagesQuery,
+  NormalizedTags,
+} from "./types/normalized-query";
+
+type SelectorPage = Pick<PageData, "category" | "tags" | "hiddenTags">;
+type CurrentPage = { category: string; tags: readonly string[] };
+
+export function matchesListPagesSelectors(
+  page: SelectorPage,
+  query: Pick<NormalizedListPagesQuery, "category" | "tags">,
+  currentPage: CurrentPage,
+): boolean {
+  return (
+    matchesCategory(page.category, query.category, currentPage.category) &&
+    matchesTags(page, query.tags, currentPage)
+  );
+}
+
+function matchesCategory(
+  category: string,
+  selector: NormalizedCategory | undefined,
+  currentCategory: string,
+): boolean {
+  if (!selector) return category === currentCategory;
+  if (selector.exclude.includes(category)) return false;
+
+  const hasPositiveSelector = selector.all || selector.current || selector.include.length > 0;
+  return (
+    !hasPositiveSelector ||
+    selector.all ||
+    (selector.current && category === currentCategory) ||
+    selector.include.includes(category)
+  );
+}
+
+function matchesTags(
+  page: Pick<SelectorPage, "tags" | "hiddenTags">,
+  selector: NormalizedTags | undefined,
+  currentPage: Pick<CurrentPage, "tags">,
+): boolean {
+  if (!selector) return true;
+
+  const allTags = new Set([...page.tags, ...page.hiddenTags]);
+  if (selector.all.some((tag) => !allTags.has(tag))) return false;
+  if (selector.any.length > 0 && !selector.any.some((tag) => allTags.has(tag))) return false;
+  if (selector.none.some((tag) => allTags.has(tag))) return false;
+
+  if (selector.special === "none") return allTags.size === 0;
+  const currentTags = new Set(currentPage.tags.filter((tag) => !tag.startsWith("_")));
+  if (selector.special === "same-visible") {
+    return page.tags.some((tag) => currentTags.has(tag));
+  }
+  if (selector.special === "same-all") {
+    return setsEqual(new Set(page.tags), currentTags);
+  }
+  return true;
+}
+
+function setsEqual(left: ReadonlySet<string>, right: ReadonlySet<string>): boolean {
+  return left.size === right.size && [...left].every((value) => right.has(value));
+}

package/src/parser/rules/block/module/listpages/types/external-data.ts

@@ -58,6 +58,28 @@ export interface PageData {
   revisions: number;
 }
 
+type PageDataInput = Pick<PageData, "fullname" | "title" | "createdAt" | "updatedAt" | "tags"> &
+  Partial<Omit<PageData, "fullname" | "title" | "createdAt" | "updatedAt" | "tags">>;
+
+export function definePageData(input: PageDataInput): PageData {
+  const separator = input.fullname.indexOf(":");
+  const category = separator === -1 ? "_default" : input.fullname.slice(0, separator);
+  const name = separator === -1 ? input.fullname : input.fullname.slice(separator + 1);
+
+  return {
+    ...input,
+    name: input.name ?? name,
+    category: input.category ?? category,
+    hiddenTags: input.hiddenTags ?? [],
+    children: input.children ?? 0,
+    comments: input.comments ?? 0,
+    size: input.size ?? 0,
+    rating: input.rating ?? 0,
+    ratingVotes: input.ratingVotes ?? 0,
+    revisions: input.revisions ?? 0,
+  };
+}
+
 /**
  * Site context information.
  */

package/src/parser/rules/contracts/parse-context.ts

@@ -3,6 +3,7 @@ import type {
   CodeBlockData,
   Diagnostic,
   Element,
+  PageRef,
   TocEntry,
   Version,
   WikitextSettings,
@@ -26,6 +27,7 @@ export interface ParseContext {
   trackPositions: boolean;
   settings: WikitextSettings;
   appendImplicitFootnoteBlock: boolean;
+  deferInclude?: (location: PageRef) => boolean;
   footnotes: Element[][];
   tocEntries: TocEntry[];
   codeBlocks: CodeBlockData[];