@wdprlib/parser 4.3.0 → 5.0.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wdprlib/parser",
-  "version": "4.3.0",
+  "version": "5.0.0",
   "description": "Parser for Wikidot markup",
   "keywords": [
     "ast",
@@ -41,6 +41,6 @@
   },
   "dependencies": {
     "@braintree/sanitize-url": "^7.1.1",
-    "@wdprlib/ast": "2.3.0"
+    "@wdprlib/ast": "3.0.0"
   }
 }

package/src/index.ts

@@ -98,6 +98,15 @@ export { Lexer, tokenize, createToken } from "./lexer";
 export type { ParserOptions } from "./parser";
 export { Parser, parse } from "./parser";
 
+// High-level parser pipeline
+export { processWikitext } from "./pipeline";
+export type {
+  ProcessedWikitextDocument,
+  ProcessWikitextCallbackContext,
+  ProcessWikitextDataProvider,
+  ProcessWikitextOptions,
+} from "./pipeline";
+
 // Modules (ListPages, ListUsers, IfTags, Include, etc.)
 export type {
   // ListPages query types
@@ -121,6 +130,7 @@ export type {
   ExtractionResult,
   // Resolution types
   ParseFunction,
+  ModuleParseResult,
   ModuleSourceTransform,
   ResolveOptions,
   // Include resolution
@@ -163,6 +173,7 @@ export {
   extractIncludeReferences,
   resolveIncludes,
   resolveIncludesAsync,
+  resolveIncludesAsyncWithTrace,
   resolveIncludesWithTrace,
   // IfTags source-level preprocessing (run between include expansion and parse)
   preprocessIftags,
@@ -174,6 +185,8 @@ export {
   parseParent,
   parseDateSelector,
   parseNumericSelector,
+  definePageData,
+  matchesListPagesSelectors,
   // ListUsers
   extractListUsersVariables,
   compileListUsersTemplate,

package/src/parser/parse/context.ts

@@ -10,6 +10,7 @@ export function createParseContext(tokens: Token[], options: ParserOptions = {})
     version: options.version ?? "wikidot",
     trackPositions: options.trackPositions ?? true,
     settings: options.settings ?? DEFAULT_SETTINGS,
+    appendImplicitFootnoteBlock: options.appendImplicitFootnoteBlock ?? true,
     footnotes: [],
     tocEntries: [],
     codeBlocks: [],

package/src/parser/parse/options.ts

@@ -31,4 +31,10 @@ export interface ParserOptions {
    * - `string[]`: every iftags block is evaluated against the given tags eagerly.
    */
   pageTags?: string[] | null;
+  /**
+   * Append the implicit document-level footnote block when no explicit block
+   * exists. Defaults to true. Fragment pipelines can disable this and add one
+   * block after all fragments have been merged.
+   */
+  appendImplicitFootnoteBlock?: boolean;
 }

package/src/parser/parse/result.ts

@@ -10,7 +10,9 @@ import { containsFootnoteBlock } from "./footnotes";
 
 export function finalizeParseResult(ctx: ParseContext, children: Element[]): ParseResult {
   const cleanedChildren = postprocessChildren(children);
-  appendImplicitFootnoteBlock(cleanedChildren);
+  if (ctx.appendImplicitFootnoteBlock) {
+    appendImplicitFootnoteBlock(cleanedChildren);
+  }
 
   return {
     ast: buildSyntaxTree(ctx, cleanedChildren),

package/src/parser/preprocess/expr/index.ts

@@ -7,8 +7,11 @@
  */
 
 import { makeUniqueSentinels, maskRawRegions, restorePlaceholders } from "../utils";
+import { matchDirectiveKind } from "./kind";
 import { expandInnermost } from "./scan";
 
+const MAX_EXPR_NESTING = 64;
+
 /**
  * Resolve every `[[#if]]` / `[[#ifexpr]]` / `[[#expr]]` that sits inside
  * another block's opener (depth > 0). Top-level directives are left for
@@ -21,6 +24,7 @@ export function preprocessExpr(source: string): string {
 
   const sentinels = makeUniqueSentinels(source);
   const { masked, placeholders } = maskRawRegions(source, sentinels);
+  if (exceedsExprNestingLimit(masked)) return source;
   const reduced = reduceExpr(masked);
   return restorePlaceholders(reduced, placeholders, sentinels);
 }
@@ -43,3 +47,25 @@ function reduceExpr(source: string): string {
   }
   return current;
 }
+
+function exceedsExprNestingLimit(source: string): boolean {
+  const expressionStack: boolean[] = [];
+  let expressionDepth = 0;
+
+  for (let i = 0; i < source.length; i++) {
+    if (source.startsWith("[[", i)) {
+      const isExpression = matchDirectiveKind(source, i) !== null;
+      expressionStack.push(isExpression);
+      if (isExpression && ++expressionDepth > MAX_EXPR_NESTING) return true;
+      i++;
+      continue;
+    }
+
+    if (source.startsWith("]]", i)) {
+      if (expressionStack.pop()) expressionDepth--;
+      i++;
+    }
+  }
+
+  return false;
+}

package/src/parser/preprocess/utils/raw-regions.ts

@@ -13,16 +13,25 @@ export interface Sentinels {
  * Choose sentinel strings that are guaranteed not to appear in `source`.
  * The placeholders we splice into the masked source have the form
  * `<open><digits><close>`, so the restore pass must not confuse them
- * with content. Extends both sentinel characters until neither appears.
+ * with content.
  */
 export function makeUniqueSentinels(source: string): Sentinels {
-  let open = BASE_PLACEHOLDER_OPEN;
-  let close = BASE_PLACEHOLDER_CLOSE;
-  while (source.includes(open) || source.includes(close)) {
-    open += BASE_PLACEHOLDER_OPEN;
-    close += BASE_PLACEHOLDER_CLOSE;
+  let openRun = 0;
+  let closeRun = 0;
+  let longestOpenRun = 0;
+  let longestCloseRun = 0;
+
+  for (const char of source) {
+    openRun = char === BASE_PLACEHOLDER_OPEN ? openRun + 1 : 0;
+    closeRun = char === BASE_PLACEHOLDER_CLOSE ? closeRun + 1 : 0;
+    longestOpenRun = Math.max(longestOpenRun, openRun);
+    longestCloseRun = Math.max(longestCloseRun, closeRun);
   }
-  return { open, close };
+
+  return {
+    open: BASE_PLACEHOLDER_OPEN.repeat(longestOpenRun + 1),
+    close: BASE_PLACEHOLDER_CLOSE.repeat(longestCloseRun + 1),
+  };
 }
 
 /**

package/src/parser/rules/block/module/include/index.ts

@@ -16,7 +16,12 @@
  * @module
  */
 
-export { resolveIncludes, resolveIncludesAsync, resolveIncludesWithTrace } from "./resolve";
+export {
+  resolveIncludes,
+  resolveIncludesAsync,
+  resolveIncludesAsyncWithTrace,
+  resolveIncludesWithTrace,
+} from "./resolve";
 export { extractIncludeReferences } from "./references";
 export type {
   IncludeFetcher,

package/src/parser/rules/block/module/include/resolve/index.ts

@@ -7,7 +7,11 @@
 
 import type { PageRef } from "@wdprlib/ast";
 import { createCachedAsyncIncludeFetcher, createCachedIncludeFetcher } from "./cache";
-import { expandIterative, expandIterativeAsync, expandIterativeWithTrace } from "./iterate";
+import {
+  expandIterative,
+  expandIterativeAsyncWithTrace,
+  expandIterativeWithTrace,
+} from "./iterate";
 import type {
   AsyncIncludeFetcher,
   IncludeFetcher,
@@ -84,13 +88,29 @@ export async function resolveIncludesAsync(
   fetcher: AsyncIncludeFetcher,
   options?: ResolveIncludesOptions,
 ): Promise<string> {
+  return (await resolveIncludesAsyncWithTrace(source, fetcher, options)).source;
+}
+
+/**
+ * Async include expansion with dependency and iteration trace data.
+ */
+export async function resolveIncludesAsyncWithTrace(
+  source: string,
+  fetcher: AsyncIncludeFetcher,
+  options?: ResolveIncludesOptions,
+): Promise<ResolveIncludesTraceResult> {
   if (options?.settings && !options.settings.enablePageSyntax) {
-    return source;
+    return {
+      source,
+      dependencies: [],
+      iterations: [],
+      reachedMaxIterations: false,
+    };
   }
 
   const maxIterations = options?.maxIterations ?? 10;
   const cachedFetcher = createCachedAsyncIncludeFetcher(fetcher, normalizePageKey);
-  return expandIterativeAsync(source, cachedFetcher, maxIterations);
+  return expandIterativeAsyncWithTrace(source, cachedFetcher, maxIterations);
 }
 
 /**

package/src/parser/rules/block/module/include/resolve/iterate.ts

@@ -102,6 +102,41 @@ export async function expandIterativeAsync(
   return current;
 }
 
+/** Async counterpart of {@link expandIterativeWithTrace}. */
+export async function expandIterativeAsyncWithTrace(
+  source: string,
+  fetcher: AsyncIncludeFetcher,
+  maxIterations: number,
+): Promise<ResolveIncludesTraceResult> {
+  let current = source;
+  const replacementCache = new Map<string, Promise<string>>();
+  const dependencies: IncludeDependency[] = [];
+  const iterations: ResolveIncludesTraceResult["iterations"] = [];
+
+  for (let i = 0; i < maxIterations; i++) {
+    const expanded = await expandOneIterationAsyncWithTrace(current, fetcher, replacementCache, i);
+    if (expanded === null) break;
+
+    dependencies.push(...expanded.dependencies);
+    iterations.push({
+      iteration: i,
+      directives: expanded.references,
+      changed: expanded.source !== current,
+    });
+
+    if (expanded.source === current) break;
+    current = expanded.source;
+  }
+
+  return {
+    source: current,
+    dependencies,
+    iterations,
+    reachedMaxIterations:
+      iterations.length === maxIterations && scanIncludeDirectives(current).length > 0,
+  };
+}
+
 function expandOneIteration(
   source: string,
   fetcher: IncludeFetcher,
@@ -179,6 +214,42 @@ async function expandOneIterationAsync(
   return parts.join("");
 }
 
+async function expandOneIterationAsyncWithTrace(
+  source: string,
+  fetcher: AsyncIncludeFetcher,
+  replacementCache: Map<string, Promise<string>>,
+  iteration: number,
+): Promise<{
+  source: string;
+  references: IncludeReference[];
+  dependencies: IncludeDependency[];
+} | null> {
+  if (!MAYBE_INCLUDE_PATTERN.test(source)) return null;
+
+  const directives = scanIncludeDirectives(source);
+  if (directives.length === 0) return null;
+
+  const references = directives.map(createIncludeReference);
+  const replacements = await Promise.all(
+    directives.map(({ inner }) => replaceCachedAsync(inner, fetcher, replacementCache)),
+  );
+  const parts: string[] = [];
+  let lastPos = 0;
+
+  for (let i = 0; i < directives.length; i++) {
+    const { start, end } = directives[i]!;
+    parts.push(source.slice(lastPos, start), replacements[i]!);
+    lastPos = end;
+  }
+  parts.push(source.slice(lastPos));
+
+  return {
+    source: parts.join(""),
+    references,
+    dependencies: references.map((reference) => ({ ...reference, iteration })),
+  };
+}
+
 function replaceCached(inner: string, fetcher: IncludeFetcher, cache: Map<string, string>): string {
   const cached = cache.get(inner);
   if (cached !== undefined) return cached;

package/src/parser/rules/block/module/index.ts

@@ -20,7 +20,7 @@
  */
 
 // Module rule types and registry
-export type { ModuleRule } from "./types";
+export type { ModuleParseResult, ModuleRule } from "./types";
 export { MODULE_RULES, getModuleRuleByName } from "./mapping";
 export { moduleRule } from "./rule";
 
@@ -82,6 +82,8 @@ export {
   parseParent,
   parseDateSelector,
   parseNumericSelector,
+  definePageData,
+  matchesListPagesSelectors,
 } from "./listpages";
 
 // IfTags module
@@ -106,6 +108,7 @@ export {
   extractIncludeReferences,
   resolveIncludes,
   resolveIncludesAsync,
+  resolveIncludesAsyncWithTrace,
   resolveIncludesWithTrace,
 } from "./include";
 

package/src/parser/rules/block/module/listpages/index.ts

@@ -59,6 +59,8 @@ export { extractDataRequirements } from "./extract";
 // Resolution
 export type { ParseFunction, ListPagesModuleData } from "./resolve";
 export { isListPagesModule, resolveListPages } from "./resolve";
+export { definePageData } from "./types/external-data";
+export { matchesListPagesSelectors } from "./selectors";
 
 // Compiler
 export { compileTemplate } from "./compiler";

package/src/parser/rules/block/module/listpages/resolution/items.ts

@@ -1,6 +1,6 @@
 import type { Element } from "@wdprlib/ast";
 import type { CompiledTemplate, ListPagesExternalData, VariableContext } from "../types";
-import type { ParseFunction } from "../../types";
+import { getModuleParseAst, type ParseFunction } from "../../types";
 import type { ListPagesModuleData } from "../resolve";
 
 export function renderListPagesItems(
@@ -23,7 +23,7 @@ export function renderListPagesItems(
       site: data.site,
     };
 
-    const itemAst = parse(compiledTemplate(ctx));
+    const itemAst = getModuleParseAst(parse(compiledTemplate(ctx)));
 
     if (module.separate) {
       items.push({

package/src/parser/rules/block/module/listpages/resolution/wrapper.ts

@@ -1,5 +1,5 @@
 import type { Element } from "@wdprlib/ast";
-import type { ParseFunction } from "../../types";
+import { getModuleParseAst, type ParseFunction } from "../../types";
 import type { ListPagesModuleData } from "../resolve";
 
 export function wrapListPagesResult(
@@ -14,14 +14,14 @@ export function wrapListPagesResult(
   const result: Element[] = [];
 
   if (module["prepend-line"] && !module.separate) {
-    const prependAst = parse(module["prepend-line"]);
+    const prependAst = getModuleParseAst(parse(module["prepend-line"]));
     result.push(...prependAst.elements);
   }
 
   result.push(...items);
 
   if (module["append-line"] && !module.separate) {
-    const appendAst = parse(module["append-line"]);
+    const appendAst = getModuleParseAst(parse(module["append-line"]));
     result.push(...appendAst.elements);
   }
 

package/src/parser/rules/block/module/listpages/selectors.ts

@@ -0,0 +1,64 @@
+import type { PageData } from "./types/external-data";
+import type {
+  NormalizedCategory,
+  NormalizedListPagesQuery,
+  NormalizedTags,
+} from "./types/normalized-query";
+
+type SelectorPage = Pick<PageData, "category" | "tags" | "hiddenTags">;
+type CurrentPage = { category: string; tags: readonly string[] };
+
+export function matchesListPagesSelectors(
+  page: SelectorPage,
+  query: Pick<NormalizedListPagesQuery, "category" | "tags">,
+  currentPage: CurrentPage,
+): boolean {
+  return (
+    matchesCategory(page.category, query.category, currentPage.category) &&
+    matchesTags(page, query.tags, currentPage)
+  );
+}
+
+function matchesCategory(
+  category: string,
+  selector: NormalizedCategory | undefined,
+  currentCategory: string,
+): boolean {
+  if (!selector) return category === currentCategory;
+  if (selector.exclude.includes(category)) return false;
+
+  const hasPositiveSelector = selector.all || selector.current || selector.include.length > 0;
+  return (
+    !hasPositiveSelector ||
+    selector.all ||
+    (selector.current && category === currentCategory) ||
+    selector.include.includes(category)
+  );
+}
+
+function matchesTags(
+  page: Pick<SelectorPage, "tags" | "hiddenTags">,
+  selector: NormalizedTags | undefined,
+  currentPage: Pick<CurrentPage, "tags">,
+): boolean {
+  if (!selector) return true;
+
+  const allTags = new Set([...page.tags, ...page.hiddenTags]);
+  if (selector.all.some((tag) => !allTags.has(tag))) return false;
+  if (selector.any.length > 0 && !selector.any.some((tag) => allTags.has(tag))) return false;
+  if (selector.none.some((tag) => allTags.has(tag))) return false;
+
+  if (selector.special === "none") return allTags.size === 0;
+  const currentTags = new Set(currentPage.tags.filter((tag) => !tag.startsWith("_")));
+  if (selector.special === "same-visible") {
+    return page.tags.some((tag) => currentTags.has(tag));
+  }
+  if (selector.special === "same-all") {
+    return setsEqual(new Set(page.tags), currentTags);
+  }
+  return true;
+}
+
+function setsEqual(left: ReadonlySet<string>, right: ReadonlySet<string>): boolean {
+  return left.size === right.size && [...left].every((value) => right.has(value));
+}

package/src/parser/rules/block/module/listpages/types/external-data.ts

@@ -58,6 +58,28 @@ export interface PageData {
   revisions: number;
 }
 
+type PageDataInput = Pick<PageData, "fullname" | "title" | "createdAt" | "updatedAt" | "tags"> &
+  Partial<Omit<PageData, "fullname" | "title" | "createdAt" | "updatedAt" | "tags">>;
+
+export function definePageData(input: PageDataInput): PageData {
+  const separator = input.fullname.indexOf(":");
+  const category = separator === -1 ? "_default" : input.fullname.slice(0, separator);
+  const name = separator === -1 ? input.fullname : input.fullname.slice(separator + 1);
+
+  return {
+    ...input,
+    name: input.name ?? name,
+    category: input.category ?? category,
+    hiddenTags: input.hiddenTags ?? [],
+    children: input.children ?? 0,
+    comments: input.comments ?? 0,
+    size: input.size ?? 0,
+    rating: input.rating ?? 0,
+    ratingVotes: input.ratingVotes ?? 0,
+    revisions: input.revisions ?? 0,
+  };
+}
+
 /**
  * Site context information.
  */

package/src/parser/rules/block/module/listusers/resolve.ts

@@ -15,7 +15,7 @@ import type {
   ListUsersCompiledTemplate,
   ListUsersVariableContext,
 } from "./types";
-import type { ParseFunction } from "../types";
+import { getModuleParseAst, type ParseFunction } from "../types";
 
 /**
  * Narrowed type for the list-users variant of the Module discriminated union.
@@ -53,6 +53,6 @@ export function resolveListUsers(
 ): Element[] {
   const ctx: ListUsersVariableContext = { user: data.user };
   const substituted = compiledTemplate(ctx);
-  const itemAst = parse(substituted);
+  const itemAst = getModuleParseAst(parse(substituted));
   return itemAst.elements;
 }