@wdprlib/parser 3.1.2 → 4.0.0

package/src/parser/rules/block/module/include/scanner.ts

@@ -0,0 +1,121 @@
+/**
+ * Matches the opening `[[include` token at the start of a line.
+ *
+ * The `m` flag makes `^` match at line boundaries, enforcing the Wikidot
+ * rule that `[[include]]` must appear at the start of a line. The trailing
+ * `\s` separates the directive name from its arguments. The actual extent
+ * of each directive is found by scanIncludeDirectives, which balances nested
+ * `[[ ... ]]` so that block markup inside a parameter value does not terminate
+ * the directive at the first `]]`.
+ */
+const INCLUDE_OPEN_PATTERN = /^\[\[include\s/gim;
+const OPEN_BRACKET = 91;
+const CLOSE_BRACKET = 93;
+
+/** A located `[[include ...]]` directive with bracket-balanced extent. */
+export interface IncludeDirectiveMatch {
+  /** Index of the opening `[[`. */
+  start: number;
+  /** Index just past the closing `]]`. */
+  end: number;
+  /** Text between `[[include ` and the closing `]]`. */
+  inner: string;
+}
+
+/**
+ * Find all `[[include ...]]` directives in `source`, choosing each
+ * closing `]]` so that block markup inside a parameter value does not
+ * end the directive prematurely.
+ */
+export function scanIncludeDirectives(source: string): IncludeDirectiveMatch[] {
+  const matches: IncludeDirectiveMatch[] = [];
+  const opener = INCLUDE_OPEN_PATTERN;
+  opener.lastIndex = 0;
+  let match: RegExpExecArray | null;
+
+  while ((match = opener.exec(source)) !== null) {
+    const start = match.index;
+    const contentStart = start + match[0].length;
+    const firstNewline = source.indexOf("\n", start);
+
+    let depth = 0;
+    let linkDepth = 0;
+    let i = start;
+    let closeEnd = -1;
+
+    while (i < source.length) {
+      const ch = source.charCodeAt(i);
+      const next = source.charCodeAt(i + 1);
+      const nextNext = source.charCodeAt(i + 2);
+
+      if (ch === OPEN_BRACKET && next === OPEN_BRACKET && nextNext === OPEN_BRACKET) {
+        linkDepth++;
+        i += 3;
+      } else if (
+        linkDepth > 0 &&
+        ch === CLOSE_BRACKET &&
+        next === CLOSE_BRACKET &&
+        nextNext === CLOSE_BRACKET
+      ) {
+        linkDepth--;
+        i += 3;
+      } else if (linkDepth > 0) {
+        i++;
+      } else if (ch === OPEN_BRACKET && next === OPEN_BRACKET) {
+        depth++;
+        i += 2;
+      } else if (ch === CLOSE_BRACKET && next === CLOSE_BRACKET) {
+        const closeStart = i;
+        depth--;
+        i += 2;
+
+        if (depth <= 0) {
+          const innerSoFar = source.slice(contentStart, closeStart);
+          if (hasAttributes(innerSoFar)) {
+            while (i < source.length && source[i] === "]") {
+              i++;
+            }
+          }
+          const onOpenerLine = firstNewline === -1 || closeStart < firstNewline;
+          if (onOpenerLine || isRestOfLineBlank(source, i)) {
+            closeEnd = i;
+            break;
+          }
+        }
+      } else {
+        i++;
+      }
+    }
+
+    if (closeEnd === -1) {
+      opener.lastIndex = start + 2;
+      continue;
+    }
+
+    matches.push({ start, end: closeEnd, inner: source.slice(contentStart, closeEnd - 2) });
+    opener.lastIndex = closeEnd;
+  }
+
+  return matches;
+}
+
+/**
+ * Returns true when the directive's inner content carries an attribute section,
+ * either pipe-delimited (`|key=value`) or space-separated after the page name.
+ */
+function hasAttributes(innerSoFar: string): boolean {
+  if (innerSoFar.includes("|")) return true;
+  const trimmed = innerSoFar.trimStart();
+  const whitespaceIndex = trimmed.search(/\s/);
+  if (whitespaceIndex === -1) return false;
+  return trimmed.slice(whitespaceIndex).trim().length > 0;
+}
+
+function isRestOfLineBlank(source: string, pos: number): boolean {
+  for (let i = pos; i < source.length; i++) {
+    const ch = source[i];
+    if (ch === "\n") return true;
+    if (ch !== " " && ch !== "\t" && ch !== "\r") return false;
+  }
+  return true;
+}

package/src/parser/rules/block/module/index.ts

@@ -0,0 +1,134 @@
+/**
+ * Module system for Wikidot dynamic constructs.
+ *
+ * Wikidot supports several "module" blocks (`[[module ListPages]]`,
+ * `[[module ListUsers]]`, `[[module CSS]]`, etc.) and pseudo-blocks
+ * (`[[include]]`, `[[iftags]]`) that require external data to resolve.
+ *
+ * This barrel module re-exports everything needed to:
+ *
+ * 1. **Extract** data requirements from a parsed AST
+ *    ({@link extractDataRequirements})
+ * 2. **Compile** body templates that contain `%%variable%%` placeholders
+ *    ({@link compileTemplate}, {@link compileListUsersTemplate})
+ * 3. **Resolve** modules by injecting fetched data back into the AST
+ *    ({@link resolveModules})
+ * 4. **Resolve includes** by fetching and inlining included pages
+ *    ({@link resolveIncludes})
+ *
+ * @module
+ */
+
+// Module rule types and registry
+export type { ModuleRule } from "./types";
+export { MODULE_RULES, getModuleRuleByName } from "./mapping";
+export { moduleRule } from "./rule";
+
+// Module parsers
+export { rateModuleRule } from "./rate/index";
+export { cssModuleRule } from "./css/index";
+export { backlinksModuleRule } from "./backlinks/index";
+export { categoriesModuleRule } from "./categories/index";
+export { joinModuleRule } from "./join/index";
+export { pageTreeModuleRule } from "./page-tree/index";
+export { listPagesModuleRule } from "./listpages/parser";
+export { listUsersModuleRule } from "./listusers/parser";
+
+// Module data types (parser-only modules)
+export type { RateModuleData } from "./rate/types";
+export type { BacklinksModuleData } from "./backlinks/types";
+export type { CategoriesModuleData } from "./categories/types";
+export type { JoinModuleData } from "./join/types";
+export type { PageTreeModuleData } from "./page-tree/types";
+
+// Common types
+export type { DataProvider } from "./types-common";
+
+// ListPages module
+export type {
+  ListPagesQuery,
+  ListPagesVariable,
+  ListPagesDataRequirement,
+  DataRequirements,
+  UserInfo,
+  PageData,
+  SiteContext,
+  ListPagesExternalData,
+  ListPagesDataFetcher,
+  VariableContext,
+  CompiledTemplate,
+  ExtractionResult,
+  ParseFunction,
+  ListPagesModuleData,
+  // Normalized types
+  NormalizedListPagesQuery,
+  NormalizedTags,
+  NormalizedCategory,
+  NormalizedOrder,
+  NormalizedParent,
+  NormalizedDateSelector,
+  NormalizedNumericSelector,
+} from "./listpages";
+export {
+  extractDataRequirements,
+  isListPagesModule,
+  resolveListPages,
+  compileTemplate,
+  // Query normalization (for advanced use cases)
+  normalizeQuery,
+  parseTags,
+  parseCategory,
+  parseOrder,
+  parseParent,
+  parseDateSelector,
+  parseNumericSelector,
+} from "./listpages";
+
+// IfTags module
+export type { TagCondition, IfTagsResolver, IfTagsData, IfTagsResolveResult } from "./iftags";
+export {
+  parseTagCondition,
+  evaluateTagCondition,
+  isIfTagsElement,
+  resolveIfTags,
+  preprocessIftags,
+} from "./iftags";
+
+// Include module
+export type { IncludeFetcher, AsyncIncludeFetcher, ResolveIncludesOptions } from "./include";
+export type {
+  IncludeReference,
+  IncludeDependency,
+  IncludeIterationTrace,
+  ResolveIncludesTraceResult,
+} from "./include";
+export {
+  extractIncludeReferences,
+  resolveIncludes,
+  resolveIncludesAsync,
+  resolveIncludesWithTrace,
+} from "./include";
+
+// ListUsers module
+export type {
+  ListUsersVariable,
+  ListUsersUserData,
+  ListUsersDataRequirement,
+  ListUsersExternalData,
+  ListUsersDataFetcher,
+  ListUsersVariableContext,
+  ListUsersCompiledTemplate,
+  ListUsersModuleData,
+} from "./listusers";
+export {
+  listUsersModuleRule as listUsersRule,
+  extractListUsersVariables,
+  compileListUsersTemplate,
+  isListUsersModule,
+  resolveListUsers,
+} from "./listusers";
+
+// Module resolver
+export type { ModuleSourceTransform, ResolveOptions } from "./resolve";
+export { resolveModules } from "./resolve";
+export { STYLE_SLOT_PREFIX } from "@wdprlib/ast";

package/src/parser/rules/block/module/join/index.ts

@@ -0,0 +1,34 @@
+/**
+ *
+ * Parser rule for the Wikidot `[[module Join]]` block.
+ *
+ * Renders a "Join this site" button. Accepts an optional `button` attribute
+ * to customize the button text.
+ *
+ * @module
+ */
+
+import type { ModuleRule } from "../types";
+import type { JoinModuleData } from "./types";
+
+/**
+ * Module rule for `[[module Join]]`.
+ *
+ * Extracts the `button` attribute as the custom button text. All other
+ * attributes are passed through in the `attributes` record for the
+ * rendering application to handle.
+ */
+export const joinModuleRule: ModuleRule = {
+  name: "module-join",
+  acceptsNames: ["join"],
+  hasBody: false,
+
+  parse(_ctx, _pos, args): JoinModuleData {
+    const { button, ...rest } = args;
+    return {
+      module: "join",
+      "button-text": button ?? null,
+      attributes: rest,
+    };
+  },
+};

package/src/parser/rules/block/module/join/types.ts

@@ -0,0 +1,23 @@
+/**
+ *
+ * Type definitions for the Join module.
+ *
+ * The `[[module Join]]` block renders a "Join this site" button that allows
+ * visitors to apply for site membership.
+ *
+ * @module
+ */
+
+/**
+ * AST data for a `[[module Join]]` element.
+ *
+ * The rendering application should display a membership application button
+ * with the specified text.
+ */
+export interface JoinModuleData {
+  module: "join";
+  /** Custom button text, or null to use the default "Join this Site" text */
+  "button-text": string | null;
+  /** Additional attributes passed to the module */
+  attributes: Record<string, string>;
+}

package/src/parser/rules/block/module/listpages/compiler.ts

@@ -0,0 +1,73 @@
+/**
+ *
+ * Template compiler for the ListPages module.
+ *
+ * Compiles ListPages template strings (e.g., `"%%title%% by %%created_by%%"`)
+ * into executable functions that can be called repeatedly with different page
+ * data for fast rendering. The compilation step splits the template into static
+ * string segments and dynamic getter functions, avoiding repeated regex matching
+ * during rendering.
+ *
+ * Supported variable syntax:
+ * - `%%name%%` - Simple variable (e.g., `%%title%%`, `%%rating%%`)
+ * - `%%name{param}%%` - Parameterized variable (e.g., `%%content{2}%%`, `%%form_data{color}%%`)
+ * - `%%name(param)%%` - Parenthesized parameter (e.g., `%%preview(100)%%`)
+ * - `%%name|format%%` - Formatted variable (e.g., `%%created_at|%Y-%m-%d%%`, `%%tags_linked|/tag/%%`)
+ *
+ * The compiled function is a closure over the parsed template parts, providing
+ * O(n) rendering time proportional to the number of template segments.
+ *
+ * @module
+ */
+
+import type { CompiledTemplate, VariableContext } from "./types";
+import { createVariableGetter } from "./template/getters";
+import { scanTemplateVariables } from "./template/syntax";
+
+/**
+ * Compile a ListPages template string into an executable function.
+ *
+ * The template is split into alternating static strings and dynamic getter
+ * functions. The returned function concatenates these parts with the getter
+ * functions evaluated against the provided variable context.
+ *
+ * @param template - The template string containing `%%variable%%` placeholders
+ * @returns A compiled function that accepts a `VariableContext` and returns the rendered string
+ */
+export function compileTemplate(template: string): CompiledTemplate {
+  const parts: (string | ((ctx: VariableContext) => string))[] = [];
+  let lastIndex = 0;
+
+  // Split template into static and dynamic parts
+  for (const match of scanTemplateVariables(template)) {
+    // Add static part before this match
+    if (match.index > lastIndex) {
+      parts.push(template.slice(lastIndex, match.index));
+    }
+
+    // Convert variable to getter function
+    const getter = createVariableGetter(
+      match.name.toLowerCase(),
+      match.braceParam,
+      match.parenParam,
+      match.format,
+    );
+    parts.push(getter);
+
+    lastIndex = match.index + match.raw.length;
+  }
+
+  // Add remaining static part
+  if (lastIndex < template.length) {
+    parts.push(template.slice(lastIndex));
+  }
+
+  // Return compiled function
+  return (ctx: VariableContext): string => {
+    let result = "";
+    for (const part of parts) {
+      result += typeof part === "string" ? part : part(ctx);
+    }
+    return result;
+  };
+}

package/src/parser/rules/block/module/listpages/extract.ts

@@ -0,0 +1,76 @@
+/**
+ *
+ * Data requirement extraction from parsed ASTs.
+ *
+ * After parsing, the AST may contain ListPages and ListUsers module nodes that
+ * need external data (page lists, user information) to be resolved. This module
+ * analyzes the AST to find all such modules, determines what variables their
+ * templates use (and therefore what data fields the external provider must supply),
+ * and pre-compiles their templates for efficient rendering during the resolution phase.
+ *
+ * The extraction result includes:
+ * - `DataRequirements` listing all ListPages/ListUsers queries with their needed variables
+ * - Pre-compiled template functions keyed by module ID for fast rendering
+ *
+ * This is the first phase of the three-phase ListPages lifecycle:
+ * 1. **Extract** (this module) - Analyze AST, determine data needs, compile templates
+ * 2. **Fetch** (external) - Application fetches data based on requirements
+ * 3. **Resolve** - Substitute fetched data into compiled templates and re-parse
+ *
+ * @module
+ */
+
+import type { SyntaxTree } from "@wdprlib/ast";
+import { walkElements } from "../walk";
+import {
+  extractListPagesModule,
+  isListPagesModule,
+  type ListPagesExtractionState,
+} from "./extraction/listpages";
+import {
+  extractListUsersModule,
+  isListUsersModule,
+  type ListUsersExtractionState,
+} from "./extraction/listusers";
+import type { ExtractionResult } from "./extraction/result";
+export type { ExtractionResult } from "./extraction/result";
+
+/**
+ * Extract all data requirements from a parsed AST.
+ *
+ * Walks the entire AST to find ListPages and ListUsers module elements,
+ * analyzes their templates to determine which variables are used, builds
+ * query objects from their attributes, and pre-compiles their templates.
+ *
+ * Each module is assigned a sequential ID (separate counters for ListPages
+ * and ListUsers) that is used to correlate requirements with fetched data
+ * and compiled templates during the resolution phase.
+ *
+ * @param ast - The parsed syntax tree to analyze
+ * @returns Extraction result containing requirements and compiled templates
+ */
+export function extractDataRequirements(ast: SyntaxTree): ExtractionResult {
+  const result: ExtractionResult = {
+    requirements: {
+      listPages: [],
+      listUsers: [],
+    },
+    compiledListPagesTemplates: new Map(),
+    compiledListUsersTemplates: new Map(),
+  };
+
+  const listPagesState: ListPagesExtractionState = { nextId: 0 };
+  const listUsersState: ListUsersExtractionState = { nextId: 0 };
+
+  walkElements(ast.elements, (element) => {
+    if (element.element !== "module") return;
+
+    if (isListPagesModule(element.data)) {
+      extractListPagesModule(element.data, listPagesState, result);
+    } else if (isListUsersModule(element.data)) {
+      extractListUsersModule(element.data, listUsersState, result);
+    }
+  });
+
+  return result;
+}

package/src/parser/rules/block/module/listpages/extraction/listpages.ts

@@ -0,0 +1,42 @@
+import type { Module } from "@wdprlib/ast";
+import type { ListPagesQuery } from "../types";
+import { compileTemplate } from "../compiler";
+import { buildQuery } from "./query";
+import { DEFAULT_BODY_TEMPLATE, extractVariablesFromTemplate } from "./template";
+import type { ExtractionResult } from "./result";
+
+export type ListPagesModuleForExtraction = Extract<Module, { module: "list-pages" }>;
+
+export interface ListPagesExtractionState {
+  nextId: number;
+}
+
+export function isListPagesModule(module: Module): module is ListPagesModuleForExtraction {
+  return module.module === "list-pages";
+}
+
+export function extractListPagesModule(
+  listPages: ListPagesModuleForExtraction,
+  state: ListPagesExtractionState,
+  result: ExtractionResult,
+): void {
+  const id = state.nextId++;
+  const body = listPages.body ?? DEFAULT_BODY_TEMPLATE;
+  const extraction = extractVariablesFromTemplate(body);
+  const query: ListPagesQuery = buildQuery(listPages);
+
+  result.requirements.listPages.push({
+    id,
+    query,
+    neededVariables: extraction.variables,
+    contentSectionIndices: extraction.contentIndices,
+    previewLengths: extraction.previewLengths,
+    formFields: extraction.formFields,
+    tagsLinkPrefix: extraction.tagsLinkPrefix,
+    hiddenTagsLinkPrefix: extraction.hiddenTagsLinkPrefix,
+    urlAttrPrefix: listPages["url-attr-prefix"],
+    rawAttributes: listPages.attributes ?? {},
+  });
+
+  result.compiledListPagesTemplates.set(id, compileTemplate(body));
+}

package/src/parser/rules/block/module/listpages/extraction/listusers.ts

@@ -0,0 +1,30 @@
+import type { Module } from "@wdprlib/ast";
+import { compileListUsersTemplate, extractListUsersVariables } from "../../listusers/extract";
+import type { ExtractionResult } from "./result";
+
+export type ListUsersModuleForExtraction = Extract<Module, { module: "list-users" }>;
+
+export interface ListUsersExtractionState {
+  nextId: number;
+}
+
+export function isListUsersModule(module: Module): module is ListUsersModuleForExtraction {
+  return module.module === "list-users";
+}
+
+export function extractListUsersModule(
+  listUsers: ListUsersModuleForExtraction,
+  state: ListUsersExtractionState,
+  result: ExtractionResult,
+): void {
+  const id = state.nextId++;
+  const body = listUsers.body ?? "";
+
+  result.requirements.listUsers.push({
+    id,
+    users: listUsers.users,
+    neededVariables: extractListUsersVariables(body),
+  });
+
+  result.compiledListUsersTemplates.set(id, compileListUsersTemplate(body));
+}

package/src/parser/rules/block/module/listpages/extraction/query.ts

@@ -0,0 +1,51 @@
+import type { Module } from "@wdprlib/ast";
+import type { ListPagesQuery } from "../types";
+
+type ListPagesModuleData = Extract<Module, { module: "list-pages" }>;
+
+/**
+ * Build a ListPagesQuery from the parsed module attributes.
+ *
+ * Maps module attribute names, which use kebab-case in the AST, to the query's
+ * camelCase property names. Data form fields prefixed with `_` are grouped under
+ * `dataFormFields`.
+ */
+export function buildQuery(module: ListPagesModuleData): ListPagesQuery {
+  return {
+    pagetype: module.pagetype as ListPagesQuery["pagetype"],
+    category: module.category,
+    tags: module.tags,
+    parent: module.parent,
+    linkTo: module["link-to"],
+    createdAt: module["created-at"],
+    updatedAt: module["updated-at"],
+    createdBy: module["created-by"],
+    rating: module.rating,
+    votes: module.votes,
+    name: module.name,
+    fullname: module.fullname,
+    range: module.range as ListPagesQuery["range"],
+    order: module.order,
+    offset: module.offset,
+    limit: module.limit,
+    perPage: module["per-page"],
+    reverse: module.reverse,
+    dataFormFields: extractDataFormFields(module.attributes),
+  };
+}
+
+function extractDataFormFields(
+  attributes: Record<string, string>,
+): Record<string, string> | undefined {
+  const fields: Record<string, string> = {};
+  let hasFields = false;
+
+  for (const [key, value] of Object.entries(attributes)) {
+    if (key.startsWith("_")) {
+      fields[key.slice(1)] = value;
+      hasFields = true;
+    }
+  }
+
+  return hasFields ? fields : undefined;
+}

package/src/parser/rules/block/module/listpages/extraction/result.ts

@@ -0,0 +1,18 @@
+import type { DataRequirements, CompiledTemplate } from "../types";
+import type { ListUsersCompiledTemplate } from "../../listusers/types";
+
+/**
+ * Complete result of extracting data requirements from an AST.
+ *
+ * Contains everything needed to fetch external data and then resolve modules:
+ * the data requirements tell the application what to fetch, and the pre-compiled
+ * templates are used during the resolution phase to efficiently render results.
+ */
+export interface ExtractionResult {
+  /** Data requirements for external fetching */
+  requirements: DataRequirements;
+  /** Pre-compiled ListPages templates keyed by module id */
+  compiledListPagesTemplates: Map<number, CompiledTemplate>;
+  /** Pre-compiled ListUsers templates keyed by module id */
+  compiledListUsersTemplates: Map<number, ListUsersCompiledTemplate>;
+}