@wdprlib/parser 3.2.0 → 4.0.0

package/src/parser/rules/block/module/listpages/extract.ts

@@ -20,78 +20,20 @@
  * @module
  */
 
-import type { SyntaxTree, Module } from "@wdprlib/ast";
-import type {
-  DataRequirements,
-  ListPagesQuery,
-  ListPagesVariable,
-  CompiledTemplate,
-} from "./types";
-import type { ListUsersCompiledTemplate } from "../listusers/types";
-import { compileTemplate } from "./compiler";
-import { extractListUsersVariables, compileListUsersTemplate } from "../listusers/extract";
+import type { SyntaxTree } from "@wdprlib/ast";
 import { walkElements } from "../walk";
-
-/**
- * Type guard to narrow a Module union to the list-pages variant.
- *
- * @param module - A Module discriminated union value
- * @returns true if the module is a list-pages module
- */
-function isListPagesModule(module: Module): module is Extract<Module, { module: "list-pages" }> {
-  return module.module === "list-pages";
-}
-
-/**
- * Regex pattern for matching ListPages template variables.
- *
- * Matches `%%name%%`, `%%name{param}%%`, `%%name(param)%%`, and `%%name|format%%`.
- * The captured groups are: [1] variable name, [2] brace parameter, [3] paren parameter,
- * [4] format/pipe parameter.
- */
-const VARIABLE_REGEX = /%%([a-z_]+)(?:\{([^}]+)\})?(?:\((\d+)\))?(?:\|([^%]+))?%%/gi;
-
-/**
- * Default template used when a ListPages module has no body specified.
- * This matches Wikidot's built-in default template that shows title, author, date, and summary.
- */
-const DEFAULT_BODY_TEMPLATE = `+ %%title_linked%%
-
-by %%created_by_linked%% %%created_at%%
-
-%%summary%%`;
-
-/**
- * Complete result of extracting data requirements from an AST.
- *
- * Contains everything needed to fetch external data and then resolve modules:
- * the data requirements tell the application what to fetch, and the pre-compiled
- * templates are used during the resolution phase to efficiently render results.
- */
-export interface ExtractionResult {
-  /** Data requirements for external fetching */
-  requirements: DataRequirements;
-  /** Pre-compiled ListPages templates keyed by module id */
-  compiledListPagesTemplates: Map<number, CompiledTemplate>;
-  /** Pre-compiled ListUsers templates keyed by module id */
-  compiledListUsersTemplates: Map<number, ListUsersCompiledTemplate>;
-}
-
-/**
- * Narrowed type for the list-pages variant of the Module union.
- * Used internally for type-safe access to list-pages-specific fields.
- */
-type ListPagesModuleData = Extract<Module, { module: "list-pages" }>;
-
-/**
- * Type guard to narrow a Module union to the list-users variant.
- *
- * @param module - A Module discriminated union value
- * @returns true if the module is a list-users module
- */
-function isListUsersModule(module: Module): module is Extract<Module, { module: "list-users" }> {
-  return module.module === "list-users";
-}
+import {
+  extractListPagesModule,
+  isListPagesModule,
+  type ListPagesExtractionState,
+} from "./extraction/listpages";
+import {
+  extractListUsersModule,
+  isListUsersModule,
+  type ListUsersExtractionState,
+} from "./extraction/listusers";
+import type { ExtractionResult } from "./extraction/result";
+export type { ExtractionResult } from "./extraction/result";
 
 /**
  * Extract all data requirements from a parsed AST.
@@ -108,303 +50,27 @@ function isListUsersModule(module: Module): module is Extract<Module, { module:
  * @returns Extraction result containing requirements and compiled templates
  */
 export function extractDataRequirements(ast: SyntaxTree): ExtractionResult {
-  const requirements: DataRequirements = {
-    listPages: [],
-    listUsers: [],
+  const result: ExtractionResult = {
+    requirements: {
+      listPages: [],
+      listUsers: [],
+    },
+    compiledListPagesTemplates: new Map(),
+    compiledListUsersTemplates: new Map(),
   };
-  const compiledListPagesTemplates = new Map<number, CompiledTemplate>();
-  const compiledListUsersTemplates = new Map<number, ListUsersCompiledTemplate>();
 
-  let listPagesIdCounter = 0;
-  let listUsersIdCounter = 0;
+  const listPagesState: ListPagesExtractionState = { nextId: 0 };
+  const listUsersState: ListUsersExtractionState = { nextId: 0 };
 
   walkElements(ast.elements, (element) => {
     if (element.element !== "module") return;
 
     if (isListPagesModule(element.data)) {
-      const listPages = element.data;
-      const id = listPagesIdCounter++;
-
-      // Get body template (use Wikidot default when not specified)
-      const body = listPages.body ?? DEFAULT_BODY_TEMPLATE;
-
-      // Extract needed variables from template
-      const extraction = extractVariablesFromTemplate(body);
-
-      // Build query from module parameters
-      const query = buildQuery(listPages);
-
-      // Get urlAttrPrefix from parsed module data (kebab-case key)
-      const urlAttrPrefix = listPages["url-attr-prefix"];
-
-      // All raw attributes (for @URL resolution by external apps)
-      const rawAttributes = listPages.attributes ?? {};
-
-      requirements.listPages.push({
-        id,
-        query,
-        neededVariables: extraction.variables,
-        contentSectionIndices: extraction.contentIndices,
-        previewLengths: extraction.previewLengths,
-        formFields: extraction.formFields,
-        tagsLinkPrefix: extraction.tagsLinkPrefix,
-        hiddenTagsLinkPrefix: extraction.hiddenTagsLinkPrefix,
-        urlAttrPrefix,
-        rawAttributes,
-      });
-
-      // Compile template
-      compiledListPagesTemplates.set(id, compileTemplate(body));
+      extractListPagesModule(element.data, listPagesState, result);
     } else if (isListUsersModule(element.data)) {
-      const listUsers = element.data;
-      const id = listUsersIdCounter++;
-      const body = listUsers.body ?? "";
-
-      const neededVariables = extractListUsersVariables(body);
-
-      requirements.listUsers.push({
-        id,
-        users: listUsers.users,
-        neededVariables,
-      });
-
-      compiledListUsersTemplates.set(id, compileListUsersTemplate(body));
+      extractListUsersModule(element.data, listUsersState, result);
     }
   });
 
-  return { requirements, compiledListPagesTemplates, compiledListUsersTemplates };
-}
-
-/**
- * Build a ListPagesQuery from the parsed module attributes.
- *
- * Maps module attribute names (which use kebab-case in the AST) to the
- * query's camelCase property names. Also extracts data form fields
- * (attributes prefixed with `_`) into a separate record.
- *
- * @param module - Parsed list-pages module data from the AST
- * @returns Query parameters ready for normalization and external fetching
- */
-function buildQuery(module: ListPagesModuleData): ListPagesQuery {
-  return {
-    pagetype: module.pagetype as ListPagesQuery["pagetype"],
-    category: module.category,
-    tags: module.tags,
-    parent: module.parent,
-    linkTo: module["link-to"],
-    createdAt: module["created-at"],
-    updatedAt: module["updated-at"],
-    createdBy: module["created-by"],
-    rating: module.rating,
-    votes: module.votes,
-    name: module.name,
-    fullname: module.fullname,
-    range: module.range as ListPagesQuery["range"],
-    order: module.order,
-    offset: module.offset,
-    limit: module.limit,
-    perPage: module["per-page"],
-    reverse: module.reverse,
-    // dataFormFields is extracted from attributes prefixed with _
-    dataFormFields: extractDataFormFields(module.attributes),
-  };
-}
-
-/**
- * Extract data form field selectors from module attributes.
- *
- * Wikidot's ListPages supports filtering by data form fields using attribute
- * names prefixed with `_` (e.g., `_color="red"`). The prefix is stripped from
- * the key in the returned record.
- *
- * @param attributes - Raw module attributes
- * @returns Record of field name to value (without `_` prefix), or undefined if no fields found
- */
-function extractDataFormFields(
-  attributes: Record<string, string>,
-): Record<string, string> | undefined {
-  const fields: Record<string, string> = {};
-  let hasFields = false;
-
-  for (const [key, value] of Object.entries(attributes)) {
-    if (key.startsWith("_")) {
-      fields[key.slice(1)] = value;
-      hasFields = true;
-    }
-  }
-
-  return hasFields ? fields : undefined;
-}
-
-/**
- * Internal result of analyzing a ListPages template string for variable usage.
- */
-interface TemplateExtraction {
-  variables: ListPagesVariable[];
-  contentIndices: number[];
-  previewLengths: number[];
-  formFields: string[];
-  tagsLinkPrefix?: string;
-  hiddenTagsLinkPrefix?: string;
-}
-
-/**
- * Extract all variable references from a ListPages template string.
- *
- * Scans the template for `%%variable%%` patterns and categorizes them:
- * - Simple variables (e.g., `%%title%%`, `%%created_at%%`)
- * - Parameterized variables (e.g., `%%content{2}%%`, `%%form_data{color}%%`)
- * - Preview with length (e.g., `%%preview(100)%%`)
- * - Tags with prefix (e.g., `%%tags_linked|/tag/%%`)
- *
- * @param template - The template string to analyze
- * @returns Structured extraction result with categorized variable information
- */
-function extractVariablesFromTemplate(template: string): TemplateExtraction {
-  const variables = new Set<ListPagesVariable>();
-  const contentIndices = new Set<number>();
-  const previewLengths = new Set<number>();
-  const formFields = new Set<string>();
-  let tagsLinkPrefix: string | undefined;
-  let hiddenTagsLinkPrefix: string | undefined;
-
-  for (const match of template.matchAll(VARIABLE_REGEX)) {
-    const [, name, braceParam, parenParam, format] = match;
-    if (!name) continue;
-    const varName = name.toLowerCase();
-
-    // Handle parameterized variables
-    if (braceParam !== undefined) {
-      switch (varName) {
-        case "content":
-          contentIndices.add(Number(braceParam));
-          variables.add("content_n");
-          continue;
-        case "form_data":
-          formFields.add(braceParam);
-          variables.add("form_data");
-          continue;
-        case "form_raw":
-          formFields.add(braceParam);
-          variables.add("form_raw");
-          continue;
-        case "form_label":
-          formFields.add(braceParam);
-          variables.add("form_label");
-          continue;
-        case "form_hint":
-          formFields.add(braceParam);
-          variables.add("form_hint");
-          continue;
-      }
-    }
-
-    if (parenParam !== undefined && varName === "preview") {
-      previewLengths.add(Number(parenParam));
-      variables.add("preview_n");
-      continue;
-    }
-
-    // Handle tags_linked with prefix
-    if (varName === "tags_linked") {
-      if (format) tagsLinkPrefix = format;
-      variables.add("tags_linked");
-      continue;
-    }
-    if (varName === "_tags_linked") {
-      if (format) hiddenTagsLinkPrefix = format;
-      variables.add("_tags_linked");
-      continue;
-    }
-
-    // Normalize and add variable
-    const normalized = normalizeVariableName(varName);
-    if (normalized) {
-      variables.add(normalized);
-    }
-  }
-
-  return {
-    variables: Array.from(variables),
-    contentIndices: Array.from(contentIndices).sort((a, b) => a - b),
-    previewLengths: Array.from(previewLengths).sort((a, b) => a - b),
-    formFields: Array.from(formFields).sort(),
-    tagsLinkPrefix,
-    hiddenTagsLinkPrefix,
-  };
-}
-
-/**
- * Normalize a variable name to its canonical `ListPagesVariable` form.
- *
- * Only known variable names are accepted. Unknown names return null,
- * causing them to be silently ignored (matching Wikidot's behavior of
- * rendering unknown variables as empty strings).
- *
- * @param name - Lowercase variable name extracted from the template
- * @returns The canonical variable name, or null if not recognized
- */
-function normalizeVariableName(name: string): ListPagesVariable | null {
-  // Direct mapping for known variables
-  const knownVariables: ListPagesVariable[] = [
-    // Lifecycle
-    "created_at",
-    "created_by",
-    "created_by_unix",
-    "created_by_id",
-    "created_by_linked",
-    "updated_at",
-    "updated_by",
-    "updated_by_unix",
-    "updated_by_id",
-    "updated_by_linked",
-    "commented_at",
-    "commented_by",
-    "commented_by_unix",
-    "commented_by_id",
-    "commented_by_linked",
-    // Structure
-    "name",
-    "category",
-    "fullname",
-    "title",
-    "title_linked",
-    "link",
-    "parent_name",
-    "parent_category",
-    "parent_fullname",
-    "parent_title",
-    "parent_title_linked",
-    // Content
-    "content",
-    "preview",
-    "summary",
-    "first_paragraph",
-    // Tags
-    "tags",
-    "_tags",
-    // Metrics
-    "children",
-    "comments",
-    "size",
-    "rating",
-    "rating_votes",
-    "rating_percent",
-    "revisions",
-    // Pagination
-    "index",
-    "total",
-    "limit",
-    "total_or_limit",
-    // Site
-    "site_title",
-    "site_name",
-    "site_domain",
-  ];
-
-  if (knownVariables.includes(name as ListPagesVariable)) {
-    return name as ListPagesVariable;
-  }
-
-  return null;
+  return result;
 }

package/src/parser/rules/block/module/listpages/extraction/listpages.ts

@@ -0,0 +1,42 @@
+import type { Module } from "@wdprlib/ast";
+import type { ListPagesQuery } from "../types";
+import { compileTemplate } from "../compiler";
+import { buildQuery } from "./query";
+import { DEFAULT_BODY_TEMPLATE, extractVariablesFromTemplate } from "./template";
+import type { ExtractionResult } from "./result";
+
+export type ListPagesModuleForExtraction = Extract<Module, { module: "list-pages" }>;
+
+export interface ListPagesExtractionState {
+  nextId: number;
+}
+
+export function isListPagesModule(module: Module): module is ListPagesModuleForExtraction {
+  return module.module === "list-pages";
+}
+
+export function extractListPagesModule(
+  listPages: ListPagesModuleForExtraction,
+  state: ListPagesExtractionState,
+  result: ExtractionResult,
+): void {
+  const id = state.nextId++;
+  const body = listPages.body ?? DEFAULT_BODY_TEMPLATE;
+  const extraction = extractVariablesFromTemplate(body);
+  const query: ListPagesQuery = buildQuery(listPages);
+
+  result.requirements.listPages.push({
+    id,
+    query,
+    neededVariables: extraction.variables,
+    contentSectionIndices: extraction.contentIndices,
+    previewLengths: extraction.previewLengths,
+    formFields: extraction.formFields,
+    tagsLinkPrefix: extraction.tagsLinkPrefix,
+    hiddenTagsLinkPrefix: extraction.hiddenTagsLinkPrefix,
+    urlAttrPrefix: listPages["url-attr-prefix"],
+    rawAttributes: listPages.attributes ?? {},
+  });
+
+  result.compiledListPagesTemplates.set(id, compileTemplate(body));
+}

package/src/parser/rules/block/module/listpages/extraction/listusers.ts

@@ -0,0 +1,30 @@
+import type { Module } from "@wdprlib/ast";
+import { compileListUsersTemplate, extractListUsersVariables } from "../../listusers/extract";
+import type { ExtractionResult } from "./result";
+
+export type ListUsersModuleForExtraction = Extract<Module, { module: "list-users" }>;
+
+export interface ListUsersExtractionState {
+  nextId: number;
+}
+
+export function isListUsersModule(module: Module): module is ListUsersModuleForExtraction {
+  return module.module === "list-users";
+}
+
+export function extractListUsersModule(
+  listUsers: ListUsersModuleForExtraction,
+  state: ListUsersExtractionState,
+  result: ExtractionResult,
+): void {
+  const id = state.nextId++;
+  const body = listUsers.body ?? "";
+
+  result.requirements.listUsers.push({
+    id,
+    users: listUsers.users,
+    neededVariables: extractListUsersVariables(body),
+  });
+
+  result.compiledListUsersTemplates.set(id, compileListUsersTemplate(body));
+}

package/src/parser/rules/block/module/listpages/extraction/query.ts

@@ -0,0 +1,51 @@
+import type { Module } from "@wdprlib/ast";
+import type { ListPagesQuery } from "../types";
+
+type ListPagesModuleData = Extract<Module, { module: "list-pages" }>;
+
+/**
+ * Build a ListPagesQuery from the parsed module attributes.
+ *
+ * Maps module attribute names, which use kebab-case in the AST, to the query's
+ * camelCase property names. Data form fields prefixed with `_` are grouped under
+ * `dataFormFields`.
+ */
+export function buildQuery(module: ListPagesModuleData): ListPagesQuery {
+  return {
+    pagetype: module.pagetype as ListPagesQuery["pagetype"],
+    category: module.category,
+    tags: module.tags,
+    parent: module.parent,
+    linkTo: module["link-to"],
+    createdAt: module["created-at"],
+    updatedAt: module["updated-at"],
+    createdBy: module["created-by"],
+    rating: module.rating,
+    votes: module.votes,
+    name: module.name,
+    fullname: module.fullname,
+    range: module.range as ListPagesQuery["range"],
+    order: module.order,
+    offset: module.offset,
+    limit: module.limit,
+    perPage: module["per-page"],
+    reverse: module.reverse,
+    dataFormFields: extractDataFormFields(module.attributes),
+  };
+}
+
+function extractDataFormFields(
+  attributes: Record<string, string>,
+): Record<string, string> | undefined {
+  const fields: Record<string, string> = {};
+  let hasFields = false;
+
+  for (const [key, value] of Object.entries(attributes)) {
+    if (key.startsWith("_")) {
+      fields[key.slice(1)] = value;
+      hasFields = true;
+    }
+  }
+
+  return hasFields ? fields : undefined;
+}

package/src/parser/rules/block/module/listpages/extraction/result.ts

@@ -0,0 +1,18 @@
+import type { DataRequirements, CompiledTemplate } from "../types";
+import type { ListUsersCompiledTemplate } from "../../listusers/types";
+
+/**
+ * Complete result of extracting data requirements from an AST.
+ *
+ * Contains everything needed to fetch external data and then resolve modules:
+ * the data requirements tell the application what to fetch, and the pre-compiled
+ * templates are used during the resolution phase to efficiently render results.
+ */
+export interface ExtractionResult {
+  /** Data requirements for external fetching */
+  requirements: DataRequirements;
+  /** Pre-compiled ListPages templates keyed by module id */
+  compiledListPagesTemplates: Map<number, CompiledTemplate>;
+  /** Pre-compiled ListUsers templates keyed by module id */
+  compiledListUsersTemplates: Map<number, ListUsersCompiledTemplate>;
+}

package/src/parser/rules/block/module/listpages/extraction/template.ts

@@ -0,0 +1,96 @@
+import type { ListPagesVariable } from "../types";
+import { scanTemplateVariables } from "../template/syntax";
+import { normalizeVariableName } from "./variables";
+
+/**
+ * Default template used when a ListPages module has no body specified.
+ */
+export const DEFAULT_BODY_TEMPLATE = `+ %%title_linked%%
+
+by %%created_by_linked%% %%created_at%%
+
+%%summary%%`;
+
+/**
+ * Internal result of analyzing a ListPages template string for variable usage.
+ */
+export interface TemplateExtraction {
+  variables: ListPagesVariable[];
+  contentIndices: number[];
+  previewLengths: number[];
+  formFields: string[];
+  tagsLinkPrefix?: string;
+  hiddenTagsLinkPrefix?: string;
+}
+
+/**
+ * Extract all variable references from a ListPages template string.
+ */
+export function extractVariablesFromTemplate(template: string): TemplateExtraction {
+  const variables = new Set<ListPagesVariable>();
+  const contentIndices = new Set<number>();
+  const previewLengths = new Set<number>();
+  const formFields = new Set<string>();
+  let tagsLinkPrefix: string | undefined;
+  let hiddenTagsLinkPrefix: string | undefined;
+
+  for (const match of scanTemplateVariables(template)) {
+    const varName = match.name.toLowerCase();
+
+    if (match.braceParam !== undefined) {
+      switch (varName) {
+        case "content":
+          contentIndices.add(Number(match.braceParam));
+          variables.add("content_n");
+          continue;
+        case "form_data":
+          formFields.add(match.braceParam);
+          variables.add("form_data");
+          continue;
+        case "form_raw":
+          formFields.add(match.braceParam);
+          variables.add("form_raw");
+          continue;
+        case "form_label":
+          formFields.add(match.braceParam);
+          variables.add("form_label");
+          continue;
+        case "form_hint":
+          formFields.add(match.braceParam);
+          variables.add("form_hint");
+          continue;
+      }
+    }
+
+    if (match.parenParam !== undefined && varName === "preview") {
+      previewLengths.add(Number(match.parenParam));
+      variables.add("preview_n");
+      continue;
+    }
+
+    if (varName === "tags_linked") {
+      if (match.format) tagsLinkPrefix = match.format;
+      variables.add("tags_linked");
+      continue;
+    }
+    if (varName === "_tags_linked") {
+      if (match.format) hiddenTagsLinkPrefix = match.format;
+      variables.add("_tags_linked");
+      continue;
+    }
+
+    const normalized = normalizeVariableName(varName);
+    if (normalized) {
+      variables.add(normalized);
+    }
+  }
+
+  return {
+    variables: Array.from(variables),
+    contentIndices: Array.from(contentIndices).sort((a, b) => a - b),
+    previewLengths: Array.from(previewLengths).sort((a, b) => a - b),
+    formFields: Array.from(formFields).sort(),
+    tagsLinkPrefix,
+    hiddenTagsLinkPrefix,
+  };
+}