@wdprlib/parser 3.1.2 → 4.0.0

package/src/parser/rules/block/module/listpages/extraction/template.ts

@@ -0,0 +1,96 @@
+import type { ListPagesVariable } from "../types";
+import { scanTemplateVariables } from "../template/syntax";
+import { normalizeVariableName } from "./variables";
+
+/**
+ * Default template used when a ListPages module has no body specified.
+ */
+export const DEFAULT_BODY_TEMPLATE = `+ %%title_linked%%
+
+by %%created_by_linked%% %%created_at%%
+
+%%summary%%`;
+
+/**
+ * Internal result of analyzing a ListPages template string for variable usage.
+ */
+export interface TemplateExtraction {
+  variables: ListPagesVariable[];
+  contentIndices: number[];
+  previewLengths: number[];
+  formFields: string[];
+  tagsLinkPrefix?: string;
+  hiddenTagsLinkPrefix?: string;
+}
+
+/**
+ * Extract all variable references from a ListPages template string.
+ */
+export function extractVariablesFromTemplate(template: string): TemplateExtraction {
+  const variables = new Set<ListPagesVariable>();
+  const contentIndices = new Set<number>();
+  const previewLengths = new Set<number>();
+  const formFields = new Set<string>();
+  let tagsLinkPrefix: string | undefined;
+  let hiddenTagsLinkPrefix: string | undefined;
+
+  for (const match of scanTemplateVariables(template)) {
+    const varName = match.name.toLowerCase();
+
+    if (match.braceParam !== undefined) {
+      switch (varName) {
+        case "content":
+          contentIndices.add(Number(match.braceParam));
+          variables.add("content_n");
+          continue;
+        case "form_data":
+          formFields.add(match.braceParam);
+          variables.add("form_data");
+          continue;
+        case "form_raw":
+          formFields.add(match.braceParam);
+          variables.add("form_raw");
+          continue;
+        case "form_label":
+          formFields.add(match.braceParam);
+          variables.add("form_label");
+          continue;
+        case "form_hint":
+          formFields.add(match.braceParam);
+          variables.add("form_hint");
+          continue;
+      }
+    }
+
+    if (match.parenParam !== undefined && varName === "preview") {
+      previewLengths.add(Number(match.parenParam));
+      variables.add("preview_n");
+      continue;
+    }
+
+    if (varName === "tags_linked") {
+      if (match.format) tagsLinkPrefix = match.format;
+      variables.add("tags_linked");
+      continue;
+    }
+    if (varName === "_tags_linked") {
+      if (match.format) hiddenTagsLinkPrefix = match.format;
+      variables.add("_tags_linked");
+      continue;
+    }
+
+    const normalized = normalizeVariableName(varName);
+    if (normalized) {
+      variables.add(normalized);
+    }
+  }
+
+  return {
+    variables: Array.from(variables),
+    contentIndices: Array.from(contentIndices).sort((a, b) => a - b),
+    previewLengths: Array.from(previewLengths).sort((a, b) => a - b),
+    formFields: Array.from(formFields).sort(),
+    tagsLinkPrefix,
+    hiddenTagsLinkPrefix,
+  };
+}

package/src/parser/rules/block/module/listpages/extraction/variables.ts

@@ -0,0 +1,58 @@
+import type { ListPagesVariable } from "../types";
+
+const KNOWN_VARIABLES: ReadonlySet<ListPagesVariable> = new Set([
+  "created_at",
+  "created_by",
+  "created_by_unix",
+  "created_by_id",
+  "created_by_linked",
+  "updated_at",
+  "updated_by",
+  "updated_by_unix",
+  "updated_by_id",
+  "updated_by_linked",
+  "commented_at",
+  "commented_by",
+  "commented_by_unix",
+  "commented_by_id",
+  "commented_by_linked",
+  "name",
+  "category",
+  "fullname",
+  "title",
+  "title_linked",
+  "link",
+  "parent_name",
+  "parent_category",
+  "parent_fullname",
+  "parent_title",
+  "parent_title_linked",
+  "content",
+  "preview",
+  "summary",
+  "first_paragraph",
+  "tags",
+  "_tags",
+  "children",
+  "comments",
+  "size",
+  "rating",
+  "rating_votes",
+  "rating_percent",
+  "revisions",
+  "index",
+  "total",
+  "limit",
+  "total_or_limit",
+  "site_title",
+  "site_name",
+  "site_domain",
+]);
+
+export function normalizeVariableName(name: string): ListPagesVariable | null {
+  if (KNOWN_VARIABLES.has(name as ListPagesVariable)) {
+    return name as ListPagesVariable;
+  }
+
+  return null;
+}

package/src/parser/rules/block/module/listpages/index.ts

@@ -0,0 +1,83 @@
+/**
+ *
+ * ListPages module for Wikidot's `[[module ListPages ...]]` block.
+ *
+ * This is the most feature-rich Wikidot module, enabling dynamic page listings
+ * with filtering, sorting, pagination, and template-based rendering. The module
+ * follows a three-phase lifecycle:
+ *
+ * 1. **Parse** - Convert `[[module ListPages ...]]` markup into an AST node
+ * 2. **Extract** - Analyze the AST to determine what data is needed (queries, template variables)
+ * 3. **Resolve** - Substitute fetched data into templates and re-parse as wikitext
+ *
+ * This barrel module re-exports all public types and functions from the sub-modules:
+ * - `parser` - Module rule for parsing ListPages markup
+ * - `types` - Query, variable, data requirement, and normalized query types
+ * - `extract` - AST analysis and data requirement extraction
+ * - `resolve` - Data substitution and template rendering
+ * - `compiler` - Template string compilation into executable functions
+ * - `url-resolver` - `@URL|default` parameter resolution for HPC support
+ * - `normalize` - Raw query string parsing into structured types
+ *
+ * @module
+ */
+
+// Parser
+export { listPagesModuleRule } from "./parser";
+
+// Types
+export type {
+  ListPagesQuery,
+  ListPagesVariable,
+  ListPagesDataRequirement,
+  DataRequirements,
+  UserInfo,
+  PageData,
+  SiteContext,
+  ListPagesExternalData,
+  ListPagesDataFetcher,
+  VariableContext,
+  CompiledTemplate,
+  // Normalized types
+  NormalizedListPagesQuery,
+  NormalizedTags,
+  NormalizedCategory,
+  NormalizedOrder,
+  NormalizedParent,
+  NormalizedDateSelector,
+  NormalizedNumericSelector,
+  OrderField,
+  OrderDirection,
+  DateComparisonOp,
+  NumericComparisonOp,
+} from "./types";
+
+// Extraction
+export type { ExtractionResult } from "./extract";
+export { extractDataRequirements } from "./extract";
+
+// Resolution
+export type { ParseFunction, ListPagesModuleData } from "./resolve";
+export { isListPagesModule, resolveListPages } from "./resolve";
+
+// Compiler
+export { compileTemplate } from "./compiler";
+
+// URL Resolution
+export {
+  parseUrlParams,
+  resolveUrlValue,
+  resolveQuery,
+  resolveAndNormalizeQuery,
+} from "./url-resolver";
+
+// Query Normalization
+export {
+  normalizeQuery,
+  parseTags,
+  parseCategory,
+  parseOrder,
+  parseParent,
+  parseDateSelector,
+  parseNumericSelector,
+} from "./normalize";

package/src/parser/rules/block/module/listpages/normalization/date-selector.ts

@@ -0,0 +1,53 @@
+import type { DateComparisonOp, NormalizedDateSelector } from "../types";
+
+/**
+ * Date comparison operators, ordered longest-first.
+ */
+const DATE_COMPARISON_OPS: DateComparisonOp[] = ["<=", ">=", "<>", "<", ">", "="];
+
+/**
+ * Pattern for relative date expressions like "last 7 days".
+ */
+const RELATIVE_DATE_PATTERN = /^last\s+(?:(\d+)\s+)?(day|week|month)s?$/i;
+
+/**
+ * Parse date selector string into structured format.
+ */
+export function parseDateSelector(value: string): NormalizedDateSelector | undefined {
+  const trimmed = value.trim();
+  if (!trimmed) return undefined;
+
+  const relativeMatch = trimmed.match(RELATIVE_DATE_PATTERN);
+  if (relativeMatch && relativeMatch[2]) {
+    const count = relativeMatch[1] ? parseInt(relativeMatch[1], 10) : 1;
+    if (count < 1) return undefined;
+    const unit = relativeMatch[2].toLowerCase() as "day" | "week" | "month";
+    return { type: "relative", unit, count };
+  }
+
+  for (const op of DATE_COMPARISON_OPS) {
+    if (trimmed.startsWith(op)) {
+      const date = trimmed.slice(op.length).trim();
+      if (date) {
+        return { type: "comparison", op, date };
+      }
+    }
+  }
+
+  if (/^\d{4}$/.test(trimmed)) {
+    return { type: "year", year: parseInt(trimmed, 10) };
+  }
+
+  const monthMatch = trimmed.match(/^(\d{4})\.(\d{1,2})$/);
+  if (monthMatch && monthMatch[1] && monthMatch[2]) {
+    const month = parseInt(monthMatch[2], 10);
+    if (month < 1 || month > 12) return undefined;
+    return {
+      type: "month",
+      year: parseInt(monthMatch[1], 10),
+      month,
+    };
+  }
+
+  return undefined;
+}

package/src/parser/rules/block/module/listpages/normalization/numeric-selector.ts

@@ -0,0 +1,32 @@
+import type { NormalizedNumericSelector, NumericComparisonOp } from "../types";
+
+/**
+ * Numeric comparison operators, ordered longest-first.
+ */
+const NUMERIC_COMPARISON_OPS: NumericComparisonOp[] = ["<=", ">=", "<", ">", "="];
+
+/**
+ * Parse numeric selector string into structured format.
+ */
+export function parseNumericSelector(value: string): NormalizedNumericSelector | undefined {
+  const trimmed = value.trim();
+  if (!trimmed) return undefined;
+
+  for (const op of NUMERIC_COMPARISON_OPS) {
+    if (trimmed.startsWith(op)) {
+      const numStr = trimmed.slice(op.length).trim();
+      const num = parseFloat(numStr);
+      if (Number.isFinite(num) && /^-?\d+(\.\d+)?$/.test(numStr)) {
+        return { op, value: num };
+      }
+      return undefined;
+    }
+  }
+
+  const num = parseFloat(trimmed);
+  if (Number.isFinite(num) && /^-?\d+(\.\d+)?$/.test(trimmed)) {
+    return { op: "=", value: num };
+  }
+
+  return undefined;
+}

package/src/parser/rules/block/module/listpages/normalization/order-parent.ts

@@ -0,0 +1,82 @@
+import type { NormalizedOrder, NormalizedParent, OrderDirection, OrderField } from "../types";
+
+/**
+ * Mapping from Wikidot's order field names to normalized OrderField values.
+ */
+const ORDER_FIELD_MAP: Record<string, OrderField> = {
+  datecreated: "created_at",
+  dateedited: "updated_at",
+  title: "title",
+  fullname: "fullname",
+  rating: "rating",
+  votes: "votes",
+  revisions: "revisions",
+  comments: "comments",
+  pagelength: "size",
+  size: "size",
+  random: "random",
+  created_at: "created_at",
+  updated_at: "updated_at",
+};
+
+/**
+ * Parse order string into structured format.
+ */
+export function parseOrder(value: string): NormalizedOrder {
+  const defaultOrder: NormalizedOrder = { field: "created_at", direction: "desc" };
+  const trimmed = value.trim().toLowerCase();
+  if (!trimmed) return defaultOrder;
+
+  const spaceParts = trimmed.split(/\s+/);
+  if (spaceParts.length >= 2 && spaceParts[0] && spaceParts[1]) {
+    const field = ORDER_FIELD_MAP[spaceParts[0]];
+    const direction = spaceParts[1] === "asc" ? "asc" : "desc";
+    if (field) {
+      return { field, direction };
+    }
+  }
+
+  let direction: OrderDirection = "desc";
+  let fieldPart = trimmed;
+
+  if (trimmed.endsWith("desc")) {
+    direction = "desc";
+    fieldPart = trimmed.slice(0, -4);
+  } else if (trimmed.endsWith("asc")) {
+    direction = "asc";
+    fieldPart = trimmed.slice(0, -3);
+  }
+
+  const field = ORDER_FIELD_MAP[fieldPart];
+  if (field) {
+    return { field, direction };
+  }
+
+  const singleField = ORDER_FIELD_MAP[trimmed];
+  if (singleField) {
+    return { field: singleField, direction: "desc" };
+  }
+
+  return defaultOrder;
+}
+
+/**
+ * Parse parent string into structured format.
+ */
+export function parseParent(value: string): NormalizedParent | undefined {
+  const trimmed = value.trim();
+  if (!trimmed) return undefined;
+
+  switch (trimmed) {
+    case "-":
+      return { type: "none" };
+    case "=":
+      return { type: "same" };
+    case "-=":
+      return { type: "different" };
+    case ".":
+      return { type: "children" };
+    default:
+      return { type: "page", name: trimmed };
+  }
+}

package/src/parser/rules/block/module/listpages/normalization/selectors.ts

@@ -0,0 +1,2 @@
+export { parseDateSelector } from "./date-selector";
+export { parseNumericSelector } from "./numeric-selector";

package/src/parser/rules/block/module/listpages/normalization/tags-category.ts

@@ -0,0 +1,86 @@
+import type { NormalizedCategory, NormalizedTags } from "../types";
+
+/**
+ * Pattern for splitting multi-value attribute strings.
+ * Wikidot allows commas, semicolons, and whitespace as separators between values.
+ */
+const TOKEN_SEPARATOR = /[,;\s]+/;
+
+/**
+ * Parse tags string into structured format.
+ */
+export function parseTags(value: string): NormalizedTags {
+  const result: NormalizedTags = {
+    all: [],
+    any: [],
+    none: [],
+    special: null,
+  };
+
+  const trimmed = value.trim();
+  if (!trimmed) return result;
+
+  if (trimmed === "-") {
+    result.special = "none";
+    return result;
+  }
+  if (trimmed === "==") {
+    result.special = "same-all";
+    return result;
+  }
+
+  const tokens = trimmed.split(TOKEN_SEPARATOR).filter(Boolean);
+
+  for (const token of tokens) {
+    if (token === "=") {
+      result.special = "same-visible";
+    } else if (token.startsWith("+")) {
+      const tag = token.slice(1);
+      if (tag) result.all.push(tag);
+    } else if (token.startsWith("-")) {
+      const tag = token.slice(1);
+      if (tag) result.none.push(tag);
+    } else {
+      result.any.push(token);
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Parse category string into structured format.
+ */
+export function parseCategory(value: string): NormalizedCategory {
+  const result: NormalizedCategory = {
+    include: [],
+    exclude: [],
+    all: false,
+    current: false,
+  };
+
+  const trimmed = value.trim().toLowerCase();
+  if (!trimmed) return result;
+
+  if (trimmed === "*") {
+    result.all = true;
+    return result;
+  }
+
+  const tokens = trimmed.split(TOKEN_SEPARATOR).filter(Boolean);
+
+  for (const token of tokens) {
+    if (token === "*") {
+      result.all = true;
+    } else if (token === ".") {
+      result.current = true;
+    } else if (token.startsWith("-")) {
+      const cat = token.slice(1);
+      if (cat) result.exclude.push(cat);
+    } else {
+      result.include.push(token);
+    }
+  }
+
+  return result;
+}

package/src/parser/rules/block/module/listpages/normalize.ts

@@ -0,0 +1,74 @@
+/**
+ *
+ * Query normalization for the ListPages module.
+ *
+ * Converts the raw string-based `ListPagesQuery` (where fields like `tags`,
+ * `category`, and `order` are plain strings) into a `NormalizedListPagesQuery`
+ * with type-safe structured objects. This makes it straightforward for external
+ * applications to build database queries from the normalized representation
+ * without having to re-parse Wikidot's query syntax.
+ *
+ * Based on Wikidot official documentation:
+ * https://www.wikidot.com/doc-modules:listpages-module
+ *
+ * @module
+ */
+
+import type { ListPagesQuery, NormalizedListPagesQuery } from "./types";
+import { parseCategory, parseTags } from "./normalization/tags-category";
+import { parseOrder, parseParent } from "./normalization/order-parent";
+import { parseDateSelector, parseNumericSelector } from "./normalization/selectors";
+
+export { parseCategory, parseTags } from "./normalization/tags-category";
+export { parseOrder, parseParent } from "./normalization/order-parent";
+export { parseDateSelector, parseNumericSelector } from "./normalization/selectors";
+
+/**
+ * Normalize a ListPagesQuery into structured types
+ *
+ * @param query - Raw query with string fields
+ * @returns Normalized query with structured types
+ */
+export function normalizeQuery(query: ListPagesQuery): NormalizedListPagesQuery {
+  const result: NormalizedListPagesQuery = {};
+
+  // Pass through simple fields
+  if (query.pagetype) result.pagetype = query.pagetype;
+  if (query.linkTo) result.linkTo = query.linkTo;
+  if (query.createdBy) result.createdBy = query.createdBy;
+  if (query.name) result.name = query.name;
+  if (query.fullname) result.fullname = query.fullname;
+  if (query.range) result.range = query.range;
+  if (query.dataFormFields) result.dataFormFields = query.dataFormFields;
+  if (query.offset !== undefined) result.offset = query.offset;
+  if (query.limit !== undefined) result.limit = query.limit;
+  if (query.perPage !== undefined) result.perPage = query.perPage;
+  if (query.reverse !== undefined) result.reverse = query.reverse;
+
+  // Parse complex fields
+  if (query.tags) result.tags = parseTags(query.tags);
+  if (query.category) result.category = parseCategory(query.category);
+  if (query.order) result.order = parseOrder(query.order);
+  if (query.parent) {
+    const parent = parseParent(query.parent);
+    if (parent) result.parent = parent;
+  }
+  if (query.createdAt) {
+    const createdAt = parseDateSelector(query.createdAt);
+    if (createdAt) result.createdAt = createdAt;
+  }
+  if (query.updatedAt) {
+    const updatedAt = parseDateSelector(query.updatedAt);
+    if (updatedAt) result.updatedAt = updatedAt;
+  }
+  if (query.rating) {
+    const rating = parseNumericSelector(query.rating);
+    if (rating) result.rating = rating;
+  }
+  if (query.votes) {
+    const votes = parseNumericSelector(query.votes);
+    if (votes) result.votes = votes;
+  }
+
+  return result;
+}

package/src/parser/rules/block/module/listpages/parser.ts

@@ -0,0 +1,106 @@
+/**
+ *
+ * Parser rule for the Wikidot `[[module ListPages ...]]` block.
+ *
+ * Parses the module's attributes into a structured `list-pages` Module AST node.
+ * Handles both hyphenated (`link-to`) and concatenated (`linkto`) attribute name
+ * formats, as Wikidot normalizes both to lowercase. The raw attribute values are
+ * preserved in the `attributes` field for `@URL` resolution by external applications.
+ *
+ * @module
+ */
+
+import type { Module } from "@wdprlib/ast";
+import type { ModuleRule } from "../types";
+import { parseBool, parseInt32 } from "../utils";
+
+/**
+ * Module rule for `[[module ListPages ...]]`.
+ *
+ * ListPages is the most complex Wikidot module. It queries pages by various
+ * criteria (tags, category, parent, date, rating, etc.) and renders each matching
+ * page using a template specified in the module body. The template uses
+ * `%%variable%%` syntax to reference page data.
+ *
+ * This rule only handles parsing; data fetching and template rendering are handled
+ * by the extract/resolve pipeline.
+ */
+export const listPagesModuleRule: ModuleRule = {
+  name: "module-listpages",
+  acceptsNames: ["listpages"],
+  hasBody: true,
+
+  parse(_ctx, _pos, args, body): Module {
+    // Extract known attributes, pass rest as additional attributes
+    // Note: attribute names are normalized to lowercase by parseAttributesRaw
+    const {
+      category,
+      tags,
+      parent,
+      rating,
+      votes,
+      name,
+      fullname,
+      range,
+      pagetype,
+      offset,
+      limit,
+      order,
+      reverse,
+      separate,
+      wrapper,
+      rss,
+    } = args;
+    // Hyphenated attributes (stored with hyphens in lowercase)
+    const linkTo = args["link-to"] ?? args.linkto;
+    const createdBy = args["created-by"] ?? args.createdby;
+    const createdAt = args["created-at"] ?? args.createdat;
+    const updatedAt = args["updated-at"] ?? args.updatedat;
+    const perPage = args["per-page"] ?? args.perpage;
+    const prependLine = args["prepend-line"] ?? args.prependline;
+    const appendLine = args["append-line"] ?? args.appendline;
+    const rssDescription = args["rss-description"] ?? args.rssdescription;
+    const rssHome = args["rss-home"] ?? args.rsshome;
+    const rssLimit = args["rss-limit"] ?? args.rsslimit;
+    const rssOnly = args["rss-only"] ?? args.rssonly;
+    const urlAttrPrefix = args["url-attr-prefix"] ?? args.urlattrprefix;
+
+    // Store all raw arguments for @URL resolution by external apps
+    const rawArgs: Record<string, string> = { ...args };
+
+    return {
+      module: "list-pages",
+      category,
+      tags,
+      parent,
+      "link-to": linkTo,
+      "created-by": createdBy,
+      "created-at": createdAt,
+      "updated-at": updatedAt,
+      rating,
+      votes,
+      name,
+      fullname,
+      range,
+      pagetype,
+      offset: parseInt32(offset),
+      limit: parseInt32(limit),
+      "per-page": parseInt32(perPage),
+      order,
+      reverse: parseBool(reverse, false),
+      separate: parseBool(separate, true),
+      wrapper: parseBool(wrapper, true),
+      "prepend-line": prependLine,
+      "append-line": appendLine,
+      rss,
+      "rss-description": rssDescription,
+      "rss-home": rssHome,
+      "rss-limit": parseInt32(rssLimit),
+      "rss-only": parseBool(rssOnly, false),
+      "url-attr-prefix": urlAttrPrefix,
+      body,
+      // All raw arguments for @URL resolution
+      attributes: rawArgs,
+    };
+  },
+};