@wdprlib/parser 5.4.0 → 6.0.0

package/src/parser/rules/block/module/listpages/types/variables.ts

@@ -2,6 +2,10 @@
  * All supported ListPages template variables.
  */
 export type ListPagesVariable =
+  | "metadata"
+  | "customrate"
+  | "customrate_votes"
+  | "customrate_percent"
   // Lifecycle - created
   | "created_at"
   | "created_by"
@@ -38,6 +42,7 @@ export type ListPagesVariable =
   | "content_n"
   | "preview"
   | "preview_n"
+  | "excerpt"
   | "summary"
   | "first_paragraph"
   // Tags

package/src/parser/rules/block/module/listpages/url-resolution/fields.ts

@@ -36,6 +36,7 @@ type StringUrlQueryKey = Extract<
   | "updatedAt"
   | "createdBy"
   | "rating"
+  | "ratingAxis"
   | "votes"
 >;
 
@@ -81,6 +82,7 @@ export const URL_RESOLVABLE_FIELDS: readonly UrlResolvableField[] = [
     aliases: ["createdby", "created_by"],
   },
   { attr: "rating", queryKey: "rating", type: "string" },
+  { attr: "rating-axis", queryKey: "ratingAxis", type: "string" },
   { attr: "votes", queryKey: "votes", type: "string" },
   { attr: "reverse", queryKey: "reverse", type: "boolean" },
 ];

package/src/parser/rules/block/module/listusers/resolve.ts

@@ -16,6 +16,7 @@ import type {
   ListUsersVariableContext,
 } from "./types";
 import { getModuleParseAst, type ParseFunction } from "../types";
+import { suppressModuleRatings } from "../rate/resolve";
 
 /**
  * Narrowed type for the list-users variant of the Module discriminated union.
@@ -53,6 +54,6 @@ export function resolveListUsers(
 ): Element[] {
   const ctx: ListUsersVariableContext = { user: data.user };
   const substituted = compiledTemplate(ctx);
-  const itemAst = getModuleParseAst(parse(substituted));
+  const itemAst = getModuleParseAst(suppressModuleRatings(parse(substituted)));
   return itemAst.elements;
 }

package/src/parser/rules/block/module/mapping.ts

@@ -11,7 +11,7 @@
  */
 
 import type { ModuleRule } from "./types";
-import { rateModuleRule } from "./rate/index";
+import { rateModuleRule, customRateModuleRule } from "./rate/index";
 import { cssModuleRule } from "./css/index";
 import { backlinksModuleRule } from "./backlinks/index";
 import { categoriesModuleRule } from "./categories/index";
@@ -29,6 +29,7 @@ import { tagCloudModuleRule } from "./tagcloud/parser";
  */
 export const MODULE_RULES: ModuleRule[] = [
   rateModuleRule,
+  customRateModuleRule,
   cssModuleRule,
   backlinksModuleRule,
   categoriesModuleRule,

package/src/parser/rules/block/module/rate/index.ts

@@ -1,28 +1,25 @@
-/**
- *
- * Parser rule for the Wikidot `[[module Rate]]` block.
- *
- * Renders a page rating widget. This is a simple module with no attributes
- * and no body content.
- *
- * @module
- */
-
+import type { CustomRateModuleData, RateModuleData } from "@wdprlib/ast";
 import type { ModuleRule } from "../types";
-import type { RateModuleData } from "./types";
 
-/**
- * Module rule for `[[module Rate]]`.
- *
- * Simply produces a `{ module: "rate" }` AST node. The rendering application
- * is responsible for displaying upvote/downvote buttons and the current rating.
- */
+/** The main rating accepts no page, axis, policy, or permission attributes. */
 export const rateModuleRule: ModuleRule = {
   name: "module-rate",
   acceptsNames: ["rate"],
   hasBody: false,
+  parse(_ctx, _pos, args): RateModuleData {
+    return { module: "rate", ref: Object.keys(args).length === 0 ? { kind: "main" } : null };
+  },
+};
 
-  parse(): RateModuleData {
-    return { module: "rate" };
+/** Reference an exact, host-registered key on the displayed page. */
+export const customRateModuleRule: ModuleRule = {
+  name: "module-custom-rate",
+  acceptsNames: ["customrate"],
+  hasBody: false,
+  parse(_ctx, _pos, args): CustomRateModuleData {
+    const key = args.key;
+    const valid =
+      key !== undefined && key !== "" && Object.keys(args).every((name) => name === "key");
+    return { module: "custom-rate", ref: valid ? { kind: "custom", axisKey: key } : null };
   },
 };

package/src/parser/rules/block/module/rate/resolve.ts

@@ -0,0 +1,108 @@
+import type { Element, RatingRef, RatingState, SyntaxTree } from "@wdprlib/ast";
+import { getModuleParseAst, type ModuleParseResult } from "../types";
+import { mapElementChildren } from "../walk";
+import type { RatingsFetcher } from "./types";
+
+function refKey(ref: RatingRef): string {
+  return ref.kind === "main" ? "main" : `custom:${ref.axisKey}`;
+}
+
+/** Resolve once after includes and module expansion, using only the displayed page. */
+export async function resolveRatings(
+  ast: SyntaxTree,
+  fetchRatings?: RatingsFetcher,
+): Promise<SyntaxTree> {
+  const refs = new Map<string, RatingRef>();
+  mapDocument(ast, (element) => {
+    if (isRating(element) && element.data.ref) refs.set(refKey(element.data.ref), element.data.ref);
+    return element;
+  });
+  const states = new Map<string, RatingState>();
+  if (fetchRatings && refs.size > 0) {
+    for (const state of await fetchRatings([...refs.values()])) {
+      const key = refKey(state.ref);
+      if (refs.has(key)) states.set(key, state);
+    }
+  }
+  return mapDocument(ast, (element) => {
+    if (!isRating(element)) return element;
+    const { state: _previous, ...data } = element.data;
+    const state = data.ref ? states.get(refKey(data.ref)) : undefined;
+    return { ...element, data: state ? { ...data, state } : data };
+  });
+}
+
+/** Module-generated source cannot declare ratings, including in its footnote side channel. */
+export function suppressModuleRatings(result: ModuleParseResult): ModuleParseResult {
+  const ast = mapDocument(getModuleParseAst(result), (element) =>
+    isRating(element) ? null : element,
+  );
+  return "ast" in result ? { ...result, ast } : ast;
+}
+
+function isRating(element: Element): element is Extract<Element, { element: "module" }> & {
+  data: Extract<
+    Extract<Element, { element: "module" }>["data"],
+    { module: "rate" | "custom-rate" }
+  >;
+} {
+  return (
+    element.element === "module" &&
+    (element.data.module === "rate" || element.data.module === "custom-rate")
+  );
+}
+
+function mapDocument(ast: SyntaxTree, transform: (element: Element) => Element | null): SyntaxTree {
+  const map = (elements: Element[]): Element[] =>
+    elements.flatMap((original) => {
+      const element = transform(original);
+      if (!element) return [];
+      if (element.element === "if") {
+        return [
+          {
+            ...element,
+            data: {
+              ...element.data,
+              // oxlint-disable-next-line unicorn/no-thenable -- public AST branch name
+              then: map(element.data.then),
+              else: map(element.data.else),
+            },
+          },
+        ];
+      }
+      if (element.element === "ifexpr") {
+        return [
+          {
+            ...element,
+            data: {
+              ...element.data,
+              // oxlint-disable-next-line unicorn/no-thenable -- public AST branch name
+              then: map(element.data.then),
+              else: map(element.data.else),
+            },
+          },
+        ];
+      }
+      if (element.element === "bibliography-block") {
+        return [
+          {
+            ...element,
+            data: {
+              ...element.data,
+              entries: element.data.entries.map((entry) => ({
+                ...entry,
+                key: map(entry.key),
+                value: map(entry.value),
+              })),
+            },
+          },
+        ];
+      }
+      return [mapElementChildren(element, map)];
+    });
+  return {
+    ...ast,
+    elements: map(ast.elements),
+    ...(ast.footnotes ? { footnotes: ast.footnotes.map(map) } : {}),
+  };
+}

package/src/parser/rules/block/module/rate/types.ts

@@ -1,19 +1,9 @@
-/**
- *
- * Type definitions for the Rate module.
- *
- * The `[[module Rate]]` block renders a page rating widget (upvote/downvote buttons).
- * It takes no attributes and has no body.
- *
- * @module
- */
+import type { RatingRef, RatingState } from "@wdprlib/ast";
+export type { RateModuleData, CustomRateModuleData } from "@wdprlib/ast";
 
 /**
- * AST data for a `[[module Rate]]` element.
- *
- * This module has no configurable properties. The rendering application
- * is responsible for displaying the appropriate rating widget.
+ * Read authorized ratings for the displayed page. Omit unknown, disabled, or
+ * inaccessible references. The host owns registration, category policies,
+ * access checks, aggregates, and persistence; rendering must never create data.
  */
-export interface RateModuleData {
-  module: "rate";
-}
+export type RatingsFetcher = (refs: readonly RatingRef[]) => Promise<readonly RatingState[]>;

package/src/parser/rules/block/module/resolution/resolve-async.ts

@@ -28,6 +28,7 @@ import { ModuleDocumentRegistry } from "./document";
 import { collectStyles, mergeCollectedStyles } from "./styles";
 import { createListPagesPager } from "../listpages/resolution/pager";
 import type { ListPagesPaginationState } from "./data-maps";
+import { suppressModuleRatings } from "../rate/resolve";
 
 export type AsyncModuleParseFunction = (source: string) => Promise<ModuleParseResult>;
 
@@ -73,7 +74,7 @@ export async function resolveModulesWithAsyncParse(
   const registry = new ModuleDocumentRegistry();
   registry.register(ast);
   const parse = async (source: string): Promise<SyntaxTree> =>
-    registry.register(await options.parse(source));
+    registry.register(suppressModuleRatings(await options.parse(source)));
 
   const [listPagesData, listUsersData, tagCloudData] = await Promise.all([
     buildListPagesDataMap(

package/src/parser/rules/block/module/resolve.ts

@@ -31,6 +31,7 @@ import {
 import { walkAndResolve } from "./resolution/walk-resolve";
 import { collectStyles, mergeCollectedStyles } from "./resolution/styles";
 import { containsSyntaxFootnoteBlock, ModuleDocumentRegistry } from "./resolution/document";
+import { resolveRatings, suppressModuleRatings } from "./rate/resolve";
 
 const MODULE_SECONDARY_INCLUDE_MAX_ITERATIONS = 5;
 
@@ -188,11 +189,9 @@ export async function resolveModules(
   if (mergedStyles.length > 0) result.styles = mergedStyles;
 
   options.onDiagnostics?.(registry.diagnostics);
-  return registry.finalize(
-    result,
-    finalElements,
-    pageTags,
-    containsSyntaxFootnoteBlock(ast.elements),
+  return resolveRatings(
+    registry.finalize(result, finalElements, pageTags, containsSyntaxFootnoteBlock(ast.elements)),
+    dataProvider.fetchRatings,
   );
 }
 
@@ -203,9 +202,12 @@ function createModuleParseFunction(
 ): ParseFunction {
   const transform = createModuleSourceTransform(options, dataProvider);
   return (source: string) =>
-    registry.register(options.parse(transform ? transform(source) : source), {
-      stripLegacyImplicitFootnoteBlock: true,
-    });
+    registry.register(
+      suppressModuleRatings(options.parse(transform ? transform(source) : source)),
+      {
+        stripLegacyImplicitFootnoteBlock: true,
+      },
+    );
 }
 
 function createModuleSourceTransform(

package/src/parser/rules/block/module/types-common.ts

@@ -17,6 +17,7 @@ import type { ListUsersDataFetcher } from "./listusers/types";
 import type { TagCloudDataFetcher } from "./tagcloud/types";
 import type { IfTagsResolver } from "./iftags/types";
 import type { IncludeFetcher } from "./include/resolve/types";
+import type { RatingsFetcher } from "./rate/types";
 
 /**
  * Callback bag for supplying external data during module resolution.
@@ -29,6 +30,9 @@ import type { IncludeFetcher } from "./include/resolve/types";
  * @group Module Resolution
  */
 export interface DataProvider {
+  /** Read registered ratings for the displayed page; omitted references stay hidden. */
+  fetchRatings?: RatingsFetcher;
+
   /**
    * Fetch page data for `[[module ListPages]]` expansion.
    *

package/src/parser/rules/inline/parsing/inline-content.ts

@@ -29,6 +29,7 @@ export interface InlineParseResult {
 
 /**
  * Parse inline content until a specific token type.
+ * EOF mode preserves literal newlines within the inherited inline boundary.
  *
  * When endType is "PARAGRAPH_BREAK", handles NEWLINEs and stops at:
  * - Double NEWLINE (paragraph break)
@@ -42,7 +43,8 @@ export function parseInlineUntil(ctx: ParseContext, endType: InlineEndType): Inl
   let consumedEmptyRaw = false;
 
   const paragraphMode = endType === "PARAGRAPH_BREAK";
-  const multiline = paragraphMode || FORMATTING_CLOSE_TOKENS.has(endType);
+  const preserveNewlines = endType === "EOF";
+  const multiline = paragraphMode || preserveNewlines || FORMATTING_CLOSE_TOKENS.has(endType);
   let inlineEnd = ctx.scope.inlineEnd ?? ctx.tokens.length;
   if (!multiline) {
     for (let end = ctx.pos; end < inlineEnd; end++) {
@@ -94,6 +96,13 @@ export function parseInlineUntil(ctx: ParseContext, endType: InlineEndType): Inl
       break;
     }
 
+    if (preserveNewlines && token.type === "NEWLINE") {
+      nodes.push({ element: "text", data: token.value });
+      pos++;
+      consumed++;
+      continue;
+    }
+
     if (multiline && token.type === "NEWLINE" && !ctx.scope.tableFormatting) {
       const boundary = getParagraphNewlineBoundary(ctx, pos, nodes.length > 0);
       if (boundary.shouldBreak) {

package/src/parser/rules/inline/size/content.ts

@@ -2,6 +2,8 @@ import type { Element } from "@wdprlib/ast";
 import type { ParseContext } from "../../types";
 import { parseBlockName } from "../../common";
 import { parseInlineUntil } from "../utils";
+import { protectedInlineRegionEnd } from "../raw/end";
+import { parseSizeOpen } from "./open";
 
 export interface SizeContentResult {
   children: Element[];
@@ -13,8 +15,9 @@ export function parseSizeContent(ctx: ParseContext, startPos: number): SizeConte
   const children: Element[] = [];
   let pos = startPos;
   let consumed = 0;
+  const inlineEnd = findSizeClose(ctx, startPos);
 
-  while (pos < ctx.tokens.length) {
+  while (pos < (ctx.scope.inlineEnd ?? ctx.tokens.length)) {
     const token = ctx.tokens[pos];
     if (!token || token.type === "EOF") {
       break;
@@ -29,9 +32,9 @@ export function parseSizeContent(ctx: ParseContext, startPos: number): SizeConte
       };
     }
 
-    const inlineCtx: ParseContext = { ...ctx, pos };
-    const inlineResult = parseInlineUntil(inlineCtx, "BLOCK_END_OPEN");
-    if (inlineResult.elements.length > 0) {
+    const inlineCtx: ParseContext = { ...ctx, pos, scope: { ...ctx.scope, inlineEnd } };
+    const inlineResult = parseInlineUntil(inlineCtx, "EOF");
+    if (inlineResult.consumed > 0) {
       for (const element of inlineResult.elements) children.push(element);
       pos += inlineResult.consumed;
       consumed += inlineResult.consumed;
@@ -45,6 +48,30 @@ export function parseSizeContent(ctx: ParseContext, startPos: number): SizeConte
   return { children, consumed, foundClose: false };
 }
 
+function findSizeClose(ctx: ParseContext, startPos: number): number {
+  const end = ctx.scope.inlineEnd ?? ctx.tokens.length;
+  let depth = 0;
+  for (let pos = startPos; pos < end; pos++) {
+    const protectedEnd = protectedInlineRegionEnd(ctx.tokens, pos, end);
+    if (protectedEnd > pos) {
+      pos = protectedEnd - 1;
+      continue;
+    }
+    const open = ctx.tokens[pos]?.type === "BLOCK_OPEN" ? parseSizeOpen({ ...ctx, pos }) : null;
+    if (open) {
+      depth++;
+      pos = open.bodyStart - 1;
+      continue;
+    }
+    const close = tryConsumeSizeClose(ctx, pos);
+    if (!close) continue;
+    if (depth === 0) return pos;
+    depth--;
+    pos += close.consumed - 1;
+  }
+  return end;
+}
+
 function tryConsumeSizeClose(ctx: ParseContext, pos: number): { consumed: number } | null {
   if (ctx.tokens[pos]?.type !== "BLOCK_END_OPEN") {
     return null;

package/src/pipeline/process.ts

@@ -1,5 +1,8 @@
 import {
   DEFAULT_SETTINGS,
+  extractReadableText,
+  extractFirstParagraph,
+  countCharacters,
   type Diagnostic,
   type PageRef,
   type WikitextPageContext,
@@ -10,6 +13,7 @@ import type { AsyncIncludeFetcher, IncludeDependency } from "../parser/rules/blo
 import { resolveIncludesAsyncWithTraceSelective } from "../parser/rules/block/module/include/resolve";
 import type { DataProvider } from "../parser/rules/block/module/types-common";
 import { resolveModulesWithAsyncParse } from "../parser/rules/block/module/resolution/resolve-async";
+import { resolveRatings } from "../parser/rules/block/module/rate/resolve";
 import type {
   ProcessedWikitextDocument,
   ProcessWikitextCallbackContext,
@@ -117,8 +121,14 @@ export async function processWikitext<TPage extends WikitextPageContext>(
     );
   }
 
+  ast = await resolveRatings(ast, dataProvider.fetchRatings);
+  const readableText = extractReadableText(ast, options.readableText);
+
   return {
     ast,
+    readableText,
+    firstParagraph: extractFirstParagraph(ast, options.readableText),
+    characterCount: countCharacters(readableText),
     page: options.page,
     settings,
     diagnostics,
@@ -167,6 +177,9 @@ function createModuleDataProvider<TPage extends WikitextPageContext>(
 ): DataProvider {
   const provider = options.dataProvider;
   return {
+    fetchRatings: provider?.fetchRatings
+      ? (refs) => provider.fetchRatings!(refs, context)
+      : undefined,
     fetchListPages: provider?.fetchListPages
       ? (query, requirement) => provider.fetchListPages!(query, requirement, context)
       : undefined,

package/src/pipeline/types.ts

@@ -4,6 +4,9 @@ import type {
   SyntaxTree,
   WikitextPageContext,
   WikitextSettings,
+  RatingRef,
+  RatingState,
+  ReadableTextOptions,
 } from "@wdprlib/ast";
 import type { IncludeDependency } from "../parser/rules/block/module/include";
 import type {
@@ -26,6 +29,17 @@ export interface ProcessWikitextCallbackContext<TPage extends WikitextPageContex
 }
 
 export interface ProcessWikitextDataProvider<TPage extends WikitextPageContext> {
+  /**
+   * Read only authorized, registered ratings in context.page, including declarations
+   * from its includes. Keys match exactly. Omit no-rate and inaccessible references.
+   * Main category policy and custom policies are independent. Votes belong to page + ref.
+   * ListPages/ListUsers output cannot declare ratings, including its nested includes.
+   * The host owns registration, aggregation and invalidation when included declarations change.
+   */
+  fetchRatings?: (
+    refs: readonly RatingRef[],
+    context: ProcessWikitextCallbackContext<TPage>,
+  ) => Promise<readonly RatingState[]>;
   fetchInclude?: (
     pageRef: PageRef,
     context: ProcessWikitextCallbackContext<TPage>,
@@ -50,12 +64,17 @@ export interface ProcessWikitextOptions<TPage extends WikitextPageContext> {
   settings?: WikitextSettings;
   dataProvider?: ProcessWikitextDataProvider<TPage>;
   includeMaxIterations?: number;
+  readableText?: ReadableTextOptions;
 }
 
 export interface ProcessedWikitextDocument<
   TPage extends WikitextPageContext = WikitextPageContext,
 > {
   ast: SyntaxTree;
+  readableText: string;
+  /** First nonempty body paragraph, excluding headings and appended footnotes. */
+  firstParagraph: string;
+  characterCount: number;
   page: TPage;
   settings: WikitextSettings;
   diagnostics: Diagnostic[];