@wdprlib/parser 5.1.6 → 5.3.0

package/src/parser/rules/block/definition-list/item-value.ts

@@ -27,17 +27,8 @@ export function parseDefinitionItemValue(
     }
 
     if (token.type === "NEWLINE") {
-      const nextToken = ctx.tokens[pos + 1];
-      if (nextToken?.type === "COLON" && nextToken.lineStart) {
-        pos++;
-        consumed++;
-        break;
-      }
-      if (nextToken?.type === "NEWLINE" || !nextToken || nextToken.type === "EOF") {
-        pos++;
-        consumed++;
-        break;
-      }
+      consumed++;
+      break;
     }
 
     const inlineCtx: ParseContext = { ...ctx, pos };

package/src/parser/rules/block/definition-list/items.ts

@@ -21,8 +21,7 @@ export interface ParsedDefinitionItem {
  *
  * The function expects `startPos` to point at a line-start COLON token.
  * It consumes the first colon, mandatory whitespace, key tokens up to
- * the second colon, then value tokens until a double newline, a new entry,
- * or end of input.
+ * the second colon, then value tokens until a newline or end of input.
  */
 export function parseDefinitionItem(
   ctx: ParseContext,

package/src/parser/rules/block/embed-block/index.ts

@@ -8,9 +8,6 @@
  * between the tags is stored verbatim as an `embed-block` element. Validation
  * and sanitisation are expected to happen at rendering time or on the server.
  *
- * The embed block is wrapped in a paragraph container in the AST, matching
- * Wikidot's rendering behaviour where embeds sit inside `<p>` tags.
- *
  * If no closing tag is found, the rule fails to prevent consuming the rest
  * of the document.
  *
@@ -18,6 +15,8 @@
  */
 import type { Element } from "@wdprlib/ast";
 import type { BlockRule, ParseContext, RuleResult } from "../../types";
+import { parseInlineUntil } from "../../inline/utils";
+import { normalizeParagraphElements } from "../paragraph/normalize";
 import { currentToken } from "../../types";
 import { collectEmbedContent, consumeEmbedClose } from "./content";
 import { parseEmbedBlockOpen } from "./open";
@@ -34,6 +33,8 @@ export const embedBlockRule: BlockRule = {
   name: "embed-block",
   startTokens: ["BLOCK_OPEN"],
   requiresLineStart: false,
+  preservesPrecedingLineBreak: true,
+  isStartPattern: (ctx, pos) => parseEmbedBlockOpen(ctx, pos) !== null,
 
   parse(ctx: ParseContext): RuleResult<Element> {
     const openToken = currentToken(ctx);
@@ -53,12 +54,17 @@ export const embedBlockRule: BlockRule = {
     consumed += contentResult.consumed;
 
     if (!contentResult.foundClose) {
-      ctx.diagnostics.push({
-        severity: "warning",
-        code: "unclosed-block",
-        message: `Missing closing tag [[/${openResult.blockName}]] for [[${openResult.blockName}]]`,
-        position: openToken.position,
-      });
+      if (
+        !ctx.diagnostics.some(
+          (d) => d.code === "unclosed-block" && d.position === openToken.position,
+        )
+      )
+        ctx.diagnostics.push({
+          severity: "warning",
+          code: "unclosed-block",
+          message: `Missing closing tag [[/${openResult.blockName}]] for [[${openResult.blockName}]]`,
+          position: openToken.position,
+        });
       return { success: false };
     }
 
@@ -66,26 +72,20 @@ export const embedBlockRule: BlockRule = {
     pos += closeConsumed;
     consumed += closeConsumed;
 
-    return {
-      success: true,
-      elements: [
-        {
-          element: "container",
-          data: {
-            type: "paragraph",
-            attributes: {},
-            elements: [
-              {
-                element: "embed-block",
-                data: {
-                  contents: contentResult.contents.trim(),
-                },
-              },
-            ],
-          },
-        },
-      ],
-      consumed,
-    };
+    const elements: Element[] = [
+      { element: "embed-block", data: { contents: contentResult.contents.trim() } },
+    ];
+    if (
+      ctx.scope.inlineEnd === undefined &&
+      ctx.tokens[pos]?.type !== "NEWLINE" &&
+      ctx.tokens[pos]?.type !== "EOF"
+    ) {
+      const after = parseInlineUntil({ ...ctx, pos }, "PARAGRAPH_BREAK");
+      const tail = normalizeParagraphElements(after.elements);
+      if (tail[0]?.element === "text") tail[0].data = tail[0].data.trimStart();
+      elements.push(...tail);
+      consumed += after.consumed;
+    }
+    return { success: true, elements, consumed };
   },
 };

package/src/parser/rules/block/heading/index.ts

@@ -55,6 +55,13 @@ export const headingRule: BlockRule = {
     const inlineCtx: ParseContext = { ...ctx, pos };
     const inlineResult = parseInlineUntil(inlineCtx, "NEWLINE");
     const children: Element[] = inlineResult.elements;
+    while (children.at(-1)?.element === "text") {
+      const last = children.at(-1)!;
+      if (last.element !== "text") break;
+      last.data = last.data.trimEnd();
+      if (last.data) break;
+      children.pop();
+    }
     consumed += inlineResult.consumed;
     pos += inlineResult.consumed;
 

package/src/parser/rules/block/module/listpages/resolution/items.ts

@@ -17,9 +17,9 @@ export function renderListPagesItems(
 
     const ctx: VariableContext = {
       page,
-      index: i + 1,
+      index: (data.pagination?.offset ?? 0) + i + 1,
       total: data.totalCount,
-      limit: module.limit,
+      limit: data.pagination ? data.pagination.limit : module.limit,
       site: data.site,
     };
 

package/src/parser/rules/block/module/listpages/resolution/pager.ts

@@ -0,0 +1,43 @@
+import type { Element } from "@wdprlib/ast";
+import type { ListPagesExternalData } from "../types";
+
+export function createListPagesPager(data: ListPagesExternalData): Element[] {
+  const pagination = data.pagination;
+  if (!pagination || pagination.totalPages <= 1) return [];
+  const { currentPage, totalPages, urlPath, parameter } = pagination;
+  const pages = new Set([1, totalPages]);
+  for (
+    let page = Math.max(1, currentPage - 2);
+    page <= Math.min(totalPages, currentPage + 2);
+    page++
+  ) {
+    pages.add(page);
+  }
+  return [
+    {
+      element: "pager",
+      data: {
+        currentPage,
+        totalPages,
+        pages: [...pages]
+          .sort((a, b) => a - b)
+          .map((page) => ({
+            page,
+            href: pageUrl(urlPath, parameter, page),
+          })),
+      },
+    },
+  ];
+}
+
+function pageUrl(urlPath: string, parameter: string, page: number): string {
+  const suffixStart = urlPath.search(/[?#]/);
+  const path = suffixStart === -1 ? urlPath : urlPath.slice(0, suffixStart);
+  const suffix = suffixStart === -1 ? "" : urlPath.slice(suffixStart);
+  const parts = path.split("/");
+  const result = parts.slice(0, 2);
+  for (let index = 2; index < parts.length; index += 2) {
+    if (parts[index] !== parameter) result.push(...parts.slice(index, index + 2));
+  }
+  return `${result.join("/").replace(/\/$/, "")}/${parameter}/${page}${suffix}`;
+}

package/src/parser/rules/block/module/listpages/resolution/wrapper.ts

@@ -1,13 +1,16 @@
 import type { Element } from "@wdprlib/ast";
 import { getModuleParseAst, type ParseFunction } from "../../types";
 import type { ListPagesModuleData } from "../resolve";
+import type { ListPagesExternalData } from "../types";
+import { createListPagesPager } from "./pager";
 
 export function wrapListPagesResult(
   module: ListPagesModuleData,
   items: Element[],
   parse: ParseFunction,
+  data?: ListPagesExternalData,
 ): Element[] {
-  if (items.length === 0) {
+  if (items.length === 0 && (!data || data.pages.length === 0)) {
     return [];
   }
 
@@ -25,6 +28,8 @@ export function wrapListPagesResult(
     result.push(...appendAst.elements);
   }
 
+  if (data) result.push(...createListPagesPager(data));
+
   if (module.wrapper) {
     return [
       {

package/src/parser/rules/block/module/listpages/resolve.ts

@@ -56,5 +56,5 @@ export function resolveListPages(
   parse: ParseFunction,
 ): Element[] {
   const items = renderListPagesItems(module, data, compiledTemplate, parse);
-  return wrapListPagesResult(module, items, parse);
+  return wrapListPagesResult(module, items, parse, data);
 }

package/src/parser/rules/block/module/listpages/types/external-data.ts

@@ -94,6 +94,19 @@ export interface SiteContext {
  */
 export interface ListPagesExternalData {
   pages: PageData[];
+  /** Matching items before offset and limit; use the same visibility filters as pages. */
   totalCount: number;
   site: SiteContext;
+  /** Filled by module resolution when a page URL is available. */
+  pagination?: {
+    currentPage: number;
+    perPage: number;
+    totalPages: number;
+    /** Actual fetch offset, including the module's starting offset. */
+    offset: number;
+    /** Total item limit after URL resolution, not the fetch batch size. */
+    limit?: number;
+    urlPath: string;
+    parameter: string;
+  };
 }

package/src/parser/rules/block/module/listpages/url-resolution/params.ts

@@ -2,12 +2,13 @@ import { URL_RESOLVABLE_FIELDS } from "./fields";
 
 /**
  * Parse URL path parameters like /offset/1/page2_limit/1.
- * Returns a map of parameter name -> value.
+ * Returns a map of parameter name -> value. Set hasPageName for a full page path;
+ * otherwise the legacy parameter-only-path heuristic is used.
  */
-export function parseUrlParams(url: string): Map<string, string> {
+export function parseUrlParams(url: string, hasPageName = false): Map<string, string> {
   const params = new Map<string, string>();
-  const parts = url.split("/").filter(Boolean);
-  const pairStart = isUrlParameter(parts[0]) ? 0 : 1;
+  const parts = url.split(/[?#]/, 1)[0]!.split("/").filter(Boolean);
+  const pairStart = !hasPageName && isUrlParameter(parts[0]) ? 0 : 1;
 
   // Skip the page name (first part), parse key/value pairs.
   for (let i = pairStart; i < parts.length - 1; i += 2) {

package/src/parser/rules/block/module/resolution/data-maps.ts

@@ -4,25 +4,97 @@ import type { ListUsersDataRequirement, ListUsersExternalData } from "../listuse
 import type { TagCloudDataRequirement, TagCloudExternalData } from "../tagcloud/types";
 import { parseUrlParams, resolveAndNormalizeQuery } from "../listpages/url-resolver";
 
+export interface ListPagesPaginationState {
+  nextUnprefixed: number;
+}
+
 export async function buildListPagesDataMap(
   dataProvider: DataProvider,
   requirements: ListPagesDataRequirement[],
   urlPath: string | undefined,
+  paginationState: ListPagesPaginationState = { nextUnprefixed: 1 },
 ): Promise<Map<number, ListPagesExternalData>> {
   const dataMap = new Map<number, ListPagesExternalData>();
-  const urlParams = parseUrlParams(urlPath ?? "");
+  const urlParams = parseUrlParams(urlPath ?? "", true);
 
   for (const req of requirements) {
     const normalizedQuery = resolveAndNormalizeQuery(req, urlParams);
-    const data = await dataProvider.fetchListPages?.(normalizedQuery, req);
+    if (!urlPath || !urlPath.startsWith("/") || urlPath.startsWith("//")) {
+      const data = await dataProvider.fetchListPages?.(normalizedQuery, req);
+      if (data) dataMap.set(req.id, data);
+      continue;
+    }
+    const ordinal = req.urlAttrPrefix ? 0 : paginationState.nextUnprefixed++;
+    const parameter = req.urlAttrPrefix
+      ? `${encodeURIComponent(req.urlAttrPrefix)}_p`
+      : ordinal === 1
+        ? "p"
+        : `p${ordinal}`;
+    const perPage = Math.min(250, positiveInteger(normalizedQuery.perPage, 20));
+    const baseOffset = positiveInteger(normalizedQuery.offset, 0);
+    const limit =
+      normalizedQuery.limit !== undefined && normalizedQuery.limit >= 0
+        ? Math.floor(normalizedQuery.limit)
+        : undefined;
+    const rawPage = urlParams.get(parameter) ?? "1";
+    let currentPage = /^\d+$/.test(rawPage) ? positiveInteger(Number(rawPage), 1) : 1;
+    currentPage = Math.min(
+      currentPage,
+      Math.floor((Number.MAX_SAFE_INTEGER - baseOffset) / perPage) + 1,
+    );
+    if (limit !== undefined)
+      currentPage = Math.min(currentPage, Math.max(1, Math.ceil(limit / perPage)));
+    const fetchPage = () => {
+      const relativeOffset = (currentPage - 1) * perPage;
+      return dataProvider.fetchListPages?.(
+        {
+          ...normalizedQuery,
+          offset: baseOffset + relativeOffset,
+          limit: Math.min(
+            perPage,
+            limit === undefined ? perPage : Math.max(0, limit - relativeOffset),
+          ),
+          perPage,
+        },
+        req,
+      );
+    };
+    let data = await fetchPage();
     if (data) {
-      dataMap.set(req.id, data);
+      const count = Math.min(Math.max(0, data.totalCount - baseOffset), limit ?? Infinity);
+      const totalPages = Math.ceil(count / perPage);
+      const lastPage = Math.max(1, totalPages);
+      if (currentPage > lastPage) {
+        currentPage = lastPage;
+        data = await fetchPage();
+      }
+      if (data)
+        dataMap.set(req.id, {
+          ...data,
+          pages: data.pages.slice(
+            0,
+            Math.min(perPage, Math.max(0, count - (currentPage - 1) * perPage)),
+          ),
+          pagination: {
+            currentPage,
+            perPage,
+            totalPages,
+            offset: baseOffset + (currentPage - 1) * perPage,
+            limit,
+            urlPath,
+            parameter,
+          },
+        });
     }
   }
 
   return dataMap;
 }
 
+function positiveInteger(value: number | undefined, fallback: number): number {
+  return value !== undefined && Number.isSafeInteger(value) && value > 0 ? value : fallback;
+}
+
 export async function buildListUsersDataMap(
   dataProvider: DataProvider,
   requirements: ListUsersDataRequirement[],

package/src/parser/rules/block/module/resolution/resolve-async.ts

@@ -26,6 +26,8 @@ import {
 import { buildListPagesDataMap, buildListUsersDataMap, buildTagCloudDataMap } from "./data-maps";
 import { ModuleDocumentRegistry } from "./document";
 import { collectStyles, mergeCollectedStyles } from "./styles";
+import { createListPagesPager } from "../listpages/resolution/pager";
+import type { ListPagesPaginationState } from "./data-maps";
 
 export type AsyncModuleParseFunction = (source: string) => Promise<ModuleParseResult>;
 
@@ -40,6 +42,7 @@ export interface ResolveModulesWithAsyncParseOptions {
   };
   urlPath?: string;
   pageTags: string[];
+  paginationState?: ListPagesPaginationState;
 }
 
 export interface AsyncModuleResolutionResult {
@@ -73,7 +76,12 @@ export async function resolveModulesWithAsyncParse(
     registry.register(await options.parse(source));
 
   const [listPagesData, listUsersData, tagCloudData] = await Promise.all([
-    buildListPagesDataMap(dataProvider, options.requirements.listPages ?? [], options.urlPath),
+    buildListPagesDataMap(
+      dataProvider,
+      options.requirements.listPages ?? [],
+      options.urlPath,
+      options.paginationState,
+    ),
     buildListUsersDataMap(dataProvider, options.requirements.listUsers ?? []),
     buildTagCloudDataMap(dataProvider, options.requirements.tagCloud ?? []),
   ]);
@@ -175,9 +183,9 @@ async function resolveListPagesAsync(
     if (!page) continue;
     const variableContext: VariableContext = {
       page,
-      index: i + 1,
+      index: (data.pagination?.offset ?? 0) + i + 1,
       total: data.totalCount,
-      limit: module.limit,
+      limit: data.pagination ? data.pagination.limit : module.limit,
       site: data.site,
     };
     const parsed = await parse(template(variableContext));
@@ -197,6 +205,7 @@ async function resolveListPagesAsync(
   if (module["append-line"] && !module.separate) {
     result.push(...(await parse(module["append-line"])).elements);
   }
+  result.push(...createListPagesPager(data));
 
   return module.wrapper
     ? [

package/src/parser/rules/block/paragraph/index.ts

@@ -44,17 +44,48 @@ export const paragraphRule: BlockRule = {
 
     return {
       success: true,
-      elements: [
-        {
-          element: "container",
-          data: {
-            type: "paragraph",
-            attributes: {},
-            elements,
-          },
-        },
-      ],
+      elements: wrapParagraphElements(elements),
       consumed: result.consumed,
     };
   },
 };
+
+/** Block images split paragraphs; ordinary images suppress the surrounding p. */
+export function wrapParagraphElements(elements: Element[]): Element[] {
+  const output: Element[] = [];
+  let group: Element[] = [];
+  let bare = false;
+  const flush = (trimBreaks = false) => {
+    const content = trimBreaks ? normalizeParagraphElements(group) : group;
+    while (content[0]?.element === "line-break") content.shift();
+    while (content.length) {
+      const last = content.at(-1)!;
+      if (last.element !== "text" || last.data.trim() !== "") break;
+      content.pop();
+    }
+    while (content[0]?.element === "text" && content[0].data.trim() === "") content.shift();
+    if (content[0]?.element === "text")
+      content[0] = { element: "text", data: content[0].data.trimStart() };
+    if (content.length)
+      output.push(
+        ...(bare || content.some((el) => el.element === "image")
+          ? content
+          : [
+              {
+                element: "container" as const,
+                data: { type: "paragraph" as const, attributes: {}, elements: content },
+              },
+            ]),
+      );
+    group = [];
+  };
+  for (const el of elements) {
+    if ((el.element === "image" && el.data.alignment !== null) || el.element === "embed-block") {
+      flush(el.element === "image");
+      output.push(el);
+      bare = el.element === "embed-block";
+    } else group.push(el);
+  }
+  flush();
+  return output;
+}

package/src/parser/rules/block/parsing/content.ts

@@ -52,7 +52,10 @@ export function parseBlocksUntil(
   let consumed = 0;
   let pos = ctx.pos;
 
-  const excluded = options?.excludedBlockNames;
+  const excluded = mergeExcludedBlockNames(
+    ctx.scope.excludedBlockNames,
+    options?.excludedBlockNames,
+  );
   const blockRules = excluded ? getExcludedBlockRules(ctx.blockRules, excluded) : ctx.blockRules;
   const blockScope = {
     ...ctx.scope,
@@ -106,6 +109,20 @@ export function parseBlocksUntil(
   return { elements, consumed };
 }
 
+/**
+ * An enclosing container's exclusions stay in force in its body: Wikidot keeps
+ * a `[[collapsible]]` nested in a div literal when the div itself sits in a
+ * collapsible.
+ */
+function mergeExcludedBlockNames(
+  inherited: ReadonlySet<string> | undefined,
+  added: ReadonlySet<string> | undefined,
+): ReadonlySet<string> | undefined {
+  if (!inherited?.size) return added;
+  if (!added?.size) return inherited;
+  return new Set([...inherited, ...added]);
+}
+
 function getExcludedBlockRules(
   blockRules: ParseContext["blockRules"],
   excluded: ReadonlySet<string>,

package/src/parser/rules/block/table/pipe/cell.ts

@@ -1,3 +1,4 @@
+import { protectedInlineRegionEnd } from "../../../inline/raw/end";
 import type { Element, TableCell } from "@wdprlib/ast";
 import type { ParseContext } from "../../../types";
 import { getCandidateInlineRules } from "../../../inline/utils";
@@ -20,7 +21,35 @@ export function parseTableCell(
   }
 
   const { inlineRules } = ctx;
-  const inlineCtx: ParseContext = { ...ctx, pos };
+  let inlineEnd = pos;
+  while (inlineEnd < ctx.tokens.length) {
+    const rawEnd = protectedInlineRegionEnd(ctx.tokens, inlineEnd, ctx.tokens.length);
+    if (rawEnd > inlineEnd) {
+      inlineEnd = rawEnd;
+      continue;
+    }
+    const token = ctx.tokens[inlineEnd];
+    if (!token || token.type === "EOF" || token.type === "NEWLINE" || isPipeTableToken(token.type))
+      break;
+    if (
+      token.type === "WHITESPACE" &&
+      ctx.tokens[inlineEnd + 1]?.type === "UNDERSCORE" &&
+      ctx.tokens[inlineEnd + 2]?.type === "NEWLINE"
+    )
+      inlineEnd += 3;
+    else inlineEnd++;
+  }
+  const inlineCtx: ParseContext = {
+    ...ctx,
+    pos,
+    scope: {
+      ...ctx.scope,
+      inlineEnd,
+      tableFormatting: isPipeTableToken(ctx.tokens[inlineEnd]?.type ?? "EOF")
+        ? ctx.scope.tableFormatting
+        : undefined,
+    },
+  };
 
   while (pos < ctx.tokens.length) {
     const token = ctx.tokens[pos];
@@ -31,6 +60,12 @@ export function parseTableCell(
       break;
     }
 
+    if (ctx.scope.tableFormatting?.suppressedClosers.has(pos)) {
+      pos++;
+      consumed++;
+      continue;
+    }
+
     if (tryConsumeUnderscoreLineBreak(ctx, pos, children)) {
       pos += 3;
       consumed += 3;

package/src/parser/rules/block/table/pipe/row.ts

@@ -1,3 +1,4 @@
+import { protectedInlineRegionEnd } from "../../../inline/raw/end";
 import type { TableCell, TableRow } from "@wdprlib/ast";
 import type { ParseContext } from "../../../types";
 import { parseCellStart } from "./cell-start";
@@ -8,6 +9,25 @@ export function parsePipeTableRows(
   ctx: ParseContext,
   startPos: number,
 ): { rows: TableRow[]; consumed: number } {
+  let end = startPos;
+  while (end < ctx.tokens.length && ctx.tokens[end]?.type !== "EOF") {
+    const protectedEnd = protectedInlineRegionEnd(ctx.tokens, end, ctx.tokens.length);
+    if (protectedEnd > end) {
+      end = protectedEnd;
+      continue;
+    }
+    if (
+      ctx.tokens[end]?.type === "NEWLINE" &&
+      !isPipeTableToken(ctx.tokens[end + 1]?.type ?? "EOF") &&
+      !(ctx.tokens[end - 1]?.type === "UNDERSCORE" && ctx.tokens[end - 2]?.type === "WHITESPACE")
+    )
+      break;
+    end++;
+  }
+  const tableCtx: ParseContext = {
+    ...ctx,
+    scope: { ...ctx.scope, tableFormatting: { end, suppressedClosers: new Set() } },
+  };
   const rows: TableRow[] = [];
   let pos = startPos;
   let consumed = 0;
@@ -19,7 +39,7 @@ export function parsePipeTableRows(
       break;
     }
 
-    const rowResult = parseTableRow(ctx, pos);
+    const rowResult = parseTableRow(tableCtx, pos);
     rows.push(rowResult.row);
     pos += rowResult.consumed;
     consumed += rowResult.consumed;

package/src/parser/rules/block/toc/element.ts

@@ -1,10 +1,10 @@
 import type { Alignment, Element } from "@wdprlib/ast";
 
-export function createTocElement(align: Alignment | null): Element {
+export function createTocElement(align: Alignment | null, title: string | undefined): Element {
   return {
     element: "table-of-contents",
     data: {
-      attributes: {},
+      attributes: title === undefined ? {} : { title },
       align,
     },
   };

package/src/parser/rules/block/toc/index.ts

@@ -6,7 +6,7 @@
  * - [[f<toc ...]] - float left
  * - [[f>toc ...]] - float right
  *
- * Note: Wikidot ignores attributes on [[toc]] (class, style, id are not applied)
+ * The title attribute overrides the heading; class, style, and id are ignored.
  * [[>toc]] and [[<toc]] are invalid in Wikidot and not supported.
  */
 import type { Element } from "@wdprlib/ast";
@@ -37,7 +37,7 @@ export const tocRule: BlockRule = {
 
     return {
       success: true,
-      elements: [createTocElement(openResult.align)],
+      elements: [createTocElement(openResult.align, openResult.title)],
       consumed: openResult.consumed,
     };
   },