@wdprlib/parser 4.2.0 → 4.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wdprlib/parser",
-  "version": "4.2.0",
+  "version": "4.4.0",
   "description": "Parser for Wikidot markup",
   "keywords": [
     "ast",
@@ -41,6 +41,6 @@
   },
   "dependencies": {
     "@braintree/sanitize-url": "^7.1.1",
-    "@wdprlib/ast": "2.2.0"
+    "@wdprlib/ast": "2.4.0"
   }
 }

package/src/index.ts

@@ -58,6 +58,12 @@ export type {
   DateItem,
   Embed,
   TocEntry,
+  // Gallery
+  GallerySize,
+  GalleryOrder,
+  GalleryItem,
+  GalleryContent,
+  GalleryData,
   // Diagnostics
   Diagnostic,
   DiagnosticSeverity,
@@ -92,6 +98,15 @@ export { Lexer, tokenize, createToken } from "./lexer";
 export type { ParserOptions } from "./parser";
 export { Parser, parse } from "./parser";
 
+// High-level parser pipeline
+export { processWikitext } from "./pipeline";
+export type {
+  ProcessedWikitextDocument,
+  ProcessWikitextCallbackContext,
+  ProcessWikitextDataProvider,
+  ProcessWikitextOptions,
+} from "./pipeline";
+
 // Modules (ListPages, ListUsers, IfTags, Include, etc.)
 export type {
   // ListPages query types
@@ -115,6 +130,7 @@ export type {
   ExtractionResult,
   // Resolution types
   ParseFunction,
+  ModuleParseResult,
   ModuleSourceTransform,
   ResolveOptions,
   // Include resolution
@@ -157,6 +173,7 @@ export {
   extractIncludeReferences,
   resolveIncludes,
   resolveIncludesAsync,
+  resolveIncludesAsyncWithTrace,
   resolveIncludesWithTrace,
   // IfTags source-level preprocessing (run between include expansion and parse)
   preprocessIftags,

package/src/parser/parse/context.ts

@@ -10,6 +10,7 @@ export function createParseContext(tokens: Token[], options: ParserOptions = {})
     version: options.version ?? "wikidot",
     trackPositions: options.trackPositions ?? true,
     settings: options.settings ?? DEFAULT_SETTINGS,
+    appendImplicitFootnoteBlock: options.appendImplicitFootnoteBlock ?? true,
     footnotes: [],
     tocEntries: [],
     codeBlocks: [],

package/src/parser/parse/options.ts

@@ -31,4 +31,10 @@ export interface ParserOptions {
    * - `string[]`: every iftags block is evaluated against the given tags eagerly.
    */
   pageTags?: string[] | null;
+  /**
+   * Append the implicit document-level footnote block when no explicit block
+   * exists. Defaults to true. Fragment pipelines can disable this and add one
+   * block after all fragments have been merged.
+   */
+  appendImplicitFootnoteBlock?: boolean;
 }

package/src/parser/parse/result.ts

@@ -10,7 +10,9 @@ import { containsFootnoteBlock } from "./footnotes";
 
 export function finalizeParseResult(ctx: ParseContext, children: Element[]): ParseResult {
   const cleanedChildren = postprocessChildren(children);
-  appendImplicitFootnoteBlock(cleanedChildren);
+  if (ctx.appendImplicitFootnoteBlock) {
+    appendImplicitFootnoteBlock(cleanedChildren);
+  }
 
   return {
     ast: buildSyntaxTree(ctx, cleanedChildren),

package/src/parser/rules/block/gallery/index.ts

@@ -0,0 +1,215 @@
+/**
+ *
+ * Block rule for the Wikidot `[[gallery]]` image gallery.
+ *
+ * ```
+ * [[gallery size="thumbnail" order="name" viewer="no"]]
+ * : first-image.jpg
+ * : *page/other-image.jpg link="some-page" alt="Alt text"
+ * [[/gallery]]
+ * ```
+ *
+ * The content form requires the `]]` to be followed immediately by one or
+ * more `: source` lines and a `[[/gallery]]` close tag (Wikidot regex
+ * `\[\[gallery(\s[^\]]*?)?\]\](?:((?:\n: [^\n]+)+)\n\[\[\/gallery\]\])?`).
+ * Anything else falls back to the standalone content-less form, which shows
+ * the current page's image attachments after data resolution
+ * (`content: { type: "auto", files: null }`); the following text then
+ * parses normally.
+ *
+ * Honored opening-tag attributes are `size` (small / medium / thumbnail /
+ * square / original, fallback thumbnail), `viewer` (`"no"`/`"false"`
+ * disable the lightbox) and `order` (auto-collection sort order, with
+ * Wikidot's deprecated aliases normalized).
+ *
+ * Deliberate differences from the Wikidot parser: the opening tag must
+ * fit on one line and attribute names are lowercased with unquoted values
+ * accepted (both shared with wdpr's other block rules), and `flickr:`
+ * sources get no special treatment — they resolve like any other
+ * filename (wdpr does not call the Flickr API).
+ *
+ * @module
+ */
+import type { Element, GalleryItem, GalleryOrder, GallerySize } from "@wdprlib/ast";
+import type { BlockRule, ParseContext, RuleResult } from "../../types";
+import { parseAttributesRaw, parseBlockName } from "../utils";
+import { parseGalleryItemLine } from "./items";
+
+export { parseGalleryItemLine } from "./items";
+
+const GALLERY_SIZES: readonly string[] = ["small", "medium", "thumbnail", "square", "original"];
+
+/** Validate a size keyword, falling back to thumbnail like Wikidot. */
+function normalizeSize(value: string | undefined): GallerySize {
+  return value !== undefined && GALLERY_SIZES.includes(value)
+    ? (value as GallerySize)
+    : "thumbnail";
+}
+
+/** `viewer="no"` / `viewer="false"` disable the lightbox; anything else enables it. */
+function normalizeViewer(value: string | undefined): boolean {
+  return value !== "no" && value !== "false";
+}
+
+/**
+ * Normalize the `order` attribute, folding Wikidot's deprecated aliases
+ * (`nameDesc`, `dateAdded`, `dateAddedDesc`) and the documented
+ * ListPages-compatibility forms (`"name desc desc"` / `"created_at desc
+ * desc"`, which mean the same as without the `desc desc`).
+ */
+function normalizeOrder(value: string | undefined): GalleryOrder {
+  switch (value) {
+    case "name":
+    case "name desc":
+    case "created_at":
+    case "created_at desc":
+      return value;
+    case "nameDesc":
+      return "name desc";
+    case "dateAdded":
+      return "created_at";
+    case "dateAddedDesc":
+      return "created_at desc";
+    case "name desc desc":
+      return "name";
+    case "created_at desc desc":
+      return "created_at";
+    default:
+      return "name";
+  }
+}
+
+interface GalleryContentResult {
+  /** Trimmed line contents (after the leading `: `) */
+  lines: string[];
+  /** Token count from the content start (the NEWLINE after `]]`) through `[[/gallery]]` */
+  consumed: number;
+}
+
+/**
+ * Try to match the content form starting at `pos` (the token right after the
+ * opening tag's `]]`): one NEWLINE, then consecutive `: source` lines, then a
+ * NEWLINE directly followed by `[[/gallery]]`. Returns null when the token
+ * stream deviates from the Wikidot regex, which makes the gallery standalone.
+ */
+function tryParseGalleryContent(ctx: ParseContext, pos: number): GalleryContentResult | null {
+  const start = pos;
+  if (ctx.tokens[pos]?.type !== "NEWLINE") {
+    return null;
+  }
+
+  const lines: string[] = [];
+  let p = pos + 1;
+
+  for (;;) {
+    const colon = ctx.tokens[p];
+    if (colon?.type !== "COLON" || !colon.lineStart) {
+      break;
+    }
+    // Wikidot requires a literal space after the colon (`\n: `)
+    const space = ctx.tokens[p + 1];
+    if (space?.type !== "WHITESPACE" || !space.value.startsWith(" ")) {
+      break;
+    }
+
+    let content = "";
+    let q = p + 1;
+    while (q < ctx.tokens.length) {
+      const token = ctx.tokens[q];
+      if (!token || token.type === "NEWLINE" || token.type === "EOF") {
+        break;
+      }
+      content += token.value;
+      q++;
+    }
+    // `: [^\n]+` needs at least one character after the space
+    if (content === " ") {
+      return null;
+    }
+    if (ctx.tokens[q]?.type !== "NEWLINE") {
+      // line hit EOF: the close tag can no longer follow on its own line
+      return null;
+    }
+
+    lines.push(content.trim());
+    p = q + 1;
+  }
+
+  if (lines.length === 0) {
+    return null;
+  }
+
+  // The last consumed NEWLINE must be directly followed by [[/gallery]]
+  if (ctx.tokens[p]?.type !== "BLOCK_END_OPEN") {
+    return null;
+  }
+  const nameResult = parseBlockName(ctx, p + 1);
+  if (!nameResult || nameResult.name !== "gallery") {
+    return null;
+  }
+  const closePos = p + 1 + nameResult.consumed;
+  if (ctx.tokens[closePos]?.type !== "BLOCK_CLOSE") {
+    return null;
+  }
+
+  return { lines, consumed: closePos + 1 - start };
+}
+
+export const galleryRule: BlockRule = {
+  name: "gallery",
+  startTokens: ["BLOCK_OPEN"],
+  requiresLineStart: true,
+
+  parse(ctx: ParseContext): RuleResult<Element> {
+    if (ctx.tokens[ctx.pos]?.type !== "BLOCK_OPEN") {
+      return { success: false };
+    }
+
+    let pos = ctx.pos + 1;
+    const nameResult = parseBlockName(ctx, pos);
+    if (!nameResult || nameResult.name !== "gallery") {
+      return { success: false };
+    }
+    pos += nameResult.consumed;
+
+    const attrResult = parseAttributesRaw(ctx, pos);
+    pos += attrResult.consumed;
+
+    if (ctx.tokens[pos]?.type !== "BLOCK_CLOSE") {
+      return { success: false };
+    }
+    pos++;
+
+    const size = normalizeSize(attrResult.attrs.size);
+    const viewer = normalizeViewer(attrResult.attrs.viewer);
+    const order = normalizeOrder(attrResult.attrs.order);
+    const openConsumed = pos - ctx.pos;
+
+    const content = tryParseGalleryContent(ctx, pos);
+    if (!content) {
+      return {
+        success: true,
+        elements: [
+          {
+            element: "gallery",
+            data: { size, order, viewer, content: { type: "auto", files: null } },
+          },
+        ],
+        consumed: openConsumed,
+      };
+    }
+
+    const items: GalleryItem[] = content.lines.map(parseGalleryItemLine);
+
+    return {
+      success: true,
+      elements: [
+        {
+          element: "gallery",
+          data: { size, order, viewer, content: { type: "items", items } },
+        },
+      ],
+      consumed: openConsumed + content.consumed,
+    };
+  },
+};

package/src/parser/rules/block/gallery/items.ts

@@ -0,0 +1,62 @@
+import type { GalleryItem } from "@wdprlib/ast";
+
+/**
+ * Parse one gallery content line (the text after the leading `: `,
+ * already trimmed) into a {@link GalleryItem}.
+ *
+ * The text before the first space is the source; the rest is parsed as
+ * `key="value"` attributes of which `link` and `alt` are honored. A
+ * leading `*` on the source or on the link value requests a new window.
+ */
+export function parseGalleryItemLine(content: string): GalleryItem {
+  const spacePos = content.indexOf(" ");
+  let source = spacePos < 0 ? content : content.slice(0, spacePos);
+  const attrText = spacePos < 0 ? "" : content.slice(spacePos + 1);
+
+  let newWindow = false;
+  if (source.startsWith("*")) {
+    source = source.slice(1);
+    newWindow = true;
+  }
+
+  const attrs = parseItemAttrs(attrText);
+  let link = attrs.get("link") ?? null;
+  if (link !== null && link.startsWith("*")) {
+    newWindow = true;
+    link = link.slice(1);
+  }
+  const alt = attrs.get("alt") ?? null;
+
+  return { source, link, alt, newWindow };
+}
+
+/**
+ * Parse `key="value"` attribute pairs from a gallery item line, following
+ * the splitting behavior of Wikidot's gallery `getAttrs()`: fragments are
+ * separated by `="`, values run to the last `"` in each fragment and are
+ * backslash-unescaped, and keys are not lowercased.
+ */
+function parseItemAttrs(text: string): Map<string, string> {
+  const attrs = new Map<string, string>();
+  const parts = text.trim().split('="');
+  let key = parts[0]?.trim() ?? "";
+
+  for (let i = 1; i < parts.length; i++) {
+    const val = parts[i] ?? "";
+    const quotePos = val.lastIndexOf('"');
+    if (quotePos < 0) {
+      attrs.set(key, "");
+      key = val.slice(1).trim();
+    } else {
+      attrs.set(key, stripslashes(val.slice(0, quotePos)));
+      key = val.slice(quotePos + 1).trim();
+    }
+  }
+
+  return attrs;
+}
+
+/** PHP-style `stripslashes()`: drop each escaping backslash, and a trailing lone one. */
+function stripslashes(value: string): string {
+  return value.replace(/\\(.)/gs, "$1").replace(/\\$/, "");
+}

package/src/parser/rules/block/index.ts

@@ -49,6 +49,7 @@ import { iftagsRule } from "./iftags";
 import { tocRule } from "./toc";
 import { orphanLiRule } from "./orphan-li";
 import { bibliographyRule } from "./bibliography";
+import { galleryRule } from "./gallery";
 
 export { headingRule } from "./heading";
 export { horizontalRuleRule } from "./horizontal-rule";
@@ -79,6 +80,7 @@ export { iftagsRule } from "./iftags";
 export { tocRule } from "./toc";
 export { orphanLiRule } from "./orphan-li";
 export { bibliographyRule } from "./bibliography";
+export { galleryRule } from "./gallery";
 
 /**
  * All block rules in priority order.
@@ -120,6 +122,7 @@ export const blockRules: BlockRule[] = [
   iframeRule,
   iftagsRule,
   bibliographyRule,
+  galleryRule,
   divRule,
   // paragraphRule is not included - used as fallback
 ];

package/src/parser/rules/block/module/include/index.ts

@@ -16,7 +16,12 @@
  * @module
  */
 
-export { resolveIncludes, resolveIncludesAsync, resolveIncludesWithTrace } from "./resolve";
+export {
+  resolveIncludes,
+  resolveIncludesAsync,
+  resolveIncludesAsyncWithTrace,
+  resolveIncludesWithTrace,
+} from "./resolve";
 export { extractIncludeReferences } from "./references";
 export type {
   IncludeFetcher,

package/src/parser/rules/block/module/include/resolve/index.ts

@@ -7,7 +7,11 @@
 
 import type { PageRef } from "@wdprlib/ast";
 import { createCachedAsyncIncludeFetcher, createCachedIncludeFetcher } from "./cache";
-import { expandIterative, expandIterativeAsync, expandIterativeWithTrace } from "./iterate";
+import {
+  expandIterative,
+  expandIterativeAsyncWithTrace,
+  expandIterativeWithTrace,
+} from "./iterate";
 import type {
   AsyncIncludeFetcher,
   IncludeFetcher,
@@ -84,13 +88,29 @@ export async function resolveIncludesAsync(
   fetcher: AsyncIncludeFetcher,
   options?: ResolveIncludesOptions,
 ): Promise<string> {
+  return (await resolveIncludesAsyncWithTrace(source, fetcher, options)).source;
+}
+
+/**
+ * Async include expansion with dependency and iteration trace data.
+ */
+export async function resolveIncludesAsyncWithTrace(
+  source: string,
+  fetcher: AsyncIncludeFetcher,
+  options?: ResolveIncludesOptions,
+): Promise<ResolveIncludesTraceResult> {
   if (options?.settings && !options.settings.enablePageSyntax) {
-    return source;
+    return {
+      source,
+      dependencies: [],
+      iterations: [],
+      reachedMaxIterations: false,
+    };
   }
 
   const maxIterations = options?.maxIterations ?? 10;
   const cachedFetcher = createCachedAsyncIncludeFetcher(fetcher, normalizePageKey);
-  return expandIterativeAsync(source, cachedFetcher, maxIterations);
+  return expandIterativeAsyncWithTrace(source, cachedFetcher, maxIterations);
 }
 
 /**

package/src/parser/rules/block/module/include/resolve/iterate.ts

@@ -102,6 +102,41 @@ export async function expandIterativeAsync(
   return current;
 }
 
+/** Async counterpart of {@link expandIterativeWithTrace}. */
+export async function expandIterativeAsyncWithTrace(
+  source: string,
+  fetcher: AsyncIncludeFetcher,
+  maxIterations: number,
+): Promise<ResolveIncludesTraceResult> {
+  let current = source;
+  const replacementCache = new Map<string, Promise<string>>();
+  const dependencies: IncludeDependency[] = [];
+  const iterations: ResolveIncludesTraceResult["iterations"] = [];
+
+  for (let i = 0; i < maxIterations; i++) {
+    const expanded = await expandOneIterationAsyncWithTrace(current, fetcher, replacementCache, i);
+    if (expanded === null) break;
+
+    dependencies.push(...expanded.dependencies);
+    iterations.push({
+      iteration: i,
+      directives: expanded.references,
+      changed: expanded.source !== current,
+    });
+
+    if (expanded.source === current) break;
+    current = expanded.source;
+  }
+
+  return {
+    source: current,
+    dependencies,
+    iterations,
+    reachedMaxIterations:
+      iterations.length === maxIterations && scanIncludeDirectives(current).length > 0,
+  };
+}
+
 function expandOneIteration(
   source: string,
   fetcher: IncludeFetcher,
@@ -179,6 +214,42 @@ async function expandOneIterationAsync(
   return parts.join("");
 }
 
+async function expandOneIterationAsyncWithTrace(
+  source: string,
+  fetcher: AsyncIncludeFetcher,
+  replacementCache: Map<string, Promise<string>>,
+  iteration: number,
+): Promise<{
+  source: string;
+  references: IncludeReference[];
+  dependencies: IncludeDependency[];
+} | null> {
+  if (!MAYBE_INCLUDE_PATTERN.test(source)) return null;
+
+  const directives = scanIncludeDirectives(source);
+  if (directives.length === 0) return null;
+
+  const references = directives.map(createIncludeReference);
+  const replacements = await Promise.all(
+    directives.map(({ inner }) => replaceCachedAsync(inner, fetcher, replacementCache)),
+  );
+  const parts: string[] = [];
+  let lastPos = 0;
+
+  for (let i = 0; i < directives.length; i++) {
+    const { start, end } = directives[i]!;
+    parts.push(source.slice(lastPos, start), replacements[i]!);
+    lastPos = end;
+  }
+  parts.push(source.slice(lastPos));
+
+  return {
+    source: parts.join(""),
+    references,
+    dependencies: references.map((reference) => ({ ...reference, iteration })),
+  };
+}
+
 function replaceCached(inner: string, fetcher: IncludeFetcher, cache: Map<string, string>): string {
   const cached = cache.get(inner);
   if (cached !== undefined) return cached;

package/src/parser/rules/block/module/index.ts

@@ -20,7 +20,7 @@
  */
 
 // Module rule types and registry
-export type { ModuleRule } from "./types";
+export type { ModuleParseResult, ModuleRule } from "./types";
 export { MODULE_RULES, getModuleRuleByName } from "./mapping";
 export { moduleRule } from "./rule";
 
@@ -106,6 +106,7 @@ export {
   extractIncludeReferences,
   resolveIncludes,
   resolveIncludesAsync,
+  resolveIncludesAsyncWithTrace,
   resolveIncludesWithTrace,
 } from "./include";
 

package/src/parser/rules/block/module/listpages/resolution/items.ts

@@ -1,6 +1,6 @@
 import type { Element } from "@wdprlib/ast";
 import type { CompiledTemplate, ListPagesExternalData, VariableContext } from "../types";
-import type { ParseFunction } from "../../types";
+import { getModuleParseAst, type ParseFunction } from "../../types";
 import type { ListPagesModuleData } from "../resolve";
 
 export function renderListPagesItems(
@@ -23,7 +23,7 @@ export function renderListPagesItems(
       site: data.site,
     };
 
-    const itemAst = parse(compiledTemplate(ctx));
+    const itemAst = getModuleParseAst(parse(compiledTemplate(ctx)));
 
     if (module.separate) {
       items.push({

package/src/parser/rules/block/module/listpages/resolution/wrapper.ts

@@ -1,5 +1,5 @@
 import type { Element } from "@wdprlib/ast";
-import type { ParseFunction } from "../../types";
+import { getModuleParseAst, type ParseFunction } from "../../types";
 import type { ListPagesModuleData } from "../resolve";
 
 export function wrapListPagesResult(
@@ -14,14 +14,14 @@ export function wrapListPagesResult(
   const result: Element[] = [];
 
   if (module["prepend-line"] && !module.separate) {
-    const prependAst = parse(module["prepend-line"]);
+    const prependAst = getModuleParseAst(parse(module["prepend-line"]));
     result.push(...prependAst.elements);
   }
 
   result.push(...items);
 
   if (module["append-line"] && !module.separate) {
-    const appendAst = parse(module["append-line"]);
+    const appendAst = getModuleParseAst(parse(module["append-line"]));
     result.push(...appendAst.elements);
   }
 

package/src/parser/rules/block/module/listusers/resolve.ts

@@ -15,7 +15,7 @@ import type {
   ListUsersCompiledTemplate,
   ListUsersVariableContext,
 } from "./types";
-import type { ParseFunction } from "../types";
+import { getModuleParseAst, type ParseFunction } from "../types";
 
 /**
  * Narrowed type for the list-users variant of the Module discriminated union.
@@ -53,6 +53,6 @@ export function resolveListUsers(
 ): Element[] {
   const ctx: ListUsersVariableContext = { user: data.user };
   const substituted = compiledTemplate(ctx);
-  const itemAst = parse(substituted);
+  const itemAst = getModuleParseAst(parse(substituted));
   return itemAst.elements;
 }