@wdprlib/parser 3.1.2 → 4.0.0

package/src/parser/rules/block/div/body.ts

@@ -0,0 +1,41 @@
+import type { Element } from "@wdprlib/ast";
+import type { ParseContext } from "../../types";
+import { parseBlocksUntil } from "../utils";
+import { isDivClose } from "./close";
+import { countDivCloses } from "./nesting";
+import { unwrapEdgeParagraphs } from "./paragraph-strip";
+
+export interface DivBodyResult {
+  elements: Element[];
+  consumed: number;
+}
+
+export function parseDivBody(
+  ctx: ParseContext,
+  startPos: number,
+  paragraphStrip: boolean,
+): DivBodyResult {
+  const bodyBudget = resolveDivBodyBudget(ctx, startPos);
+  const bodyCtx: ParseContext = {
+    ...ctx,
+    pos: startPos,
+    scope: { ...ctx.scope, divClosesBudget: bodyBudget },
+  };
+
+  const bodyResult = parseBlocksUntil(bodyCtx, isDivClose);
+  const elements = paragraphStrip ? unwrapEdgeParagraphs(bodyResult.elements) : bodyResult.elements;
+
+  return {
+    elements,
+    consumed: bodyResult.consumed,
+  };
+}
+
+function resolveDivBodyBudget(ctx: ParseContext, startPos: number): number {
+  if (ctx.scope.divClosesBudget !== undefined) {
+    return ctx.scope.divClosesBudget - 1;
+  }
+
+  const closesInScope = countDivCloses(ctx, startPos);
+  return closesInScope > 0 ? closesInScope - 1 : 0;
+}

package/src/parser/rules/block/div/close.ts

@@ -0,0 +1,41 @@
+import type { ParseContext } from "../../types";
+import { parseBlockName } from "../utils";
+
+export function isDivClose(ctx: ParseContext): boolean {
+  const token = ctx.tokens[ctx.pos];
+  if (token?.type !== "BLOCK_END_OPEN") return false;
+
+  const closeNameResult = parseBlockName(ctx, ctx.pos + 1);
+  return closeNameResult?.name === "div";
+}
+
+export function consumeDivClose(
+  ctx: ParseContext,
+  startPos: number,
+): { pos: number; consumed: number } {
+  let pos = startPos;
+  let consumed = 0;
+
+  if (ctx.tokens[pos]?.type !== "BLOCK_END_OPEN") {
+    return { pos, consumed };
+  }
+
+  pos++;
+  consumed++;
+
+  const closeNameResult = parseBlockName(ctx, pos);
+  if (closeNameResult) {
+    pos += closeNameResult.consumed;
+    consumed += closeNameResult.consumed;
+  }
+  if (ctx.tokens[pos]?.type === "BLOCK_CLOSE") {
+    pos++;
+    consumed++;
+  }
+  if (ctx.tokens[pos]?.type === "NEWLINE") {
+    pos++;
+    consumed++;
+  }
+
+  return { pos, consumed };
+}

package/src/parser/rules/block/div/failed.ts

@@ -0,0 +1,117 @@
+import type { Element } from "@wdprlib/ast";
+import type { ParseContext, RuleResult } from "../../types";
+import { parseAttributes, parseBlockName } from "../utils";
+
+/**
+ * Handles the case where `[[div]]` fails as a block element because
+ * the closing `]]` is not followed by a NEWLINE.
+ */
+export function consumeFailedDiv(ctx: ParseContext): RuleResult<Element> {
+  const elements: Element[] = [];
+  let pos = ctx.pos;
+  let consumed = 0;
+  let lastClosePos = -1;
+  let lastCloseConsumed = 0;
+
+  let scanPos = pos;
+  while (scanPos < ctx.tokens.length) {
+    const t = ctx.tokens[scanPos];
+    if (!t || t.type === "EOF") break;
+
+    if (t.type === "BLOCK_OPEN" && t.lineStart && scanPos > pos) {
+      const nameResult = parseBlockName(ctx, scanPos + 1);
+      if (nameResult?.name === "div" || nameResult?.name === "div_") {
+        let checkPos = scanPos + 1 + nameResult.consumed;
+        const attrResult = parseAttributes(ctx, checkPos);
+        checkPos += attrResult.consumed;
+        if (ctx.tokens[checkPos]?.type === "BLOCK_CLOSE") {
+          checkPos++;
+          if (ctx.tokens[checkPos]?.type === "NEWLINE" || ctx.tokens[checkPos]?.type === "EOF") {
+            break;
+          }
+        }
+      }
+    }
+
+    if (t.type === "BLOCK_END_OPEN") {
+      const nameResult = parseBlockName(ctx, scanPos + 1);
+      if (nameResult?.name === "div") {
+        lastClosePos = scanPos;
+        lastCloseConsumed = 1 + nameResult.consumed;
+        const closeToken = ctx.tokens[scanPos + 1 + nameResult.consumed];
+        if (closeToken?.type === "BLOCK_CLOSE") {
+          lastCloseConsumed++;
+        }
+      }
+    }
+    scanPos++;
+  }
+
+  if (lastClosePos === -1) {
+    return { success: false };
+  }
+
+  const endPosForDiag = lastClosePos;
+  for (let diagPos = ctx.pos; diagPos < endPosForDiag; diagPos++) {
+    const t = ctx.tokens[diagPos];
+    if (t?.type === "BLOCK_OPEN") {
+      const nameResult = parseBlockName(ctx, diagPos + 1);
+      if (nameResult?.name === "div" || nameResult?.name === "div_") {
+        if (t.position) {
+          ctx.diagnostics.push({
+            severity: "error",
+            code: "inline-block-element",
+            message: `[[${nameResult.name}]] must be followed by a newline to be a block element`,
+            position: t.position,
+          });
+        }
+      }
+    }
+  }
+
+  const endPos = lastClosePos + lastCloseConsumed;
+  while (pos < endPos && pos < ctx.tokens.length) {
+    const t = ctx.tokens[pos];
+    if (!t || t.type === "EOF") break;
+
+    if (t.type === "NEWLINE") {
+      let peekPos = pos + 1;
+      while (ctx.tokens[peekPos]?.type === "WHITESPACE") peekPos++;
+      if (ctx.tokens[peekPos]?.type === "NEWLINE") {
+        while (ctx.tokens[pos]?.type === "NEWLINE" || ctx.tokens[pos]?.type === "WHITESPACE") {
+          pos++;
+          consumed++;
+        }
+        continue;
+      }
+      elements.push({ element: "line-break" });
+      pos++;
+      consumed++;
+      continue;
+    }
+
+    elements.push({ element: "text", data: t.value });
+    pos++;
+    consumed++;
+  }
+
+  if (ctx.tokens[pos]?.type === "NEWLINE") {
+    pos++;
+    consumed++;
+  }
+
+  return {
+    success: true,
+    elements: [
+      {
+        element: "container",
+        data: {
+          type: "paragraph",
+          attributes: {},
+          elements,
+        },
+      },
+    ],
+    consumed,
+  };
+}

package/src/parser/rules/block/div/index.ts

@@ -0,0 +1,112 @@
+/**
+ *
+ * Block rule for Wikidot `[[div]]` and `[[div_]]` container blocks.
+ *
+ * `[[div]]` wraps its body content in a `<div>` element, with full
+ * paragraph processing for the body. `[[div_]]` (paragraph strip mode)
+ * unwraps the first and last paragraphs so their content appears directly
+ * inside the `<div>`, while middle paragraphs keep their `<p>` wrappers.
+ *
+ * Both variants accept HTML attributes (class, style, id, etc.) on the
+ * opening tag.
+ *
+ * Wikidot-specific edge cases:
+ * - The opening `]]` MUST be followed by a NEWLINE for the block to be
+ *   recognised. `[[div]]inline[[/div]]` is NOT a valid div -- it becomes
+ *   a failed div (see `consumeFailedDiv()`).
+ * - When a div fails, everything from the opening `[[div]]` through the
+ *   last `[[/div]]` is collected as a single paragraph of text/line-break
+ *   elements. Blank lines within that span are silently removed.
+ * - `[[div_]]` uses `unwrapEdgeParagraphs()` to strip paragraph
+ *   wrappers from the first and last elements.
+ *
+ * @module
+ */
+import type { Element } from "@wdprlib/ast";
+import type { BlockRule, ParseContext, RuleResult } from "../../types";
+import { currentToken } from "../../types";
+import { parseDivBody } from "./body";
+import { consumeDivClose } from "./close";
+import { consumeFailedDiv } from "./failed";
+import { parseDivOpen } from "./open";
+
+/**
+ * Block rule for `[[div]]`/`[[div_]]` container blocks.
+ *
+ * `requiresLineStart` is `false` because nested `[[div_]]` inside another
+ * `[[div_]]` may appear after inline content.
+ */
+export const divRule: BlockRule = {
+  name: "div",
+  startTokens: ["BLOCK_OPEN"],
+  requiresLineStart: false, // Allow nested [[div_]] inside [[div_]]
+
+  parse(ctx: ParseContext): RuleResult<Element> {
+    const openToken = currentToken(ctx);
+    if (openToken.type !== "BLOCK_OPEN") {
+      return { success: false };
+    }
+
+    const openResult = parseDivOpen(ctx);
+    if (!openResult) {
+      return { success: false };
+    }
+
+    // Wikidot: [[div]] must be followed by newline to be recognized as block
+    // [[div]]inline[[/div]] is NOT recognized as div
+    // When this fails, Wikidot consumes everything up to the last [[/div]]
+    // as text in a single paragraph (blank lines are ignored)
+    if (!openResult.hasRequiredNewline) {
+      return consumeFailedDiv(ctx);
+    }
+
+    // Wikidot matches [[div]]/[[/div]] pairs from outside-in. When there are
+    // more opens than closes, the innermost excess opens become text. We enforce
+    // this with a "closes budget": the number of additional nested divs that can
+    // open. When budget reaches 0, this div cannot open.
+    if (ctx.scope.divClosesBudget === 0) {
+      return { success: false };
+    }
+
+    // Record opening tag position for diagnostics
+    const openPosition = openToken.position;
+
+    let pos = openResult.bodyStart;
+    let consumed = openResult.consumed;
+    const bodyResult = parseDivBody(ctx, pos, openResult.paragraphStrip);
+    consumed += bodyResult.consumed;
+    pos += bodyResult.consumed;
+
+    // Check for missing close tag
+    if (ctx.tokens[pos]?.type !== "BLOCK_END_OPEN") {
+      ctx.diagnostics.push({
+        severity: "warning",
+        code: "unclosed-block",
+        message: `Missing closing tag [[/div]] for [[${openResult.blockName}]]`,
+        position: openPosition,
+      });
+    }
+
+    // Consume [[/div]]
+    if (ctx.tokens[pos]?.type === "BLOCK_END_OPEN") {
+      const closeResult = consumeDivClose(ctx, pos);
+      pos = closeResult.pos;
+      consumed += closeResult.consumed;
+    }
+
+    return {
+      success: true,
+      elements: [
+        {
+          element: "container",
+          data: {
+            type: "div",
+            attributes: openResult.attributes,
+            elements: bodyResult.elements,
+          },
+        },
+      ],
+      consumed,
+    };
+  },
+};

package/src/parser/rules/block/div/nesting.ts

@@ -0,0 +1,37 @@
+import type { ParseContext } from "../../types";
+
+const divCloseCountCache = new WeakMap<ParseContext["tokens"], Uint32Array>();
+
+/**
+ * Counts `[[/div]]` close tags from a given position to the end of the
+ * token stream. Used to calculate the nesting budget for div blocks.
+ */
+export function countDivCloses(ctx: ParseContext, startPos: number): number {
+  let counts = divCloseCountCache.get(ctx.tokens);
+  if (!counts) {
+    counts = new Uint32Array(ctx.tokens.length + 1);
+    for (let i = ctx.tokens.length - 1; i >= 0; i--) {
+      let count = counts[i + 1]!;
+      if (isDivCloseToken(ctx, i)) {
+        count++;
+      }
+      counts[i] = count;
+    }
+    divCloseCountCache.set(ctx.tokens, counts);
+  }
+  return counts[startPos] ?? 0;
+}
+
+function isDivCloseToken(ctx: ParseContext, pos: number): boolean {
+  if (ctx.tokens[pos]?.type !== "BLOCK_END_OPEN") return false;
+  const name = ctx.tokens[pos + 1];
+  if (!name || (name.type !== "TEXT" && name.type !== "IDENTIFIER")) return false;
+  if (name.value.length !== 3) return false;
+  const d = name.value.charCodeAt(0);
+  const i = name.value.charCodeAt(1);
+  const v = name.value.charCodeAt(2);
+  if (!((d === 100 || d === 68) && (i === 105 || i === 73) && (v === 118 || v === 86))) {
+    return false;
+  }
+  return ctx.tokens[pos + 2]?.type !== "UNDERSCORE";
+}

package/src/parser/rules/block/div/open.ts

@@ -0,0 +1,59 @@
+import type { AttributeMap } from "@wdprlib/ast";
+import type { ParseContext } from "../../types";
+import { parseAttributes, parseBlockName } from "../utils";
+
+export interface DivOpenResult {
+  blockName: "div" | "div_";
+  paragraphStrip: boolean;
+  attributes: AttributeMap;
+  bodyStart: number;
+  consumed: number;
+  hasRequiredNewline: boolean;
+}
+
+export function parseDivOpen(ctx: ParseContext): DivOpenResult | null {
+  if (ctx.tokens[ctx.pos]?.type !== "BLOCK_OPEN") {
+    return null;
+  }
+
+  let pos = ctx.pos + 1;
+  let consumed = 1;
+
+  const nameResult = parseBlockName(ctx, pos);
+  if (!nameResult || !isDivBlockName(nameResult.name)) {
+    return null;
+  }
+
+  pos += nameResult.consumed;
+  consumed += nameResult.consumed;
+
+  const attrResult = parseAttributes(ctx, pos);
+  pos += attrResult.consumed;
+  consumed += attrResult.consumed;
+
+  if (ctx.tokens[pos]?.type !== "BLOCK_CLOSE") {
+    return null;
+  }
+
+  pos++;
+  consumed++;
+
+  const hasRequiredNewline = ctx.tokens[pos]?.type === "NEWLINE";
+  if (hasRequiredNewline) {
+    pos++;
+    consumed++;
+  }
+
+  return {
+    blockName: nameResult.name,
+    paragraphStrip: nameResult.name === "div_",
+    attributes: attrResult.attrs,
+    bodyStart: pos,
+    consumed,
+    hasRequiredNewline,
+  };
+}
+
+function isDivBlockName(name: string): name is "div" | "div_" {
+  return name === "div" || name === "div_";
+}

package/src/parser/rules/block/div/paragraph-strip.ts

@@ -0,0 +1,44 @@
+import type { ContainerData, Element } from "@wdprlib/ast";
+
+/**
+ * Implements the `[[div_]]` paragraph-strip behaviour.
+ *
+ * In Wikidot's `div_` mode, the first and last paragraph containers have
+ * their `<p>` wrappers removed, leaving the inner elements bare. Any
+ * middle paragraphs retain their wrapping.
+ */
+export function unwrapEdgeParagraphs(elements: Element[]): Element[] {
+  if (elements.length === 0) return elements;
+
+  const result = [...elements];
+
+  const first = result[0];
+  if (isParagraphContainer(first)) {
+    const inner = first.data.elements;
+    result.splice(0, 1, ...inner);
+  }
+
+  const lastIdx = result.length - 1;
+  const last = result[lastIdx];
+  if (lastIdx >= 0 && isParagraphContainer(last)) {
+    const inner = last.data.elements;
+    result.splice(lastIdx, 1, ...inner);
+  }
+
+  return result;
+}
+
+type ParagraphContainer = Extract<Element, { element: "container" }> & {
+  data: ContainerData & { type: "paragraph" };
+};
+
+function isParagraphContainer(el: Element | undefined): el is ParagraphContainer {
+  return (
+    el !== undefined &&
+    el.element === "container" &&
+    typeof el.data === "object" &&
+    el.data !== null &&
+    "type" in el.data &&
+    el.data.type === "paragraph"
+  );
+}

package/src/parser/rules/block/embed-block/content.ts

@@ -0,0 +1,53 @@
+import type { ParseContext } from "../../types";
+import { parseBlockName } from "../utils";
+import { isEmbedBlockName } from "./tags";
+
+export interface EmbedContentResult {
+  contents: string;
+  consumed: number;
+  foundClose: boolean;
+}
+
+export function collectEmbedContent(ctx: ParseContext, startPos: number): EmbedContentResult {
+  const contentParts: string[] = [];
+  let pos = startPos;
+  let consumed = 0;
+
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (!token) break;
+
+    if (token.type === "BLOCK_END_OPEN") {
+      const closeNameResult = parseBlockName(ctx, pos + 1);
+      if (closeNameResult && isEmbedBlockName(closeNameResult.name.toLowerCase())) {
+        return { contents: contentParts.join(""), consumed, foundClose: true };
+      }
+    }
+
+    contentParts.push(token.value);
+    pos++;
+    consumed++;
+  }
+
+  return { contents: contentParts.join(""), consumed, foundClose: false };
+}
+
+export function consumeEmbedClose(ctx: ParseContext, startPos: number): number {
+  let pos = startPos + 1;
+  let consumed = 1;
+
+  const closeNameResult = parseBlockName(ctx, pos);
+  if (closeNameResult) {
+    pos += closeNameResult.consumed;
+    consumed += closeNameResult.consumed;
+  }
+  if (ctx.tokens[pos]?.type === "BLOCK_CLOSE") {
+    pos++;
+    consumed++;
+  }
+  if (ctx.tokens[pos]?.type === "NEWLINE") {
+    consumed++;
+  }
+
+  return consumed;
+}

package/src/parser/rules/block/embed-block/index.ts

@@ -0,0 +1,91 @@
+/**
+ *
+ * Block rule for Wikidot embed blocks: `[[embed]]`, `[[embedvideo]]`,
+ * and `[[embedaudio]]` (each with a matching closing tag).
+ *
+ * In original Wikidot, only HTML that matches a server-side allow-list is
+ * rendered. This parser does not perform that filtering -- the raw content
+ * between the tags is stored verbatim as an `embed-block` element. Validation
+ * and sanitisation are expected to happen at rendering time or on the server.
+ *
+ * The embed block is wrapped in a paragraph container in the AST, matching
+ * Wikidot's rendering behaviour where embeds sit inside `<p>` tags.
+ *
+ * If no closing tag is found, the rule fails to prevent consuming the rest
+ * of the document.
+ *
+ * @module
+ */
+import type { Element } from "@wdprlib/ast";
+import type { BlockRule, ParseContext, RuleResult } from "../../types";
+import { currentToken } from "../../types";
+import { collectEmbedContent, consumeEmbedClose } from "./content";
+import { parseEmbedBlockOpen } from "./open";
+
+/**
+ * Block rule for `[[embed]]`, `[[embedvideo]]`, and `[[embedaudio]]`.
+ *
+ * Content between the opening and closing tags is captured as raw text.
+ * The block name matching is case-insensitive; the closing tag may use
+ * any of the three names (`embed`, `embedvideo`, `embedaudio`) regardless
+ * of which was used to open.
+ */
+export const embedBlockRule: BlockRule = {
+  name: "embed-block",
+  startTokens: ["BLOCK_OPEN"],
+  requiresLineStart: false,
+
+  parse(ctx: ParseContext): RuleResult<Element> {
+    const openToken = currentToken(ctx);
+    if (openToken.type !== "BLOCK_OPEN") {
+      return { success: false };
+    }
+
+    const openResult = parseEmbedBlockOpen(ctx, ctx.pos);
+    if (!openResult) {
+      return { success: false };
+    }
+
+    let pos = openResult.pos;
+    let consumed = openResult.consumed;
+    const contentResult = collectEmbedContent(ctx, pos);
+    pos += contentResult.consumed;
+    consumed += contentResult.consumed;
+
+    if (!contentResult.foundClose) {
+      ctx.diagnostics.push({
+        severity: "warning",
+        code: "unclosed-block",
+        message: `Missing closing tag [[/${openResult.blockName}]] for [[${openResult.blockName}]]`,
+        position: openToken.position,
+      });
+      return { success: false };
+    }
+
+    const closeConsumed = consumeEmbedClose(ctx, pos);
+    pos += closeConsumed;
+    consumed += closeConsumed;
+
+    return {
+      success: true,
+      elements: [
+        {
+          element: "container",
+          data: {
+            type: "paragraph",
+            attributes: {},
+            elements: [
+              {
+                element: "embed-block",
+                data: {
+                  contents: contentResult.contents.trim(),
+                },
+              },
+            ],
+          },
+        },
+      ],
+      consumed,
+    };
+  },
+};

package/src/parser/rules/block/embed-block/open.ts

@@ -0,0 +1,52 @@
+import type { ParseContext } from "../../types";
+import { parseBlockName } from "../utils";
+import { isEmbedBlockName } from "./tags";
+
+export interface EmbedBlockOpenResult {
+  blockName: string;
+  pos: number;
+  consumed: number;
+}
+
+export function parseEmbedBlockOpen(
+  ctx: ParseContext,
+  startPos: number,
+): EmbedBlockOpenResult | null {
+  let pos = startPos;
+  let consumed = 0;
+
+  if (ctx.tokens[pos]?.type !== "BLOCK_OPEN") {
+    return null;
+  }
+  pos++;
+  consumed++;
+
+  const nameResult = parseBlockName(ctx, pos);
+  if (!nameResult) {
+    return null;
+  }
+
+  const blockName = nameResult.name.toLowerCase();
+  if (!isEmbedBlockName(blockName)) {
+    return null;
+  }
+  pos += nameResult.consumed;
+  consumed += nameResult.consumed;
+
+  while (ctx.tokens[pos]?.type === "WHITESPACE") {
+    pos++;
+    consumed++;
+  }
+
+  if (ctx.tokens[pos]?.type !== "BLOCK_CLOSE") {
+    return null;
+  }
+  pos++;
+  consumed++;
+
+  return {
+    blockName,
+    pos,
+    consumed,
+  };
+}

package/src/parser/rules/block/embed-block/tags.ts

@@ -0,0 +1,5 @@
+const EMBED_BLOCK_NAMES = new Set(["embed", "embedvideo", "embedaudio"]);
+
+export function isEmbedBlockName(name: string): boolean {
+  return EMBED_BLOCK_NAMES.has(name);
+}