@wdprlib/parser 3.2.0 → 4.0.0

package/src/parser/rules/block/definition-list/item-key.ts

@@ -0,0 +1,95 @@
+import type { Element } from "@wdprlib/ast";
+import type { ParseContext } from "../../types";
+import { parseInlineUntil } from "../../inline/utils";
+
+export interface DefinitionItemKeyResult {
+  keyString: string;
+  key: Element[];
+  consumed: number;
+}
+
+export function parseDefinitionItemKey(
+  ctx: ParseContext,
+  startPos: number,
+): DefinitionItemKeyResult | null {
+  let pos = startPos;
+  let consumed = 0;
+
+  const colonToken = ctx.tokens[pos];
+  if (!colonToken || colonToken.type !== "COLON" || !colonToken.lineStart) {
+    return null;
+  }
+  pos++;
+  consumed++;
+
+  const whitespaceAfterColon = ctx.tokens[pos];
+  if (!whitespaceAfterColon || whitespaceAfterColon.type !== "WHITESPACE") {
+    return null;
+  }
+
+  while (ctx.tokens[pos]?.type === "WHITESPACE") {
+    pos++;
+    consumed++;
+  }
+
+  const keyTokens: string[] = [];
+  const key: Element[] = [];
+  let foundSecondColon = false;
+
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (!token || token.type === "NEWLINE" || token.type === "EOF") {
+      break;
+    }
+    if (token.type === "COLON") {
+      foundSecondColon = true;
+      pos++;
+      consumed++;
+      break;
+    }
+
+    const inlineCtx: ParseContext = { ...ctx, pos };
+    const result = parseInlineUntil(inlineCtx, "COLON");
+    if (result.elements.length > 0) {
+      key.push(...result.elements);
+      for (let i = 0; i < result.consumed; i++) {
+        const t = ctx.tokens[pos + i];
+        if (t) keyTokens.push(t.value);
+      }
+      pos += result.consumed;
+      consumed += result.consumed;
+    } else {
+      keyTokens.push(token.value);
+      pos++;
+      consumed++;
+    }
+  }
+
+  if (!foundSecondColon) {
+    return null;
+  }
+
+  trimTrailingWhitespaceText(key);
+
+  return {
+    keyString: keyTokens.join("").trim(),
+    key,
+    consumed,
+  };
+}
+
+function trimTrailingWhitespaceText(nodes: Element[]): void {
+  while (nodes.length > 0) {
+    const lastNode = nodes[nodes.length - 1];
+    if (
+      lastNode &&
+      lastNode.element === "text" &&
+      typeof lastNode.data === "string" &&
+      lastNode.data.trim() === ""
+    ) {
+      nodes.pop();
+    } else {
+      break;
+    }
+  }
+}

package/src/parser/rules/block/definition-list/item-value.ts

@@ -0,0 +1,56 @@
+import type { Element } from "@wdprlib/ast";
+import type { ParseContext } from "../../types";
+import { parseInlineUntil } from "../../inline/utils";
+
+export interface DefinitionItemValueResult {
+  value: Element[];
+  consumed: number;
+}
+
+export function parseDefinitionItemValue(
+  ctx: ParseContext,
+  startPos: number,
+): DefinitionItemValueResult {
+  const value: Element[] = [];
+  let pos = startPos;
+  let consumed = 0;
+
+  while (ctx.tokens[pos]?.type === "WHITESPACE") {
+    pos++;
+    consumed++;
+  }
+
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (!token || token.type === "EOF") {
+      break;
+    }
+
+    if (token.type === "NEWLINE") {
+      const nextToken = ctx.tokens[pos + 1];
+      if (nextToken?.type === "COLON" && nextToken.lineStart) {
+        pos++;
+        consumed++;
+        break;
+      }
+      if (nextToken?.type === "NEWLINE" || !nextToken || nextToken.type === "EOF") {
+        pos++;
+        consumed++;
+        break;
+      }
+    }
+
+    const inlineCtx: ParseContext = { ...ctx, pos };
+    const result = parseInlineUntil(inlineCtx, "NEWLINE");
+    if (result.elements.length > 0) {
+      value.push(...result.elements);
+      pos += result.consumed;
+      consumed += result.consumed;
+    } else {
+      pos++;
+      consumed++;
+    }
+  }
+
+  return { value, consumed };
+}

package/src/parser/rules/block/definition-list/items.ts

@@ -0,0 +1,54 @@
+import type { DefinitionListItem, Element } from "@wdprlib/ast";
+import type { ParseContext } from "../../types";
+import { parseDefinitionItemKey } from "./item-key";
+import { parseDefinitionItemValue } from "./item-value";
+
+/**
+ * Internal representation of one definition list item before conversion
+ * to the AST's DefinitionListItem format.
+ */
+export interface ParsedDefinitionItem {
+  /** Raw string of the key, used for `key_string` in the AST. */
+  keyString: string;
+  /** Parsed inline elements representing the key / term. */
+  key: Element[];
+  /** Parsed inline elements representing the value / definition. */
+  value: Element[];
+}
+
+/**
+ * Parses a single definition list entry of the form `: key : value`.
+ *
+ * The function expects `startPos` to point at a line-start COLON token.
+ * It consumes the first colon, mandatory whitespace, key tokens up to
+ * the second colon, then value tokens until a double newline, a new entry,
+ * or end of input.
+ */
+export function parseDefinitionItem(
+  ctx: ParseContext,
+  startPos: number,
+): { item: ParsedDefinitionItem; consumed: number } | null {
+  const keyResult = parseDefinitionItemKey(ctx, startPos);
+  if (!keyResult) {
+    return null;
+  }
+
+  const valueResult = parseDefinitionItemValue(ctx, startPos + keyResult.consumed);
+
+  return {
+    item: {
+      keyString: keyResult.keyString,
+      key: keyResult.key,
+      value: valueResult.value,
+    },
+    consumed: keyResult.consumed + valueResult.consumed,
+  };
+}
+
+export function toDefinitionListItems(items: ParsedDefinitionItem[]): DefinitionListItem[] {
+  return items.map((item) => ({
+    key_string: item.keyString,
+    key: item.key,
+    value: item.value,
+  }));
+}

package/src/parser/rules/block/div/body.ts

@@ -0,0 +1,41 @@
+import type { Element } from "@wdprlib/ast";
+import type { ParseContext } from "../../types";
+import { parseBlocksUntil } from "../utils";
+import { isDivClose } from "./close";
+import { countDivCloses } from "./nesting";
+import { unwrapEdgeParagraphs } from "./paragraph-strip";
+
+export interface DivBodyResult {
+  elements: Element[];
+  consumed: number;
+}
+
+export function parseDivBody(
+  ctx: ParseContext,
+  startPos: number,
+  paragraphStrip: boolean,
+): DivBodyResult {
+  const bodyBudget = resolveDivBodyBudget(ctx, startPos);
+  const bodyCtx: ParseContext = {
+    ...ctx,
+    pos: startPos,
+    scope: { ...ctx.scope, divClosesBudget: bodyBudget },
+  };
+
+  const bodyResult = parseBlocksUntil(bodyCtx, isDivClose);
+  const elements = paragraphStrip ? unwrapEdgeParagraphs(bodyResult.elements) : bodyResult.elements;
+
+  return {
+    elements,
+    consumed: bodyResult.consumed,
+  };
+}
+
+function resolveDivBodyBudget(ctx: ParseContext, startPos: number): number {
+  if (ctx.scope.divClosesBudget !== undefined) {
+    return ctx.scope.divClosesBudget - 1;
+  }
+
+  const closesInScope = countDivCloses(ctx, startPos);
+  return closesInScope > 0 ? closesInScope - 1 : 0;
+}

package/src/parser/rules/block/div/close.ts

@@ -0,0 +1,41 @@
+import type { ParseContext } from "../../types";
+import { parseBlockName } from "../utils";
+
+export function isDivClose(ctx: ParseContext): boolean {
+  const token = ctx.tokens[ctx.pos];
+  if (token?.type !== "BLOCK_END_OPEN") return false;
+
+  const closeNameResult = parseBlockName(ctx, ctx.pos + 1);
+  return closeNameResult?.name === "div";
+}
+
+export function consumeDivClose(
+  ctx: ParseContext,
+  startPos: number,
+): { pos: number; consumed: number } {
+  let pos = startPos;
+  let consumed = 0;
+
+  if (ctx.tokens[pos]?.type !== "BLOCK_END_OPEN") {
+    return { pos, consumed };
+  }
+
+  pos++;
+  consumed++;
+
+  const closeNameResult = parseBlockName(ctx, pos);
+  if (closeNameResult) {
+    pos += closeNameResult.consumed;
+    consumed += closeNameResult.consumed;
+  }
+  if (ctx.tokens[pos]?.type === "BLOCK_CLOSE") {
+    pos++;
+    consumed++;
+  }
+  if (ctx.tokens[pos]?.type === "NEWLINE") {
+    pos++;
+    consumed++;
+  }
+
+  return { pos, consumed };
+}

package/src/parser/rules/block/div/failed.ts

@@ -0,0 +1,117 @@
+import type { Element } from "@wdprlib/ast";
+import type { ParseContext, RuleResult } from "../../types";
+import { parseAttributes, parseBlockName } from "../utils";
+
+/**
+ * Handles the case where `[[div]]` fails as a block element because
+ * the closing `]]` is not followed by a NEWLINE.
+ */
+export function consumeFailedDiv(ctx: ParseContext): RuleResult<Element> {
+  const elements: Element[] = [];
+  let pos = ctx.pos;
+  let consumed = 0;
+  let lastClosePos = -1;
+  let lastCloseConsumed = 0;
+
+  let scanPos = pos;
+  while (scanPos < ctx.tokens.length) {
+    const t = ctx.tokens[scanPos];
+    if (!t || t.type === "EOF") break;
+
+    if (t.type === "BLOCK_OPEN" && t.lineStart && scanPos > pos) {
+      const nameResult = parseBlockName(ctx, scanPos + 1);
+      if (nameResult?.name === "div" || nameResult?.name === "div_") {
+        let checkPos = scanPos + 1 + nameResult.consumed;
+        const attrResult = parseAttributes(ctx, checkPos);
+        checkPos += attrResult.consumed;
+        if (ctx.tokens[checkPos]?.type === "BLOCK_CLOSE") {
+          checkPos++;
+          if (ctx.tokens[checkPos]?.type === "NEWLINE" || ctx.tokens[checkPos]?.type === "EOF") {
+            break;
+          }
+        }
+      }
+    }
+
+    if (t.type === "BLOCK_END_OPEN") {
+      const nameResult = parseBlockName(ctx, scanPos + 1);
+      if (nameResult?.name === "div") {
+        lastClosePos = scanPos;
+        lastCloseConsumed = 1 + nameResult.consumed;
+        const closeToken = ctx.tokens[scanPos + 1 + nameResult.consumed];
+        if (closeToken?.type === "BLOCK_CLOSE") {
+          lastCloseConsumed++;
+        }
+      }
+    }
+    scanPos++;
+  }
+
+  if (lastClosePos === -1) {
+    return { success: false };
+  }
+
+  const endPosForDiag = lastClosePos;
+  for (let diagPos = ctx.pos; diagPos < endPosForDiag; diagPos++) {
+    const t = ctx.tokens[diagPos];
+    if (t?.type === "BLOCK_OPEN") {
+      const nameResult = parseBlockName(ctx, diagPos + 1);
+      if (nameResult?.name === "div" || nameResult?.name === "div_") {
+        if (t.position) {
+          ctx.diagnostics.push({
+            severity: "error",
+            code: "inline-block-element",
+            message: `[[${nameResult.name}]] must be followed by a newline to be a block element`,
+            position: t.position,
+          });
+        }
+      }
+    }
+  }
+
+  const endPos = lastClosePos + lastCloseConsumed;
+  while (pos < endPos && pos < ctx.tokens.length) {
+    const t = ctx.tokens[pos];
+    if (!t || t.type === "EOF") break;
+
+    if (t.type === "NEWLINE") {
+      let peekPos = pos + 1;
+      while (ctx.tokens[peekPos]?.type === "WHITESPACE") peekPos++;
+      if (ctx.tokens[peekPos]?.type === "NEWLINE") {
+        while (ctx.tokens[pos]?.type === "NEWLINE" || ctx.tokens[pos]?.type === "WHITESPACE") {
+          pos++;
+          consumed++;
+        }
+        continue;
+      }
+      elements.push({ element: "line-break" });
+      pos++;
+      consumed++;
+      continue;
+    }
+
+    elements.push({ element: "text", data: t.value });
+    pos++;
+    consumed++;
+  }
+
+  if (ctx.tokens[pos]?.type === "NEWLINE") {
+    pos++;
+    consumed++;
+  }
+
+  return {
+    success: true,
+    elements: [
+      {
+        element: "container",
+        data: {
+          type: "paragraph",
+          attributes: {},
+          elements,
+        },
+      },
+    ],
+    consumed,
+  };
+}

package/src/parser/rules/block/div/index.ts

@@ -0,0 +1,112 @@
+/**
+ *
+ * Block rule for Wikidot `[[div]]` and `[[div_]]` container blocks.
+ *
+ * `[[div]]` wraps its body content in a `<div>` element, with full
+ * paragraph processing for the body. `[[div_]]` (paragraph strip mode)
+ * unwraps the first and last paragraphs so their content appears directly
+ * inside the `<div>`, while middle paragraphs keep their `<p>` wrappers.
+ *
+ * Both variants accept HTML attributes (class, style, id, etc.) on the
+ * opening tag.
+ *
+ * Wikidot-specific edge cases:
+ * - The opening `]]` MUST be followed by a NEWLINE for the block to be
+ *   recognised. `[[div]]inline[[/div]]` is NOT a valid div -- it becomes
+ *   a failed div (see `consumeFailedDiv()`).
+ * - When a div fails, everything from the opening `[[div]]` through the
+ *   last `[[/div]]` is collected as a single paragraph of text/line-break
+ *   elements. Blank lines within that span are silently removed.
+ * - `[[div_]]` uses `unwrapEdgeParagraphs()` to strip paragraph
+ *   wrappers from the first and last elements.
+ *
+ * @module
+ */
+import type { Element } from "@wdprlib/ast";
+import type { BlockRule, ParseContext, RuleResult } from "../../types";
+import { currentToken } from "../../types";
+import { parseDivBody } from "./body";
+import { consumeDivClose } from "./close";
+import { consumeFailedDiv } from "./failed";
+import { parseDivOpen } from "./open";
+
+/**
+ * Block rule for `[[div]]`/`[[div_]]` container blocks.
+ *
+ * `requiresLineStart` is `false` because nested `[[div_]]` inside another
+ * `[[div_]]` may appear after inline content.
+ */
+export const divRule: BlockRule = {
+  name: "div",
+  startTokens: ["BLOCK_OPEN"],
+  requiresLineStart: false, // Allow nested [[div_]] inside [[div_]]
+
+  parse(ctx: ParseContext): RuleResult<Element> {
+    const openToken = currentToken(ctx);
+    if (openToken.type !== "BLOCK_OPEN") {
+      return { success: false };
+    }
+
+    const openResult = parseDivOpen(ctx);
+    if (!openResult) {
+      return { success: false };
+    }
+
+    // Wikidot: [[div]] must be followed by newline to be recognized as block
+    // [[div]]inline[[/div]] is NOT recognized as div
+    // When this fails, Wikidot consumes everything up to the last [[/div]]
+    // as text in a single paragraph (blank lines are ignored)
+    if (!openResult.hasRequiredNewline) {
+      return consumeFailedDiv(ctx);
+    }
+
+    // Wikidot matches [[div]]/[[/div]] pairs from outside-in. When there are
+    // more opens than closes, the innermost excess opens become text. We enforce
+    // this with a "closes budget": the number of additional nested divs that can
+    // open. When budget reaches 0, this div cannot open.
+    if (ctx.scope.divClosesBudget === 0) {
+      return { success: false };
+    }
+
+    // Record opening tag position for diagnostics
+    const openPosition = openToken.position;
+
+    let pos = openResult.bodyStart;
+    let consumed = openResult.consumed;
+    const bodyResult = parseDivBody(ctx, pos, openResult.paragraphStrip);
+    consumed += bodyResult.consumed;
+    pos += bodyResult.consumed;
+
+    // Check for missing close tag
+    if (ctx.tokens[pos]?.type !== "BLOCK_END_OPEN") {
+      ctx.diagnostics.push({
+        severity: "warning",
+        code: "unclosed-block",
+        message: `Missing closing tag [[/div]] for [[${openResult.blockName}]]`,
+        position: openPosition,
+      });
+    }
+
+    // Consume [[/div]]
+    if (ctx.tokens[pos]?.type === "BLOCK_END_OPEN") {
+      const closeResult = consumeDivClose(ctx, pos);
+      pos = closeResult.pos;
+      consumed += closeResult.consumed;
+    }
+
+    return {
+      success: true,
+      elements: [
+        {
+          element: "container",
+          data: {
+            type: "div",
+            attributes: openResult.attributes,
+            elements: bodyResult.elements,
+          },
+        },
+      ],
+      consumed,
+    };
+  },
+};

package/src/parser/rules/block/div/nesting.ts

@@ -0,0 +1,37 @@
+import type { ParseContext } from "../../types";
+
+const divCloseCountCache = new WeakMap<ParseContext["tokens"], Uint32Array>();
+
+/**
+ * Counts `[[/div]]` close tags from a given position to the end of the
+ * token stream. Used to calculate the nesting budget for div blocks.
+ */
+export function countDivCloses(ctx: ParseContext, startPos: number): number {
+  let counts = divCloseCountCache.get(ctx.tokens);
+  if (!counts) {
+    counts = new Uint32Array(ctx.tokens.length + 1);
+    for (let i = ctx.tokens.length - 1; i >= 0; i--) {
+      let count = counts[i + 1]!;
+      if (isDivCloseToken(ctx, i)) {
+        count++;
+      }
+      counts[i] = count;
+    }
+    divCloseCountCache.set(ctx.tokens, counts);
+  }
+  return counts[startPos] ?? 0;
+}
+
+function isDivCloseToken(ctx: ParseContext, pos: number): boolean {
+  if (ctx.tokens[pos]?.type !== "BLOCK_END_OPEN") return false;
+  const name = ctx.tokens[pos + 1];
+  if (!name || (name.type !== "TEXT" && name.type !== "IDENTIFIER")) return false;
+  if (name.value.length !== 3) return false;
+  const d = name.value.charCodeAt(0);
+  const i = name.value.charCodeAt(1);
+  const v = name.value.charCodeAt(2);
+  if (!((d === 100 || d === 68) && (i === 105 || i === 73) && (v === 118 || v === 86))) {
+    return false;
+  }
+  return ctx.tokens[pos + 2]?.type !== "UNDERSCORE";
+}

package/src/parser/rules/block/div/open.ts

@@ -0,0 +1,59 @@
+import type { AttributeMap } from "@wdprlib/ast";
+import type { ParseContext } from "../../types";
+import { parseAttributes, parseBlockName } from "../utils";
+
+export interface DivOpenResult {
+  blockName: "div" | "div_";
+  paragraphStrip: boolean;
+  attributes: AttributeMap;
+  bodyStart: number;
+  consumed: number;
+  hasRequiredNewline: boolean;
+}
+
+export function parseDivOpen(ctx: ParseContext): DivOpenResult | null {
+  if (ctx.tokens[ctx.pos]?.type !== "BLOCK_OPEN") {
+    return null;
+  }
+
+  let pos = ctx.pos + 1;
+  let consumed = 1;
+
+  const nameResult = parseBlockName(ctx, pos);
+  if (!nameResult || !isDivBlockName(nameResult.name)) {
+    return null;
+  }
+
+  pos += nameResult.consumed;
+  consumed += nameResult.consumed;
+
+  const attrResult = parseAttributes(ctx, pos);
+  pos += attrResult.consumed;
+  consumed += attrResult.consumed;
+
+  if (ctx.tokens[pos]?.type !== "BLOCK_CLOSE") {
+    return null;
+  }
+
+  pos++;
+  consumed++;
+
+  const hasRequiredNewline = ctx.tokens[pos]?.type === "NEWLINE";
+  if (hasRequiredNewline) {
+    pos++;
+    consumed++;
+  }
+
+  return {
+    blockName: nameResult.name,
+    paragraphStrip: nameResult.name === "div_",
+    attributes: attrResult.attrs,
+    bodyStart: pos,
+    consumed,
+    hasRequiredNewline,
+  };
+}
+
+function isDivBlockName(name: string): name is "div" | "div_" {
+  return name === "div" || name === "div_";
+}

package/src/parser/rules/block/div/paragraph-strip.ts

@@ -0,0 +1,44 @@
+import type { ContainerData, Element } from "@wdprlib/ast";
+
+/**
+ * Implements the `[[div_]]` paragraph-strip behaviour.
+ *
+ * In Wikidot's `div_` mode, the first and last paragraph containers have
+ * their `<p>` wrappers removed, leaving the inner elements bare. Any
+ * middle paragraphs retain their wrapping.
+ */
+export function unwrapEdgeParagraphs(elements: Element[]): Element[] {
+  if (elements.length === 0) return elements;
+
+  const result = [...elements];
+
+  const first = result[0];
+  if (isParagraphContainer(first)) {
+    const inner = first.data.elements;
+    result.splice(0, 1, ...inner);
+  }
+
+  const lastIdx = result.length - 1;
+  const last = result[lastIdx];
+  if (lastIdx >= 0 && isParagraphContainer(last)) {
+    const inner = last.data.elements;
+    result.splice(lastIdx, 1, ...inner);
+  }
+
+  return result;
+}
+
+type ParagraphContainer = Extract<Element, { element: "container" }> & {
+  data: ContainerData & { type: "paragraph" };
+};
+
+function isParagraphContainer(el: Element | undefined): el is ParagraphContainer {
+  return (
+    el !== undefined &&
+    el.element === "container" &&
+    typeof el.data === "object" &&
+    el.data !== null &&
+    "type" in el.data &&
+    el.data.type === "paragraph"
+  );
+}