@wdprlib/parser 3.1.2 → 3.2.0

package/src/parser/rules/block/definition-list.ts

@@ -0,0 +1,270 @@
+/**
+ *
+ * Block rule for Wikidot definition lists written with the `: key : value` syntax.
+ *
+ * Each item starts at the beginning of a line with a COLON, followed by
+ * mandatory whitespace, the key (term), a second COLON, and then the value
+ * (definition). Multiple consecutive items form a single `<dl>` block.
+ *
+ * ```
+ * : Apple : A fruit that grows on trees.
+ * : Banana : A yellow curved fruit.
+ * ```
+ *
+ * Key parsing details:
+ * - Whitespace after the first colon is required (`": key"` not `":key"`).
+ * - The key portion supports inline markup (bold, links, etc.).
+ * - The value continues until a double newline, a new definition entry, or
+ *   the end of the document.
+ * - A single newline within the value does NOT break the entry -- parsing
+ *   continues on the next line.
+ *
+ * @module
+ */
+import type { Element, DefinitionListItem } from "@wdprlib/ast";
+import type { BlockRule, ParseContext, RuleResult } from "../types";
+import { parseInlineUntil } from "../inline/utils";
+
+/**
+ * Internal representation of one definition list item before conversion
+ * to the AST's {@link DefinitionListItem} format.
+ */
+interface ParsedDefinitionItem {
+  /** Raw string of the key, used for `key_string` in the AST. */
+  keyString: string;
+  /** Parsed inline elements representing the key / term. */
+  key: Element[];
+  /** Parsed inline elements representing the value / definition. */
+  value: Element[];
+}
+
+/**
+ * Parses a single definition list entry of the form `: key : value`.
+ *
+ * The function expects `startPos` to point at a line-start COLON token.
+ * It consumes the first colon, mandatory whitespace, key tokens up to
+ * the second colon, then value tokens until one of the following:
+ * - A double newline (paragraph break).
+ * - A new entry (COLON at line start).
+ * - End of input.
+ *
+ * The key is parsed for inline content using {@link parseInlineUntil}
+ * (stopping at COLON), and the value uses the same utility (stopping at
+ * NEWLINE).
+ *
+ * @param ctx      - Parse context.
+ * @param startPos - Token index of the expected line-start COLON.
+ * @returns The parsed item and token count, or `null` on failure.
+ */
+function parseDefinitionItem(
+  ctx: ParseContext,
+  startPos: number,
+): { item: ParsedDefinitionItem; consumed: number } | null {
+  let pos = startPos;
+  let consumed = 0;
+
+  // Expect COLON at line start
+  const colonToken = ctx.tokens[pos];
+  if (!colonToken || colonToken.type !== "COLON" || !colonToken.lineStart) {
+    return null;
+  }
+  pos++;
+  consumed++;
+
+  // Wikidot requires whitespace after first colon: ": key : value"
+  const whitespaceAfterColon = ctx.tokens[pos];
+  if (!whitespaceAfterColon || whitespaceAfterColon.type !== "WHITESPACE") {
+    return null;
+  }
+
+  // Skip whitespace after first colon
+  while (ctx.tokens[pos]?.type === "WHITESPACE") {
+    pos++;
+    consumed++;
+  }
+
+  // Collect key tokens until second COLON
+  const keyTokens: string[] = [];
+  const keyNodes: Element[] = [];
+  let foundSecondColon = false;
+
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (!token || token.type === "NEWLINE" || token.type === "EOF") {
+      break;
+    }
+    if (token.type === "COLON") {
+      foundSecondColon = true;
+      pos++;
+      consumed++;
+      break;
+    }
+
+    // Parse inline content for key
+    const inlineCtx: ParseContext = { ...ctx, pos };
+    const result = parseInlineUntil(inlineCtx, "COLON");
+    if (result.elements.length > 0) {
+      keyNodes.push(...result.elements);
+      // Collect raw key string
+      for (let i = 0; i < result.consumed; i++) {
+        const t = ctx.tokens[pos + i];
+        if (t) keyTokens.push(t.value);
+      }
+      pos += result.consumed;
+      consumed += result.consumed;
+    } else {
+      keyTokens.push(token.value);
+      pos++;
+      consumed++;
+    }
+  }
+
+  if (!foundSecondColon) {
+    return null;
+  }
+
+  // Skip whitespace after second colon
+  while (ctx.tokens[pos]?.type === "WHITESPACE") {
+    pos++;
+    consumed++;
+  }
+
+  // Parse value (rest of line, can continue with line breaks)
+  const valueNodes: Element[] = [];
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (!token || token.type === "EOF") {
+      break;
+    }
+
+    // Check for end of definition (double newline or new definition)
+    if (token.type === "NEWLINE") {
+      const nextToken = ctx.tokens[pos + 1];
+      // Look ahead for continuation with underscore line break
+      if (nextToken?.type === "COLON" && nextToken.lineStart) {
+        // New definition item starts
+        pos++;
+        consumed++;
+        break;
+      }
+      if (nextToken?.type === "NEWLINE" || !nextToken || nextToken.type === "EOF") {
+        // Double newline or end - stop
+        pos++;
+        consumed++;
+        break;
+      }
+      // Single newline - continue parsing (becomes line break)
+    }
+
+    // Parse inline content
+    const inlineCtx: ParseContext = { ...ctx, pos };
+    const result = parseInlineUntil(inlineCtx, "NEWLINE");
+    if (result.elements.length > 0) {
+      valueNodes.push(...result.elements);
+      pos += result.consumed;
+      consumed += result.consumed;
+    } else {
+      pos++;
+      consumed++;
+    }
+  }
+
+  // Remove trailing whitespace from key
+  const keyString = keyTokens.join("").trim();
+
+  // Remove trailing whitespace nodes from key
+  while (keyNodes.length > 0) {
+    const lastNode = keyNodes[keyNodes.length - 1];
+    if (
+      lastNode &&
+      lastNode.element === "text" &&
+      typeof lastNode.data === "string" &&
+      lastNode.data.trim() === ""
+    ) {
+      keyNodes.pop();
+    } else {
+      break;
+    }
+  }
+
+  return {
+    item: {
+      keyString,
+      key: keyNodes,
+      value: valueNodes,
+    },
+    consumed,
+  };
+}
+
+/**
+ * Block rule for Wikidot definition lists (`: key : value`).
+ *
+ * Parsing strategy:
+ * 1. Verify the first token is a line-start COLON.
+ * 2. Repeatedly call `parseDefinitionItem()` to collect entries.
+ * 3. Stop when the current token is no longer a line-start COLON (i.e.
+ *    the definition list block has ended).
+ * 4. Convert internal items into the AST {@link DefinitionListItem} format.
+ * 5. Emit a single `definition-list` element.
+ */
+export const definitionListRule: BlockRule = {
+  name: "definitionList",
+  startTokens: ["COLON"],
+  requiresLineStart: true,
+
+  parse(ctx: ParseContext): RuleResult<Element> {
+    const items: ParsedDefinitionItem[] = [];
+    let pos = ctx.pos;
+    let totalConsumed = 0;
+
+    while (pos < ctx.tokens.length) {
+      const token = ctx.tokens[pos];
+      if (!token || token.type === "EOF") {
+        break;
+      }
+
+      // Only parse lines starting with :
+      if (token.type !== "COLON" || !token.lineStart) {
+        break;
+      }
+
+      const result = parseDefinitionItem(ctx, pos);
+      if (!result) {
+        break;
+      }
+
+      items.push(result.item);
+      pos += result.consumed;
+      totalConsumed += result.consumed;
+
+      // Skip any whitespace between items
+      while (ctx.tokens[pos]?.type === "WHITESPACE") {
+        pos++;
+        totalConsumed++;
+      }
+    }
+
+    if (items.length === 0) {
+      return { success: false };
+    }
+
+    // Convert items to DefinitionListItem format
+    const definitionItems: DefinitionListItem[] = items.map((item) => ({
+      key_string: item.keyString,
+      key: item.key,
+      value: item.value,
+    }));
+
+    return {
+      success: true,
+      elements: [
+        {
+          element: "definition-list",
+          data: definitionItems,
+        },
+      ],
+      consumed: totalConsumed,
+    };
+  },
+};

package/src/parser/rules/block/div.ts

@@ -0,0 +1,400 @@
+/**
+ *
+ * Block rule for Wikidot `[[div]]` and `[[div_]]` container blocks.
+ *
+ * `[[div]]` wraps its body content in a `<div>` element, with full
+ * paragraph processing for the body. `[[div_]]` (paragraph strip mode)
+ * unwraps the first and last paragraphs so their content appears directly
+ * inside the `<div>`, while middle paragraphs keep their `<p>` wrappers.
+ *
+ * Both variants accept HTML attributes (class, style, id, etc.) on the
+ * opening tag.
+ *
+ * Wikidot-specific edge cases:
+ * - The opening `]]` MUST be followed by a NEWLINE for the block to be
+ *   recognised. `[[div]]inline[[/div]]` is NOT a valid div -- it becomes
+ *   a failed div (see `consumeFailedDiv()`).
+ * - When a div fails, everything from the opening `[[div]]` through the
+ *   last `[[/div]]` is collected as a single paragraph of text/line-break
+ *   elements. Blank lines within that span are silently removed.
+ * - `[[div_]]` uses `unwrapEdgeParagraphs()` to strip paragraph
+ *   wrappers from the first and last elements.
+ *
+ * @module
+ */
+import type { Element } from "@wdprlib/ast";
+import type { BlockRule, ParseContext, RuleResult } from "../types";
+import { currentToken } from "../types";
+import { parseBlockName, parseAttributes, parseBlocksUntil } from "./utils";
+
+/**
+ * Block rule for `[[div]]`/`[[div_]]` container blocks.
+ *
+ * `requiresLineStart` is `false` because nested `[[div_]]` inside another
+ * `[[div_]]` may appear after inline content.
+ */
+export const divRule: BlockRule = {
+  name: "div",
+  startTokens: ["BLOCK_OPEN"],
+  requiresLineStart: false, // Allow nested [[div_]] inside [[div_]]
+
+  parse(ctx: ParseContext): RuleResult<Element> {
+    const openToken = currentToken(ctx);
+    if (openToken.type !== "BLOCK_OPEN") {
+      return { success: false };
+    }
+
+    let pos = ctx.pos + 1;
+    let consumed = 1;
+
+    // Parse block name
+    const nameResult = parseBlockName(ctx, pos);
+    if (!nameResult) {
+      return { success: false };
+    }
+
+    const blockName = nameResult.name;
+    // Check if it's a div or div_
+    if (blockName !== "div" && blockName !== "div_") {
+      return { success: false };
+    }
+
+    // div_ means paragraph strip (no paragraph wrapping)
+    const paragraphStrip = blockName === "div_";
+
+    pos += nameResult.consumed;
+    consumed += nameResult.consumed;
+
+    // Parse attributes
+    const attrResult = parseAttributes(ctx, pos);
+    pos += attrResult.consumed;
+    consumed += attrResult.consumed;
+
+    // Expect ]]
+    if (ctx.tokens[pos]?.type !== "BLOCK_CLOSE") {
+      return { success: false };
+    }
+    pos++;
+    consumed++;
+
+    // Wikidot: [[div]] must be followed by newline to be recognized as block
+    // [[div]]inline[[/div]] is NOT recognized as div
+    // When this fails, Wikidot consumes everything up to the last [[/div]]
+    // as text in a single paragraph (blank lines are ignored)
+    if (ctx.tokens[pos]?.type !== "NEWLINE") {
+      return consumeFailedDiv(ctx);
+    }
+
+    // Wikidot matches [[div]]/[[/div]] pairs from outside-in. When there are
+    // more opens than closes, the innermost excess opens become text. We enforce
+    // this with a "closes budget": the number of additional nested divs that can
+    // open. When budget reaches 0, this div cannot open.
+    if (ctx.scope.divClosesBudget === 0) {
+      return { success: false };
+    }
+
+    pos++;
+    consumed++;
+
+    // Record opening tag position for diagnostics
+    const openPosition = openToken.position;
+
+    // Calculate closes budget for nested divs in the body.
+    // Count [[/div]] from body start to scope boundary, subtract 1 (for self).
+    let bodyBudget: number | undefined;
+    if (ctx.scope.divClosesBudget !== undefined) {
+      bodyBudget = ctx.scope.divClosesBudget - 1;
+    } else {
+      const closesInScope = countDivCloses(ctx, pos);
+      bodyBudget = closesInScope > 0 ? closesInScope - 1 : 0;
+    }
+
+    // Close condition for [[/div]]
+    const closeCondition = (checkCtx: ParseContext): boolean => {
+      const token = checkCtx.tokens[checkCtx.pos];
+      if (token?.type === "BLOCK_END_OPEN") {
+        const closeNameResult = parseBlockName(checkCtx, checkCtx.pos + 1);
+        if (closeNameResult?.name === "div") {
+          return true;
+        }
+      }
+      return false;
+    };
+
+    const bodyCtx: ParseContext = {
+      ...ctx,
+      pos,
+      scope: { ...ctx.scope, divClosesBudget: bodyBudget },
+    };
+    let children: Element[];
+
+    if (paragraphStrip) {
+      // div_ - parse as blocks, then unwrap first/last paragraphs
+      // Wikidot: blank lines create <p> for middle blocks only
+      const bodyResult = parseBlocksUntil(bodyCtx, closeCondition);
+      consumed += bodyResult.consumed;
+      pos += bodyResult.consumed;
+      children = unwrapEdgeParagraphs(bodyResult.elements);
+    } else {
+      // div - parse blocks with paragraph wrapping
+      const bodyResult = parseBlocksUntil(bodyCtx, closeCondition);
+      consumed += bodyResult.consumed;
+      pos += bodyResult.consumed;
+      children = bodyResult.elements;
+    }
+
+    // Check for missing close tag
+    if (ctx.tokens[pos]?.type !== "BLOCK_END_OPEN") {
+      ctx.diagnostics.push({
+        severity: "warning",
+        code: "unclosed-block",
+        message: `Missing closing tag [[/div]] for [[${blockName}]]`,
+        position: openPosition,
+      });
+    }
+
+    // Consume [[/div]]
+    if (ctx.tokens[pos]?.type === "BLOCK_END_OPEN") {
+      pos++;
+      consumed++;
+      const closeNameResult = parseBlockName(ctx, pos);
+      if (closeNameResult) {
+        pos += closeNameResult.consumed;
+        consumed += closeNameResult.consumed;
+      }
+      if (ctx.tokens[pos]?.type === "BLOCK_CLOSE") {
+        pos++;
+        consumed++;
+      }
+      if (ctx.tokens[pos]?.type === "NEWLINE") {
+        pos++;
+        consumed++;
+      }
+    }
+
+    return {
+      success: true,
+      elements: [
+        {
+          element: "container",
+          data: {
+            type: "div",
+            attributes: attrResult.attrs,
+            elements: children,
+          },
+        },
+      ],
+      consumed,
+    };
+  },
+};
+
+/**
+ * Counts `[[/div]]` close tags from a given position to the end of the
+ * token stream. Used to calculate the nesting budget for div blocks.
+ */
+function countDivCloses(ctx: ParseContext, startPos: number): number {
+  let count = 0;
+  for (let i = startPos; i < ctx.tokens.length; i++) {
+    const t = ctx.tokens[i];
+    if (!t || t.type === "EOF") break;
+    if (t.type === "BLOCK_END_OPEN") {
+      const nameResult = parseBlockName(ctx, i + 1);
+      if (nameResult?.name === "div") {
+        count++;
+      }
+    }
+  }
+  return count;
+}
+
+/**
+ * Handles the case where `[[div]]` fails as a block element because
+ * the closing `]]` is not followed by a NEWLINE.
+ *
+ * In Wikidot, this scenario causes the parser to scan forward for the
+ * LAST `[[/div]]` in the contiguous token stream and collect everything
+ * from the current position through that close tag as a single paragraph.
+ * Blank lines (double newlines) within the range are silently collapsed,
+ * and single newlines become `<br />`.
+ *
+ * If no `[[/div]]` is found at all, the rule fails entirely.
+ *
+ * @param ctx - Parse context, positioned at the opening `[[div...]]` tag.
+ * @returns A paragraph container with text/line-break elements, or failure.
+ */
+function consumeFailedDiv(ctx: ParseContext): RuleResult<Element> {
+  const elements: Element[] = [];
+  let pos = ctx.pos;
+  let consumed = 0;
+  let lastClosePos = -1;
+  let lastCloseConsumed = 0;
+
+  // Find the last [[/div]] before the next valid div block.
+  // A valid div block is [[div]]/[[div_]] at line start followed by ]] + NEWLINE.
+  // When a valid div block is found, stop scanning — it should be parsed as a
+  // separate block element, not absorbed into this failed div's text.
+  let scanPos = pos;
+  while (scanPos < ctx.tokens.length) {
+    const t = ctx.tokens[scanPos];
+    if (!t || t.type === "EOF") break;
+
+    // Check for a valid div block opening (skip the initial failed div at pos)
+    if (t.type === "BLOCK_OPEN" && t.lineStart && scanPos > pos) {
+      const nameResult = parseBlockName(ctx, scanPos + 1);
+      if (nameResult?.name === "div" || nameResult?.name === "div_") {
+        let checkPos = scanPos + 1 + nameResult.consumed;
+        const attrResult = parseAttributes(ctx, checkPos);
+        checkPos += attrResult.consumed;
+        if (ctx.tokens[checkPos]?.type === "BLOCK_CLOSE") {
+          checkPos++;
+          if (ctx.tokens[checkPos]?.type === "NEWLINE" || ctx.tokens[checkPos]?.type === "EOF") {
+            // Valid div block found — stop scanning here
+            break;
+          }
+        }
+      }
+    }
+
+    if (t.type === "BLOCK_END_OPEN") {
+      const nameResult = parseBlockName(ctx, scanPos + 1);
+      if (nameResult?.name === "div") {
+        // Found [[/div]] - record position after ]]
+        lastClosePos = scanPos;
+        lastCloseConsumed = 1 + nameResult.consumed; // [[/ + div
+        const closeToken = ctx.tokens[scanPos + 1 + nameResult.consumed];
+        if (closeToken?.type === "BLOCK_CLOSE") {
+          lastCloseConsumed++;
+        }
+      }
+    }
+    scanPos++;
+  }
+
+  if (lastClosePos === -1) {
+    // No [[/div]] found, fall back to normal failure
+    return { success: false };
+  }
+
+  // Emit diagnostics for all inline [[div]] patterns in the absorbed range.
+  // The initial [[div]] at ctx.pos is always included; any additional [[div]]
+  // patterns within the range also get diagnostics.
+  const endPosForDiag = lastClosePos;
+  for (let diagPos = ctx.pos; diagPos < endPosForDiag; diagPos++) {
+    const t = ctx.tokens[diagPos];
+    if (t?.type === "BLOCK_OPEN") {
+      const nameResult = parseBlockName(ctx, diagPos + 1);
+      if (nameResult?.name === "div" || nameResult?.name === "div_") {
+        if (t.position) {
+          ctx.diagnostics.push({
+            severity: "error",
+            code: "inline-block-element",
+            message: `[[${nameResult.name}]] must be followed by a newline to be a block element`,
+            position: t.position,
+          });
+        }
+      }
+    }
+  }
+
+  // Consume everything from current position to after the last [[/div]]
+  const endPos = lastClosePos + lastCloseConsumed;
+  while (pos < endPos && pos < ctx.tokens.length) {
+    const t = ctx.tokens[pos];
+    if (!t || t.type === "EOF") break;
+
+    if (t.type === "NEWLINE") {
+      // Check if this is a blank line (NEWLINE+NEWLINE or NEWLINE+WHITESPACE+NEWLINE)
+      let peekPos = pos + 1;
+      while (ctx.tokens[peekPos]?.type === "WHITESPACE") peekPos++;
+      if (ctx.tokens[peekPos]?.type === "NEWLINE") {
+        // Blank line — skip all newlines and whitespace
+        while (ctx.tokens[pos]?.type === "NEWLINE" || ctx.tokens[pos]?.type === "WHITESPACE") {
+          pos++;
+          consumed++;
+        }
+        continue;
+      }
+      // Single newline → line-break
+      elements.push({ element: "line-break" });
+      pos++;
+      consumed++;
+      continue;
+    }
+
+    elements.push({ element: "text", data: t.value });
+    pos++;
+    consumed++;
+  }
+
+  // Consume trailing newline after [[/div]] if present
+  if (ctx.tokens[pos]?.type === "NEWLINE") {
+    pos++;
+    consumed++;
+  }
+
+  return {
+    success: true,
+    elements: [
+      {
+        element: "container",
+        data: {
+          type: "paragraph",
+          attributes: {},
+          elements,
+        },
+      },
+    ],
+    consumed,
+  };
+}
+
+/**
+ * Implements the `[[div_]]` paragraph-strip behaviour.
+ *
+ * In Wikidot's `div_` mode, the first and last paragraph containers have
+ * their `<p>` wrappers removed, leaving the inner elements bare. Any
+ * middle paragraphs retain their wrapping. This produces output where the
+ * opening and closing text sit directly inside the `<div>`.
+ *
+ * @param elements - Block elements produced by body parsing.
+ * @returns A new array with edge paragraphs unwrapped.
+ */
+function unwrapEdgeParagraphs(elements: Element[]): Element[] {
+  if (elements.length === 0) return elements;
+
+  const result = [...elements];
+
+  // Unwrap first element if paragraph
+  if (isParagraphContainer(result[0])) {
+    const inner = (result[0] as any).data.elements as Element[];
+    result.splice(0, 1, ...inner);
+  }
+
+  // Unwrap last element if paragraph (find new last index after splice)
+  const lastIdx = result.length - 1;
+  if (lastIdx >= 0 && isParagraphContainer(result[lastIdx])) {
+    const inner = (result[lastIdx] as any).data.elements as Element[];
+    result.splice(lastIdx, 1, ...inner);
+  }
+
+  return result;
+}
+
+/**
+ * Checks whether an element is a paragraph container
+ * (i.e. `{ element: "container", data: { type: "paragraph" } }`).
+ *
+ * @param el - Element to test, or `undefined`.
+ * @returns `true` if the element is a paragraph container.
+ */
+function isParagraphContainer(el: Element | undefined): boolean {
+  return (
+    el !== undefined &&
+    el.element === "container" &&
+    typeof el.data === "object" &&
+    el.data !== null &&
+    "type" in el.data &&
+    el.data.type === "paragraph"
+  );
+}