@wdprlib/parser 3.1.2 → 4.0.0

package/src/parser/rules/block/parsing/inline-content.ts

@@ -0,0 +1,105 @@
+import type { Element } from "@wdprlib/ast";
+import type { ParseContext } from "../../types";
+import { getCandidateInlineRules } from "../../inline/utils";
+import type { BlockParseResult } from "./content";
+import { consumeInlineContentNewlines, removeTrailingLineBreaks } from "./inline-newline";
+import { getCandidateBlockRules } from "./rule-dispatch";
+
+/**
+ * Parses mixed inline/block content until a close condition is met,
+ * WITHOUT paragraph wrapping.
+ *
+ * This is used for `div_` (paragraph strip mode) where newlines become
+ * `<br />` elements rather than paragraph separators. Blank lines
+ * (multiple consecutive newlines) are collapsed into a single `<br />`.
+ *
+ * Block-level elements (nested div, collapsible, etc.) are mixed directly
+ * into the inline element stream. Newlines immediately before a BLOCK_OPEN
+ * or BLOCK_END_OPEN are silently consumed (no `<br />` generated).
+ *
+ * Trailing line-break elements are stripped from the result.
+ *
+ * @param ctx            - Parse context positioned at the start of the body.
+ * @param closeCondition - Predicate that signals the end of the content.
+ * @returns Parsed elements and total tokens consumed.
+ */
+export function parseInlineContentUntil(
+  ctx: ParseContext,
+  closeCondition: (ctx: ParseContext) => boolean,
+): BlockParseResult {
+  const elements: Element[] = [];
+  let consumed = 0;
+  let pos = ctx.pos;
+
+  const { blockRules, inlineRules } = ctx;
+  const checkCtx: ParseContext = { ...ctx, pos };
+  const blockCtx: ParseContext = { ...ctx, pos };
+  const inlineCtx: ParseContext = { ...ctx, pos };
+
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (!token || token.type === "EOF") {
+      break;
+    }
+
+    checkCtx.pos = pos;
+    if (closeCondition(checkCtx)) {
+      break;
+    }
+
+    if (token.type === "WHITESPACE" && token.lineStart) {
+      pos++;
+      consumed++;
+      continue;
+    }
+
+    if (token.type === "NEWLINE") {
+      const newlineResult = consumeInlineContentNewlines(ctx, pos);
+      consumed += newlineResult.consumed;
+      pos += newlineResult.consumed;
+      if (newlineResult.addLineBreak) {
+        elements.push({ element: "line-break" });
+      }
+      continue;
+    }
+
+    let matched = false;
+    blockCtx.pos = pos;
+
+    for (const rule of getCandidateBlockRules(blockRules, token)) {
+      const result = rule.parse(blockCtx);
+      if (result.success) {
+        elements.push(...result.elements);
+        consumed += result.consumed;
+        pos += result.consumed;
+        matched = true;
+        break;
+      }
+    }
+
+    if (matched) continue;
+
+    inlineCtx.pos = pos;
+
+    for (const rule of getCandidateInlineRules(inlineRules, token.type)) {
+      const result = rule.parse(inlineCtx);
+      if (result.success) {
+        elements.push(...result.elements);
+        consumed += result.consumed;
+        pos += result.consumed;
+        matched = true;
+        break;
+      }
+    }
+
+    if (!matched) {
+      elements.push({ element: "text", data: token.value });
+      consumed++;
+      pos++;
+    }
+  }
+
+  removeTrailingLineBreaks(elements);
+
+  return { elements, consumed };
+}

package/src/parser/rules/block/parsing/inline-newline.ts

@@ -0,0 +1,41 @@
+import type { Element } from "@wdprlib/ast";
+import type { ParseContext } from "../../types";
+import { isNonBoundaryBlockToken } from "./non-boundary";
+
+export interface InlineNewlineResult {
+  consumed: number;
+  addLineBreak: boolean;
+}
+
+export function consumeInlineContentNewlines(
+  ctx: ParseContext,
+  startPos: number,
+): InlineNewlineResult {
+  let pos = startPos + 1;
+  let consumed = 1;
+
+  while (ctx.tokens[pos]?.type === "NEWLINE") {
+    pos++;
+    consumed++;
+  }
+
+  const nextToken = ctx.tokens[pos];
+  if (!nextToken || nextToken.type === "EOF") {
+    return { consumed, addLineBreak: false };
+  }
+
+  if (nextToken.type === "BLOCK_OPEN" || nextToken.type === "BLOCK_END_OPEN") {
+    const peekCtx: ParseContext = { ...ctx, pos };
+    if (!isNonBoundaryBlockToken(peekCtx, pos)) {
+      return { consumed, addLineBreak: false };
+    }
+  }
+
+  return { consumed, addLineBreak: true };
+}
+
+export function removeTrailingLineBreaks(elements: Element[]): void {
+  while (elements.length > 0 && elements[elements.length - 1]?.element === "line-break") {
+    elements.pop();
+  }
+}

package/src/parser/rules/block/parsing/non-boundary.ts

@@ -0,0 +1,24 @@
+import type { ParseContext } from "../../types";
+import { KNOWN_BLOCK_NAMES } from "../../../constants";
+import { parseBlockName } from "../utils";
+
+/**
+ * Whether the BLOCK_OPEN / BLOCK_END_OPEN token at `pos` opens a block name
+ * that should not end the surrounding paragraph / inline run.
+ */
+export function isNonBoundaryBlockToken(ctx: ParseContext, pos: number): boolean {
+  const token = ctx.tokens[pos];
+  if (token?.type !== "BLOCK_OPEN" && token?.type !== "BLOCK_END_OPEN") {
+    return false;
+  }
+
+  const nameResult = parseBlockName(ctx, pos + 1);
+  if (nameResult === null) {
+    return ctx.tokens[pos + 1]?.type !== "EQUALS";
+  }
+
+  if (ctx.scope.excludedBlockNames?.has(nameResult.name)) {
+    return true;
+  }
+  return !KNOWN_BLOCK_NAMES.has(nameResult.name);
+}

package/src/parser/rules/block/parsing/rule-dispatch.ts

@@ -0,0 +1,44 @@
+import type { Token } from "../../../../lexer";
+import type { BlockRule } from "../../types";
+
+/**
+ * Determines whether a block rule is eligible for the current token.
+ *
+ * A rule is eligible if:
+ * 1. The token is at line start (when `rule.requiresLineStart` is true).
+ * 2. The token's type is in the rule's `startTokens` list (or the list
+ *    is empty, meaning the rule is a universal fallback).
+ *
+ * @param rule  - The block rule to check.
+ * @param token - The current token.
+ * @returns `true` if the rule may be attempted.
+ */
+export function canApplyBlockRule(rule: BlockRule, token: Token): boolean {
+  if (rule.requiresLineStart && !token.lineStart) {
+    return false;
+  }
+  if (rule.startTokens.length === 0) {
+    return true; // fallback rule
+  }
+  return rule.startTokens.includes(token.type);
+}
+
+const blockRuleCandidateCache = new WeakMap<BlockRule[], Map<string, BlockRule[]>>();
+
+export function getCandidateBlockRules(blockRules: BlockRule[], token: Token): BlockRule[] {
+  let byToken = blockRuleCandidateCache.get(blockRules);
+  if (!byToken) {
+    byToken = new Map();
+    blockRuleCandidateCache.set(blockRules, byToken);
+  }
+
+  const key = `${token.type}:${token.lineStart ? "1" : "0"}`;
+  const cached = byToken.get(key);
+  if (cached) {
+    return cached;
+  }
+
+  const candidates = blockRules.filter((rule) => canApplyBlockRule(rule, token));
+  byToken.set(key, candidates);
+  return candidates;
+}

package/src/parser/rules/block/table/index.ts

@@ -0,0 +1,80 @@
+/**
+ *
+ * Block rule for Wikidot pipe-syntax tables.
+ *
+ * Wikidot tables are written using `||` delimiters at the start of a line:
+ *
+ * ```
+ * || Cell 1 || Cell 2 ||
+ * || Cell 3 || Cell 4 ||
+ * ```
+ *
+ * Cell variants:
+ * - `||` -- normal cell (`<td>`)
+ * - `||~` -- header cell (`<th>`)
+ * - `||<` -- left-aligned cell
+ * - `||>` -- right-aligned cell (TABLE_RIGHT)
+ * - `||=` -- center-aligned cell
+ *
+ * Colspan is achieved by using multiple consecutive `||` before content:
+ * `||||` = colspan 2, `||||||` = colspan 3, etc.
+ *
+ * Key Wikidot behaviour:
+ * - Cells MUST be terminated by another `||` (or variant). Unterminated
+ *   cells (reaching end of line without a closing `||`) are discarded.
+ * - If all cells in a row are unterminated, one empty cell is kept.
+ * - Content within cells supports inline markup (bold, links, etc.).
+ * - Leading and trailing whitespace in cell content is trimmed.
+ *
+ * The table element carries `_source: "pipe"` in its attributes to
+ * distinguish it from block-syntax tables (`[[table]]`).
+ *
+ * @module
+ */
+import type { Element, TableData } from "@wdprlib/ast";
+import type { BlockRule, ParseContext, RuleResult } from "../../types";
+import { currentToken } from "../../types";
+import { isPipeTableToken, parsePipeTableRows } from "./pipe";
+
+/**
+ * Block rule for pipe-syntax tables.
+ *
+ * Parsing strategy:
+ * 1. Verify the first token is a table column token at line start.
+ * 2. Parse consecutive rows (each row is a line starting with a table
+ *    column token).
+ * 3. Each row is parsed by `parseTableRow()`, which iterates cells
+ *    via `parseCellStart()` and `parseTableCell()`.
+ * 4. Emit a `table` element with `_source: "pipe"`.
+ */
+export const tableRule: BlockRule = {
+  name: "table",
+  startTokens: ["TABLE_MARKER", "TABLE_HEADER", "TABLE_LEFT", "TABLE_CENTER", "TABLE_RIGHT"],
+  requiresLineStart: true,
+
+  parse(ctx: ParseContext): RuleResult<Element> {
+    const firstToken = currentToken(ctx);
+
+    if (!firstToken.lineStart || !isPipeTableToken(firstToken.type)) {
+      return { success: false };
+    }
+
+    const rowResult = parsePipeTableRows(ctx, ctx.pos);
+
+    const tableData: TableData = {
+      attributes: { _source: "pipe" },
+      rows: rowResult.rows,
+    };
+
+    return {
+      success: true,
+      elements: [
+        {
+          element: "table",
+          data: tableData,
+        },
+      ],
+      consumed: rowResult.consumed,
+    };
+  },
+};

package/src/parser/rules/block/table/pipe/cell-start.ts

@@ -0,0 +1,69 @@
+import type { Alignment } from "@wdprlib/ast";
+import type { ParseContext } from "../../../types";
+
+export interface CellStart {
+  align?: Alignment;
+  header: boolean;
+  colspan: number;
+}
+
+export function parseCellStart(
+  ctx: ParseContext,
+  startPos: number,
+): { cellStart: CellStart; consumed: number } | null {
+  let pos = startPos;
+  let colspan = 0;
+  let align: Alignment | undefined;
+  let header = false;
+
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (!token) break;
+
+    if (token.type === "TABLE_HEADER") {
+      colspan++;
+      header = true;
+      pos++;
+      break;
+    }
+    if (token.type === "TABLE_LEFT") {
+      colspan++;
+      align = "left";
+      pos++;
+      break;
+    }
+    if (token.type === "TABLE_CENTER") {
+      colspan++;
+      align = "center";
+      pos++;
+      break;
+    }
+    if (token.type === "TABLE_RIGHT") {
+      colspan++;
+      align = "right";
+      pos++;
+      break;
+    }
+    if (token.type === "TABLE_MARKER") {
+      colspan++;
+      pos++;
+      continue;
+    }
+    if (colspan > 0) {
+      return {
+        cellStart: { align, header, colspan },
+        consumed: pos - startPos,
+      };
+    }
+    return null;
+  }
+
+  if (colspan > 0) {
+    return {
+      cellStart: { align, header, colspan },
+      consumed: pos - startPos,
+    };
+  }
+
+  return null;
+}

package/src/parser/rules/block/table/pipe/cell.ts

@@ -0,0 +1,106 @@
+import type { Element, TableCell } from "@wdprlib/ast";
+import type { ParseContext } from "../../../types";
+import { getCandidateInlineRules } from "../../../inline/utils";
+import type { CellStart } from "./cell-start";
+import { isPipeTableToken } from "./tokens";
+import { trimElements } from "./trim";
+
+export function parseTableCell(
+  ctx: ParseContext,
+  startPos: number,
+  cellStart: CellStart,
+): { cell: TableCell; consumed: number; terminatedProperly: boolean } {
+  let pos = startPos;
+  let consumed = 0;
+  const children: Element[] = [];
+
+  while (ctx.tokens[pos]?.type === "WHITESPACE") {
+    pos++;
+    consumed++;
+  }
+
+  const { inlineRules } = ctx;
+  const inlineCtx: ParseContext = { ...ctx, pos };
+
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (!token || token.type === "NEWLINE" || token.type === "EOF") {
+      break;
+    }
+    if (isPipeTableToken(token.type)) {
+      break;
+    }
+
+    if (tryConsumeUnderscoreLineBreak(ctx, pos, children)) {
+      pos += 3;
+      consumed += 3;
+      continue;
+    }
+
+    if (token.type === "WHITESPACE") {
+      children.push({ element: "text", data: token.value });
+      pos++;
+      consumed++;
+      continue;
+    }
+
+    inlineCtx.pos = pos;
+    let matched = false;
+
+    for (const rule of getCandidateInlineRules(inlineRules, token.type)) {
+      const result = rule.parse(inlineCtx);
+      if (result.success) {
+        children.push(...result.elements);
+        consumed += result.consumed;
+        pos += result.consumed;
+        matched = true;
+        break;
+      }
+    }
+
+    if (!matched) {
+      children.push({ element: "text", data: token.value });
+      consumed++;
+      pos++;
+    }
+  }
+
+  const currentToken = ctx.tokens[pos];
+  const terminatedProperly = currentToken ? isPipeTableToken(currentToken.type) : false;
+
+  return {
+    cell: {
+      header: cellStart.header,
+      "column-span": cellStart.colspan,
+      align: terminatedProperly ? (cellStart.align ?? null) : null,
+      attributes: {},
+      elements: terminatedProperly ? trimElements(children) : [],
+    },
+    consumed,
+    terminatedProperly,
+  };
+}
+
+function tryConsumeUnderscoreLineBreak(
+  ctx: ParseContext,
+  pos: number,
+  children: Element[],
+): boolean {
+  const token = ctx.tokens[pos];
+  if (token?.type !== "WHITESPACE") {
+    return false;
+  }
+
+  const nextTok = ctx.tokens[pos + 1];
+  const afterTok = ctx.tokens[pos + 2];
+  if (
+    nextTok?.type === "UNDERSCORE" &&
+    afterTok &&
+    (afterTok.type === "NEWLINE" || afterTok.type === "EOF")
+  ) {
+    children.push({ element: "line-break" });
+    return true;
+  }
+
+  return false;
+}

package/src/parser/rules/block/table/pipe/index.ts

@@ -0,0 +1,2 @@
+export { isPipeTableToken } from "./tokens";
+export { parsePipeTableRows } from "./row";

package/src/parser/rules/block/table/pipe/row.ts

@@ -0,0 +1,88 @@
+import type { TableCell, TableRow } from "@wdprlib/ast";
+import type { ParseContext } from "../../../types";
+import { parseCellStart } from "./cell-start";
+import { parseTableCell } from "./cell";
+import { isPipeTableToken } from "./tokens";
+
+export function parsePipeTableRows(
+  ctx: ParseContext,
+  startPos: number,
+): { rows: TableRow[]; consumed: number } {
+  const rows: TableRow[] = [];
+  let pos = startPos;
+  let consumed = 0;
+
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+
+    if (!token || !token.lineStart || !isPipeTableToken(token.type)) {
+      break;
+    }
+
+    const rowResult = parseTableRow(ctx, pos);
+    rows.push(rowResult.row);
+    pos += rowResult.consumed;
+    consumed += rowResult.consumed;
+  }
+
+  return { rows, consumed };
+}
+
+function parseTableRow(ctx: ParseContext, startPos: number): { row: TableRow; consumed: number } {
+  const cells: TableCell[] = [];
+  let pos = startPos;
+  let consumed = 0;
+
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (!token || token.type === "NEWLINE" || token.type === "EOF") {
+      break;
+    }
+
+    const startResult = parseCellStart(ctx, pos);
+    if (!startResult) {
+      break;
+    }
+
+    pos += startResult.consumed;
+    consumed += startResult.consumed;
+
+    const nextToken = ctx.tokens[pos];
+    if (!nextToken || nextToken.type === "NEWLINE" || nextToken.type === "EOF") {
+      break;
+    }
+
+    const cellResult = parseTableCell(ctx, pos, startResult.cellStart);
+    if (cellResult.terminatedProperly) {
+      cells.push(cellResult.cell);
+    }
+    pos += cellResult.consumed;
+    consumed += cellResult.consumed;
+  }
+
+  if (ctx.tokens[pos]?.type === "NEWLINE") {
+    consumed++;
+  }
+
+  if (cells.length === 0) {
+    cells.push(createEmptyTableCell());
+  }
+
+  return {
+    row: {
+      attributes: {},
+      cells,
+    },
+    consumed,
+  };
+}
+
+function createEmptyTableCell(): TableCell {
+  return {
+    header: false,
+    "column-span": 1,
+    align: null,
+    attributes: {},
+    elements: [],
+  };
+}

package/src/parser/rules/block/table/pipe/tokens.ts

@@ -0,0 +1,14 @@
+import type { TokenType } from "../../../../../lexer/tokens";
+
+export function isPipeTableToken(type: TokenType): boolean {
+  switch (type) {
+    case "TABLE_MARKER":
+    case "TABLE_HEADER":
+    case "TABLE_LEFT":
+    case "TABLE_CENTER":
+    case "TABLE_RIGHT":
+      return true;
+    default:
+      return false;
+  }
+}

package/src/parser/rules/block/table/pipe/trim.ts

@@ -0,0 +1,50 @@
+import type { Element } from "@wdprlib/ast";
+
+export function trimElements(elements: Element[]): Element[] {
+  let start = 0;
+  let end = elements.length;
+  let firstReplacement: Element | null = null;
+  let lastReplacement: Element | null = null;
+
+  while (start < end) {
+    const first = elements[start];
+    if (first?.element !== "text" || typeof first.data !== "string") {
+      break;
+    }
+
+    const trimmed = first.data.trimStart();
+    if (trimmed === "") {
+      start++;
+    } else {
+      firstReplacement = trimmed === first.data ? null : { element: "text", data: trimmed };
+      break;
+    }
+  }
+
+  while (start < end) {
+    const last = elements[end - 1];
+    if (last?.element !== "text" || typeof last.data !== "string") {
+      break;
+    }
+
+    const trimmed = last.data.trimEnd();
+    if (trimmed === "") {
+      end--;
+    } else {
+      lastReplacement = trimmed === last.data ? null : { element: "text", data: trimmed };
+      break;
+    }
+  }
+
+  const result = elements.slice(start, end);
+  if (result.length > 0) {
+    if (firstReplacement) {
+      result[0] = firstReplacement;
+    }
+    if (lastReplacement) {
+      result[result.length - 1] = lastReplacement;
+    }
+  }
+
+  return result;
+}