@wdprlib/parser 3.1.2 → 4.0.0

package/src/parser/rules/inline/text/index.ts

@@ -0,0 +1,85 @@
+/**
+ *
+ * Provides the two lowest-priority inline rules: `textRule` and `fallbackRule`.
+ *
+ * These rules act as catch-alls that convert unrecognized tokens into
+ * plain `"text"` AST elements, ensuring no token is ever silently dropped
+ * during inline parsing.
+ *
+ * `textRule` handles `TEXT` and `WHITESPACE` tokens specifically and is
+ * included in the main `inlineRules` array as the last entry before
+ * the fallback.
+ *
+ * `fallbackRule` has an empty `startTokens` array, which means it matches
+ * ANY token type. It is exported separately as `inlineFallbackRule` and
+ * is NOT included in the `inlineRules` array to prevent it from
+ * short-circuiting more specific rules. Instead, it is invoked explicitly
+ * by the parser when no other rule matches.
+ *
+ * @module
+ */
+import type { Element } from "@wdprlib/ast";
+import type { InlineRule, ParseContext, RuleResult } from "../../types";
+import { currentToken } from "../../types";
+import { textElement } from "./element";
+
+/**
+ * Inline rule for plain text and whitespace tokens.
+ *
+ * Matches `TEXT` and `WHITESPACE` token types and converts them
+ * directly to `"text"` AST elements. This rule always succeeds.
+ *
+ * Placed last (before the fallback) in the inline rules array so
+ * that all formatting and structural rules are tried first.
+ */
+export const textRule: InlineRule = {
+  name: "text",
+  startTokens: ["TEXT", "WHITESPACE"],
+
+  /**
+   * Converts a TEXT or WHITESPACE token into a text element.
+   *
+   * @param ctx - Parse context with token stream and current position
+   * @returns Always returns `{ success: true }` with a single `"text"` element
+   */
+  parse(ctx: ParseContext): RuleResult<Element> {
+    const token = currentToken(ctx);
+
+    return {
+      success: true,
+      elements: [textElement(token.value)],
+      consumed: 1,
+    };
+  },
+};
+
+/**
+ * Universal fallback rule for any token type not matched by other rules.
+ *
+ * The empty `startTokens` array signals to the parser that this rule
+ * can match any token. It converts the token's value to a `"text"`
+ * element, ensuring no token is silently dropped.
+ *
+ * This rule is used as a last-resort handler and is intentionally
+ * excluded from the main `inlineRules` array.
+ */
+export const fallbackRule: InlineRule = {
+  name: "fallback",
+  startTokens: [], // matches anything not matched by other rules
+
+  /**
+   * Converts any unrecognized token into a text element.
+   *
+   * @param ctx - Parse context with token stream and current position
+   * @returns Always returns `{ success: true }` with a single `"text"` element
+   */
+  parse(ctx: ParseContext): RuleResult<Element> {
+    const token = currentToken(ctx);
+
+    return {
+      success: true,
+      elements: [textElement(token.value)],
+      consumed: 1,
+    };
+  },
+};

package/src/parser/rules/inline/underline/child.ts

@@ -0,0 +1,26 @@
+import type { Element } from "@wdprlib/ast";
+import type { ParseContext } from "../../types";
+import { parseInlineUntil } from "../utils";
+
+export interface UnderlineChildResult {
+  elements: Element[];
+  consumed: number;
+}
+
+export function parseUnderlineChild(ctx: ParseContext, pos: number): UnderlineChildResult {
+  const token = ctx.tokens[pos];
+  if (!token) {
+    return { elements: [], consumed: 0 };
+  }
+
+  if (token.type === "NEWLINE") {
+    return { elements: [{ element: "line-break" }], consumed: 1 };
+  }
+
+  const result = parseInlineUntil({ ...ctx, pos }, "UNDERLINE_MARKER");
+  if (result.elements.length > 0) {
+    return { elements: result.elements, consumed: result.consumed };
+  }
+
+  return { elements: [{ element: "text", data: token.value }], consumed: 1 };
+}

package/src/parser/rules/inline/underline/content.ts

@@ -0,0 +1,29 @@
+import type { Element } from "@wdprlib/ast";
+import type { ParseContext } from "../../types";
+import { parseUnderlineChild } from "./child";
+
+export function parseUnderlineContent(
+  ctx: ParseContext,
+  startPos: number,
+): { children: Element[]; consumed: number } {
+  const children: Element[] = [];
+  let pos = startPos;
+  let consumed = 1;
+
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (!token || token.type === "EOF") break;
+
+    if (token.type === "UNDERLINE_MARKER") {
+      consumed++;
+      break;
+    }
+
+    const child = parseUnderlineChild(ctx, pos);
+    children.push(...child.elements);
+    pos += child.consumed;
+    consumed += child.consumed;
+  }
+
+  return { children, consumed };
+}

package/src/parser/rules/inline/underline/index.ts

@@ -0,0 +1,84 @@
+/**
+ *
+ * Parses the Wikidot underline formatting syntax: `__text__`.
+ *
+ * Underline text is delimited by double underscores. Unlike most
+ * inline formatting markers (bold, italic, etc.) which require the
+ * closing marker on the same line, underline markers can span
+ * multiple lines within the same paragraph. The closing marker
+ * must appear before a paragraph break (blank line).
+ *
+ * Single newlines within underlined content are converted to
+ * `<br />` elements, matching Wikidot's multiline underline behavior.
+ *
+ * If no closing `__` is found before a paragraph break, the opening
+ * marker is emitted as literal text.
+ *
+ * Empty underline (`____`) is silently discarded by Wikidot (produces
+ * no output).
+ *
+ * Renders as a `<u>` element in HTML.
+ *
+ * Produces a `"container"` AST element with `type: "underline"`.
+ *
+ * @module
+ */
+import type { Element } from "@wdprlib/ast";
+import type { InlineRule, ParseContext, RuleResult } from "../../types";
+import { currentToken, hasClosingMarkerBeforeParagraphBreak } from "../../types";
+import { createInlineContainer } from "../formatting/container";
+import { parseUnderlineContent } from "./content";
+
+/**
+ * Inline rule for parsing `__underline__` formatting.
+ *
+ * Triggered by an `UNDERLINE_MARKER` token (`__`). Uses
+ * {@link hasClosingMarkerBeforeParagraphBreak} instead of the
+ * single-line variant because Wikidot allows underline to span
+ * multiple lines within a paragraph.
+ *
+ * When no closing marker is found before a paragraph break, the
+ * opening `__` is treated as literal text.
+ */
+export const underlineRule: InlineRule = {
+  name: "underline",
+  startTokens: ["UNDERLINE_MARKER"],
+
+  /**
+   * Attempts to parse underline formatting at the current position.
+   *
+   * @param ctx - Parse context with token stream and current position
+   * @returns A successful result containing either a `"container"` element
+   *          with `type: "underline"`, an empty array (for `____`), or a
+   *          text fallback for unmatched markers
+   */
+  parse(ctx: ParseContext): RuleResult<Element> {
+    const startToken = currentToken(ctx);
+
+    // Check if closing marker exists before paragraph break
+    if (!hasClosingMarkerBeforeParagraphBreak({ ...ctx, pos: ctx.pos + 1 }, "UNDERLINE_MARKER")) {
+      return {
+        success: true,
+        elements: [{ element: "text", data: startToken.value }],
+        consumed: 1,
+      };
+    }
+
+    const { children, consumed } = parseUnderlineContent(ctx, ctx.pos + 1);
+
+    // Empty underline (____) is discarded entirely in Wikidot
+    if (children.length === 0) {
+      return {
+        success: true,
+        elements: [],
+        consumed,
+      };
+    }
+
+    return {
+      success: true,
+      elements: [createInlineContainer("underline", children)],
+      consumed,
+    };
+  },
+};

package/src/parser/rules/inline/user/element.ts

@@ -0,0 +1,11 @@
+import type { Element } from "@wdprlib/ast";
+
+export function userElement(name: string, showAvatar: boolean): Element {
+  return {
+    element: "user",
+    data: {
+      name,
+      "show-avatar": showAvatar,
+    },
+  };
+}

package/src/parser/rules/inline/user/index.ts

@@ -0,0 +1,34 @@
+/**
+ *
+ * Parses the Wikidot user reference syntax: `[[user name]]` and `[[*user name]]`.
+ *
+ * @module
+ */
+import type { Element } from "@wdprlib/ast";
+import type { InlineRule, ParseContext, RuleResult } from "../../types";
+import { currentToken } from "../../types";
+import { userElement } from "./element";
+import { parseUserReference } from "./syntax";
+
+export const userRule: InlineRule = {
+  name: "user",
+  startTokens: ["BLOCK_OPEN"],
+
+  parse(ctx: ParseContext): RuleResult<Element> {
+    const openToken = currentToken(ctx);
+    if (openToken.type !== "BLOCK_OPEN") {
+      return { success: false };
+    }
+
+    const parsed = parseUserReference(ctx, ctx.pos + 1);
+    if (!parsed) {
+      return { success: false };
+    }
+
+    return {
+      success: true,
+      elements: [userElement(parsed.name, parsed.showAvatar)],
+      consumed: 1 + parsed.consumed,
+    };
+  },
+};

package/src/parser/rules/inline/user/syntax.ts

@@ -0,0 +1,67 @@
+import type { ParseContext } from "../../types";
+
+export function parseUserReference(
+  ctx: ParseContext,
+  startPos: number,
+): { name: string; showAvatar: boolean; consumed: number } | null {
+  let pos = startPos;
+  let consumed = 0;
+
+  if (ctx.tokens[pos]?.type === "WHITESPACE") {
+    return null;
+  }
+
+  let showAvatar = false;
+  if (ctx.tokens[pos]?.type === "STAR") {
+    showAvatar = true;
+    pos++;
+    consumed++;
+  }
+
+  while (ctx.tokens[pos]?.type === "WHITESPACE") {
+    pos++;
+    consumed++;
+  }
+
+  const nameToken = ctx.tokens[pos];
+  if (!nameToken || (nameToken.type !== "TEXT" && nameToken.type !== "IDENTIFIER")) {
+    return null;
+  }
+  if (nameToken.value.toLowerCase() !== "user") {
+    return null;
+  }
+  pos++;
+  consumed++;
+
+  while (ctx.tokens[pos]?.type === "WHITESPACE") {
+    pos++;
+    consumed++;
+  }
+
+  let username = "";
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (
+      !token ||
+      token.type === "BLOCK_CLOSE" ||
+      token.type === "NEWLINE" ||
+      token.type === "EOF"
+    ) {
+      break;
+    }
+    username += token.value;
+    pos++;
+    consumed++;
+  }
+
+  username = username.trim();
+  if (!username || ctx.tokens[pos]?.type !== "BLOCK_CLOSE") {
+    return null;
+  }
+
+  return {
+    name: username,
+    showAvatar,
+    consumed: consumed + 1,
+  };
+}

package/src/parser/rules/inline/utils.ts

@@ -0,0 +1,4 @@
+export { canApplyInlineRule } from "./parsing/rules";
+export { collectUntilNewline } from "./parsing/collect";
+export { getCandidateInlineRules } from "./parsing/rules";
+export { parseInlineUntil, type InlineParseResult } from "./parsing/inline-content";

package/src/parser/rules/tokens.ts

@@ -0,0 +1,106 @@
+import type { Token, TokenType } from "../../lexer";
+import type { ParseContext } from "./contracts";
+
+/**
+ * Helper to get current token
+ */
+export function currentToken(ctx: ParseContext): Token {
+  return ctx.tokens[ctx.pos] ?? eofToken();
+}
+
+/**
+ * Helper to peek ahead
+ */
+export function peekToken(ctx: ParseContext, n = 1): Token {
+  return ctx.tokens[ctx.pos + n] ?? eofToken();
+}
+
+/**
+ * Helper to check token type
+ */
+export function checkToken(ctx: ParseContext, type: TokenType): boolean {
+  return currentToken(ctx).type === type;
+}
+
+/**
+ * Helper to check if at end
+ */
+export function isAtEnd(ctx: ParseContext): boolean {
+  return ctx.pos >= ctx.tokens.length || currentToken(ctx).type === "EOF";
+}
+
+/**
+ * Create EOF token
+ */
+function eofToken(): Token {
+  return {
+    type: "EOF",
+    value: "",
+    position: { start: { line: 0, column: 0, offset: 0 }, end: { line: 0, column: 0, offset: 0 } },
+    lineStart: false,
+  };
+}
+
+/**
+ * Check if closing marker exists before newline.
+ * If markerValue is provided, also check that the token value matches.
+ */
+export function hasClosingMarkerBeforeNewline(
+  ctx: ParseContext,
+  markerType: TokenType,
+  markerValue?: string,
+): boolean {
+  let pos = ctx.pos;
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (!token || token.type === "NEWLINE" || token.type === "EOF") {
+      return false;
+    }
+    if (token.type === markerType) {
+      if (markerValue === undefined || token.value === markerValue) {
+        return true;
+      }
+    }
+    pos++;
+  }
+  return false;
+}
+
+/**
+ * Check if closing marker exists before paragraph break (double newline).
+ * Allows inline formatting to span multiple lines within a paragraph.
+ */
+export function hasClosingMarkerBeforeParagraphBreak(
+  ctx: ParseContext,
+  markerType: TokenType,
+  markerValue?: string,
+): boolean {
+  let pos = ctx.pos;
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (!token || token.type === "EOF") {
+      return false;
+    }
+    // Check for paragraph break (NEWLINE followed by NEWLINE after optional whitespace)
+    if (token.type === "NEWLINE") {
+      let lookAhead = 1;
+      while (ctx.tokens[pos + lookAhead]?.type === "WHITESPACE") {
+        lookAhead++;
+      }
+      if (
+        ctx.tokens[pos + lookAhead]?.type === "NEWLINE" ||
+        ctx.tokens[pos + lookAhead]?.type === "EOF" ||
+        !ctx.tokens[pos + lookAhead]
+      ) {
+        return false;
+      }
+    }
+    if (token.type === markerType) {
+      if (markerValue === undefined || token.value === markerValue) {
+        return true;
+      }
+    }
+    pos++;
+  }
+  return false;
+}

package/src/parser/rules/types.ts

@@ -0,0 +1,9 @@
+export type { BlockRule, InlineRule, ParseContext, RuleResult, ScopeContext } from "./contracts";
+export {
+  checkToken,
+  currentToken,
+  hasClosingMarkerBeforeNewline,
+  hasClosingMarkerBeforeParagraphBreak,
+  isAtEnd,
+  peekToken,
+} from "./tokens";

package/src/parser/toc.ts

@@ -0,0 +1,130 @@
+/**
+ *
+ * Table of Contents (TOC) generation for Wikidot markup.
+ *
+ * Converts a flat array of `TocEntry` items (collected from heading elements
+ * during parsing) into nested bullet-list `Element` nodes suitable for rendering
+ * as `[[toc]]`. Uses the depth module to transform flat heading levels (h1-h6)
+ * into a properly nested list hierarchy.
+ *
+ * Each TOC entry becomes an anchor link (`#toc0`, `#toc1`, ...) pointing to the
+ * corresponding heading in the rendered page, matching Wikidot's original
+ * anchor naming scheme.
+ *
+ * @module
+ */
+
+import type { Element, TocEntry, ListItem } from "@wdprlib/ast";
+import { processDepths, type DepthList, type DepthItem } from "./depth";
+
+/**
+ * Sequential counter for generating unique TOC anchor IDs.
+ *
+ * Wikidot assigns sequential `#toc0`, `#toc1`, ... anchors to headings in
+ * document order. This class maintains a monotonically increasing counter
+ * that is shared across all TOC trees to ensure globally unique anchors.
+ */
+class TocIndexer {
+  private index = 0;
+
+  /**
+   * Returns the next sequential index and advances the counter.
+   * @returns The current index value (0-based) before incrementing
+   */
+  next(): number {
+    return this.index++;
+  }
+}
+
+/**
+ * Build a nested bullet-list Element from depth-processed TOC items.
+ *
+ * Each item in the depth list is converted to a `ListItem`, with nested lists
+ * becoming sub-list items and leaf items becoming anchor links.
+ *
+ * @param indexer - Shared counter for generating sequential `#tocN` anchors
+ * @param items - Depth-processed list of heading text strings
+ * @returns A `list` Element with type "bullet" containing the TOC hierarchy
+ */
+function buildTocList(indexer: TocIndexer, items: DepthList<null, string>): Element {
+  const listItems: ListItem[] = items.map((item) => buildTocListItem(indexer, item));
+
+  return {
+    element: "list",
+    data: {
+      type: "bullet",
+      attributes: {},
+      items: listItems,
+    },
+  };
+}
+
+/**
+ * Build a single TOC list item from a depth item.
+ *
+ * For leaf items, creates an anchor link element with a `#tocN` href.
+ * For nested list items, recursively builds a sub-list.
+ *
+ * @param indexer - Shared counter for generating sequential `#tocN` anchors
+ * @param item - A single depth item (either a leaf heading or a nested list)
+ * @returns A `ListItem` for inclusion in the TOC list
+ */
+function buildTocListItem(indexer: TocIndexer, item: DepthItem<null, string>): ListItem {
+  if (item.kind === "list") {
+    return {
+      "item-type": "sub-list",
+      element: "list",
+      data: {
+        type: "bullet",
+        attributes: {},
+        items: item.children.map((child) => buildTocListItem(indexer, child)),
+      },
+    };
+  }
+
+  // item.kind === "item"
+  const anchor = `#toc${indexer.next()}`;
+  const linkElement: Element = {
+    element: "link",
+    data: {
+      type: "table-of-contents",
+      link: anchor,
+      extra: null,
+      label: { text: item.value },
+      target: null,
+    },
+  };
+
+  return {
+    "item-type": "elements",
+    attributes: {},
+    elements: [linkElement],
+  };
+}
+
+/**
+ * Convert flat TocEntry[] to nested List elements
+ *
+ * @param entries - Flat list of TOC entries with level and text
+ * @returns Array of List elements (usually one, but can be multiple if levels reset)
+ */
+export function buildTableOfContents(entries: TocEntry[]): Element[] {
+  if (entries.length === 0) {
+    return [];
+  }
+
+  // Convert entries to depth-annotated items
+  // level is 1-based (h1=1, h2=2, ...), convert to 0-based depth
+  const depthItems = entries.map((entry) => ({
+    depth: entry.level - 1,
+    ltype: null as null, // We don't differentiate list types for TOC
+    value: entry.text,
+  }));
+
+  // Process into nested structure
+  const trees = processDepths<null, string>(null, depthItems);
+
+  // Build List elements from each tree
+  const indexer = new TocIndexer();
+  return trees.map((tree) => buildTocList(indexer, tree.list));
+}