@wdprlib/parser 3.2.0 → 4.0.0

package/src/parser/rules/inline/{underline.ts → underline/index.ts}

@@ -24,9 +24,10 @@
  * @module
  */
 import type { Element } from "@wdprlib/ast";
-import type { InlineRule, ParseContext, RuleResult } from "../types";
-import { currentToken, hasClosingMarkerBeforeParagraphBreak } from "../types";
-import { parseInlineUntil } from "./utils";
+import type { InlineRule, ParseContext, RuleResult } from "../../types";
+import { currentToken, hasClosingMarkerBeforeParagraphBreak } from "../../types";
+import { createInlineContainer } from "../formatting/container";
+import { parseUnderlineContent } from "./content";
 
 /**
  * Inline rule for parsing `__underline__` formatting.
@@ -63,42 +64,7 @@ export const underlineRule: InlineRule = {
       };
     }
 
-    // Parse content between markers, handling newlines as line-breaks
-    const children: Element[] = [];
-    let pos = ctx.pos + 1;
-    let consumed = 1; // opening marker
-
-    while (pos < ctx.tokens.length) {
-      const token = ctx.tokens[pos];
-      if (!token || token.type === "EOF") break;
-
-      // Found closing marker
-      if (token.type === "UNDERLINE_MARKER") {
-        consumed++;
-        break;
-      }
-
-      // Handle newlines as line-breaks
-      if (token.type === "NEWLINE") {
-        children.push({ element: "line-break" });
-        pos++;
-        consumed++;
-        continue;
-      }
-
-      // Parse inline content until NEWLINE or closing marker
-      const inlineCtx = { ...ctx, pos };
-      const result = parseInlineUntil(inlineCtx, "UNDERLINE_MARKER");
-      if (result.elements.length > 0) {
-        children.push(...result.elements);
-        pos += result.consumed;
-        consumed += result.consumed;
-      } else {
-        children.push({ element: "text", data: token.value });
-        pos++;
-        consumed++;
-      }
-    }
+    const { children, consumed } = parseUnderlineContent(ctx, ctx.pos + 1);
 
     // Empty underline (____) is discarded entirely in Wikidot
     if (children.length === 0) {
@@ -111,16 +77,7 @@ export const underlineRule: InlineRule = {
 
     return {
       success: true,
-      elements: [
-        {
-          element: "container",
-          data: {
-            type: "underline",
-            attributes: {},
-            elements: children,
-          },
-        },
-      ],
+      elements: [createInlineContainer("underline", children)],
       consumed,
     };
   },

package/src/parser/rules/inline/user/element.ts

@@ -0,0 +1,11 @@
+import type { Element } from "@wdprlib/ast";
+
+export function userElement(name: string, showAvatar: boolean): Element {
+  return {
+    element: "user",
+    data: {
+      name,
+      "show-avatar": showAvatar,
+    },
+  };
+}

package/src/parser/rules/inline/user/index.ts

@@ -0,0 +1,34 @@
+/**
+ *
+ * Parses the Wikidot user reference syntax: `[[user name]]` and `[[*user name]]`.
+ *
+ * @module
+ */
+import type { Element } from "@wdprlib/ast";
+import type { InlineRule, ParseContext, RuleResult } from "../../types";
+import { currentToken } from "../../types";
+import { userElement } from "./element";
+import { parseUserReference } from "./syntax";
+
+export const userRule: InlineRule = {
+  name: "user",
+  startTokens: ["BLOCK_OPEN"],
+
+  parse(ctx: ParseContext): RuleResult<Element> {
+    const openToken = currentToken(ctx);
+    if (openToken.type !== "BLOCK_OPEN") {
+      return { success: false };
+    }
+
+    const parsed = parseUserReference(ctx, ctx.pos + 1);
+    if (!parsed) {
+      return { success: false };
+    }
+
+    return {
+      success: true,
+      elements: [userElement(parsed.name, parsed.showAvatar)],
+      consumed: 1 + parsed.consumed,
+    };
+  },
+};

package/src/parser/rules/inline/user/syntax.ts

@@ -0,0 +1,67 @@
+import type { ParseContext } from "../../types";
+
+export function parseUserReference(
+  ctx: ParseContext,
+  startPos: number,
+): { name: string; showAvatar: boolean; consumed: number } | null {
+  let pos = startPos;
+  let consumed = 0;
+
+  if (ctx.tokens[pos]?.type === "WHITESPACE") {
+    return null;
+  }
+
+  let showAvatar = false;
+  if (ctx.tokens[pos]?.type === "STAR") {
+    showAvatar = true;
+    pos++;
+    consumed++;
+  }
+
+  while (ctx.tokens[pos]?.type === "WHITESPACE") {
+    pos++;
+    consumed++;
+  }
+
+  const nameToken = ctx.tokens[pos];
+  if (!nameToken || (nameToken.type !== "TEXT" && nameToken.type !== "IDENTIFIER")) {
+    return null;
+  }
+  if (nameToken.value.toLowerCase() !== "user") {
+    return null;
+  }
+  pos++;
+  consumed++;
+
+  while (ctx.tokens[pos]?.type === "WHITESPACE") {
+    pos++;
+    consumed++;
+  }
+
+  let username = "";
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (
+      !token ||
+      token.type === "BLOCK_CLOSE" ||
+      token.type === "NEWLINE" ||
+      token.type === "EOF"
+    ) {
+      break;
+    }
+    username += token.value;
+    pos++;
+    consumed++;
+  }
+
+  username = username.trim();
+  if (!username || ctx.tokens[pos]?.type !== "BLOCK_CLOSE") {
+    return null;
+  }
+
+  return {
+    name: username,
+    showAvatar,
+    consumed: consumed + 1,
+  };
+}

package/src/parser/rules/inline/utils.ts

@@ -1,344 +1,4 @@
-import type { TokenType, Token } from "../../../lexer";
-import type { Element } from "@wdprlib/ast";
-import type { ParseContext, InlineRule } from "../types";
-import {
-  BLOCK_START_TOKENS,
-  INDENT_ACCEPTING_BLOCK_NAMES,
-  KNOWN_BLOCK_NAMES,
-} from "../../constants";
-import { parseBlockName } from "../utils";
-
-/**
- * Checks whether the block token at `tokenPos` (BLOCK_OPEN or BLOCK_END_OPEN)
- * names a block in the excluded set.
- */
-function isExcludedBlockToken(ctx: ParseContext, tokenPos: number): boolean {
-  const excluded = ctx.scope.excludedBlockNames;
-  if (!excluded?.size) return false;
-  const token = ctx.tokens[tokenPos];
-  if (token?.type !== "BLOCK_OPEN" && token?.type !== "BLOCK_END_OPEN") return false;
-  const nameResult = parseBlockName(ctx, tokenPos + 1);
-  return nameResult !== null && excluded.has(nameResult.name);
-}
-
-/**
- * Checks whether the block token at `tokenPos` names a block that no rule
- * recognizes (e.g. `[[foo]]`). Wikidot leaves such tokens inside paragraphs
- * rather than treating them as paragraph boundaries.
- *
- * Align blocks (`[[=]]`, `[[==]]`) are recognized as a special case: their
- * marker tokens are `EQUALS`, not `TEXT`/`IDENTIFIER`, so `parseBlockName`
- * cannot extract a name. They are still real block boundaries.
- */
-function isUnknownBlockToken(ctx: ParseContext, tokenPos: number): boolean {
-  const token = ctx.tokens[tokenPos];
-  if (token?.type !== "BLOCK_OPEN" && token?.type !== "BLOCK_END_OPEN") return false;
-  const nameResult = parseBlockName(ctx, tokenPos + 1);
-  if (nameResult === null) {
-    // `[[=]]` / `[[==]]` align markers tokenize as EQUALS, not TEXT/IDENTIFIER.
-    if (ctx.tokens[tokenPos + 1]?.type === "EQUALS") {
-      return false;
-    }
-    // No recognizable identifier after [[ / [[/ — treat as inline.
-    return true;
-  }
-  return !KNOWN_BLOCK_NAMES.has(nameResult.name);
-}
-
-/**
- * Checks whether the block token at `tokenPos` names a block whose rule
- * accepts leading whitespace before the opener (`requiresLineStart: false`).
- *
- * Used to decide whether a `\n<indent>[[name]]` sequence should end a
- * paragraph: only when the matching block rule would actually consume
- * the indented token. Otherwise the boundary check would split the
- * paragraph for tokens that the block dispatcher then refuses, leaving
- * literal `[[toc]]` text in a fresh paragraph.
- */
-function isIndentAcceptingBlock(ctx: ParseContext, tokenPos: number): boolean {
-  const token = ctx.tokens[tokenPos];
-  if (token?.type !== "BLOCK_OPEN" && token?.type !== "BLOCK_END_OPEN") return false;
-  const nameResult = parseBlockName(ctx, tokenPos + 1);
-  if (nameResult === null) return false;
-  return INDENT_ACCEPTING_BLOCK_NAMES.has(nameResult.name);
-}
-
-/**
- * Result of parsing inline content
- */
-export interface InlineParseResult {
-  elements: Element[];
-  consumed: number;
-}
-
-/**
- * Check if an inline rule can be applied
- */
-export function canApplyInlineRule(rule: InlineRule, token: { type: TokenType }): boolean {
-  if (rule.startTokens.length === 0) {
-    return true; // fallback rule
-  }
-  return rule.startTokens.includes(token.type);
-}
-
-/**
- * Parse inline content until a specific token type
- *
- * When endType is "PARAGRAPH_BREAK", handles NEWLINEs and stops at:
- * - Double NEWLINE (paragraph break)
- * - NEWLINE followed by block-start token
- * - EOF
- */
-export function parseInlineUntil(ctx: ParseContext, endType: TokenType): InlineParseResult {
-  const nodes: Element[] = [];
-  let consumed = 0;
-  let pos = ctx.pos;
-
-  // Check if we're in paragraph mode (handle NEWLINEs inline)
-  const paragraphMode = endType === ("PARAGRAPH_BREAK" as TokenType);
-
-  const { inlineRules } = ctx;
-
-  while (pos < ctx.tokens.length) {
-    const token = ctx.tokens[pos];
-    if (!token || token.type === "EOF") {
-      break;
-    }
-
-    // Stop at block close condition if set in context
-    // This allows paragraph parser to respect parent block's close condition
-    if (paragraphMode && ctx.scope.blockCloseCondition) {
-      const checkCtx: ParseContext = { ...ctx, pos };
-      if (ctx.scope.blockCloseCondition(checkCtx)) {
-        break;
-      }
-    }
-
-    // Standard mode: stop at NEWLINE
-    if (!paragraphMode && token.type === "NEWLINE") {
-      break;
-    }
-
-    // Paragraph mode: check for paragraph break or block start
-    if (paragraphMode && token.type === "NEWLINE") {
-      // Look ahead to check what's after the newline
-      let lookAhead = 1;
-      while (ctx.tokens[pos + lookAhead]?.type === "WHITESPACE") {
-        lookAhead++;
-      }
-      const nextMeaningfulToken = ctx.tokens[pos + lookAhead];
-
-      // Check if this is [[/span]] - don't treat as block start, handle inline
-      let isOrphanCloseSpan = false;
-      if (nextMeaningfulToken?.type === "BLOCK_END_OPEN") {
-        // Check if it's [[/span]]
-        const namePos = pos + lookAhead + 1;
-        let nameLookAhead = 0;
-        while (ctx.tokens[namePos + nameLookAhead]?.type === "WHITESPACE") {
-          nameLookAhead++;
-        }
-        const nameToken = ctx.tokens[namePos + nameLookAhead];
-        if (nameToken?.type === "IDENTIFIER" && nameToken.value.toLowerCase() === "span") {
-          isOrphanCloseSpan = true;
-        }
-      }
-
-      // Check if this is [[# name]] - anchor name is inline, not block start
-      let isAnchorName = false;
-      if (nextMeaningfulToken?.type === "BLOCK_OPEN") {
-        const namePos = pos + lookAhead + 1;
-        let nameLookAhead = 0;
-        while (ctx.tokens[namePos + nameLookAhead]?.type === "WHITESPACE") {
-          nameLookAhead++;
-        }
-        const hashToken = ctx.tokens[namePos + nameLookAhead];
-        if (hashToken?.type === "HASH" || (hashToken?.type === "TEXT" && hashToken.value === "#")) {
-          isAnchorName = true;
-        }
-      }
-
-      // Check if this is [[>X or [[<X (where X is not ]]) - not a valid block opener
-      // [[>]] and [[<]] are valid align blocks, but [[>toc]] etc. are invalid
-      let isInvalidBlockOpen = false;
-      if (nextMeaningfulToken?.type === "BLOCK_OPEN") {
-        const afterOpen = pos + lookAhead + 1;
-        const firstAfter = ctx.tokens[afterOpen];
-        if (firstAfter?.type === "TEXT" && (firstAfter.value === ">" || firstAfter.value === "<")) {
-          const secondAfter = ctx.tokens[afterOpen + 1];
-          if (secondAfter && secondAfter.type !== "BLOCK_CLOSE") {
-            isInvalidBlockOpen = true;
-          }
-        }
-        // Check if this is [[footnoteblock]] but already parsed (2nd+ occurrence)
-        let skipWhitespace = 0;
-        while (ctx.tokens[afterOpen + skipWhitespace]?.type === "WHITESPACE") {
-          skipWhitespace++;
-        }
-        const blockNameToken = ctx.tokens[afterOpen + skipWhitespace];
-        if (
-          blockNameToken &&
-          (blockNameToken.type === "TEXT" || blockNameToken.type === "IDENTIFIER") &&
-          blockNameToken.value.toLowerCase() === "footnoteblock" &&
-          ctx.scope.footnoteBlockParsed
-        ) {
-          isInvalidBlockOpen = true;
-        }
-      }
-
-      // Check if HEADING_MARKER would actually succeed as a heading
-      // Wikidot requires: 1-6 plus signs + whitespace. Otherwise it's plain text.
-      let isInvalidHeading = false;
-      if (nextMeaningfulToken?.type === "HEADING_MARKER") {
-        const markerLen = nextMeaningfulToken.value.length;
-        const afterMarkerPos = pos + lookAhead + 1;
-        const afterMarker = ctx.tokens[afterMarkerPos];
-        // Invalid if: 7+ plus signs, or no whitespace after marker (or after optional *)
-        if (markerLen > 6) {
-          isInvalidHeading = true;
-        } else if (afterMarker?.type === "STAR") {
-          // +* pattern - check whitespace after *
-          const afterStar = ctx.tokens[afterMarkerPos + 1];
-          if (afterStar?.type !== "WHITESPACE") {
-            isInvalidHeading = true;
-          }
-        } else if (afterMarker?.type !== "WHITESPACE") {
-          isInvalidHeading = true;
-        }
-      }
-
-      // Check if this block token names an excluded block (e.g. nested collapsible)
-      const isExcludedBlock =
-        (nextMeaningfulToken?.type === "BLOCK_OPEN" ||
-          nextMeaningfulToken?.type === "BLOCK_END_OPEN") &&
-        isExcludedBlockToken(ctx, pos + lookAhead);
-
-      // Wikidot treats `[[foo]]` (where `foo` is not a known block name) as
-      // inline text rather than a paragraph-breaking block. Mirror that here.
-      const isUnknownBlock =
-        (nextMeaningfulToken?.type === "BLOCK_OPEN" ||
-          nextMeaningfulToken?.type === "BLOCK_END_OPEN") &&
-        isUnknownBlockToken(ctx, pos + lookAhead);
-
-      // Stop at double NEWLINE, EOF, or block start token (at line start)
-      // But don't stop at [[/span]], [[# name]], [[>/[[<, invalid headings,
-      // excluded block names, or unrecognized block names.
-      //
-      // Most block-start tokens require the strict `lineStart` flag (no
-      // leading whitespace at all): `   # one` is NOT a list item in
-      // Wikidot, `  + Heading` is NOT a heading, etc. We preserve that.
-      //
-      // A subset of `[[...]]` block constructs is the exception:
-      // their rules declare `requiresLineStart: false`, so Wikidot
-      // accepts leading whitespace before them and `[[/<name>]]` at
-      // arbitrary indentation also has to close such a block. The
-      // `lookAhead` walk above already consumed the NEWLINE and any
-      // leading WHITESPACE, so we know `nextMeaningfulToken` sits at
-      // the semantic start of the next line. We relax the `lineStart`
-      // check only when the block name's rule will actually accept the
-      // indented opener ({@link INDENT_ACCEPTING_BLOCK_NAMES});
-      // otherwise (e.g. `[[toc]]`, `[[footnoteblock]]`, align markers)
-      // the dispatcher would reject the indented token anyway and we
-      // would end up splitting the paragraph only to leave literal
-      // `[[…]]` text behind.
-      const isIndentedBlockOpener =
-        nextMeaningfulToken &&
-        (nextMeaningfulToken.type === "BLOCK_OPEN" ||
-          nextMeaningfulToken.type === "BLOCK_END_OPEN") &&
-        isIndentAcceptingBlock(ctx, pos + lookAhead);
-      const isBlockStart =
-        nextMeaningfulToken &&
-        BLOCK_START_TOKENS.includes(nextMeaningfulToken.type) &&
-        (nextMeaningfulToken.lineStart || isIndentedBlockOpener) &&
-        !isOrphanCloseSpan &&
-        !isAnchorName &&
-        !isInvalidBlockOpen &&
-        !isInvalidHeading &&
-        !isExcludedBlock &&
-        !isUnknownBlock;
-      if (
-        !nextMeaningfulToken ||
-        nextMeaningfulToken.type === "NEWLINE" ||
-        nextMeaningfulToken.type === "EOF" ||
-        isBlockStart
-      ) {
-        // Check if a block rule with preservesPrecedingLineBreak matches at the next position.
-        // Wikidot's Divalign expands content inline, so \n before it becomes <br />.
-        // Other blocks (Code, Div, etc.) suppress this by prepending \n\n to their token.
-        if (isBlockStart && nodes.length > 0) {
-          const nextPos = pos + lookAhead;
-          const shouldPreserve = ctx.blockRules.some(
-            (rule) => rule.preservesPrecedingLineBreak && rule.isStartPattern?.(ctx, nextPos),
-          );
-          if (shouldPreserve) {
-            const lb: any = { element: "line-break" };
-            lb._preservedTrailingBreak = true;
-            nodes.push(lb);
-          }
-        }
-        // Consume the NEWLINE and stop
-        consumed++;
-        if (nextMeaningfulToken?.type === "NEWLINE") {
-          consumed++; // Also consume second newline for paragraph break
-        }
-        break;
-      }
-    }
-
-    if (token.type === endType) {
-      break;
-    }
-
-    const inlineCtx: ParseContext = {
-      ...ctx,
-      pos,
-    };
-
-    let matched = false;
-    for (const rule of inlineRules) {
-      // Skip the rule that would match the end type to avoid infinite recursion
-      if (rule.startTokens.includes(endType)) {
-        continue;
-      }
-      if (canApplyInlineRule(rule, token)) {
-        const result = rule.parse(inlineCtx);
-        if (result.success) {
-          nodes.push(...result.elements);
-          consumed += result.consumed;
-          pos += result.consumed;
-          matched = true;
-          break;
-        }
-      }
-    }
-
-    if (!matched) {
-      // Fallback to text
-      nodes.push({ element: "text", data: token.value });
-      consumed++;
-      pos++;
-    }
-  }
-
-  return { elements: nodes, consumed };
-}
-
-/**
- * Collect tokens until newline or EOF
- */
-export function collectUntilNewline(ctx: ParseContext): { tokens: Token[]; consumed: number } {
-  const tokens: Token[] = [];
-  let consumed = 0;
-  let pos = ctx.pos;
-
-  while (pos < ctx.tokens.length) {
-    const token = ctx.tokens[pos];
-    if (!token || token.type === "NEWLINE" || token.type === "EOF") {
-      break;
-    }
-    tokens.push(token);
-    consumed++;
-    pos++;
-  }
-
-  return { tokens, consumed };
-}
+export { canApplyInlineRule } from "./parsing/rules";
+export { collectUntilNewline } from "./parsing/collect";
+export { getCandidateInlineRules } from "./parsing/rules";
+export { parseInlineUntil, type InlineParseResult } from "./parsing/inline-content";

package/src/parser/rules/tokens.ts

@@ -0,0 +1,106 @@
+import type { Token, TokenType } from "../../lexer";
+import type { ParseContext } from "./contracts";
+
+/**
+ * Helper to get current token
+ */
+export function currentToken(ctx: ParseContext): Token {
+  return ctx.tokens[ctx.pos] ?? eofToken();
+}
+
+/**
+ * Helper to peek ahead
+ */
+export function peekToken(ctx: ParseContext, n = 1): Token {
+  return ctx.tokens[ctx.pos + n] ?? eofToken();
+}
+
+/**
+ * Helper to check token type
+ */
+export function checkToken(ctx: ParseContext, type: TokenType): boolean {
+  return currentToken(ctx).type === type;
+}
+
+/**
+ * Helper to check if at end
+ */
+export function isAtEnd(ctx: ParseContext): boolean {
+  return ctx.pos >= ctx.tokens.length || currentToken(ctx).type === "EOF";
+}
+
+/**
+ * Create EOF token
+ */
+function eofToken(): Token {
+  return {
+    type: "EOF",
+    value: "",
+    position: { start: { line: 0, column: 0, offset: 0 }, end: { line: 0, column: 0, offset: 0 } },
+    lineStart: false,
+  };
+}
+
+/**
+ * Check if closing marker exists before newline.
+ * If markerValue is provided, also check that the token value matches.
+ */
+export function hasClosingMarkerBeforeNewline(
+  ctx: ParseContext,
+  markerType: TokenType,
+  markerValue?: string,
+): boolean {
+  let pos = ctx.pos;
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (!token || token.type === "NEWLINE" || token.type === "EOF") {
+      return false;
+    }
+    if (token.type === markerType) {
+      if (markerValue === undefined || token.value === markerValue) {
+        return true;
+      }
+    }
+    pos++;
+  }
+  return false;
+}
+
+/**
+ * Check if closing marker exists before paragraph break (double newline).
+ * Allows inline formatting to span multiple lines within a paragraph.
+ */
+export function hasClosingMarkerBeforeParagraphBreak(
+  ctx: ParseContext,
+  markerType: TokenType,
+  markerValue?: string,
+): boolean {
+  let pos = ctx.pos;
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (!token || token.type === "EOF") {
+      return false;
+    }
+    // Check for paragraph break (NEWLINE followed by NEWLINE after optional whitespace)
+    if (token.type === "NEWLINE") {
+      let lookAhead = 1;
+      while (ctx.tokens[pos + lookAhead]?.type === "WHITESPACE") {
+        lookAhead++;
+      }
+      if (
+        ctx.tokens[pos + lookAhead]?.type === "NEWLINE" ||
+        ctx.tokens[pos + lookAhead]?.type === "EOF" ||
+        !ctx.tokens[pos + lookAhead]
+      ) {
+        return false;
+      }
+    }
+    if (token.type === markerType) {
+      if (markerValue === undefined || token.value === markerValue) {
+        return true;
+      }
+    }
+    pos++;
+  }
+  return false;
+}