@wdprlib/parser 3.1.2 → 3.2.0

package/src/parser/rules/inline/text.ts

@@ -0,0 +1,84 @@
+/**
+ *
+ * Provides the two lowest-priority inline rules: `textRule` and `fallbackRule`.
+ *
+ * These rules act as catch-alls that convert unrecognized tokens into
+ * plain `"text"` AST elements, ensuring no token is ever silently dropped
+ * during inline parsing.
+ *
+ * `textRule` handles `TEXT` and `WHITESPACE` tokens specifically and is
+ * included in the main `inlineRules` array as the last entry before
+ * the fallback.
+ *
+ * `fallbackRule` has an empty `startTokens` array, which means it matches
+ * ANY token type. It is exported separately as `inlineFallbackRule` and
+ * is NOT included in the `inlineRules` array to prevent it from
+ * short-circuiting more specific rules. Instead, it is invoked explicitly
+ * by the parser when no other rule matches.
+ *
+ * @module
+ */
+import type { Element } from "@wdprlib/ast";
+import type { InlineRule, ParseContext, RuleResult } from "../types";
+import { currentToken } from "../types";
+
+/**
+ * Inline rule for plain text and whitespace tokens.
+ *
+ * Matches `TEXT` and `WHITESPACE` token types and converts them
+ * directly to `"text"` AST elements. This rule always succeeds.
+ *
+ * Placed last (before the fallback) in the inline rules array so
+ * that all formatting and structural rules are tried first.
+ */
+export const textRule: InlineRule = {
+  name: "text",
+  startTokens: ["TEXT", "WHITESPACE"],
+
+  /**
+   * Converts a TEXT or WHITESPACE token into a text element.
+   *
+   * @param ctx - Parse context with token stream and current position
+   * @returns Always returns `{ success: true }` with a single `"text"` element
+   */
+  parse(ctx: ParseContext): RuleResult<Element> {
+    const token = currentToken(ctx);
+
+    return {
+      success: true,
+      elements: [{ element: "text", data: token.value }],
+      consumed: 1,
+    };
+  },
+};
+
+/**
+ * Universal fallback rule for any token type not matched by other rules.
+ *
+ * The empty `startTokens` array signals to the parser that this rule
+ * can match any token. It converts the token's value to a `"text"`
+ * element, ensuring no token is silently dropped.
+ *
+ * This rule is used as a last-resort handler and is intentionally
+ * excluded from the main `inlineRules` array.
+ */
+export const fallbackRule: InlineRule = {
+  name: "fallback",
+  startTokens: [], // matches anything not matched by other rules
+
+  /**
+   * Converts any unrecognized token into a text element.
+   *
+   * @param ctx - Parse context with token stream and current position
+   * @returns Always returns `{ success: true }` with a single `"text"` element
+   */
+  parse(ctx: ParseContext): RuleResult<Element> {
+    const token = currentToken(ctx);
+
+    return {
+      success: true,
+      elements: [{ element: "text", data: token.value }],
+      consumed: 1,
+    };
+  },
+};

package/src/parser/rules/inline/underline.ts

@@ -0,0 +1,127 @@
+/**
+ *
+ * Parses the Wikidot underline formatting syntax: `__text__`.
+ *
+ * Underline text is delimited by double underscores. Unlike most
+ * inline formatting markers (bold, italic, etc.) which require the
+ * closing marker on the same line, underline markers can span
+ * multiple lines within the same paragraph. The closing marker
+ * must appear before a paragraph break (blank line).
+ *
+ * Single newlines within underlined content are converted to
+ * `<br />` elements, matching Wikidot's multiline underline behavior.
+ *
+ * If no closing `__` is found before a paragraph break, the opening
+ * marker is emitted as literal text.
+ *
+ * Empty underline (`____`) is silently discarded by Wikidot (produces
+ * no output).
+ *
+ * Renders as a `<u>` element in HTML.
+ *
+ * Produces a `"container"` AST element with `type: "underline"`.
+ *
+ * @module
+ */
+import type { Element } from "@wdprlib/ast";
+import type { InlineRule, ParseContext, RuleResult } from "../types";
+import { currentToken, hasClosingMarkerBeforeParagraphBreak } from "../types";
+import { parseInlineUntil } from "./utils";
+
+/**
+ * Inline rule for parsing `__underline__` formatting.
+ *
+ * Triggered by an `UNDERLINE_MARKER` token (`__`). Uses
+ * {@link hasClosingMarkerBeforeParagraphBreak} instead of the
+ * single-line variant because Wikidot allows underline to span
+ * multiple lines within a paragraph.
+ *
+ * When no closing marker is found before a paragraph break, the
+ * opening `__` is treated as literal text.
+ */
+export const underlineRule: InlineRule = {
+  name: "underline",
+  startTokens: ["UNDERLINE_MARKER"],
+
+  /**
+   * Attempts to parse underline formatting at the current position.
+   *
+   * @param ctx - Parse context with token stream and current position
+   * @returns A successful result containing either a `"container"` element
+   *          with `type: "underline"`, an empty array (for `____`), or a
+   *          text fallback for unmatched markers
+   */
+  parse(ctx: ParseContext): RuleResult<Element> {
+    const startToken = currentToken(ctx);
+
+    // Check if closing marker exists before paragraph break
+    if (!hasClosingMarkerBeforeParagraphBreak({ ...ctx, pos: ctx.pos + 1 }, "UNDERLINE_MARKER")) {
+      return {
+        success: true,
+        elements: [{ element: "text", data: startToken.value }],
+        consumed: 1,
+      };
+    }
+
+    // Parse content between markers, handling newlines as line-breaks
+    const children: Element[] = [];
+    let pos = ctx.pos + 1;
+    let consumed = 1; // opening marker
+
+    while (pos < ctx.tokens.length) {
+      const token = ctx.tokens[pos];
+      if (!token || token.type === "EOF") break;
+
+      // Found closing marker
+      if (token.type === "UNDERLINE_MARKER") {
+        consumed++;
+        break;
+      }
+
+      // Handle newlines as line-breaks
+      if (token.type === "NEWLINE") {
+        children.push({ element: "line-break" });
+        pos++;
+        consumed++;
+        continue;
+      }
+
+      // Parse inline content until NEWLINE or closing marker
+      const inlineCtx = { ...ctx, pos };
+      const result = parseInlineUntil(inlineCtx, "UNDERLINE_MARKER");
+      if (result.elements.length > 0) {
+        children.push(...result.elements);
+        pos += result.consumed;
+        consumed += result.consumed;
+      } else {
+        children.push({ element: "text", data: token.value });
+        pos++;
+        consumed++;
+      }
+    }
+
+    // Empty underline (____) is discarded entirely in Wikidot
+    if (children.length === 0) {
+      return {
+        success: true,
+        elements: [],
+        consumed,
+      };
+    }
+
+    return {
+      success: true,
+      elements: [
+        {
+          element: "container",
+          data: {
+            type: "underline",
+            attributes: {},
+            elements: children,
+          },
+        },
+      ],
+      consumed,
+    };
+  },
+};

package/src/parser/rules/inline/user.ts

@@ -0,0 +1,147 @@
+/**
+ *
+ * Parses the Wikidot user reference syntax: `[[user name]]` and
+ * `[[*user name]]`.
+ *
+ * A user reference displays a linked username (typically linking to
+ * the user's profile page). The variant with a star prefix (`[[*user]]`)
+ * also displays the user's avatar alongside the username.
+ *
+ * Wikidot syntax:
+ * - `[[user some-user]]` -- displays username as a link
+ * - `[[*user some-user]]` -- displays avatar and username
+ *
+ * Note: Wikidot requires no whitespace immediately after `[[`. This
+ * means `[[ user name]]` is invalid, but `[[user name]]` and
+ * `[[*user name]]` are valid.
+ *
+ * The username may contain any characters except `]]` and newlines.
+ * Leading/trailing whitespace around the username is trimmed.
+ *
+ * Produces a `"user"` AST element with `data.name` (the username)
+ * and `data["show-avatar"]` (boolean).
+ *
+ * @module
+ */
+import type { Element } from "@wdprlib/ast";
+import type { InlineRule, ParseContext, RuleResult } from "../types";
+import { currentToken } from "../types";
+
+/**
+ * Inline rule for parsing `[[user name]]` and `[[*user name]]` references.
+ *
+ * Triggered by a `BLOCK_OPEN` (`[[`) token. Optionally detects a `*`
+ * prefix for avatar display, then verifies the keyword `user`, and
+ * collects the username until `]]`.
+ *
+ * Fails if:
+ * - Whitespace immediately follows `[[` (Wikidot requires no leading space)
+ * - The keyword is not `user`
+ * - The username is empty
+ * - No closing `]]` is found
+ */
+export const userRule: InlineRule = {
+  name: "user",
+  startTokens: ["BLOCK_OPEN"],
+
+  /**
+   * Attempts to parse a user reference at the current position.
+   *
+   * @param ctx - Parse context with token stream and current position
+   * @returns A successful result with a `"user"` element, or `{ success: false }`
+   */
+  parse(ctx: ParseContext): RuleResult<Element> {
+    const openToken = currentToken(ctx);
+    if (openToken.type !== "BLOCK_OPEN") {
+      return { success: false };
+    }
+
+    let pos = ctx.pos + 1;
+    let consumed = 1;
+
+    // Wikidot requires no whitespace immediately after [[
+    // [[ user]] is invalid, [[user]] is valid
+    if (ctx.tokens[pos]?.type === "WHITESPACE") {
+      return { success: false };
+    }
+
+    // Check for star (avatar flag)
+    let showAvatar = false;
+    if (ctx.tokens[pos]?.type === "STAR") {
+      showAvatar = true;
+      pos++;
+      consumed++;
+    }
+
+    // Skip whitespace after star
+    while (ctx.tokens[pos]?.type === "WHITESPACE") {
+      pos++;
+      consumed++;
+    }
+
+    // Parse block name
+    const nameToken = ctx.tokens[pos];
+    if (!nameToken || (nameToken.type !== "TEXT" && nameToken.type !== "IDENTIFIER")) {
+      return { success: false };
+    }
+
+    const blockName = nameToken.value.toLowerCase();
+    if (blockName !== "user") {
+      return { success: false };
+    }
+    pos++;
+    consumed++;
+
+    // Skip whitespace
+    while (ctx.tokens[pos]?.type === "WHITESPACE") {
+      pos++;
+      consumed++;
+    }
+
+    // Parse username - collect all tokens until ]]
+    let username = "";
+    while (pos < ctx.tokens.length) {
+      const token = ctx.tokens[pos];
+      if (
+        !token ||
+        token.type === "BLOCK_CLOSE" ||
+        token.type === "NEWLINE" ||
+        token.type === "EOF"
+      ) {
+        break;
+      }
+      username += token.value;
+      pos++;
+      consumed++;
+    }
+
+    // Trim whitespace from username
+    username = username.trim();
+
+    // Username is required
+    if (!username) {
+      return { success: false };
+    }
+
+    // Expect ]]
+    if (ctx.tokens[pos]?.type !== "BLOCK_CLOSE") {
+      return { success: false };
+    }
+    pos++;
+    consumed++;
+
+    return {
+      success: true,
+      elements: [
+        {
+          element: "user",
+          data: {
+            name: username,
+            "show-avatar": showAvatar,
+          },
+        },
+      ],
+      consumed,
+    };
+  },
+};

package/src/parser/rules/inline/utils.ts

@@ -0,0 +1,344 @@
+import type { TokenType, Token } from "../../../lexer";
+import type { Element } from "@wdprlib/ast";
+import type { ParseContext, InlineRule } from "../types";
+import {
+  BLOCK_START_TOKENS,
+  INDENT_ACCEPTING_BLOCK_NAMES,
+  KNOWN_BLOCK_NAMES,
+} from "../../constants";
+import { parseBlockName } from "../utils";
+
+/**
+ * Checks whether the block token at `tokenPos` (BLOCK_OPEN or BLOCK_END_OPEN)
+ * names a block in the excluded set.
+ */
+function isExcludedBlockToken(ctx: ParseContext, tokenPos: number): boolean {
+  const excluded = ctx.scope.excludedBlockNames;
+  if (!excluded?.size) return false;
+  const token = ctx.tokens[tokenPos];
+  if (token?.type !== "BLOCK_OPEN" && token?.type !== "BLOCK_END_OPEN") return false;
+  const nameResult = parseBlockName(ctx, tokenPos + 1);
+  return nameResult !== null && excluded.has(nameResult.name);
+}
+
+/**
+ * Checks whether the block token at `tokenPos` names a block that no rule
+ * recognizes (e.g. `[[foo]]`). Wikidot leaves such tokens inside paragraphs
+ * rather than treating them as paragraph boundaries.
+ *
+ * Align blocks (`[[=]]`, `[[==]]`) are recognized as a special case: their
+ * marker tokens are `EQUALS`, not `TEXT`/`IDENTIFIER`, so `parseBlockName`
+ * cannot extract a name. They are still real block boundaries.
+ */
+function isUnknownBlockToken(ctx: ParseContext, tokenPos: number): boolean {
+  const token = ctx.tokens[tokenPos];
+  if (token?.type !== "BLOCK_OPEN" && token?.type !== "BLOCK_END_OPEN") return false;
+  const nameResult = parseBlockName(ctx, tokenPos + 1);
+  if (nameResult === null) {
+    // `[[=]]` / `[[==]]` align markers tokenize as EQUALS, not TEXT/IDENTIFIER.
+    if (ctx.tokens[tokenPos + 1]?.type === "EQUALS") {
+      return false;
+    }
+    // No recognizable identifier after [[ / [[/ — treat as inline.
+    return true;
+  }
+  return !KNOWN_BLOCK_NAMES.has(nameResult.name);
+}
+
+/**
+ * Checks whether the block token at `tokenPos` names a block whose rule
+ * accepts leading whitespace before the opener (`requiresLineStart: false`).
+ *
+ * Used to decide whether a `\n<indent>[[name]]` sequence should end a
+ * paragraph: only when the matching block rule would actually consume
+ * the indented token. Otherwise the boundary check would split the
+ * paragraph for tokens that the block dispatcher then refuses, leaving
+ * literal `[[toc]]` text in a fresh paragraph.
+ */
+function isIndentAcceptingBlock(ctx: ParseContext, tokenPos: number): boolean {
+  const token = ctx.tokens[tokenPos];
+  if (token?.type !== "BLOCK_OPEN" && token?.type !== "BLOCK_END_OPEN") return false;
+  const nameResult = parseBlockName(ctx, tokenPos + 1);
+  if (nameResult === null) return false;
+  return INDENT_ACCEPTING_BLOCK_NAMES.has(nameResult.name);
+}
+
+/**
+ * Result of parsing inline content
+ */
+export interface InlineParseResult {
+  elements: Element[];
+  consumed: number;
+}
+
+/**
+ * Check if an inline rule can be applied
+ */
+export function canApplyInlineRule(rule: InlineRule, token: { type: TokenType }): boolean {
+  if (rule.startTokens.length === 0) {
+    return true; // fallback rule
+  }
+  return rule.startTokens.includes(token.type);
+}
+
+/**
+ * Parse inline content until a specific token type
+ *
+ * When endType is "PARAGRAPH_BREAK", handles NEWLINEs and stops at:
+ * - Double NEWLINE (paragraph break)
+ * - NEWLINE followed by block-start token
+ * - EOF
+ */
+export function parseInlineUntil(ctx: ParseContext, endType: TokenType): InlineParseResult {
+  const nodes: Element[] = [];
+  let consumed = 0;
+  let pos = ctx.pos;
+
+  // Check if we're in paragraph mode (handle NEWLINEs inline)
+  const paragraphMode = endType === ("PARAGRAPH_BREAK" as TokenType);
+
+  const { inlineRules } = ctx;
+
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (!token || token.type === "EOF") {
+      break;
+    }
+
+    // Stop at block close condition if set in context
+    // This allows paragraph parser to respect parent block's close condition
+    if (paragraphMode && ctx.scope.blockCloseCondition) {
+      const checkCtx: ParseContext = { ...ctx, pos };
+      if (ctx.scope.blockCloseCondition(checkCtx)) {
+        break;
+      }
+    }
+
+    // Standard mode: stop at NEWLINE
+    if (!paragraphMode && token.type === "NEWLINE") {
+      break;
+    }
+
+    // Paragraph mode: check for paragraph break or block start
+    if (paragraphMode && token.type === "NEWLINE") {
+      // Look ahead to check what's after the newline
+      let lookAhead = 1;
+      while (ctx.tokens[pos + lookAhead]?.type === "WHITESPACE") {
+        lookAhead++;
+      }
+      const nextMeaningfulToken = ctx.tokens[pos + lookAhead];
+
+      // Check if this is [[/span]] - don't treat as block start, handle inline
+      let isOrphanCloseSpan = false;
+      if (nextMeaningfulToken?.type === "BLOCK_END_OPEN") {
+        // Check if it's [[/span]]
+        const namePos = pos + lookAhead + 1;
+        let nameLookAhead = 0;
+        while (ctx.tokens[namePos + nameLookAhead]?.type === "WHITESPACE") {
+          nameLookAhead++;
+        }
+        const nameToken = ctx.tokens[namePos + nameLookAhead];
+        if (nameToken?.type === "IDENTIFIER" && nameToken.value.toLowerCase() === "span") {
+          isOrphanCloseSpan = true;
+        }
+      }
+
+      // Check if this is [[# name]] - anchor name is inline, not block start
+      let isAnchorName = false;
+      if (nextMeaningfulToken?.type === "BLOCK_OPEN") {
+        const namePos = pos + lookAhead + 1;
+        let nameLookAhead = 0;
+        while (ctx.tokens[namePos + nameLookAhead]?.type === "WHITESPACE") {
+          nameLookAhead++;
+        }
+        const hashToken = ctx.tokens[namePos + nameLookAhead];
+        if (hashToken?.type === "HASH" || (hashToken?.type === "TEXT" && hashToken.value === "#")) {
+          isAnchorName = true;
+        }
+      }
+
+      // Check if this is [[>X or [[<X (where X is not ]]) - not a valid block opener
+      // [[>]] and [[<]] are valid align blocks, but [[>toc]] etc. are invalid
+      let isInvalidBlockOpen = false;
+      if (nextMeaningfulToken?.type === "BLOCK_OPEN") {
+        const afterOpen = pos + lookAhead + 1;
+        const firstAfter = ctx.tokens[afterOpen];
+        if (firstAfter?.type === "TEXT" && (firstAfter.value === ">" || firstAfter.value === "<")) {
+          const secondAfter = ctx.tokens[afterOpen + 1];
+          if (secondAfter && secondAfter.type !== "BLOCK_CLOSE") {
+            isInvalidBlockOpen = true;
+          }
+        }
+        // Check if this is [[footnoteblock]] but already parsed (2nd+ occurrence)
+        let skipWhitespace = 0;
+        while (ctx.tokens[afterOpen + skipWhitespace]?.type === "WHITESPACE") {
+          skipWhitespace++;
+        }
+        const blockNameToken = ctx.tokens[afterOpen + skipWhitespace];
+        if (
+          blockNameToken &&
+          (blockNameToken.type === "TEXT" || blockNameToken.type === "IDENTIFIER") &&
+          blockNameToken.value.toLowerCase() === "footnoteblock" &&
+          ctx.scope.footnoteBlockParsed
+        ) {
+          isInvalidBlockOpen = true;
+        }
+      }
+
+      // Check if HEADING_MARKER would actually succeed as a heading
+      // Wikidot requires: 1-6 plus signs + whitespace. Otherwise it's plain text.
+      let isInvalidHeading = false;
+      if (nextMeaningfulToken?.type === "HEADING_MARKER") {
+        const markerLen = nextMeaningfulToken.value.length;
+        const afterMarkerPos = pos + lookAhead + 1;
+        const afterMarker = ctx.tokens[afterMarkerPos];
+        // Invalid if: 7+ plus signs, or no whitespace after marker (or after optional *)
+        if (markerLen > 6) {
+          isInvalidHeading = true;
+        } else if (afterMarker?.type === "STAR") {
+          // +* pattern - check whitespace after *
+          const afterStar = ctx.tokens[afterMarkerPos + 1];
+          if (afterStar?.type !== "WHITESPACE") {
+            isInvalidHeading = true;
+          }
+        } else if (afterMarker?.type !== "WHITESPACE") {
+          isInvalidHeading = true;
+        }
+      }
+
+      // Check if this block token names an excluded block (e.g. nested collapsible)
+      const isExcludedBlock =
+        (nextMeaningfulToken?.type === "BLOCK_OPEN" ||
+          nextMeaningfulToken?.type === "BLOCK_END_OPEN") &&
+        isExcludedBlockToken(ctx, pos + lookAhead);
+
+      // Wikidot treats `[[foo]]` (where `foo` is not a known block name) as
+      // inline text rather than a paragraph-breaking block. Mirror that here.
+      const isUnknownBlock =
+        (nextMeaningfulToken?.type === "BLOCK_OPEN" ||
+          nextMeaningfulToken?.type === "BLOCK_END_OPEN") &&
+        isUnknownBlockToken(ctx, pos + lookAhead);
+
+      // Stop at double NEWLINE, EOF, or block start token (at line start)
+      // But don't stop at [[/span]], [[# name]], [[>/[[<, invalid headings,
+      // excluded block names, or unrecognized block names.
+      //
+      // Most block-start tokens require the strict `lineStart` flag (no
+      // leading whitespace at all): `   # one` is NOT a list item in
+      // Wikidot, `  + Heading` is NOT a heading, etc. We preserve that.
+      //
+      // A subset of `[[...]]` block constructs is the exception:
+      // their rules declare `requiresLineStart: false`, so Wikidot
+      // accepts leading whitespace before them and `[[/<name>]]` at
+      // arbitrary indentation also has to close such a block. The
+      // `lookAhead` walk above already consumed the NEWLINE and any
+      // leading WHITESPACE, so we know `nextMeaningfulToken` sits at
+      // the semantic start of the next line. We relax the `lineStart`
+      // check only when the block name's rule will actually accept the
+      // indented opener ({@link INDENT_ACCEPTING_BLOCK_NAMES});
+      // otherwise (e.g. `[[toc]]`, `[[footnoteblock]]`, align markers)
+      // the dispatcher would reject the indented token anyway and we
+      // would end up splitting the paragraph only to leave literal
+      // `[[…]]` text behind.
+      const isIndentedBlockOpener =
+        nextMeaningfulToken &&
+        (nextMeaningfulToken.type === "BLOCK_OPEN" ||
+          nextMeaningfulToken.type === "BLOCK_END_OPEN") &&
+        isIndentAcceptingBlock(ctx, pos + lookAhead);
+      const isBlockStart =
+        nextMeaningfulToken &&
+        BLOCK_START_TOKENS.includes(nextMeaningfulToken.type) &&
+        (nextMeaningfulToken.lineStart || isIndentedBlockOpener) &&
+        !isOrphanCloseSpan &&
+        !isAnchorName &&
+        !isInvalidBlockOpen &&
+        !isInvalidHeading &&
+        !isExcludedBlock &&
+        !isUnknownBlock;
+      if (
+        !nextMeaningfulToken ||
+        nextMeaningfulToken.type === "NEWLINE" ||
+        nextMeaningfulToken.type === "EOF" ||
+        isBlockStart
+      ) {
+        // Check if a block rule with preservesPrecedingLineBreak matches at the next position.
+        // Wikidot's Divalign expands content inline, so \n before it becomes <br />.
+        // Other blocks (Code, Div, etc.) suppress this by prepending \n\n to their token.
+        if (isBlockStart && nodes.length > 0) {
+          const nextPos = pos + lookAhead;
+          const shouldPreserve = ctx.blockRules.some(
+            (rule) => rule.preservesPrecedingLineBreak && rule.isStartPattern?.(ctx, nextPos),
+          );
+          if (shouldPreserve) {
+            const lb: any = { element: "line-break" };
+            lb._preservedTrailingBreak = true;
+            nodes.push(lb);
+          }
+        }
+        // Consume the NEWLINE and stop
+        consumed++;
+        if (nextMeaningfulToken?.type === "NEWLINE") {
+          consumed++; // Also consume second newline for paragraph break
+        }
+        break;
+      }
+    }
+
+    if (token.type === endType) {
+      break;
+    }
+
+    const inlineCtx: ParseContext = {
+      ...ctx,
+      pos,
+    };
+
+    let matched = false;
+    for (const rule of inlineRules) {
+      // Skip the rule that would match the end type to avoid infinite recursion
+      if (rule.startTokens.includes(endType)) {
+        continue;
+      }
+      if (canApplyInlineRule(rule, token)) {
+        const result = rule.parse(inlineCtx);
+        if (result.success) {
+          nodes.push(...result.elements);
+          consumed += result.consumed;
+          pos += result.consumed;
+          matched = true;
+          break;
+        }
+      }
+    }
+
+    if (!matched) {
+      // Fallback to text
+      nodes.push({ element: "text", data: token.value });
+      consumed++;
+      pos++;
+    }
+  }
+
+  return { elements: nodes, consumed };
+}
+
+/**
+ * Collect tokens until newline or EOF
+ */
+export function collectUntilNewline(ctx: ParseContext): { tokens: Token[]; consumed: number } {
+  const tokens: Token[] = [];
+  let consumed = 0;
+  let pos = ctx.pos;
+
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (!token || token.type === "NEWLINE" || token.type === "EOF") {
+      break;
+    }
+    tokens.push(token);
+    consumed++;
+    pos++;
+  }
+
+  return { tokens, consumed };
+}