@wdprlib/parser 5.1.6 → 5.2.0

package/src/parser/rules/inline/parsing/block-start-predicates.ts

@@ -1,3 +1,4 @@
+import { parseImageOpen } from "../image/open";
 import type { ParseContext } from "../../types";
 import { BLOCK_START_TOKEN_SET } from "../../../constants";
 import {
@@ -26,6 +27,7 @@ export function isParagraphBreakingBlockStart(
   }
 
   return (
+    !parseImageOpen({ ...ctx, pos: nextPos }) &&
     !isOrphanCloseSpan(ctx, nextPos) &&
     !isAnchorName(ctx, nextPos) &&
     !isInvalidBlockOpen(ctx, nextPos) &&

package/src/parser/rules/inline/parsing/inline-content.ts

@@ -1,3 +1,4 @@
+import { protectedInlineRegionEnd } from "../raw/end";
 import type { Element } from "@wdprlib/ast";
 import type { ParseContext } from "../../types";
 import {
@@ -32,31 +33,53 @@ export function parseInlineUntil(ctx: ParseContext, endType: InlineEndType): Inl
   let pos = ctx.pos;
 
   const paragraphMode = endType === "PARAGRAPH_BREAK";
+  const multiline = paragraphMode || FORMATTING_CLOSE_TOKENS.has(endType);
+  let inlineEnd = ctx.scope.inlineEnd ?? ctx.tokens.length;
+  if (!multiline) {
+    for (let end = ctx.pos; end < inlineEnd; end++) {
+      const protectedEnd = protectedInlineRegionEnd(ctx.tokens, end, inlineEnd);
+      if (protectedEnd > end) {
+        end = protectedEnd - 1;
+        continue;
+      }
+      if (
+        ctx.tokens[end]?.type === "NEWLINE" &&
+        ctx.tokens[end - 1]?.type === "UNDERSCORE" &&
+        ctx.tokens[end - 2]?.type === "WHITESPACE"
+      )
+        continue;
+      if (ctx.tokens[end]?.type === "NEWLINE" || ctx.tokens[end]?.type === endType) {
+        inlineEnd = end;
+        break;
+      }
+    }
+  }
   const { inlineRules } = ctx;
   const inlineCtx: ParseContext = {
     ...ctx,
     pos,
+    scope: { ...ctx.scope, inlineEnd },
   };
   const canCollectLongPlainTextRuns = ctx.tokens.length >= MIN_INLINE_TEXT_RUN_DOCUMENT_TOKENS;
 
-  while (pos < ctx.tokens.length) {
+  while (pos < inlineEnd) {
     const token = ctx.tokens[pos];
     if (!token || token.type === "EOF") {
       break;
     }
 
-    if (paragraphMode && ctx.scope.blockCloseCondition) {
+    if (ctx.scope.blockCloseCondition) {
       const checkCtx: ParseContext = { ...ctx, pos };
       if (ctx.scope.blockCloseCondition(checkCtx)) {
         break;
       }
     }
 
-    if (!paragraphMode && token.type === "NEWLINE") {
+    if (!multiline && token.type === "NEWLINE") {
       break;
     }
 
-    if (paragraphMode && token.type === "NEWLINE") {
+    if (multiline && token.type === "NEWLINE" && !ctx.scope.tableFormatting) {
       const boundary = getParagraphNewlineBoundary(ctx, pos, nodes.length > 0);
       if (boundary.shouldBreak) {
         if (boundary.preservePrecedingLineBreak) {
@@ -68,6 +91,12 @@ export function parseInlineUntil(ctx: ParseContext, endType: InlineEndType): Inl
       }
     }
 
+    if (ctx.scope.tableFormatting?.suppressedClosers.has(pos)) {
+      pos++;
+      consumed++;
+      continue;
+    }
+
     if (token.type === endType) {
       break;
     }
@@ -96,6 +125,20 @@ export function parseInlineUntil(ctx: ParseContext, endType: InlineEndType): Inl
     for (const rule of getCandidateInlineRules(inlineRules, token.type)) {
       const result = rule.parse(inlineCtx);
       if (result.success) {
+        if (rule.name === "comment") {
+          let after = pos + result.consumed;
+          while (ctx.tokens[after]?.type === "WHITESPACE") after++;
+          if (ctx.tokens[after]?.type === "NEWLINE" || ctx.tokens[after]?.type === "EOF") {
+            while (nodes.at(-1)?.element === "text") {
+              const last = nodes.at(-1)!;
+              if (last.element !== "text") break;
+              last.data = last.data.trimEnd();
+              if (last.data) break;
+              nodes.pop();
+            }
+            if (nodes.at(-1)?.element === "line-break") nodes.pop();
+          }
+        }
         nodes.push(...result.elements);
         consumed += result.consumed;
         pos += result.consumed;
@@ -113,3 +156,14 @@ export function parseInlineUntil(ctx: ParseContext, endType: InlineEndType): Inl
 
   return { elements: nodes, consumed };
 }
+
+const FORMATTING_CLOSE_TOKENS: ReadonlySet<string> = new Set([
+  "BOLD_MARKER",
+  "ITALIC_MARKER",
+  "UNDERLINE_MARKER",
+  "STRIKE_MARKER",
+  "SUPER_MARKER",
+  "SUB_MARKER",
+  "MONO_CLOSE",
+  "COLOR_MARKER",
+]);

package/src/parser/rules/inline/raw/end.ts

@@ -0,0 +1,28 @@
+import type { Token } from "../../../../lexer";
+
+/** Exclusive end of a complete single-line raw region, or the original position. */
+export function rawRegionEnd(tokens: readonly Token[], start: number, end: number): number {
+  const type = tokens[start]?.type;
+  const close =
+    type === "RAW_OPEN" ? "RAW_OPEN" : type === "RAW_BLOCK_OPEN" ? "RAW_BLOCK_CLOSE" : null;
+  if (!close) return start;
+  for (let pos = start + 1; pos < end; pos++) {
+    if (tokens[pos]?.type === "NEWLINE" || tokens[pos]?.type === "EOF") break;
+    if (tokens[pos]?.type === close) return pos + 1;
+  }
+  return start;
+}
+
+/** Raw text and complete comments shield delimiters from their enclosing rule. */
+export function protectedInlineRegionEnd(
+  tokens: readonly Token[],
+  start: number,
+  end: number,
+): number {
+  const rawEnd = rawRegionEnd(tokens, start, end);
+  if (rawEnd > start || tokens[start]?.type !== "COMMENT_OPEN") return rawEnd;
+  for (let pos = start + 1; pos < end; pos++) {
+    if (tokens[pos]?.type === "COMMENT_CLOSE") return pos + 1;
+  }
+  return start;
+}

package/src/parser/rules/inline/span/content.ts

@@ -1,3 +1,4 @@
+import { rawRegionEnd } from "../raw/end";
 import type { Element } from "@wdprlib/ast";
 import type { ParseContext } from "../../types";
 import { inlineRules } from "../../index";
@@ -24,16 +25,21 @@ export function parseSpanContent(
   const escapedChildren: Element[] = [];
   const splitSpans: Element[][] = [];
   let foundClose = false;
+  let forcedClose = false;
   let afterBlankLine = false;
   let consumed = 0;
   let pos = startPos;
 
-  while (pos < ctx.tokens.length) {
+  while (pos < (ctx.scope.inlineEnd ?? ctx.tokens.length)) {
     const token = ctx.tokens[pos];
     if (!token || token.type === "EOF") {
       break;
     }
 
+    if (ctx.scope.tableFormatting?.suppressedClosers.has(pos)) {
+      forcedClose = true;
+      break;
+    }
     const close = parseCloseSpan(ctx, pos);
     if (close.success) {
       pos += close.consumed;
@@ -70,6 +76,31 @@ export function parseSpanContent(
     consumed += parsed.consumed;
   }
 
+  if (!foundClose && ctx.scope.tableFormatting && (pos === ctx.scope.inlineEnd || forcedClose)) {
+    let depth = 0;
+    for (let next = pos; next < ctx.scope.tableFormatting.end; next++) {
+      const rawEnd = rawRegionEnd(ctx.tokens, next, ctx.scope.tableFormatting.end);
+      if (rawEnd > next) {
+        next = rawEnd - 1;
+        continue;
+      }
+      if (
+        ctx.tokens[next]?.type === "BLOCK_OPEN" &&
+        /^span_?$/i.test(ctx.tokens[next + 1]?.value ?? "")
+      )
+        depth++;
+      const close = parseCloseSpan(ctx, next);
+      if (!close.success) continue;
+      if (depth > 0) {
+        depth--;
+        continue;
+      }
+      for (let offset = 0; offset < close.consumed; offset++)
+        ctx.scope.tableFormatting.suppressedClosers.add(next + offset);
+      foundClose = true;
+      break;
+    }
+  }
   return { children, escapedChildren, splitSpans, consumed, foundClose };
 }
 

package/src/parser/rules/inline/strikethrough/index.ts

@@ -6,7 +6,7 @@
  * token has dual meaning in Wikidot: it can be either a strikethrough
  * marker or an em-dash. The disambiguation rule is:
  *
- * - If a matching closing `--` is found on the same line AND the closing
+ * - If a matching closing `--` is found within the same paragraph AND the closing
  *   marker is NOT preceded by whitespace, it is treated as strikethrough.
  * - Otherwise, the `--` is converted to an em-dash character (U+2014).
  *

package/src/parser/rules/inline/strikethrough/parse.ts

@@ -1,14 +1,7 @@
 import type { Element } from "@wdprlib/ast";
 import type { ParseContext, RuleResult } from "../../types";
-import { createInlineContainer } from "../formatting/container";
-import { parseInlineUntil } from "../utils";
+import { parseDelimitedContainer } from "../formatting/container";
 
 export function parseStrikethroughContent(ctx: ParseContext): RuleResult<Element> {
-  const result = parseInlineUntil({ ...ctx, pos: ctx.pos + 1 }, "STRIKE_MARKER");
-
-  return {
-    success: true,
-    elements: [createInlineContainer("strikethrough", result.elements)],
-    consumed: 1 + result.consumed + 1,
-  };
+  return parseDelimitedContainer(ctx, "STRIKE_MARKER", "strikethrough");
 }

package/src/parser/rules/inline/strikethrough/syntax.ts

@@ -1,24 +1,7 @@
 import type { ParseContext } from "../../types";
+import { findFormattingClose } from "../formatting/close";
 
-/**
- * Returns true when the current `--` can be parsed as strikethrough.
- */
 export function hasValidStrikethroughClose(ctx: ParseContext): boolean {
-  let pos = ctx.pos + 1;
-  let prevWasWhitespace = false;
-
-  while (pos < ctx.tokens.length) {
-    const token = ctx.tokens[pos];
-    if (!token || token.type === "NEWLINE" || token.type === "EOF") {
-      return false;
-    }
-
-    if (token.type === "STRIKE_MARKER") {
-      return !prevWasWhitespace;
-    }
-
-    prevWasWhitespace = token.type === "WHITESPACE";
-    pos++;
-  }
-  return false;
+  const close = findFormattingClose(ctx, ctx.pos + 1, "STRIKE_MARKER");
+  return close !== null && close > ctx.pos + 1 && ctx.tokens[close - 1]?.type !== "WHITESPACE";
 }

package/src/parser/rules/inline/subscript.ts

@@ -3,8 +3,8 @@
  * Parses the Wikidot subscript formatting syntax: `,,text,,`.
  *
  * Subscript text is delimited by double commas. The opening and closing
- * markers must appear on the same line. If no closing `,,` is found
- * before a newline, the opening marker is emitted as literal text.
+ * markers must appear within the same paragraph. If no closing `,,` is found
+ * before a block boundary, the opening marker is emitted as literal text.
  *
  * Empty subscript (`,,,,`) is silently discarded by Wikidot (produces
  * no output), matching the behavior of bold and superscript.
@@ -17,13 +17,13 @@
  */
 import type { Element } from "@wdprlib/ast";
 import type { InlineRule, ParseContext, RuleResult } from "../types";
-import { parseSameLineDelimitedContainer } from "./formatting/container";
+import { parseDelimitedContainer } from "./formatting/container";
 
 /**
  * Inline rule for parsing `,,subscript,,` formatting.
  *
  * Triggered by a `SUB_MARKER` token (`,,`). Checks for a matching
- * closing marker on the same line, then recursively parses inline
+ * closing marker within the same paragraph, then recursively parses inline
  * content between the markers.
  *
  * When no closing marker is found, the opening `,,` is treated as
@@ -42,6 +42,6 @@ export const subscriptRule: InlineRule = {
    *          text fallback for unmatched markers
    */
   parse(ctx: ParseContext): RuleResult<Element> {
-    return parseSameLineDelimitedContainer(ctx, "SUB_MARKER", "subscript", { discardEmpty: true });
+    return parseDelimitedContainer(ctx, "SUB_MARKER", "subscript", { discardEmpty: true });
   },
 };

package/src/parser/rules/inline/superscript.ts

@@ -3,8 +3,8 @@
  * Parses the Wikidot superscript formatting syntax: `^^text^^`.
  *
  * Superscript text is delimited by double carets. The opening and
- * closing markers must appear on the same line. If no closing `^^`
- * is found before a newline, the opening marker is emitted as literal text.
+ * closing markers must appear within the same paragraph. If no closing `^^`
+ * is found before a block boundary, the opening marker is emitted as literal text.
  *
  * Empty superscript (`^^^^`) is silently discarded by Wikidot (produces
  * no output), matching the behavior of bold and subscript.
@@ -17,13 +17,13 @@
  */
 import type { Element } from "@wdprlib/ast";
 import type { InlineRule, ParseContext, RuleResult } from "../types";
-import { parseSameLineDelimitedContainer } from "./formatting/container";
+import { parseDelimitedContainer } from "./formatting/container";
 
 /**
  * Inline rule for parsing `^^superscript^^` formatting.
  *
  * Triggered by a `SUPER_MARKER` token (`^^`). Checks for a matching
- * closing marker on the same line, then recursively parses inline
+ * closing marker within the same paragraph, then recursively parses inline
  * content between the markers.
  *
  * When no closing marker is found, the opening `^^` is treated as
@@ -42,7 +42,7 @@ export const superscriptRule: InlineRule = {
    *          text fallback for unmatched markers
    */
   parse(ctx: ParseContext): RuleResult<Element> {
-    return parseSameLineDelimitedContainer(ctx, "SUPER_MARKER", "superscript", {
+    return parseDelimitedContainer(ctx, "SUPER_MARKER", "superscript", {
       discardEmpty: true,
     });
   },

package/src/parser/rules/inline/underline/index.ts

@@ -1,84 +1,11 @@
-/**
- *
- * Parses the Wikidot underline formatting syntax: `__text__`.
- *
- * Underline text is delimited by double underscores. Unlike most
- * inline formatting markers (bold, italic, etc.) which require the
- * closing marker on the same line, underline markers can span
- * multiple lines within the same paragraph. The closing marker
- * must appear before a paragraph break (blank line).
- *
- * Single newlines within underlined content are converted to
- * `<br />` elements, matching Wikidot's multiline underline behavior.
- *
- * If no closing `__` is found before a paragraph break, the opening
- * marker is emitted as literal text.
- *
- * Empty underline (`____`) is silently discarded by Wikidot (produces
- * no output).
- *
- * Renders as a `<u>` element in HTML.
- *
- * Produces a `"container"` AST element with `type: "underline"`.
- *
- * @module
- */
-import type { Element } from "@wdprlib/ast";
-import type { InlineRule, ParseContext, RuleResult } from "../../types";
-import { currentToken, hasClosingMarkerBeforeParagraphBreak } from "../../types";
-import { createInlineContainer } from "../formatting/container";
-import { parseUnderlineContent } from "./content";
+import type { InlineRule } from "../../types";
+import { parseDelimitedContainer } from "../formatting/container";
 
-/**
- * Inline rule for parsing `__underline__` formatting.
- *
- * Triggered by an `UNDERLINE_MARKER` token (`__`). Uses
- * {@link hasClosingMarkerBeforeParagraphBreak} instead of the
- * single-line variant because Wikidot allows underline to span
- * multiple lines within a paragraph.
- *
- * When no closing marker is found before a paragraph break, the
- * opening `__` is treated as literal text.
- */
+/** Underline can span ordinary newlines, but cannot cross its enclosing block. */
 export const underlineRule: InlineRule = {
   name: "underline",
   startTokens: ["UNDERLINE_MARKER"],
-
-  /**
-   * Attempts to parse underline formatting at the current position.
-   *
-   * @param ctx - Parse context with token stream and current position
-   * @returns A successful result containing either a `"container"` element
-   *          with `type: "underline"`, an empty array (for `____`), or a
-   *          text fallback for unmatched markers
-   */
-  parse(ctx: ParseContext): RuleResult<Element> {
-    const startToken = currentToken(ctx);
-
-    // Check if closing marker exists before paragraph break
-    if (!hasClosingMarkerBeforeParagraphBreak({ ...ctx, pos: ctx.pos + 1 }, "UNDERLINE_MARKER")) {
-      return {
-        success: true,
-        elements: [{ element: "text", data: startToken.value }],
-        consumed: 1,
-      };
-    }
-
-    const { children, consumed } = parseUnderlineContent(ctx, ctx.pos + 1);
-
-    // Empty underline (____) is discarded entirely in Wikidot
-    if (children.length === 0) {
-      return {
-        success: true,
-        elements: [],
-        consumed,
-      };
-    }
-
-    return {
-      success: true,
-      elements: [createInlineContainer("underline", children)],
-      consumed,
-    };
+  parse(ctx) {
+    return parseDelimitedContainer(ctx, "UNDERLINE_MARKER", "underline", { discardEmpty: true });
   },
 };

package/src/parser/rules/tokens.ts

@@ -1,5 +1,6 @@
 import type { Token, TokenType } from "../../lexer";
 import type { ParseContext } from "./contracts";
+import { getParagraphNewlineBoundary } from "./inline/parsing/paragraph-boundary";
 
 /**
  * Helper to get current token
@@ -51,7 +52,7 @@ export function hasClosingMarkerBeforeNewline(
   markerValue?: string,
 ): boolean {
   let pos = ctx.pos;
-  while (pos < ctx.tokens.length) {
+  while (pos < (ctx.scope.inlineEnd ?? ctx.tokens.length)) {
     const token = ctx.tokens[pos];
     if (!token || token.type === "NEWLINE" || token.type === "EOF") {
       return false;
@@ -76,24 +77,14 @@ export function hasClosingMarkerBeforeParagraphBreak(
   markerValue?: string,
 ): boolean {
   let pos = ctx.pos;
-  while (pos < ctx.tokens.length) {
+  while (pos < (ctx.scope.inlineEnd ?? ctx.tokens.length)) {
     const token = ctx.tokens[pos];
     if (!token || token.type === "EOF") {
       return false;
     }
-    // Check for paragraph break (NEWLINE followed by NEWLINE after optional whitespace)
-    if (token.type === "NEWLINE") {
-      let lookAhead = 1;
-      while (ctx.tokens[pos + lookAhead]?.type === "WHITESPACE") {
-        lookAhead++;
-      }
-      if (
-        ctx.tokens[pos + lookAhead]?.type === "NEWLINE" ||
-        ctx.tokens[pos + lookAhead]?.type === "EOF" ||
-        !ctx.tokens[pos + lookAhead]
-      ) {
-        return false;
-      }
+    if (ctx.scope.blockCloseCondition?.({ ...ctx, pos })) return false;
+    if (token.type === "NEWLINE" && getParagraphNewlineBoundary(ctx, pos, true).shouldBreak) {
+      return false;
     }
     if (token.type === markerType) {
       if (markerValue === undefined || token.value === markerValue) {

package/src/parser/rules/inline/underline/child.ts

@@ -1,26 +0,0 @@
-import type { Element } from "@wdprlib/ast";
-import type { ParseContext } from "../../types";
-import { parseInlineUntil } from "../utils";
-
-export interface UnderlineChildResult {
-  elements: Element[];
-  consumed: number;
-}
-
-export function parseUnderlineChild(ctx: ParseContext, pos: number): UnderlineChildResult {
-  const token = ctx.tokens[pos];
-  if (!token) {
-    return { elements: [], consumed: 0 };
-  }
-
-  if (token.type === "NEWLINE") {
-    return { elements: [{ element: "line-break" }], consumed: 1 };
-  }
-
-  const result = parseInlineUntil({ ...ctx, pos }, "UNDERLINE_MARKER");
-  if (result.elements.length > 0) {
-    return { elements: result.elements, consumed: result.consumed };
-  }
-
-  return { elements: [{ element: "text", data: token.value }], consumed: 1 };
-}

package/src/parser/rules/inline/underline/content.ts

@@ -1,29 +0,0 @@
-import type { Element } from "@wdprlib/ast";
-import type { ParseContext } from "../../types";
-import { parseUnderlineChild } from "./child";
-
-export function parseUnderlineContent(
-  ctx: ParseContext,
-  startPos: number,
-): { children: Element[]; consumed: number } {
-  const children: Element[] = [];
-  let pos = startPos;
-  let consumed = 1;
-
-  while (pos < ctx.tokens.length) {
-    const token = ctx.tokens[pos];
-    if (!token || token.type === "EOF") break;
-
-    if (token.type === "UNDERLINE_MARKER") {
-      consumed++;
-      break;
-    }
-
-    const child = parseUnderlineChild(ctx, pos);
-    children.push(...child.elements);
-    pos += child.consumed;
-    consumed += child.consumed;
-  }
-
-  return { children, consumed };
-}