@wdprlib/parser 3.2.0 → 4.0.0

package/src/parser/rules/inline/line-break.ts

@@ -1,326 +0,0 @@
-/**
- *
- * Parses the various Wikidot line-break syntaxes.
- *
- * Wikidot supports three distinct mechanisms for producing `<br />` elements:
- *
- * 1. Implicit newline: a single `NEWLINE` token within a paragraph
- *    becomes a `<br />`, unless it precedes a block-level element
- *    (heading, list, blockquote, etc.) or another newline (paragraph break).
- *
- * 2. Backslash at end of line: `\` followed by newline. The preprocessor
- *    converts `\\\n` to a `BACKSLASH_BREAK` token (U+E000), which this
- *    rule then handles. Wikidot preserves a space after the line break
- *    in this case.
- *
- * 3. Underscore at end of line: ` _` followed by newline, or `_` at the
- *    start of a line followed by newline. This is a more explicit
- *    line-break syntax.
- *
- * All three rules mark their line-break elements with `_preservedTrailingBreak`
- * when the break was explicitly requested (backslash or underscore syntax),
- * so the paragraph postprocessor knows not to strip trailing breaks.
- *
- * The newline rule suppresses line-breaks in several situations to avoid
- * spurious `<br />` elements before block-level constructs.
- *
- * @module
- */
-import type { Element } from "@wdprlib/ast";
-import type { InlineRule, ParseContext, RuleResult } from "../types";
-import type { TokenType } from "../../../lexer";
-
-/**
- * Token types that indicate the start of a block-level element.
- *
- * When a NEWLINE is followed (after optional whitespace) by one of
- * these token types, the newline line-break rule suppresses the
- * `<br />` to prevent extra whitespace before block elements.
- */
-const BLOCK_START_TOKENS: TokenType[] = [
-  "BLOCKQUOTE_MARKER", // >
-  "LIST_BULLET", // *
-  "LIST_NUMBER", // #
-  "HEADING_MARKER", // + ++ +++
-  "HR_MARKER", // ----
-  "TABLE_MARKER", // ||
-];
-
-/**
- * Checks whether a token type represents the start of a block-level element.
- *
- * @param type - The token type to check
- * @returns `true` if the token type is in the {@link BLOCK_START_TOKENS} list
- */
-function isBlockStartToken(type: TokenType): boolean {
-  return BLOCK_START_TOKENS.includes(type);
-}
-
-/**
- * Inline rule for implicit newline-to-line-break conversion.
- *
- * A single `NEWLINE` token within inline content typically becomes a
- * `<br />` element. However, the line break is suppressed in several
- * situations to match Wikidot's behavior:
- *
- * - End of input (no meaningful token follows)
- * - Another NEWLINE follows (this is a paragraph break, not a line break)
- * - A valid block-start token follows at line start (heading, list, etc.)
- * - A `BACKSLASH_BREAK` token follows (the backslash rule handles the break)
- *
- * Additional validation is performed for heading and list markers to ensure
- * they actually form valid block structures (e.g. a heading marker of 7+
- * characters is not a valid heading).
- */
-export const newlineLineBreakRule: InlineRule = {
-  name: "newlineLineBreak",
-  startTokens: ["NEWLINE"],
-
-  /**
-   * Attempts to convert a NEWLINE token into a line-break element.
-   *
-   * @param ctx - Parse context with token stream and current position
-   * @returns A successful result with either a `"line-break"` element or
-   *          an empty array (when the break is suppressed)
-   */
-  parse(ctx: ParseContext): RuleResult<Element> {
-    const currentTok = ctx.tokens[ctx.pos];
-    if (!currentTok || currentTok.type !== "NEWLINE") {
-      return { success: false };
-    }
-
-    // Check what comes after the newline
-    let lookAhead = 1;
-
-    // Skip optional whitespace
-    while (ctx.tokens[ctx.pos + lookAhead]?.type === "WHITESPACE") {
-      lookAhead++;
-    }
-
-    const nextMeaningfulToken = ctx.tokens[ctx.pos + lookAhead];
-
-    // Check if HEADING_MARKER would actually form a valid heading
-    // Block-start tokens are only valid when at actual line start
-    let isValidBlock = isBlockStartToken(nextMeaningfulToken?.type as TokenType);
-    if (isValidBlock && !nextMeaningfulToken?.lineStart) {
-      isValidBlock = false;
-    }
-    if (isValidBlock && nextMeaningfulToken?.type === "HEADING_MARKER") {
-      const markerLen = nextMeaningfulToken.value.length;
-      const afterPos = ctx.pos + lookAhead + 1;
-      const afterMarker = ctx.tokens[afterPos];
-      if (markerLen > 6) {
-        isValidBlock = false;
-      } else if (afterMarker?.type === "STAR") {
-        if (ctx.tokens[afterPos + 1]?.type !== "WHITESPACE") isValidBlock = false;
-      } else if (afterMarker?.type !== "WHITESPACE") {
-        isValidBlock = false;
-      }
-    }
-
-    // Check if there's a BACKSLASH_BREAK ahead (skip whitespace)
-    // Pattern: NEWLINE + WHITESPACE? + BACKSLASH_BREAK
-    // In this case, the BACKSLASH_BREAK rule will handle the line-break
-    let hasBackslashBreak = false;
-    {
-      let ahead = 1;
-      while (ctx.tokens[ctx.pos + ahead]?.type === "WHITESPACE") {
-        ahead++;
-      }
-      if (ctx.tokens[ctx.pos + ahead]?.type === "BACKSLASH_BREAK") {
-        hasBackslashBreak = true;
-      }
-    }
-
-    // Skip line-break if:
-    // - End of input
-    // - Another NEWLINE (paragraph break will handle this)
-    // - Valid block start token
-    // - BACKSLASH_BREAK ahead (that rule will create the line-break)
-    if (
-      !nextMeaningfulToken ||
-      nextMeaningfulToken.type === "EOF" ||
-      nextMeaningfulToken.type === "NEWLINE" ||
-      isValidBlock ||
-      hasBackslashBreak
-    ) {
-      // Don't generate line-break, return empty array
-      return {
-        success: true,
-        elements: [],
-        consumed: 1,
-      };
-    }
-
-    return {
-      success: true,
-      elements: [{ element: "line-break" }],
-      consumed: 1,
-    };
-  },
-};
-
-/**
- * Inline rule for backslash-at-end-of-line line breaks.
- *
- * In Wikidot, a backslash at the end of a line (`\` followed by newline)
- * creates a line break. The preprocessor converts this `\\\n` sequence
- * into a special `BACKSLASH_BREAK` token (U+E000).
- *
- * This rule handles two token patterns:
- * - `WHITESPACE + BACKSLASH_BREAK`: produces a line-break followed by a
- *   space text element (Wikidot preserves the space after the break)
- * - Standalone `BACKSLASH_BREAK`: produces only a line-break
- *
- * A special case exists when the backslash break is followed by an
- * underscore line-break pattern (` _\n`): in that case, the trailing
- * space is omitted to avoid doubled spacing.
- *
- * All line-break elements produced by this rule are marked with
- * `_preservedTrailingBreak = true` so the paragraph postprocessor
- * does not strip them.
- */
-export const backslashLineBreakRule: InlineRule = {
-  name: "backslashLineBreak",
-  startTokens: ["WHITESPACE", "BACKSLASH_BREAK"],
-
-  /**
-   * Attempts to parse a backslash line break at the current position.
-   *
-   * @param ctx - Parse context with token stream and current position
-   * @returns A successful result with line-break elements (and possibly a
-   *          trailing space), or `{ success: false }` if the pattern does not match
-   */
-  parse(ctx: ParseContext): RuleResult<Element> {
-    const currentTok = ctx.tokens[ctx.pos];
-    if (!currentTok) {
-      return { success: false };
-    }
-
-    // Pattern: WHITESPACE + BACKSLASH_BREAK → line-break + text(" ")
-    // But if followed by underscore line-break pattern, don't include the space
-    if (currentTok.type === "WHITESPACE") {
-      const nextTok = ctx.tokens[ctx.pos + 1];
-      if (nextTok?.type === "BACKSLASH_BREAK") {
-        // Check if followed by " _\n" pattern (underscore line-break)
-        const afterBreak = ctx.tokens[ctx.pos + 2];
-        const afterAfter = ctx.tokens[ctx.pos + 3];
-        const afterAfterAfter = ctx.tokens[ctx.pos + 4];
-
-        const isFollowedByUnderscoreBreak =
-          afterBreak?.type === "WHITESPACE" &&
-          afterAfter?.type === "UNDERSCORE" &&
-          (afterAfterAfter?.type === "NEWLINE" || afterAfterAfter?.type === "EOF");
-
-        if (isFollowedByUnderscoreBreak) {
-          // Don't include the space, let underscore rule handle the rest
-          // Mark as explicit line-break to preserve at paragraph end
-          const lb: any = { element: "line-break" };
-          lb._preservedTrailingBreak = true;
-          return {
-            success: true,
-            elements: [lb],
-            consumed: 2,
-          };
-        }
-
-        // Mark as explicit line-break to preserve at paragraph end
-        const lb: any = { element: "line-break" };
-        lb._preservedTrailingBreak = true;
-        return {
-          success: true,
-          elements: [lb, { element: "text", data: " " }],
-          consumed: 2,
-        };
-      }
-      return { success: false };
-    }
-
-    // Standalone BACKSLASH_BREAK
-    // Mark as explicit line-break to preserve at paragraph end
-    if (currentTok.type === "BACKSLASH_BREAK") {
-      const lb: any = { element: "line-break" };
-      lb._preservedTrailingBreak = true;
-      return {
-        success: true,
-        elements: [lb],
-        consumed: 1,
-      };
-    }
-
-    return { success: false };
-  },
-};
-
-/**
- * Inline rule for underscore-at-end-of-line line breaks.
- *
- * Wikidot syntax: ` _` followed by newline (space + underscore + newline),
- * or `_` at the start of a line followed by newline.
- *
- * This rule handles two token patterns:
- * - Pattern 1: `WHITESPACE + UNDERSCORE + NEWLINE/EOF`
- * - Pattern 2: `UNDERSCORE (at lineStart) + NEWLINE/EOF`
- *
- * Both patterns consume the newline as part of the line-break to prevent
- * the newline rule from producing a duplicate break.
- *
- * All line-break elements are marked with `_preservedTrailingBreak = true`
- * so the paragraph postprocessor does not strip them.
- */
-export const underscoreLineBreakRule: InlineRule = {
-  name: "underscoreLineBreak",
-  startTokens: ["WHITESPACE", "UNDERSCORE"],
-
-  /**
-   * Attempts to parse an underscore line break at the current position.
-   *
-   * @param ctx - Parse context with token stream and current position
-   * @returns A successful result with a `"line-break"` element,
-   *          or `{ success: false }` if the pattern does not match
-   */
-  parse(ctx: ParseContext): RuleResult<Element> {
-    const currentTok = ctx.tokens[ctx.pos];
-    if (!currentTok) {
-      return { success: false };
-    }
-
-    // Pattern 1: WHITESPACE followed by UNDERSCORE, then NEWLINE
-    // Mark as explicit line-break to preserve at paragraph end
-    if (currentTok.type === "WHITESPACE") {
-      const nextTok = ctx.tokens[ctx.pos + 1];
-      const afterTok = ctx.tokens[ctx.pos + 2];
-
-      if (
-        nextTok?.type === "UNDERSCORE" &&
-        afterTok &&
-        (afterTok.type === "NEWLINE" || afterTok.type === "EOF")
-      ) {
-        const lb: any = { element: "line-break" };
-        lb._preservedTrailingBreak = true;
-        return {
-          success: true,
-          elements: [lb],
-          consumed: 3, // WHITESPACE + UNDERSCORE + NEWLINE
-        };
-      }
-    }
-
-    // Pattern 2: UNDERSCORE at start of line, then NEWLINE
-    // Mark as explicit line-break to preserve at paragraph end
-    if (currentTok.type === "UNDERSCORE" && currentTok.lineStart) {
-      const nextTok = ctx.tokens[ctx.pos + 1];
-      if (nextTok && (nextTok.type === "NEWLINE" || nextTok.type === "EOF")) {
-        const lb: any = { element: "line-break" };
-        lb._preservedTrailingBreak = true;
-        return {
-          success: true,
-          elements: [lb],
-          consumed: 2, // UNDERSCORE + NEWLINE
-        };
-      }
-    }
-
-    return { success: false };
-  },
-};

package/src/parser/rules/inline/link-triple.ts

@@ -1,267 +0,0 @@
-/**
- *
- * Parses the Wikidot triple-bracket link syntax: `[[[target | label]]]`.
- *
- * Triple-bracket links are Wikidot's primary page-linking mechanism.
- * They support several target formats:
- *
- * - Page links: `[[[page-name]]]` or `[[[page-name | Label]]]`
- * - Category pages: `[[[category:page-name]]]` (display shows text after colon)
- * - Anchor links: `[[[#anchor-name]]]`
- * - External URLs: `[[[https://example.com | Label]]]`
- * - Interwiki links: `[[[wikipedia:Article]]]` (for known prefixes)
- *
- * Special syntax:
- * - `[[[*target]]]` -- `*` prefix is stripped from target; for external URLs,
- *   adds `target="_blank"` (new tab)
- * - `[[[*|label]]]` -- links to root `/` with the given label
- * - `[[[page|]]]` -- empty label after pipe defaults to the page name
- *
- * Multi-line support: a single newline within the link is converted to
- * a space (in both target and label portions), but a double newline
- * (paragraph break) or a newline directly before `]]]` invalidates the link.
- *
- * When the opening `[[[` has no valid closing `]]]`, it falls through
- * as literal text rather than failing.
- *
- * Produces a `"link"` AST element with an appropriate `type` field
- * (`"page"`, `"anchor"`, `"direct"`, or `"interwiki"`).
- *
- * @module
- */
-import type { Element, LinkType, LinkLocation, LinkLabel } from "@wdprlib/ast";
-import type { InlineRule, ParseContext, RuleResult } from "../types";
-import { currentToken } from "../types";
-
-/**
- * Scans ahead to check whether a valid `LINK_CLOSE` (`]]]`) token
- * exists, respecting Wikidot's multiline link rules.
- *
- * Allows at most one newline within the link content (typically after
- * the pipe separator). Rejects the link if:
- * - A double newline (paragraph break) is found
- * - A newline appears directly before the closing `]]]`
- * - EOF is reached without finding `]]]`
- *
- * @param ctx - The current parse context
- * @param startPos - Token index at which to begin scanning (after `[[[`)
- * @returns `true` if a valid closing `]]]` is found
- */
-function hasClosingLinkMarker(ctx: ParseContext, startPos: number): boolean {
-  let pos = startPos;
-  while (pos < ctx.tokens.length) {
-    const token = ctx.tokens[pos];
-    if (!token || token.type === "EOF") {
-      return false;
-    }
-    if (token.type === "LINK_CLOSE") {
-      return true;
-    }
-    // Allow at most one newline, but not directly before LINK_CLOSE
-    if (token.type === "NEWLINE") {
-      const next = ctx.tokens[pos + 1];
-      if (next?.type === "NEWLINE") {
-        return false; // Double newline = paragraph break
-      }
-      // Newline directly before close = invalid
-      if (next?.type === "LINK_CLOSE") {
-        return false;
-      }
-    }
-    pos++;
-  }
-  return false;
-}
-
-/**
- * Inline rule for parsing `[[[target | label]]]` triple-bracket links.
- *
- * Triggered by a `LINK_OPEN` (`[[[`) token. Collects the target string
- * and optional pipe-separated label, then determines the link type
- * (page, anchor, direct URL, or interwiki) based on the target format.
- *
- * When no valid closing `]]]` is found, the opening `[[[` is emitted
- * as literal text.
- *
- * Edge cases handled:
- * - Empty target with pipe (`[[[|text]]]`) is invalid
- * - Multiple consecutive `#` in the target (`[[[page##anchor]]]`) is invalid
- * - `[[[*|label]]]` links to root `/`
- * - `[[[*target]]]` strips `*`; adds `target="_blank"` for external URLs
- * - Category pages show only the text after the colon when no label is given
- */
-export const linkTripleRule: InlineRule = {
-  name: "linkTriple",
-  startTokens: ["LINK_OPEN"],
-
-  /**
-   * Attempts to parse a triple-bracket link at the current position.
-   *
-   * @param ctx - Parse context with token stream and current position
-   * @returns A successful result with a `"link"` element, or a text
-   *          fallback when the syntax is invalid
-   */
-  parse(ctx: ParseContext): RuleResult<Element> {
-    const startToken = currentToken(ctx);
-
-    if (!hasClosingLinkMarker(ctx, ctx.pos + 1)) {
-      return {
-        success: true,
-        elements: [{ element: "text", data: startToken.value }],
-        consumed: 1,
-      };
-    }
-
-    // Collect tokens until LINK_CLOSE (newlines converted to spaces)
-    let target = "";
-    let labelText = "";
-    let foundPipe = false;
-    let consumed = 1; // opening [[[
-    let pos = ctx.pos + 1;
-
-    while (pos < ctx.tokens.length) {
-      const token = ctx.tokens[pos];
-      if (!token || token.type === "LINK_CLOSE" || token.type === "EOF") {
-        break;
-      }
-
-      // Convert newlines to spaces in link content (Wikidot allows single newlines)
-      if (token.type === "NEWLINE") {
-        if (foundPipe) {
-          labelText += " ";
-        } else {
-          target += " ";
-        }
-        consumed++;
-        pos++;
-        continue;
-      }
-
-      if (token.type === "PIPE" && !foundPipe) {
-        foundPipe = true;
-      } else if (foundPipe) {
-        labelText += token.value;
-      } else {
-        target += token.value;
-      }
-
-      consumed++;
-      pos++;
-    }
-
-    // Consume closing ]]]
-    if (ctx.tokens[pos]?.type === "LINK_CLOSE") {
-      consumed++;
-    }
-
-    const trimmedTarget = target.trim();
-
-    // Invalid: empty target with pipe (e.g., [[[|some-page]]])
-    if (trimmedTarget === "" && foundPipe) {
-      return {
-        success: true,
-        elements: [{ element: "text", data: startToken.value }],
-        consumed: 1,
-      };
-    }
-
-    // Invalid: multiple consecutive # in target (e.g., [[[home###|Home]]], [[[page##anchor]]])
-    // Wikidot rejects these as invalid link syntax
-    if (/#{2,}/.test(trimmedTarget)) {
-      return {
-        success: true,
-        elements: [{ element: "text", data: startToken.value }],
-        consumed: 1,
-      };
-    }
-
-    // `*` prefix: stripped from target; sets target="_blank" for external URLs
-    let finalTarget = trimmedTarget;
-    let hasStar = false;
-    if (trimmedTarget.startsWith("*")) {
-      hasStar = true;
-      finalTarget = trimmedTarget.slice(1);
-    }
-
-    const { linkType, link } = determineLinkTypeAndLocation(finalTarget);
-    const trimmedLabel = labelText.trim();
-
-    // Determine display text
-    let displayText: string;
-    if (foundPipe) {
-      // If label is empty (e.g., [[[page|]]]), use page name
-      displayText = trimmedLabel || finalTarget;
-    } else {
-      // For category pages (system:Recent Changes), use only the part after colon
-      // Use trimmedTarget (preserves * prefix) for display when no pipe
-      const colonIdx = trimmedTarget.indexOf(":");
-      if (colonIdx !== -1 && !trimmedTarget.startsWith("http") && !trimmedTarget.startsWith("*")) {
-        displayText = trimmedTarget.slice(colonIdx + 1).trim();
-      } else {
-        displayText = trimmedTarget;
-      }
-    }
-
-    const label: LinkLabel = { text: displayText };
-
-    return {
-      success: true,
-      elements: [
-        {
-          element: "link",
-          data: {
-            type: linkType,
-            link,
-            extra: null,
-            label,
-            target: hasStar && linkType === "direct" ? "new-tab" : null,
-          },
-        },
-      ],
-      consumed,
-    };
-  },
-};
-
-/**
- * Known interwiki prefixes recognized by Wikidot.
- *
- * Links whose target starts with one of these prefixes followed by a colon
- * (e.g. `wikipedia:Article`) are classified as interwiki links rather than
- * category page links.
- */
-const INTERWIKI_PREFIXES = new Set(["wikipedia", "google", "dictionary", "wikidot"]);
-
-/**
- * Determines the link type and structured location data from a raw
- * triple-bracket link target string.
- *
- * Classification order:
- * 1. Targets starting with `#` are anchor links
- * 2. Targets starting with `http://` or `https://` are direct (external) links
- * 3. Targets with a colon and a known interwiki prefix (without slashes)
- *    are interwiki links
- * 4. Everything else is a page link (including category pages like
- *    `system:Recent Changes`)
- *
- * @param target - The trimmed, processed link target string
- * @returns An object with `linkType` and `link` (the structured location data)
- */
-function determineLinkTypeAndLocation(target: string): { linkType: LinkType; link: LinkLocation } {
-  if (target.startsWith("#")) {
-    return { linkType: "anchor", link: target };
-  }
-  if (target.startsWith("http://") || target.startsWith("https://")) {
-    return { linkType: "direct", link: target };
-  }
-  // Check for interwiki links (only known prefixes)
-  const colonIdx = target.indexOf(":");
-  if (colonIdx > 0 && !target.includes("/")) {
-    const prefix = target.slice(0, colonIdx).toLowerCase();
-    if (INTERWIKI_PREFIXES.has(prefix)) {
-      return { linkType: "interwiki", link: target };
-    }
-  }
-  // Page link (includes category pages like "system:Recent Changes")
-  return { linkType: "page", link: { site: null, page: target } };
-}