@wdprlib/parser 3.2.0 → 4.0.0

package/src/parser/rules/inline/parsing/paragraph-boundary.ts

@@ -0,0 +1,47 @@
+import type { ParseContext } from "../../types";
+import { isParagraphBreakingBlockStart } from "./block-start-predicates";
+
+export interface ParagraphNewlineBoundary {
+  shouldBreak: boolean;
+  consumed: number;
+  preservePrecedingLineBreak: boolean;
+}
+
+export function getParagraphNewlineBoundary(
+  ctx: ParseContext,
+  newlinePos: number,
+  hasContent: boolean,
+): ParagraphNewlineBoundary {
+  const lookAhead = skipWhitespaceAfterNewline(ctx, newlinePos);
+  const nextMeaningfulToken = ctx.tokens[newlinePos + lookAhead];
+  const nextPos = newlinePos + lookAhead;
+  const isBlockStart = isParagraphBreakingBlockStart(ctx, newlinePos, lookAhead);
+
+  if (
+    nextMeaningfulToken &&
+    nextMeaningfulToken.type !== "NEWLINE" &&
+    nextMeaningfulToken.type !== "EOF" &&
+    !isBlockStart
+  ) {
+    return { shouldBreak: false, consumed: 0, preservePrecedingLineBreak: false };
+  }
+
+  return {
+    shouldBreak: true,
+    consumed: nextMeaningfulToken?.type === "NEWLINE" ? 2 : 1,
+    preservePrecedingLineBreak:
+      hasContent &&
+      isBlockStart &&
+      ctx.blockRules.some(
+        (rule) => rule.preservesPrecedingLineBreak && rule.isStartPattern?.(ctx, nextPos),
+      ),
+  };
+}
+
+function skipWhitespaceAfterNewline(ctx: ParseContext, newlinePos: number): number {
+  let lookAhead = 1;
+  while (ctx.tokens[newlinePos + lookAhead]?.type === "WHITESPACE") {
+    lookAhead++;
+  }
+  return lookAhead;
+}

package/src/parser/rules/inline/parsing/plain-text.ts

@@ -0,0 +1,69 @@
+import type { TokenType } from "../../../../lexer";
+import type { ParseContext } from "../../types";
+
+const MIN_INLINE_TEXT_RUN_LENGTH = 32;
+export const MIN_INLINE_TEXT_RUN_DOCUMENT_TOKENS = 100_000;
+
+export type InlineEndType = TokenType | "PARAGRAPH_BREAK";
+
+export interface PlainTextRun {
+  value: string;
+  consumed: number;
+}
+
+export function collectLongPlainTextRun(
+  ctx: ParseContext,
+  startPos: number,
+  endType: InlineEndType,
+): PlainTextRun | null {
+  const firstToken = ctx.tokens[startPos];
+  if (
+    firstToken?.type === "TEXT" &&
+    firstToken.value.length >= MIN_INLINE_TEXT_RUN_LENGTH &&
+    firstToken.value !== "("
+  ) {
+    return { value: firstToken.value, consumed: 1 };
+  }
+
+  let pos = startPos;
+  let totalLength = 0;
+  const parts: string[] = [];
+
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (!token || token.type === "EOF" || token.type === "NEWLINE" || token.type === endType) {
+      break;
+    }
+    if (!isPlainTextRunToken(ctx, pos)) {
+      break;
+    }
+
+    parts.push(token.value);
+    totalLength += token.value.length;
+    pos++;
+  }
+
+  return totalLength >= MIN_INLINE_TEXT_RUN_LENGTH
+    ? { value: parts.join(""), consumed: pos - startPos }
+    : null;
+}
+
+function isPlainTextRunToken(ctx: ParseContext, pos: number): boolean {
+  const token = ctx.tokens[pos];
+  if (!token) return false;
+
+  if (token.type === "IDENTIFIER") {
+    return true;
+  }
+
+  if (token.type === "WHITESPACE") {
+    const next = ctx.tokens[pos + 1];
+    return next?.type !== "BACKSLASH_BREAK" && next?.type !== "UNDERSCORE";
+  }
+
+  if (token.type !== "TEXT") {
+    return false;
+  }
+
+  return token.value !== "(";
+}

package/src/parser/rules/inline/parsing/preserved-line-break.ts

@@ -0,0 +1,11 @@
+import type { Element } from "@wdprlib/ast";
+
+type PreservedLineBreak = Extract<Element, { element: "line-break" }> & {
+  _preservedTrailingBreak?: boolean;
+};
+
+export function createPreservedTrailingLineBreak(): Element {
+  const lineBreak: PreservedLineBreak = { element: "line-break" };
+  lineBreak._preservedTrailingBreak = true;
+  return lineBreak;
+}

package/src/parser/rules/inline/parsing/rules.ts

@@ -0,0 +1,34 @@
+import type { TokenType } from "../../../../lexer";
+import type { InlineRule } from "../../types";
+
+/**
+ * Check if an inline rule can be applied.
+ */
+export function canApplyInlineRule(rule: InlineRule, token: { type: TokenType }): boolean {
+  if (rule.startTokens.length === 0) {
+    return true;
+  }
+  return rule.startTokens.includes(token.type);
+}
+
+const inlineRuleCandidateCache = new WeakMap<InlineRule[], Map<TokenType, InlineRule[]>>();
+
+export function getCandidateInlineRules(
+  inlineRules: InlineRule[],
+  tokenType: TokenType,
+): InlineRule[] {
+  let byType = inlineRuleCandidateCache.get(inlineRules);
+  if (!byType) {
+    byType = new Map();
+    inlineRuleCandidateCache.set(inlineRules, byType);
+  }
+
+  const cached = byType.get(tokenType);
+  if (cached) {
+    return cached;
+  }
+
+  const candidates = inlineRules.filter((rule) => canApplyInlineRule(rule, { type: tokenType }));
+  byType.set(tokenType, candidates);
+  return candidates;
+}

package/src/parser/rules/inline/parsing/simple-token.ts

@@ -0,0 +1,26 @@
+import type { Element } from "@wdprlib/ast";
+import type { Token } from "../../../../lexer";
+
+export interface SimpleInlineTokenResult {
+  element: Element;
+  consumed: number;
+}
+
+export function parseSimpleInlineToken(
+  token: Token,
+  nextToken: Token | undefined,
+): SimpleInlineTokenResult | null {
+  if (token.type === "TEXT" && token.value !== "(") {
+    return { element: { element: "text", data: token.value }, consumed: 1 };
+  }
+
+  if (
+    token.type === "WHITESPACE" &&
+    nextToken?.type !== "BACKSLASH_BREAK" &&
+    nextToken?.type !== "UNDERSCORE"
+  ) {
+    return { element: { element: "text", data: token.value }, consumed: 1 };
+  }
+
+  return null;
+}

package/src/parser/rules/inline/raw/angle.ts

@@ -0,0 +1,40 @@
+import type { Element } from "@wdprlib/ast";
+import type { ParseContext, RuleResult } from "../../types";
+import { currentToken, hasClosingMarkerBeforeNewline } from "../../types";
+import { rawElement, textElement } from "./result";
+
+export function parseAngleRaw(ctx: ParseContext): RuleResult<Element> {
+  const startToken = currentToken(ctx);
+  let pos = ctx.pos + 1;
+
+  if (!hasClosingMarkerBeforeNewline({ ...ctx, pos }, "RAW_BLOCK_CLOSE")) {
+    if (ctx.tokens[pos]?.type === "NEWLINE" && ctx.tokens[pos + 1]?.type === "RAW_BLOCK_CLOSE") {
+      return textElement(startToken.value, 3);
+    }
+    return textElement(startToken.value, 1);
+  }
+
+  let value = "";
+  let consumed = 1;
+
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (
+      !token ||
+      token.type === "RAW_BLOCK_CLOSE" ||
+      token.type === "NEWLINE" ||
+      token.type === "EOF"
+    ) {
+      break;
+    }
+    value += token.value;
+    consumed++;
+    pos++;
+  }
+
+  if (ctx.tokens[pos]?.type === "RAW_BLOCK_CLOSE") {
+    consumed++;
+  }
+
+  return rawElement(value, consumed);
+}

package/src/parser/rules/inline/raw/double-at.ts

@@ -0,0 +1,78 @@
+import type { Element } from "@wdprlib/ast";
+import type { ParseContext, RuleResult } from "../../types";
+import { currentToken, hasClosingMarkerBeforeNewline } from "../../types";
+import { emptyRaw, rawElement, textElement } from "./result";
+
+export function parseDoubleAtRaw(ctx: ParseContext): RuleResult<Element> {
+  const startToken = currentToken(ctx);
+  let pos = ctx.pos + 1;
+
+  if (ctx.tokens[pos]?.type === "RAW_OPEN") {
+    return emptyRaw(2);
+  }
+
+  if (!hasClosingMarkerBeforeNewline({ ...ctx, pos }, "RAW_OPEN")) {
+    if (ctx.tokens[pos]?.type === "NEWLINE" && ctx.tokens[pos + 1]?.type === "RAW_OPEN") {
+      return emptyRaw(3);
+    }
+    return textElement(startToken.value, 1);
+  }
+
+  let value = "";
+  let consumed = 1;
+  let hasBlockOpen = false;
+  let hasBlockClose = false;
+
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (!token || token.type === "RAW_OPEN" || token.type === "NEWLINE" || token.type === "EOF") {
+      break;
+    }
+
+    if (token.type === "RAW_BLOCK_CLOSE") {
+      const splitResult = tryConsumeSplitBlockClose(ctx, pos, value, consumed);
+      if (splitResult) {
+        return splitResult;
+      }
+      hasBlockClose = true;
+    }
+    if (token.type === "RAW_BLOCK_OPEN") {
+      hasBlockOpen = true;
+    }
+
+    value += token.value;
+    consumed++;
+    pos++;
+  }
+
+  if (ctx.tokens[pos]?.type === "RAW_OPEN") {
+    consumed++;
+  }
+
+  if (hasBlockOpen && hasBlockClose) {
+    return emptyRaw(consumed);
+  }
+
+  return rawElement(value, consumed);
+}
+
+function tryConsumeSplitBlockClose(
+  ctx: ParseContext,
+  pos: number,
+  value: string,
+  consumed: number,
+): RuleResult<Element> | null {
+  const nextToken = ctx.tokens[pos + 1];
+  if (nextToken?.type !== "RAW_OPEN") {
+    return null;
+  }
+
+  return {
+    success: true,
+    elements: [
+      { element: "raw", data: `${value}>` },
+      { element: "text", data: "@" },
+    ],
+    consumed: consumed + 2,
+  };
+}

package/src/parser/rules/inline/raw/index.ts

@@ -0,0 +1,26 @@
+/**
+ *
+ * Parses the Wikidot raw (verbatim) text syntaxes: `@@...@@` and `@<...>@`.
+ *
+ * @module
+ */
+import type { Element } from "@wdprlib/ast";
+import type { InlineRule, ParseContext, RuleResult } from "../../types";
+import { currentToken } from "../../types";
+import { parseAngleRaw } from "./angle";
+import { parseDoubleAtRaw } from "./double-at";
+
+export const rawRule: InlineRule = {
+  name: "raw",
+  startTokens: ["RAW_OPEN", "RAW_BLOCK_OPEN"],
+
+  parse(ctx: ParseContext): RuleResult<Element> {
+    const startToken = currentToken(ctx);
+
+    if (startToken.type === "RAW_BLOCK_OPEN") {
+      return parseAngleRaw(ctx);
+    }
+
+    return parseDoubleAtRaw(ctx);
+  },
+};

package/src/parser/rules/inline/raw/result.ts

@@ -0,0 +1,26 @@
+import type { Element } from "@wdprlib/ast";
+import type { RuleResult } from "../../types";
+
+export function rawElement(value: string, consumed: number): RuleResult<Element> {
+  return {
+    success: true,
+    elements: [{ element: "raw", data: value }],
+    consumed,
+  };
+}
+
+export function textElement(value: string, consumed: number): RuleResult<Element> {
+  return {
+    success: true,
+    elements: [{ element: "text", data: value }],
+    consumed,
+  };
+}
+
+export function emptyRaw(consumed: number): RuleResult<Element> {
+  return {
+    success: true,
+    elements: [],
+    consumed,
+  };
+}

package/src/parser/rules/inline/size/content.ts

@@ -0,0 +1,65 @@
+import type { Element } from "@wdprlib/ast";
+import type { ParseContext } from "../../types";
+import { parseBlockName } from "../../common";
+import { parseInlineUntil } from "../utils";
+
+export interface SizeContentResult {
+  children: Element[];
+  consumed: number;
+  foundClose: boolean;
+}
+
+export function parseSizeContent(ctx: ParseContext, startPos: number): SizeContentResult {
+  const children: Element[] = [];
+  let pos = startPos;
+  let consumed = 0;
+
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (!token || token.type === "EOF") {
+      break;
+    }
+
+    const closeResult = tryConsumeSizeClose(ctx, pos);
+    if (closeResult) {
+      return {
+        children,
+        consumed: consumed + closeResult.consumed,
+        foundClose: true,
+      };
+    }
+
+    const inlineCtx: ParseContext = { ...ctx, pos };
+    const inlineResult = parseInlineUntil(inlineCtx, "BLOCK_END_OPEN");
+    if (inlineResult.elements.length > 0) {
+      children.push(...inlineResult.elements);
+      pos += inlineResult.consumed;
+      consumed += inlineResult.consumed;
+    } else {
+      children.push({ element: "text", data: token.value });
+      pos++;
+      consumed++;
+    }
+  }
+
+  return { children, consumed, foundClose: false };
+}
+
+function tryConsumeSizeClose(ctx: ParseContext, pos: number): { consumed: number } | null {
+  if (ctx.tokens[pos]?.type !== "BLOCK_END_OPEN") {
+    return null;
+  }
+
+  const closeNameResult = parseBlockName(ctx, pos + 1);
+  if (!closeNameResult || closeNameResult.name !== "size") {
+    return null;
+  }
+
+  let closePos = pos + 1 + closeNameResult.consumed;
+  let consumed = 1 + closeNameResult.consumed;
+  if (ctx.tokens[closePos]?.type === "BLOCK_CLOSE") {
+    consumed++;
+  }
+
+  return { consumed };
+}

package/src/parser/rules/inline/size/index.ts

@@ -0,0 +1,55 @@
+/**
+ *
+ * Parses the Wikidot font size syntax: `[[size value]]text[[/size]]`.
+ *
+ * @module
+ */
+import type { Element } from "@wdprlib/ast";
+import type { InlineRule, ParseContext, RuleResult } from "../../types";
+import { currentToken } from "../../types";
+import { parseSizeContent } from "./content";
+import { parseSizeOpen } from "./open";
+
+export const sizeRule: InlineRule = {
+  name: "size",
+  startTokens: ["BLOCK_OPEN"],
+
+  parse(ctx: ParseContext): RuleResult<Element> {
+    const openToken = currentToken(ctx);
+    if (openToken.type !== "BLOCK_OPEN") {
+      return { success: false };
+    }
+
+    const openResult = parseSizeOpen(ctx);
+    if (!openResult) {
+      return { success: false };
+    }
+
+    const contentResult = parseSizeContent(ctx, openResult.bodyStart);
+    const consumed = openResult.consumed + contentResult.consumed;
+
+    if (!contentResult.foundClose) {
+      ctx.diagnostics.push({
+        severity: "warning",
+        code: "unclosed-block",
+        message: "Missing closing tag [[/size]] for [[size]]",
+        position: openToken.position,
+      });
+    }
+
+    return {
+      success: true,
+      elements: [
+        {
+          element: "container",
+          data: {
+            type: "size",
+            attributes: { style: `font-size:${openResult.size};` },
+            elements: contentResult.children,
+          },
+        },
+      ],
+      consumed,
+    };
+  },
+};

package/src/parser/rules/inline/size/open.ts

@@ -0,0 +1,43 @@
+import type { ParseContext } from "../../types";
+import { parseBlockName } from "../../common";
+import { parseSizeValue } from "./value";
+
+export interface SizeOpenResult {
+  size: string;
+  bodyStart: number;
+  consumed: number;
+}
+
+export function parseSizeOpen(ctx: ParseContext): SizeOpenResult | null {
+  if (ctx.tokens[ctx.pos]?.type !== "BLOCK_OPEN") {
+    return null;
+  }
+
+  let pos = ctx.pos + 1;
+  let consumed = 1;
+
+  const nameResult = parseBlockName(ctx, pos);
+  if (!nameResult || nameResult.name !== "size") {
+    return null;
+  }
+
+  pos += nameResult.consumed;
+  consumed += nameResult.consumed;
+
+  const sizeResult = parseSizeValue(ctx, pos);
+  if (!sizeResult) {
+    return null;
+  }
+
+  pos += sizeResult.consumed;
+  consumed += sizeResult.consumed;
+
+  if (ctx.tokens[pos]?.type !== "BLOCK_CLOSE") {
+    return null;
+  }
+
+  pos++;
+  consumed++;
+
+  return { size: sizeResult.size, bodyStart: pos, consumed };
+}

package/src/parser/rules/inline/size/value.ts

@@ -0,0 +1,45 @@
+import type { ParseContext } from "../../types";
+
+const VALID_SIZE_UNITS = ["px", "em", "rem", "ex", "%", "cm", "mm", "in", "pc"];
+
+export function parseSizeValue(
+  ctx: ParseContext,
+  startPos: number,
+): { size: string; consumed: number } | null {
+  let pos = startPos;
+  let consumed = 0;
+
+  while (ctx.tokens[pos]?.type === "WHITESPACE") {
+    pos++;
+    consumed++;
+  }
+
+  const parts: string[] = [];
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (
+      !token ||
+      token.type === "BLOCK_CLOSE" ||
+      token.type === "NEWLINE" ||
+      token.type === "EOF" ||
+      token.type === "WHITESPACE"
+    ) {
+      break;
+    }
+    parts.push(token.value);
+    pos++;
+    consumed++;
+  }
+
+  if (parts.length === 0) {
+    return null;
+  }
+
+  const size = parts.join("");
+  return isValidSizeValue(size) ? { size, consumed } : null;
+}
+
+function isValidSizeValue(size: string): boolean {
+  const unitPattern = VALID_SIZE_UNITS.join("|");
+  return new RegExp(`^(\\d+(?:\\.\\d+)?)(${unitPattern})$`, "i").test(size);
+}

package/src/parser/rules/inline/span/content.ts

@@ -0,0 +1,97 @@
+import type { Element } from "@wdprlib/ast";
+import type { ParseContext } from "../../types";
+import { inlineRules } from "../../index";
+import { getCandidateInlineRules } from "../utils";
+import { consumeSpanNewline } from "./newline";
+import { parseCloseSpan } from "./syntax";
+import type { SpanBlockName } from "./syntax";
+
+export interface SpanContent {
+  children: Element[];
+  escapedChildren: Element[];
+  splitSpans: Element[][];
+  consumed: number;
+  foundClose: boolean;
+}
+
+export function parseSpanContent(
+  ctx: ParseContext,
+  startPos: number,
+  blockName: SpanBlockName,
+): SpanContent {
+  const paragraphStrip = blockName === "span_";
+  const children: Element[] = [];
+  const escapedChildren: Element[] = [];
+  const splitSpans: Element[][] = [];
+  let foundClose = false;
+  let afterBlankLine = false;
+  let consumed = 0;
+  let pos = startPos;
+
+  while (pos < ctx.tokens.length) {
+    const token = ctx.tokens[pos];
+    if (!token || token.type === "EOF") {
+      break;
+    }
+
+    const close = parseCloseSpan(ctx, pos);
+    if (close.success) {
+      pos += close.consumed;
+      consumed += close.consumed;
+      foundClose = true;
+      break;
+    }
+
+    if (token.type === "NEWLINE") {
+      const newline = consumeSpanNewline(
+        ctx,
+        pos,
+        paragraphStrip,
+        afterBlankLine,
+        children,
+        escapedChildren,
+        splitSpans,
+      );
+      pos += newline.consumed;
+      consumed += newline.consumed;
+      afterBlankLine = afterBlankLine || newline.afterBlankLine;
+      continue;
+    }
+
+    if (token.type === "WHITESPACE" && token.lineStart) {
+      pos++;
+      consumed++;
+      continue;
+    }
+
+    const targetChildren = afterBlankLine ? escapedChildren : children;
+    const parsed = parseOneSpanChild(ctx, pos, targetChildren);
+    pos += parsed.consumed;
+    consumed += parsed.consumed;
+  }
+
+  return { children, escapedChildren, splitSpans, consumed, foundClose };
+}
+
+function parseOneSpanChild(
+  ctx: ParseContext,
+  pos: number,
+  targetChildren: Element[],
+): { consumed: number } {
+  const token = ctx.tokens[pos];
+  if (!token) {
+    return { consumed: 0 };
+  }
+
+  const inlineCtx: ParseContext = { ...ctx, pos };
+  for (const rule of getCandidateInlineRules(inlineRules, token.type)) {
+    const result = rule.parse(inlineCtx);
+    if (result.success) {
+      targetChildren.push(...result.elements);
+      return { consumed: result.consumed };
+    }
+  }
+
+  targetChildren.push({ element: "text", data: token.value });
+  return { consumed: 1 };
+}