@wdprlib/parser 3.1.2 → 4.0.0

package/src/parser/postprocess/spanStrip/merge.ts

@@ -0,0 +1,117 @@
+import type { Element } from "@wdprlib/ast";
+import { extractEscapedSpans, removeLineBreaksAroundSpanStrip } from "./escaped";
+import { paragraphElement } from "./factory";
+import {
+  getContainerData,
+  hasParagraphStripSpan,
+  isContainer,
+  isEmptyExpr,
+  isSplitSpan,
+} from "./predicates";
+import { splitParagraphAtBlankLineSpans, splitParagraphAtEmptyExpr } from "./split";
+
+/**
+ * Merge and split paragraphs according to Wikidot's `span_` and expr behaviors.
+ */
+export function mergeSpanStripParagraphs(children: Element[]): Element[] {
+  const expandedChildren = expandSplitParagraphs(children);
+  const result: Element[] = [];
+  let i = 0;
+
+  while (i < expandedChildren.length) {
+    const node = expandedChildren[i];
+
+    if (!node || !isContainer(node, "paragraph")) {
+      if (node) result.push(node);
+      i++;
+      continue;
+    }
+
+    if (!hasParagraphStripSpan(node)) {
+      result.push(node);
+      i++;
+      continue;
+    }
+
+    const paraData = getContainerData(node);
+    if (!paraData) {
+      result.push(node);
+      i++;
+      continue;
+    }
+
+    const mergedChildren: Element[] = [...paraData.elements];
+    i++;
+
+    while (i < expandedChildren.length) {
+      const nextPara = expandedChildren[i];
+      if (!nextPara || !isContainer(nextPara, "paragraph")) {
+        break;
+      }
+
+      const nextParaData = getContainerData(nextPara);
+      if (!nextParaData) {
+        break;
+      }
+
+      const hasSpanStrip = hasParagraphStripSpan(nextPara);
+      mergedChildren.push(...nextParaData.elements);
+      i++;
+
+      if (!hasSpanStrip) {
+        const peekNext = expandedChildren[i];
+        if (!peekNext || !isContainer(peekNext, "paragraph") || !hasParagraphStripSpan(peekNext)) {
+          break;
+        }
+      }
+    }
+
+    const escapedSpans = extractEscapedSpans(mergedChildren);
+    removeLineBreaksAroundSpanStrip(mergedChildren);
+
+    if (escapedSpans.length > 0) {
+      if (mergedChildren.length > 0) {
+        result.push(paragraphElement(mergedChildren));
+      }
+    } else {
+      result.push(...mergedChildren);
+    }
+
+    result.push(...escapedSpans);
+  }
+
+  return result;
+}
+
+function expandSplitParagraphs(children: Element[]): Element[] {
+  const expandedChildren: Element[] = [];
+  for (const child of children) {
+    if (isContainer(child, "paragraph")) {
+      const data = getContainerData(child);
+      if (data) {
+        const split = getParagraphSplitKind(data.elements);
+        if (split === "blank-line-span") {
+          expandedChildren.push(...splitParagraphAtBlankLineSpans(child));
+          continue;
+        }
+        if (split === "empty-expr") {
+          expandedChildren.push(...splitParagraphAtEmptyExpr(child));
+          continue;
+        }
+      }
+    }
+    expandedChildren.push(child);
+  }
+  return expandedChildren;
+}
+
+function getParagraphSplitKind(elements: Element[]): "blank-line-span" | "empty-expr" | null {
+  let hasEmptyExpr = false;
+  for (const element of elements) {
+    if (isSplitSpan(element)) {
+      return "blank-line-span";
+    }
+    hasEmptyExpr ||= isEmptyExpr(element);
+  }
+  return hasEmptyExpr ? "empty-expr" : null;
+}

package/src/parser/postprocess/spanStrip/predicates.ts

@@ -0,0 +1,59 @@
+import type { ContainerData, Element, ExprData } from "@wdprlib/ast";
+
+export type InternalContainerData = ContainerData & {
+  _paragraphStrip?: boolean;
+  _emptyParagraphStrip?: boolean;
+  _escapedFromParagraph?: boolean;
+  _splitByBlankLine?: boolean;
+};
+
+export function isContainer(el: Element, type: string): boolean {
+  if (el.element !== "container") return false;
+  return el.data.type === type;
+}
+
+export function getContainerData(el: Element): ContainerData | null {
+  if (el.element !== "container") return null;
+  return el.data;
+}
+
+export function isSpanStripMarker(el: Element | undefined): boolean {
+  if (!el || el.element !== "container") return false;
+  const data = el.data as InternalContainerData;
+  return (
+    data.type === "span" && (data._paragraphStrip === true || data._emptyParagraphStrip === true)
+  );
+}
+
+export function hasParagraphStripSpan(para: Element): boolean {
+  const data = getContainerData(para);
+  if (!data || data.type !== "paragraph") return false;
+  return data.elements.some((child) => isSpanStripMarker(child));
+}
+
+export function isEscapedSpan(el: Element): boolean {
+  if (el.element !== "container") return false;
+  const data = el.data as InternalContainerData;
+  return data.type === "span" && data._escapedFromParagraph === true;
+}
+
+export function isSplitSpan(el: Element): boolean {
+  if (el.element !== "container") return false;
+  const data = el.data as InternalContainerData;
+  return data.type === "span" && data._splitByBlankLine === true;
+}
+
+export function isEmptyExpr(el: Element): boolean {
+  if (el.element !== "expr") return false;
+  const data = el.data as ExprData;
+  return data.expression === "";
+}
+
+export function isEmptySpan(el: Element): boolean {
+  if (el.element !== "container") return false;
+  return el.data.type === "span" && el.data.elements.length === 0;
+}
+
+export function isWhitespaceText(el: Element): boolean {
+  return el.element === "text" && typeof el.data === "string" && /^\s+$/.test(el.data);
+}

package/src/parser/postprocess/spanStrip/split.ts

@@ -0,0 +1,67 @@
+import type { Element } from "@wdprlib/ast";
+import { paragraphElement } from "./factory";
+import { getContainerData, isEmptyExpr, isSplitSpan } from "./predicates";
+
+export function splitParagraphAtBlankLineSpans(para: Element): Element[] {
+  const data = getContainerData(para);
+  if (!data || data.type !== "paragraph") return [para];
+
+  const result: Element[] = [];
+  let currentElements: Element[] = [];
+
+  for (const child of data.elements) {
+    if (isSplitSpan(child)) {
+      if (currentElements.length > 0) {
+        result.push(paragraphElement(currentElements));
+        currentElements = [];
+      }
+      currentElements.push(child);
+    } else {
+      currentElements.push(child);
+    }
+  }
+
+  if (currentElements.length > 0) {
+    result.push(paragraphElement(currentElements));
+  }
+
+  return result.length > 0 ? result : [para];
+}
+
+export function splitParagraphAtEmptyExpr(para: Element): Element[] {
+  const data = getContainerData(para);
+  if (!data || data.type !== "paragraph") return [para];
+  if (!data.elements.some(isEmptyExpr)) return [para];
+
+  const result: Element[] = [];
+  let currentElements: Element[] = [];
+
+  for (let i = 0; i < data.elements.length; i++) {
+    const child = data.elements[i];
+    if (!child) continue;
+
+    if (isEmptyExpr(child)) {
+      if (
+        currentElements.length > 0 &&
+        currentElements[currentElements.length - 1]?.element === "line-break"
+      ) {
+        currentElements.pop();
+      }
+      if (currentElements.length > 0) {
+        result.push(paragraphElement(currentElements));
+        currentElements = [];
+      }
+      if (i + 1 < data.elements.length && data.elements[i + 1]?.element === "line-break") {
+        i++;
+      }
+    } else {
+      currentElements.push(child);
+    }
+  }
+
+  if (currentElements.length > 0) {
+    result.push(paragraphElement(currentElements));
+  }
+
+  return result.length > 0 ? result : [];
+}

package/src/parser/preprocess/expr/chars.ts

@@ -0,0 +1,15 @@
+export function isWhitespace(ch: string | undefined): boolean {
+  return ch === " " || ch === "\t" || ch === "\n" || ch === "\r";
+}
+
+export function isIdentChar(ch: string | undefined): boolean {
+  if (!ch) return false;
+  const code = ch.charCodeAt(0);
+  return (
+    (code >= 48 && code <= 57) ||
+    (code >= 65 && code <= 90) ||
+    (code >= 97 && code <= 122) ||
+    code === 95 ||
+    code === 45
+  );
+}

package/src/parser/preprocess/expr/evaluate.ts

@@ -0,0 +1,22 @@
+import { evaluateExpression, formatExprValue, isTruthy } from "@wdprlib/ast";
+import type { DirectiveKind, DirectiveMatch } from "./types";
+
+/** Evaluate a parsed directive into its replacement string. */
+export function evaluateDirective(kind: DirectiveKind, match: DirectiveMatch): string {
+  if (kind === "expr") {
+    const result = evaluateExpression(match.head);
+    if (result.success) return formatExprValue(result.value);
+    if (result.error === "empty expression") return "";
+    return "ERROR";
+  }
+
+  if (kind === "if") {
+    if (!match.hasPipe) return "";
+    return isTruthy(match.head) ? match.thenText : match.elseText;
+  }
+
+  if (!match.hasPipe) return "";
+  const result = evaluateExpression(match.head);
+  if (!result.success) return "ERROR";
+  return result.value !== 0 && !Number.isNaN(result.value) ? match.thenText : match.elseText;
+}

package/src/parser/preprocess/expr/index.ts

@@ -0,0 +1,45 @@
+/**
+ *
+ * Text-level expansion of `[[#if ...]]`, `[[#ifexpr ...]]`, and
+ * `[[#expr ...]]` directives that sit *inside* another block's opener.
+ *
+ * @module
+ */
+
+import { makeUniqueSentinels, maskRawRegions, restorePlaceholders } from "../utils";
+import { expandInnermost } from "./scan";
+
+/**
+ * Resolve every `[[#if]]` / `[[#ifexpr]]` / `[[#expr]]` that sits inside
+ * another block's opener (depth > 0). Top-level directives are left for
+ * the inline parser. Innermost-first reduction lets an outer directive
+ * re-process the flattened body on the next pass. Unmatched / malformed
+ * directives are left untouched.
+ */
+export function preprocessExpr(source: string): string {
+  if (!source.includes("[[#")) return source;
+
+  const sentinels = makeUniqueSentinels(source);
+  const { masked, placeholders } = maskRawRegions(source, sentinels);
+  const reduced = reduceExpr(masked);
+  return restorePlaceholders(reduced, placeholders, sentinels);
+}
+
+/**
+ * Backwards-compatible alias for the older `preprocessIf` name (used by
+ * external callers that target the previous, `[[#if]]`-only behaviour).
+ * Both names point at the same implementation, which now also resolves
+ * `[[#ifexpr]]` and `[[#expr]]` in opener context.
+ */
+export const preprocessIf: (source: string) => string = preprocessExpr;
+
+function reduceExpr(source: string): string {
+  let current = source;
+  const maxIterations = source.length + 1;
+  for (let i = 0; i < maxIterations; i++) {
+    const next = expandInnermost(current);
+    if (next === current) return current;
+    current = next;
+  }
+  return current;
+}

package/src/parser/preprocess/expr/kind.ts

@@ -0,0 +1,19 @@
+import type { DirectiveKind } from "./types";
+import { isIdentChar } from "./chars";
+
+/** Return the kind of `[[#xxx` directive at `i`, or null if none matches. */
+export function matchDirectiveKind(source: string, i: number): DirectiveKind | null {
+  if (!source.startsWith("[[#", i)) return null;
+  // Order matters: `ifexpr` must be checked before `if` because the
+  // shorter `if` prefix would otherwise consume `ifexpr` openings.
+  if (source.startsWith("ifexpr", i + 3) && !isIdentChar(source[i + 9])) {
+    return "ifexpr";
+  }
+  if (source.startsWith("if", i + 3) && !isIdentChar(source[i + 5])) {
+    return "if";
+  }
+  if (source.startsWith("expr", i + 3) && !isIdentChar(source[i + 7])) {
+    return "expr";
+  }
+  return null;
+}

package/src/parser/preprocess/expr/parse.ts

@@ -0,0 +1,103 @@
+import { isWhitespace } from "./chars";
+import { matchDirectiveKind } from "./kind";
+import type { DirectiveKind, DirectiveMatch } from "./types";
+
+/**
+ * Try to parse a single `[[#kind ...]]` directive starting at `start`.
+ * Returns `null` when the directive is malformed (no closing `]]`) or
+ * when its body contains another `[[#kind]]` of the same family
+ * (so the caller should keep descending). The substrings are returned
+ * raw; callers decide how to evaluate them.
+ */
+export function tryParseInnermostDirective(
+  source: string,
+  start: number,
+  kind: DirectiveKind,
+): DirectiveMatch | null {
+  const keywordLen = kind === "ifexpr" ? 6 : kind === "expr" ? 4 : 2;
+  let pos = start + 3 + keywordLen;
+  while (pos < source.length && isWhitespace(source[pos])) pos++;
+
+  const headStart = pos;
+  let blockDepth = 0;
+  let linkDepth = 0;
+  const pipes: number[] = [];
+  let closeStart = -1;
+
+  while (pos < source.length) {
+    if (matchDirectiveKind(source, pos) !== null) {
+      return null;
+    }
+    if (source.startsWith("[[[", pos)) {
+      linkDepth++;
+      pos += 3;
+      continue;
+    }
+    if (linkDepth > 0 && source.startsWith("]]]", pos)) {
+      linkDepth--;
+      pos += 3;
+      continue;
+    }
+    if (linkDepth > 0) {
+      pos++;
+      continue;
+    }
+    if (source.startsWith("[[", pos)) {
+      blockDepth++;
+      pos += 2;
+      continue;
+    }
+    if (source.startsWith("]]", pos)) {
+      if (blockDepth === 0) {
+        closeStart = pos;
+        break;
+      }
+      blockDepth--;
+      pos += 2;
+      continue;
+    }
+    if (source[pos] === "|" && blockDepth === 0 && linkDepth === 0) {
+      pipes.push(pos);
+    }
+    pos++;
+  }
+
+  if (closeStart === -1) return null;
+  const hasPipe = pipes.length > 0;
+  if (!hasPipe && (kind === "if" || kind === "ifexpr")) return null;
+
+  return buildDirectiveMatch(source, headStart, closeStart, pipes, hasPipe);
+}
+
+function buildDirectiveMatch(
+  source: string,
+  headStart: number,
+  closeStart: number,
+  pipes: number[],
+  hasPipe: boolean,
+): DirectiveMatch {
+  if (!hasPipe) {
+    return {
+      end: closeStart + 2,
+      head: source.slice(headStart, closeStart).trim(),
+      thenText: "",
+      elseText: "",
+      hasPipe,
+    };
+  }
+
+  const head = source.slice(headStart, pipes[0]!).trim();
+  const thenText =
+    pipes.length >= 2
+      ? source.slice(pipes[0]! + 1, pipes[1]!).trim()
+      : source.slice(pipes[0]! + 1, closeStart).trim();
+  const elseText = pipes.length >= 2 ? source.slice(pipes[1]! + 1, closeStart).trim() : "";
+
+  return {
+    end: closeStart + 2,
+    head,
+    thenText,
+    elseText,
+    hasPipe,
+  };
+}

package/src/parser/preprocess/expr/scan.ts

@@ -0,0 +1,34 @@
+import { computeBracketDepths } from "../utils";
+import { evaluateDirective } from "./evaluate";
+import { matchDirectiveKind } from "./kind";
+import { tryParseInnermostDirective } from "./parse";
+
+/**
+ * Walk `source`, locate every innermost `[[#if]]` / `[[#ifexpr]]` /
+ * `[[#expr]]` directive that sits inside an unclosed `[[`, and replace
+ * it with its evaluated string. Returns the source unchanged when no
+ * replacements were made.
+ */
+export function expandInnermost(source: string): string {
+  const depths = computeBracketDepths(source);
+  let result = "";
+  let i = 0;
+  let replaced = false;
+
+  while (i < source.length) {
+    const kind = matchDirectiveKind(source, i);
+    if (kind !== null && depths[i]! > 0) {
+      const match = tryParseInnermostDirective(source, i, kind);
+      if (match !== null) {
+        result += evaluateDirective(kind, match);
+        i = match.end;
+        replaced = true;
+        continue;
+      }
+    }
+    result += source[i];
+    i++;
+  }
+
+  return replaced ? result : source;
+}

package/src/parser/preprocess/expr/types.ts

@@ -0,0 +1,14 @@
+export type DirectiveKind = "if" | "ifexpr" | "expr";
+
+export interface DirectiveMatch {
+  /** Position just past the closing `]]`. */
+  end: number;
+  /** Raw condition / expression. */
+  head: string;
+  /** Raw `then` branch. */
+  thenText: string;
+  /** Raw `else` branch. */
+  elseText: string;
+  /** Whether the directive supplied a `|` at all. */
+  hasPipe: boolean;
+}

package/src/parser/preprocess/index.ts

@@ -0,0 +1,38 @@
+/**
+ *
+ * Preprocessing pipeline that transforms raw wikitext before tokenization.
+ *
+ * Wikidot applies two categories of text substitutions before the main parser
+ * sees the input. This module orchestrates those substitutions in the correct
+ * order: whitespace normalization first (to establish consistent line structure),
+ * then typographic transformations (to convert ASCII quote/ellipsis patterns
+ * into Unicode equivalents).
+ *
+ * The preprocessing step is essential because the lexer and parser assume
+ * normalized input (Unix newlines, no tabs, consistent whitespace).
+ *
+ * @module
+ */
+
+import { substitute as whitespaceSubstitute } from "./whitespace";
+import { substitute as typographySubstitute } from "./typography";
+
+export { substitute as whitespace } from "./whitespace";
+export { substitute as typography } from "./typography";
+
+/**
+ * Run the full preprocessing pipeline on raw wikitext.
+ *
+ * Applies the following transformations in order:
+ * 1. Whitespace normalization (DOS/Mac newlines, tabs, leading spaces, etc.)
+ * 2. Typographic substitutions (curly quotes, ellipsis)
+ *
+ * @param text - Raw wikitext input
+ * @returns Preprocessed text ready for tokenization
+ */
+export function preprocess(text: string): string {
+  let result = text;
+  result = whitespaceSubstitute(result);
+  result = typographySubstitute(result);
+  return result;
+}

package/src/parser/preprocess/typography.ts

@@ -0,0 +1,132 @@
+/**
+ *
+ * Typographic preprocessing for Wikidot markup.
+ *
+ * Wikidot converts certain ASCII character sequences into their Unicode
+ * typographic equivalents before parsing. This module handles the following
+ * conversions:
+ *
+ * - ` `` ... '' ` becomes left/right double curly quotes (U+201C / U+201D)
+ * - ` ,, ... '' ` becomes low-9 double quote + right double quote (U+201E / U+201D)
+ * - `` ` ... ' `` becomes left/right single curly quotes (U+2018 / U+2019)
+ * - `...` (three dots) and `. . .` (spaced dots) become an ellipsis (U+2026)
+ *
+ * Em dash conversion (`--` to U+2014) is intentionally NOT handled here.
+ * It is performed in the parser instead, because the `--` sequence also appears
+ * in HTML comment markers (`[!--` and `--]`), and converting it during
+ * preprocessing would break comment detection.
+ *
+ * @module
+ */
+
+/** Unicode left single quotation mark (U+2018) */
+const LEFT_SINGLE_QUOTE = "\u2018"; // '
+/** Unicode right single quotation mark (U+2019) */
+const RIGHT_SINGLE_QUOTE = "\u2019"; // '
+/** Unicode left double quotation mark (U+201C) */
+const LEFT_DOUBLE_QUOTE = "\u201c"; // "
+/** Unicode right double quotation mark (U+201D) */
+const RIGHT_DOUBLE_QUOTE = "\u201d"; // "
+/** Unicode double low-9 quotation mark (U+201E), used in German/Polish typography */
+const LOW_DOUBLE_QUOTE = "\u201e"; // „
+/** Unicode horizontal ellipsis (U+2026) */
+const ELLIPSIS = "\u2026"; // …
+
+function replaceExactEllipsisPattern(text: string, pattern: string): string {
+  let searchFrom = 0;
+  let result = "";
+  let lastCopied = 0;
+  const patternLength = pattern.length;
+
+  while (searchFrom < text.length) {
+    const index = text.indexOf(pattern, searchFrom);
+    if (index === -1) break;
+
+    const prev = index > 0 ? text[index - 1] : "";
+    const next = index + patternLength < text.length ? text[index + patternLength] : "";
+    if (prev !== "." && next !== ".") {
+      result += text.slice(lastCopied, index) + ELLIPSIS;
+      lastCopied = index + patternLength;
+      searchFrom = lastCopied;
+    } else {
+      searchFrom = index + 1;
+    }
+  }
+
+  return lastCopied === 0 ? text : result + text.slice(lastCopied);
+}
+
+function replaceDelimitedTypography(
+  text: string,
+  opener: string,
+  closer: string,
+  leftQuote: string,
+  rightQuote: string,
+): string {
+  let searchFrom = 0;
+  let result = "";
+  let lastCopied = 0;
+
+  while (searchFrom < text.length) {
+    const openIndex = text.indexOf(opener, searchFrom);
+    if (openIndex === -1) break;
+
+    const contentStart = openIndex + opener.length;
+    const closeIndex = text.indexOf(closer, contentStart);
+    if (closeIndex === -1) break;
+
+    result += text.slice(lastCopied, openIndex);
+    result += leftQuote;
+    result += text.slice(contentStart, closeIndex);
+    result += rightQuote;
+
+    lastCopied = closeIndex + closer.length;
+    searchFrom = lastCopied;
+  }
+
+  return lastCopied === 0 ? text : result + text.slice(lastCopied);
+}
+
+/**
+ * Apply all typographic substitutions to the given text.
+ *
+ * Substitutions are applied in a specific order: double quotes first,
+ * then low double quotes, then single quotes, then ellipsis. This order
+ * matters because the backtick and apostrophe characters are shared
+ * between single and double quote patterns.
+ *
+ * @param text - Text to transform
+ * @returns Text with ASCII typography patterns replaced by Unicode equivalents
+ */
+export function substitute(text: string): string {
+  let result = text;
+
+  // Double quotes: ``...'' -> "..."
+  if (result.includes("``") && result.includes("''")) {
+    result = replaceDelimitedTypography(result, "``", "''", LEFT_DOUBLE_QUOTE, RIGHT_DOUBLE_QUOTE);
+  }
+
+  // Low double quotes: ,,..'' -> „..."
+  if (result.includes(",,") && result.includes("''")) {
+    result = replaceDelimitedTypography(result, ",,", "''", LOW_DOUBLE_QUOTE, RIGHT_DOUBLE_QUOTE);
+  }
+
+  // Single quotes: `...' -> '...'
+  if (result.includes("`") && result.includes("'")) {
+    result = replaceDelimitedTypography(result, "`", "'", LEFT_SINGLE_QUOTE, RIGHT_SINGLE_QUOTE);
+  }
+
+  // Ellipsis: ... or . . . -> …
+  // Must be exactly 3 dots, not preceded or followed by more dots
+  // Handle continuous dots: ...
+  if (result.includes("...")) {
+    result = replaceExactEllipsisPattern(result, "...");
+  }
+
+  // Handle spaced dots: . . .
+  if (result.includes(". . .")) {
+    result = replaceExactEllipsisPattern(result, ". . .");
+  }
+
+  return result;
+}