@wdprlib/parser 3.1.1 → 3.2.0

package/src/parser/preprocess/expr.ts

@@ -0,0 +1,265 @@
+/**
+ *
+ * Text-level expansion of `[[#if ...]]`, `[[#ifexpr ...]]`, and
+ * `[[#expr ...]]` directives that sit *inside* another block's opener.
+ *
+ * The inline rules in `rules/inline/expr.ts` parse these forms as regular
+ * inline elements, but that only works when the directive appears in
+ * parseable inline text. When one is embedded inside a block opener's
+ * attribute string, e.g.
+ *
+ * ```wikitext
+ * [[div class="x [[#if 1 | a | b ]]"]]
+ * [[li class="[[#if 1 | folded | unfolded ]] [[#ifexpr 1>0 | hot | cold ]]"]]
+ * [[div col="[[#expr 1+1]]"]]
+ * ```
+ *
+ * the lexer cannot recover a well-formed opener from the input. The
+ * embedded directive has to collapse to a plain string before the parser
+ * sees the outer tag.
+ *
+ * This pass only resolves directives whose `[[#` sits inside an unclosed
+ * `[[` (depth > 0). Top-level directives are left untouched so the inline
+ * parser / AST renderer keeps its full evaluator + element support.
+ *
+ * Truthiness rules match the inline `ifRule` / `ifExprRule`: an empty
+ * string, `"0"`, `"false"`, `"null"` (case-insensitive) are falsy.
+ *
+ * @module
+ */
+
+import { evaluateExpression, formatExprValue, isTruthy } from "@wdprlib/ast";
+import {
+  computeBracketDepths,
+  makeUniqueSentinels,
+  maskRawRegions,
+  restorePlaceholders,
+} from "./utils";
+
+/**
+ * Resolve every `[[#if]]` / `[[#ifexpr]]` / `[[#expr]]` that sits inside
+ * another block's opener (depth > 0). Top-level directives are left for
+ * the inline parser. Innermost-first reduction lets an outer directive
+ * re-process the flattened body on the next pass. Unmatched / malformed
+ * directives are left untouched.
+ */
+export function preprocessExpr(source: string): string {
+  if (!source.includes("[[#")) return source;
+
+  const sentinels = makeUniqueSentinels(source);
+  const { masked, placeholders } = maskRawRegions(source, sentinels);
+  const reduced = reduceExpr(masked);
+  return restorePlaceholders(reduced, placeholders, sentinels);
+}
+
+/**
+ * Backwards-compatible alias for the older `preprocessIf` name (used by
+ * external callers that target the previous, `[[#if]]`-only behaviour).
+ * Both names point at the same implementation, which now also resolves
+ * `[[#ifexpr]]` and `[[#expr]]` in opener context.
+ */
+export const preprocessIf: (source: string) => string = preprocessExpr;
+
+function reduceExpr(source: string): string {
+  let current = source;
+  const maxIterations = source.length + 1;
+  for (let i = 0; i < maxIterations; i++) {
+    const next = expandInnermost(current);
+    if (next === current) return current;
+    current = next;
+  }
+  return current;
+}
+
+/**
+ * Walk `source`, locate every innermost `[[#if]]` / `[[#ifexpr]]` /
+ * `[[#expr]]` directive that sits inside an unclosed `[[`, and replace
+ * it with its evaluated string. Returns the source unchanged when no
+ * replacements were made.
+ */
+function expandInnermost(source: string): string {
+  const depths = computeBracketDepths(source);
+  let result = "";
+  let i = 0;
+  let replaced = false;
+
+  while (i < source.length) {
+    const kind = matchDirectiveKind(source, i);
+    if (kind !== null && depths[i]! > 0) {
+      const match = tryParseInnermostDirective(source, i, kind);
+      if (match !== null) {
+        result += evaluateDirective(kind, match);
+        i = match.end;
+        replaced = true;
+        continue;
+      }
+    }
+    result += source[i];
+    i++;
+  }
+
+  return replaced ? result : source;
+}
+
+type DirectiveKind = "if" | "ifexpr" | "expr";
+
+/** Return the kind of `[[#xxx` directive at `i`, or null if none matches. */
+function matchDirectiveKind(source: string, i: number): DirectiveKind | null {
+  if (!source.startsWith("[[#", i)) return null;
+  // Order matters: `ifexpr` must be checked before `if` because the
+  // shorter `if` prefix would otherwise consume `ifexpr` openings.
+  if (source.startsWith("ifexpr", i + 3) && !isIdentChar(source[i + 9])) {
+    return "ifexpr";
+  }
+  if (source.startsWith("if", i + 3) && !isIdentChar(source[i + 5])) {
+    return "if";
+  }
+  if (source.startsWith("expr", i + 3) && !isIdentChar(source[i + 7])) {
+    return "expr";
+  }
+  return null;
+}
+
+interface DirectiveMatch {
+  /** Position just past the closing `]]`. */
+  end: number;
+  /** Raw condition / expression (everything between the keyword and the first top-level `|` or `]]`). */
+  head: string;
+  /** Raw `then` branch (empty when no `|` appeared). */
+  thenText: string;
+  /** Raw `else` branch (empty when only one `|` appeared). */
+  elseText: string;
+  /** Whether the directive supplied a `|` at all. */
+  hasPipe: boolean;
+}
+
+/**
+ * Try to parse a single `[[#kind ...]]` directive starting at `start`.
+ * Returns `null` when the directive is malformed (no closing `]]`) or
+ * when its body contains another `[[#kind]]` of the same family
+ * (so the caller should keep descending). The substrings are returned
+ * raw; callers decide how to evaluate them.
+ */
+function tryParseInnermostDirective(
+  source: string,
+  start: number,
+  kind: DirectiveKind,
+): DirectiveMatch | null {
+  const keywordLen = kind === "ifexpr" ? 6 : kind === "expr" ? 4 : 2;
+  // start + 3 ("[[#") + keywordLen → first char after the keyword.
+  let pos = start + 3 + keywordLen;
+  // The inline rule does not require a whitespace separator here — it
+  // accepts e.g. `[[#expr(1+1)]]` and `[[#ifexpr(1)|yes|no]]`. Skip any
+  // optional leading whitespace and let the body scan handle the rest.
+  while (pos < source.length && isWhitespace(source[pos])) pos++;
+
+  const headStart = pos;
+  let blockDepth = 0;
+  let linkDepth = 0;
+  const pipes: number[] = [];
+  let closeStart = -1;
+
+  while (pos < source.length) {
+    // Reject any nested directive of the same family so we resolve
+    // innermost-first.
+    if (matchDirectiveKind(source, pos) !== null) {
+      return null;
+    }
+    if (source.startsWith("[[[", pos)) {
+      linkDepth++;
+      pos += 3;
+      continue;
+    }
+    if (linkDepth > 0 && source.startsWith("]]]", pos)) {
+      linkDepth--;
+      pos += 3;
+      continue;
+    }
+    if (linkDepth > 0) {
+      pos++;
+      continue;
+    }
+    if (source.startsWith("[[", pos)) {
+      blockDepth++;
+      pos += 2;
+      continue;
+    }
+    if (source.startsWith("]]", pos)) {
+      if (blockDepth === 0) {
+        closeStart = pos;
+        break;
+      }
+      blockDepth--;
+      pos += 2;
+      continue;
+    }
+    if (source[pos] === "|" && blockDepth === 0 && linkDepth === 0) {
+      pipes.push(pos);
+    }
+    pos++;
+  }
+
+  if (closeStart === -1) return null;
+  const hasPipe = pipes.length > 0;
+  // `[[#if]]` / `[[#ifexpr]]` require a `then` branch separated by `|`.
+  // A directive without a pipe is malformed; leave it for the inline
+  // parser to report rather than silently dropping it.
+  if (!hasPipe && (kind === "if" || kind === "ifexpr")) return null;
+
+  let head: string;
+  let thenText = "";
+  let elseText = "";
+
+  if (!hasPipe) {
+    head = source.slice(headStart, closeStart).trim();
+  } else {
+    head = source.slice(headStart, pipes[0]!).trim();
+    if (pipes.length >= 2) {
+      thenText = source.slice(pipes[0]! + 1, pipes[1]!).trim();
+      elseText = source.slice(pipes[1]! + 1, closeStart).trim();
+    } else {
+      thenText = source.slice(pipes[0]! + 1, closeStart).trim();
+    }
+  }
+
+  return {
+    end: closeStart + 2,
+    head,
+    thenText,
+    elseText,
+    hasPipe,
+  };
+}
+
+/** Evaluate a parsed directive into its replacement string. */
+function evaluateDirective(kind: DirectiveKind, m: DirectiveMatch): string {
+  if (kind === "expr") {
+    const result = evaluateExpression(m.head);
+    if (result.success) return formatExprValue(result.value);
+    // The inline renderer emits nothing for an empty `[[#expr ]]`; mirror
+    // that so an opener-embedded empty expr collapses to an empty
+    // attribute value rather than the literal "ERROR" placeholder.
+    if (result.error === "empty expression") return "";
+    return "ERROR";
+  }
+  if (kind === "if") {
+    if (!m.hasPipe) return "";
+    return isTruthy(m.head) ? m.thenText : m.elseText;
+  }
+  // ifexpr — the inline renderer treats every error (including empty
+  // expression) as a "run-time error" string, so we keep the placeholder
+  // here to avoid silently swallowing a malformed conditional.
+  if (!m.hasPipe) return "";
+  const result = evaluateExpression(m.head);
+  if (!result.success) return "ERROR";
+  return result.value !== 0 && !Number.isNaN(result.value) ? m.thenText : m.elseText;
+}
+
+function isWhitespace(ch: string | undefined): boolean {
+  return ch === " " || ch === "\t" || ch === "\n" || ch === "\r";
+}
+
+function isIdentChar(ch: string | undefined): boolean {
+  if (!ch) return false;
+  return /[a-z0-9_-]/i.test(ch);
+}

package/src/parser/preprocess/index.ts

@@ -0,0 +1,38 @@
+/**
+ *
+ * Preprocessing pipeline that transforms raw wikitext before tokenization.
+ *
+ * Wikidot applies two categories of text substitutions before the main parser
+ * sees the input. This module orchestrates those substitutions in the correct
+ * order: whitespace normalization first (to establish consistent line structure),
+ * then typographic transformations (to convert ASCII quote/ellipsis patterns
+ * into Unicode equivalents).
+ *
+ * The preprocessing step is essential because the lexer and parser assume
+ * normalized input (Unix newlines, no tabs, consistent whitespace).
+ *
+ * @module
+ */
+
+import { substitute as whitespaceSubstitute } from "./whitespace";
+import { substitute as typographySubstitute } from "./typography";
+
+export { substitute as whitespace } from "./whitespace";
+export { substitute as typography } from "./typography";
+
+/**
+ * Run the full preprocessing pipeline on raw wikitext.
+ *
+ * Applies the following transformations in order:
+ * 1. Whitespace normalization (DOS/Mac newlines, tabs, leading spaces, etc.)
+ * 2. Typographic substitutions (curly quotes, ellipsis)
+ *
+ * @param text - Raw wikitext input
+ * @returns Preprocessed text ready for tokenization
+ */
+export function preprocess(text: string): string {
+  let result = text;
+  result = whitespaceSubstitute(result);
+  result = typographySubstitute(result);
+  return result;
+}

package/src/parser/preprocess/typography.ts

@@ -0,0 +1,67 @@
+/**
+ *
+ * Typographic preprocessing for Wikidot markup.
+ *
+ * Wikidot converts certain ASCII character sequences into their Unicode
+ * typographic equivalents before parsing. This module handles the following
+ * conversions:
+ *
+ * - ` `` ... '' ` becomes left/right double curly quotes (U+201C / U+201D)
+ * - ` ,, ... '' ` becomes low-9 double quote + right double quote (U+201E / U+201D)
+ * - `` ` ... ' `` becomes left/right single curly quotes (U+2018 / U+2019)
+ * - `...` (three dots) and `. . .` (spaced dots) become an ellipsis (U+2026)
+ *
+ * Em dash conversion (`--` to U+2014) is intentionally NOT handled here.
+ * It is performed in the parser instead, because the `--` sequence also appears
+ * in HTML comment markers (`[!--` and `--]`), and converting it during
+ * preprocessing would break comment detection.
+ *
+ * @module
+ */
+
+/** Unicode left single quotation mark (U+2018) */
+const LEFT_SINGLE_QUOTE = "\u2018"; // '
+/** Unicode right single quotation mark (U+2019) */
+const RIGHT_SINGLE_QUOTE = "\u2019"; // '
+/** Unicode left double quotation mark (U+201C) */
+const LEFT_DOUBLE_QUOTE = "\u201c"; // "
+/** Unicode right double quotation mark (U+201D) */
+const RIGHT_DOUBLE_QUOTE = "\u201d"; // "
+/** Unicode double low-9 quotation mark (U+201E), used in German/Polish typography */
+const LOW_DOUBLE_QUOTE = "\u201e"; // „
+/** Unicode horizontal ellipsis (U+2026) */
+const ELLIPSIS = "\u2026"; // …
+
+/**
+ * Apply all typographic substitutions to the given text.
+ *
+ * Substitutions are applied in a specific order: double quotes first,
+ * then low double quotes, then single quotes, then ellipsis. This order
+ * matters because the backtick and apostrophe characters are shared
+ * between single and double quote patterns.
+ *
+ * @param text - Text to transform
+ * @returns Text with ASCII typography patterns replaced by Unicode equivalents
+ */
+export function substitute(text: string): string {
+  let result = text;
+
+  // Double quotes: ``...'' -> "..."
+  result = result.replace(/``(.*?)''/g, `${LEFT_DOUBLE_QUOTE}$1${RIGHT_DOUBLE_QUOTE}`);
+
+  // Low double quotes: ,,..'' -> „..."
+  result = result.replace(/,,(.*?)''/g, `${LOW_DOUBLE_QUOTE}$1${RIGHT_DOUBLE_QUOTE}`);
+
+  // Single quotes: `...' -> '...'
+  result = result.replace(/`(.*?)'/g, `${LEFT_SINGLE_QUOTE}$1${RIGHT_SINGLE_QUOTE}`);
+
+  // Ellipsis: ... or . . . -> …
+  // Must be exactly 3 dots, not preceded or followed by more dots
+  // Handle continuous dots: ...
+  result = result.replace(/(?<![.])\.\.\.(?![.])/g, ELLIPSIS);
+
+  // Handle spaced dots: . . .
+  result = result.replace(/(?<![.])\. \. \.(?![.])/g, ELLIPSIS);
+
+  return result;
+}

package/src/parser/preprocess/utils.ts

@@ -0,0 +1,250 @@
+/**
+ *
+ * Shared helpers for text-level preprocess passes that run before
+ * tokenization (e.g. `[[iftags]]` collapse, opener-embedded `[[#if]]`
+ * collapse).
+ *
+ * Each pass needs to:
+ * - mask raw regions (`[[code]]`, `[[html]]`, `@@..@@`, `@<..>@`) so a
+ *   pattern they enclose is not transformed
+ * - know the bracket-opener depth at every offset so it can distinguish
+ *   directives at the top level from ones nested inside another block's
+ *   opener attribute string
+ *
+ * The depth tracking mirrors the lexer's `blockOpenerDepth`:
+ * - `[[` increments, `]]` decrements (clamped at 0)
+ * - `[[[ ... ]]]` triple links do not affect block depth
+ * - quoted attribute values (`= "..."`) are skipped to the next `"` /
+ *   newline, matching the lexer's `QUOTED_STRING` recognition
+ * - newlines reset depth to 0 (block openers are single-line constructs)
+ *
+ * @module
+ */
+
+const BASE_PLACEHOLDER_OPEN = "\uE000";
+const BASE_PLACEHOLDER_CLOSE = "\uE001";
+
+const RAW_BLOCK_OPEN_PATTERN = /\[\[\s*(code|html)\b[^\]]*\]\]/iy;
+
+/** Unique sentinel characters used to wrap raw-region placeholders. */
+export interface Sentinels {
+  open: string;
+  close: string;
+}
+
+/**
+ * Choose sentinel strings that are guaranteed not to appear in `source`.
+ * The placeholders we splice into the masked source have the form
+ * `<open><digits><close>`, so the restore pass must not confuse them
+ * with content. Extends both sentinel characters until neither appears.
+ */
+export function makeUniqueSentinels(source: string): Sentinels {
+  let open = BASE_PLACEHOLDER_OPEN;
+  let close = BASE_PLACEHOLDER_CLOSE;
+  while (source.includes(open) || source.includes(close)) {
+    open += BASE_PLACEHOLDER_OPEN;
+    close += BASE_PLACEHOLDER_CLOSE;
+  }
+  return { open, close };
+}
+
+/**
+ * Walk `source` and replace each raw region with a placeholder token so
+ * downstream passes (regex / scan) do not transform their bodies. The
+ * original substrings are kept in `placeholders` for {@link restorePlaceholders}
+ * to splice back at the end.
+ *
+ * Raw regions handled:
+ * - `[[code ...]]...[[/code]]` — consumes to EOF when the closing tag
+ *   is missing (mirroring the block parser's behaviour for unclosed
+ *   code blocks).
+ * - `[[html ...]]...[[/html]]` — only masked when the closing tag is
+ *   present; an unclosed `[[html]]` is left in place so a later directive
+ *   is not incorrectly hidden behind the mask.
+ * - `@<...>@` (single-line balanced raw — `>@` must be on the same line).
+ * - `@@...@@` (single-line inline raw — must not span newlines).
+ *
+ * Genuinely unclosed `@@` / `@<` are left in place (the parser treats
+ * them as literal text anyway). Comments `[!-- ... --]` are intentionally
+ * not masked: Wikidot's legacy Text_Wiki evaluates `[[iftags]]` before
+ * comments, so masking here would invert that order.
+ */
+export function maskRawRegions(
+  source: string,
+  sentinels: Sentinels,
+): { masked: string; placeholders: string[] } {
+  const placeholders: string[] = [];
+  let masked = "";
+  let i = 0;
+
+  while (i < source.length) {
+    if (source[i] === "[" && source[i + 1] === "[") {
+      RAW_BLOCK_OPEN_PATTERN.lastIndex = i;
+      const openMatch = RAW_BLOCK_OPEN_PATTERN.exec(source);
+      if (openMatch) {
+        const name = openMatch[1]!.toLowerCase();
+        const openLen = openMatch[0].length;
+        const closePattern = new RegExp(`\\[\\[\\/\\s*${name}\\s*\\]\\]`, "ig");
+        closePattern.lastIndex = i + openLen;
+        const closeMatch = closePattern.exec(source);
+        if (closeMatch) {
+          const regionEnd = closeMatch.index + closeMatch[0].length;
+          masked += pushPlaceholder(placeholders, source.slice(i, regionEnd), sentinels);
+          i = regionEnd;
+          continue;
+        }
+        if (name === "code") {
+          masked += pushPlaceholder(placeholders, source.slice(i), sentinels);
+          i = source.length;
+          continue;
+        }
+      }
+    }
+
+    if (source[i] === "@" && source[i + 1] === "<") {
+      const close = source.indexOf(">@", i + 2);
+      const newline = source.indexOf("\n", i + 2);
+      if (close !== -1 && (newline === -1 || close < newline)) {
+        const regionEnd = close + 2;
+        masked += pushPlaceholder(placeholders, source.slice(i, regionEnd), sentinels);
+        i = regionEnd;
+        continue;
+      }
+    }
+
+    if (source[i] === "@" && source[i + 1] === "@") {
+      const close = source.indexOf("@@", i + 2);
+      const newline = source.indexOf("\n", i + 2);
+      if (close !== -1 && (newline === -1 || close < newline)) {
+        const regionEnd = close + 2;
+        masked += pushPlaceholder(placeholders, source.slice(i, regionEnd), sentinels);
+        i = regionEnd;
+        continue;
+      }
+    }
+
+    masked += source[i];
+    i++;
+  }
+
+  return { masked, placeholders };
+}
+
+function pushPlaceholder(placeholders: string[], text: string, sentinels: Sentinels): string {
+  const idx = placeholders.length;
+  placeholders.push(text);
+  return `${sentinels.open}${idx}${sentinels.close}`;
+}
+
+function escapeRegex(str: string): string {
+  return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+/** Inverse of {@link maskRawRegions}: replace placeholders with originals. */
+export function restorePlaceholders(
+  source: string,
+  placeholders: string[],
+  sentinels: Sentinels,
+): string {
+  const pattern = new RegExp(
+    `${escapeRegex(sentinels.open)}(\\d+)${escapeRegex(sentinels.close)}`,
+    "g",
+  );
+  return source.replace(pattern, (_, idx: string) => placeholders[Number(idx)] ?? "");
+}
+
+/**
+ * Compute the unmatched-`[[` depth at each character offset of `source`.
+ * Mirrors the lexer's `blockOpenerDepth`. Returns `Int32Array` of length
+ * `source.length + 1`; `depths[k]` is the depth immediately before the
+ * character at offset `k` is consumed.
+ */
+export function computeBracketDepths(source: string): Int32Array {
+  const n = source.length;
+  const depths = new Int32Array(n + 1);
+  let depth = 0;
+  let i = 0;
+  while (i < n) {
+    depths[i] = depth;
+    const c = source.charCodeAt(i);
+    const c1 = i + 1 < n ? source.charCodeAt(i + 1) : -1;
+    const c2 = i + 2 < n ? source.charCodeAt(i + 2) : -1;
+
+    if (depth > 0 && c === 0x22 /* " */ && precededByEqualsAttr(source, i)) {
+      const end = findQuoteEnd(source, i + 1);
+      for (let k = i; k <= end; k++) depths[k] = depth;
+      i = end + 1;
+      continue;
+    }
+
+    if (c === 0x5b /* [ */ && c1 === 0x5b && c2 === 0x5b) {
+      const end = findTripleLinkEnd(source, i + 3);
+      for (let k = i; k <= end; k++) depths[k] = depth;
+      i = end + 1;
+      continue;
+    }
+
+    if (c === 0x5b && c1 === 0x5b) {
+      depth++;
+      depths[i + 1] = depth;
+      i += 2;
+      continue;
+    }
+
+    if (c === 0x5d /* ] */ && c1 === 0x5d) {
+      depth = Math.max(0, depth - 1);
+      depths[i + 1] = depth;
+      i += 2;
+      continue;
+    }
+
+    if (c === 0x0a /* \n */) {
+      // Block openers are single-line; reset depth at line boundaries so
+      // an unterminated `[[xxx` does not keep subsequent directives
+      // inside its (imaginary) opener context.
+      depth = 0;
+    }
+
+    i++;
+  }
+  depths[n] = depth;
+  return depths;
+}
+
+function precededByEqualsAttr(s: string, i: number): boolean {
+  let j = i - 1;
+  while (j >= 0) {
+    const ch = s.charCodeAt(j);
+    if (ch === 0x20 /* space */ || ch === 0x09 /* tab */) {
+      j--;
+      continue;
+    }
+    return ch === 0x3d; /* = */
+  }
+  return false;
+}
+
+function findQuoteEnd(s: string, from: number): number {
+  for (let i = from; i < s.length; i++) {
+    const ch = s.charCodeAt(i);
+    if (ch === 0x22 /* " */ || ch === 0x0a /* \n */) return i;
+  }
+  return s.length - 1;
+}
+
+function findTripleLinkEnd(s: string, from: number): number {
+  for (let i = from; i < s.length; i++) {
+    if (
+      s.charCodeAt(i) === 0x5d &&
+      i + 2 < s.length &&
+      s.charCodeAt(i + 1) === 0x5d &&
+      s.charCodeAt(i + 2) === 0x5d
+    ) {
+      return i + 2;
+    }
+    if (s.charCodeAt(i) === 0x0a && i + 1 < s.length && s.charCodeAt(i + 1) === 0x0a) {
+      return i;
+    }
+  }
+  return s.length - 1;
+}