@wdprlib/parser 3.2.0 → 4.0.0

package/src/parser/preprocess/expr.ts

@@ -1,265 +0,0 @@
-/**
- *
- * Text-level expansion of `[[#if ...]]`, `[[#ifexpr ...]]`, and
- * `[[#expr ...]]` directives that sit *inside* another block's opener.
- *
- * The inline rules in `rules/inline/expr.ts` parse these forms as regular
- * inline elements, but that only works when the directive appears in
- * parseable inline text. When one is embedded inside a block opener's
- * attribute string, e.g.
- *
- * ```wikitext
- * [[div class="x [[#if 1 | a | b ]]"]]
- * [[li class="[[#if 1 | folded | unfolded ]] [[#ifexpr 1>0 | hot | cold ]]"]]
- * [[div col="[[#expr 1+1]]"]]
- * ```
- *
- * the lexer cannot recover a well-formed opener from the input. The
- * embedded directive has to collapse to a plain string before the parser
- * sees the outer tag.
- *
- * This pass only resolves directives whose `[[#` sits inside an unclosed
- * `[[` (depth > 0). Top-level directives are left untouched so the inline
- * parser / AST renderer keeps its full evaluator + element support.
- *
- * Truthiness rules match the inline `ifRule` / `ifExprRule`: an empty
- * string, `"0"`, `"false"`, `"null"` (case-insensitive) are falsy.
- *
- * @module
- */
-
-import { evaluateExpression, formatExprValue, isTruthy } from "@wdprlib/ast";
-import {
-  computeBracketDepths,
-  makeUniqueSentinels,
-  maskRawRegions,
-  restorePlaceholders,
-} from "./utils";
-
-/**
- * Resolve every `[[#if]]` / `[[#ifexpr]]` / `[[#expr]]` that sits inside
- * another block's opener (depth > 0). Top-level directives are left for
- * the inline parser. Innermost-first reduction lets an outer directive
- * re-process the flattened body on the next pass. Unmatched / malformed
- * directives are left untouched.
- */
-export function preprocessExpr(source: string): string {
-  if (!source.includes("[[#")) return source;
-
-  const sentinels = makeUniqueSentinels(source);
-  const { masked, placeholders } = maskRawRegions(source, sentinels);
-  const reduced = reduceExpr(masked);
-  return restorePlaceholders(reduced, placeholders, sentinels);
-}
-
-/**
- * Backwards-compatible alias for the older `preprocessIf` name (used by
- * external callers that target the previous, `[[#if]]`-only behaviour).
- * Both names point at the same implementation, which now also resolves
- * `[[#ifexpr]]` and `[[#expr]]` in opener context.
- */
-export const preprocessIf: (source: string) => string = preprocessExpr;
-
-function reduceExpr(source: string): string {
-  let current = source;
-  const maxIterations = source.length + 1;
-  for (let i = 0; i < maxIterations; i++) {
-    const next = expandInnermost(current);
-    if (next === current) return current;
-    current = next;
-  }
-  return current;
-}
-
-/**
- * Walk `source`, locate every innermost `[[#if]]` / `[[#ifexpr]]` /
- * `[[#expr]]` directive that sits inside an unclosed `[[`, and replace
- * it with its evaluated string. Returns the source unchanged when no
- * replacements were made.
- */
-function expandInnermost(source: string): string {
-  const depths = computeBracketDepths(source);
-  let result = "";
-  let i = 0;
-  let replaced = false;
-
-  while (i < source.length) {
-    const kind = matchDirectiveKind(source, i);
-    if (kind !== null && depths[i]! > 0) {
-      const match = tryParseInnermostDirective(source, i, kind);
-      if (match !== null) {
-        result += evaluateDirective(kind, match);
-        i = match.end;
-        replaced = true;
-        continue;
-      }
-    }
-    result += source[i];
-    i++;
-  }
-
-  return replaced ? result : source;
-}
-
-type DirectiveKind = "if" | "ifexpr" | "expr";
-
-/** Return the kind of `[[#xxx` directive at `i`, or null if none matches. */
-function matchDirectiveKind(source: string, i: number): DirectiveKind | null {
-  if (!source.startsWith("[[#", i)) return null;
-  // Order matters: `ifexpr` must be checked before `if` because the
-  // shorter `if` prefix would otherwise consume `ifexpr` openings.
-  if (source.startsWith("ifexpr", i + 3) && !isIdentChar(source[i + 9])) {
-    return "ifexpr";
-  }
-  if (source.startsWith("if", i + 3) && !isIdentChar(source[i + 5])) {
-    return "if";
-  }
-  if (source.startsWith("expr", i + 3) && !isIdentChar(source[i + 7])) {
-    return "expr";
-  }
-  return null;
-}
-
-interface DirectiveMatch {
-  /** Position just past the closing `]]`. */
-  end: number;
-  /** Raw condition / expression (everything between the keyword and the first top-level `|` or `]]`). */
-  head: string;
-  /** Raw `then` branch (empty when no `|` appeared). */
-  thenText: string;
-  /** Raw `else` branch (empty when only one `|` appeared). */
-  elseText: string;
-  /** Whether the directive supplied a `|` at all. */
-  hasPipe: boolean;
-}
-
-/**
- * Try to parse a single `[[#kind ...]]` directive starting at `start`.
- * Returns `null` when the directive is malformed (no closing `]]`) or
- * when its body contains another `[[#kind]]` of the same family
- * (so the caller should keep descending). The substrings are returned
- * raw; callers decide how to evaluate them.
- */
-function tryParseInnermostDirective(
-  source: string,
-  start: number,
-  kind: DirectiveKind,
-): DirectiveMatch | null {
-  const keywordLen = kind === "ifexpr" ? 6 : kind === "expr" ? 4 : 2;
-  // start + 3 ("[[#") + keywordLen → first char after the keyword.
-  let pos = start + 3 + keywordLen;
-  // The inline rule does not require a whitespace separator here — it
-  // accepts e.g. `[[#expr(1+1)]]` and `[[#ifexpr(1)|yes|no]]`. Skip any
-  // optional leading whitespace and let the body scan handle the rest.
-  while (pos < source.length && isWhitespace(source[pos])) pos++;
-
-  const headStart = pos;
-  let blockDepth = 0;
-  let linkDepth = 0;
-  const pipes: number[] = [];
-  let closeStart = -1;
-
-  while (pos < source.length) {
-    // Reject any nested directive of the same family so we resolve
-    // innermost-first.
-    if (matchDirectiveKind(source, pos) !== null) {
-      return null;
-    }
-    if (source.startsWith("[[[", pos)) {
-      linkDepth++;
-      pos += 3;
-      continue;
-    }
-    if (linkDepth > 0 && source.startsWith("]]]", pos)) {
-      linkDepth--;
-      pos += 3;
-      continue;
-    }
-    if (linkDepth > 0) {
-      pos++;
-      continue;
-    }
-    if (source.startsWith("[[", pos)) {
-      blockDepth++;
-      pos += 2;
-      continue;
-    }
-    if (source.startsWith("]]", pos)) {
-      if (blockDepth === 0) {
-        closeStart = pos;
-        break;
-      }
-      blockDepth--;
-      pos += 2;
-      continue;
-    }
-    if (source[pos] === "|" && blockDepth === 0 && linkDepth === 0) {
-      pipes.push(pos);
-    }
-    pos++;
-  }
-
-  if (closeStart === -1) return null;
-  const hasPipe = pipes.length > 0;
-  // `[[#if]]` / `[[#ifexpr]]` require a `then` branch separated by `|`.
-  // A directive without a pipe is malformed; leave it for the inline
-  // parser to report rather than silently dropping it.
-  if (!hasPipe && (kind === "if" || kind === "ifexpr")) return null;
-
-  let head: string;
-  let thenText = "";
-  let elseText = "";
-
-  if (!hasPipe) {
-    head = source.slice(headStart, closeStart).trim();
-  } else {
-    head = source.slice(headStart, pipes[0]!).trim();
-    if (pipes.length >= 2) {
-      thenText = source.slice(pipes[0]! + 1, pipes[1]!).trim();
-      elseText = source.slice(pipes[1]! + 1, closeStart).trim();
-    } else {
-      thenText = source.slice(pipes[0]! + 1, closeStart).trim();
-    }
-  }
-
-  return {
-    end: closeStart + 2,
-    head,
-    thenText,
-    elseText,
-    hasPipe,
-  };
-}
-
-/** Evaluate a parsed directive into its replacement string. */
-function evaluateDirective(kind: DirectiveKind, m: DirectiveMatch): string {
-  if (kind === "expr") {
-    const result = evaluateExpression(m.head);
-    if (result.success) return formatExprValue(result.value);
-    // The inline renderer emits nothing for an empty `[[#expr ]]`; mirror
-    // that so an opener-embedded empty expr collapses to an empty
-    // attribute value rather than the literal "ERROR" placeholder.
-    if (result.error === "empty expression") return "";
-    return "ERROR";
-  }
-  if (kind === "if") {
-    if (!m.hasPipe) return "";
-    return isTruthy(m.head) ? m.thenText : m.elseText;
-  }
-  // ifexpr — the inline renderer treats every error (including empty
-  // expression) as a "run-time error" string, so we keep the placeholder
-  // here to avoid silently swallowing a malformed conditional.
-  if (!m.hasPipe) return "";
-  const result = evaluateExpression(m.head);
-  if (!result.success) return "ERROR";
-  return result.value !== 0 && !Number.isNaN(result.value) ? m.thenText : m.elseText;
-}
-
-function isWhitespace(ch: string | undefined): boolean {
-  return ch === " " || ch === "\t" || ch === "\n" || ch === "\r";
-}
-
-function isIdentChar(ch: string | undefined): boolean {
-  if (!ch) return false;
-  return /[a-z0-9_-]/i.test(ch);
-}

package/src/parser/preprocess/utils.ts

@@ -1,250 +0,0 @@
-/**
- *
- * Shared helpers for text-level preprocess passes that run before
- * tokenization (e.g. `[[iftags]]` collapse, opener-embedded `[[#if]]`
- * collapse).
- *
- * Each pass needs to:
- * - mask raw regions (`[[code]]`, `[[html]]`, `@@..@@`, `@<..>@`) so a
- *   pattern they enclose is not transformed
- * - know the bracket-opener depth at every offset so it can distinguish
- *   directives at the top level from ones nested inside another block's
- *   opener attribute string
- *
- * The depth tracking mirrors the lexer's `blockOpenerDepth`:
- * - `[[` increments, `]]` decrements (clamped at 0)
- * - `[[[ ... ]]]` triple links do not affect block depth
- * - quoted attribute values (`= "..."`) are skipped to the next `"` /
- *   newline, matching the lexer's `QUOTED_STRING` recognition
- * - newlines reset depth to 0 (block openers are single-line constructs)
- *
- * @module
- */
-
-const BASE_PLACEHOLDER_OPEN = "\uE000";
-const BASE_PLACEHOLDER_CLOSE = "\uE001";
-
-const RAW_BLOCK_OPEN_PATTERN = /\[\[\s*(code|html)\b[^\]]*\]\]/iy;
-
-/** Unique sentinel characters used to wrap raw-region placeholders. */
-export interface Sentinels {
-  open: string;
-  close: string;
-}
-
-/**
- * Choose sentinel strings that are guaranteed not to appear in `source`.
- * The placeholders we splice into the masked source have the form
- * `<open><digits><close>`, so the restore pass must not confuse them
- * with content. Extends both sentinel characters until neither appears.
- */
-export function makeUniqueSentinels(source: string): Sentinels {
-  let open = BASE_PLACEHOLDER_OPEN;
-  let close = BASE_PLACEHOLDER_CLOSE;
-  while (source.includes(open) || source.includes(close)) {
-    open += BASE_PLACEHOLDER_OPEN;
-    close += BASE_PLACEHOLDER_CLOSE;
-  }
-  return { open, close };
-}
-
-/**
- * Walk `source` and replace each raw region with a placeholder token so
- * downstream passes (regex / scan) do not transform their bodies. The
- * original substrings are kept in `placeholders` for {@link restorePlaceholders}
- * to splice back at the end.
- *
- * Raw regions handled:
- * - `[[code ...]]...[[/code]]` — consumes to EOF when the closing tag
- *   is missing (mirroring the block parser's behaviour for unclosed
- *   code blocks).
- * - `[[html ...]]...[[/html]]` — only masked when the closing tag is
- *   present; an unclosed `[[html]]` is left in place so a later directive
- *   is not incorrectly hidden behind the mask.
- * - `@<...>@` (single-line balanced raw — `>@` must be on the same line).
- * - `@@...@@` (single-line inline raw — must not span newlines).
- *
- * Genuinely unclosed `@@` / `@<` are left in place (the parser treats
- * them as literal text anyway). Comments `[!-- ... --]` are intentionally
- * not masked: Wikidot's legacy Text_Wiki evaluates `[[iftags]]` before
- * comments, so masking here would invert that order.
- */
-export function maskRawRegions(
-  source: string,
-  sentinels: Sentinels,
-): { masked: string; placeholders: string[] } {
-  const placeholders: string[] = [];
-  let masked = "";
-  let i = 0;
-
-  while (i < source.length) {
-    if (source[i] === "[" && source[i + 1] === "[") {
-      RAW_BLOCK_OPEN_PATTERN.lastIndex = i;
-      const openMatch = RAW_BLOCK_OPEN_PATTERN.exec(source);
-      if (openMatch) {
-        const name = openMatch[1]!.toLowerCase();
-        const openLen = openMatch[0].length;
-        const closePattern = new RegExp(`\\[\\[\\/\\s*${name}\\s*\\]\\]`, "ig");
-        closePattern.lastIndex = i + openLen;
-        const closeMatch = closePattern.exec(source);
-        if (closeMatch) {
-          const regionEnd = closeMatch.index + closeMatch[0].length;
-          masked += pushPlaceholder(placeholders, source.slice(i, regionEnd), sentinels);
-          i = regionEnd;
-          continue;
-        }
-        if (name === "code") {
-          masked += pushPlaceholder(placeholders, source.slice(i), sentinels);
-          i = source.length;
-          continue;
-        }
-      }
-    }
-
-    if (source[i] === "@" && source[i + 1] === "<") {
-      const close = source.indexOf(">@", i + 2);
-      const newline = source.indexOf("\n", i + 2);
-      if (close !== -1 && (newline === -1 || close < newline)) {
-        const regionEnd = close + 2;
-        masked += pushPlaceholder(placeholders, source.slice(i, regionEnd), sentinels);
-        i = regionEnd;
-        continue;
-      }
-    }
-
-    if (source[i] === "@" && source[i + 1] === "@") {
-      const close = source.indexOf("@@", i + 2);
-      const newline = source.indexOf("\n", i + 2);
-      if (close !== -1 && (newline === -1 || close < newline)) {
-        const regionEnd = close + 2;
-        masked += pushPlaceholder(placeholders, source.slice(i, regionEnd), sentinels);
-        i = regionEnd;
-        continue;
-      }
-    }
-
-    masked += source[i];
-    i++;
-  }
-
-  return { masked, placeholders };
-}
-
-function pushPlaceholder(placeholders: string[], text: string, sentinels: Sentinels): string {
-  const idx = placeholders.length;
-  placeholders.push(text);
-  return `${sentinels.open}${idx}${sentinels.close}`;
-}
-
-function escapeRegex(str: string): string {
-  return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-}
-
-/** Inverse of {@link maskRawRegions}: replace placeholders with originals. */
-export function restorePlaceholders(
-  source: string,
-  placeholders: string[],
-  sentinels: Sentinels,
-): string {
-  const pattern = new RegExp(
-    `${escapeRegex(sentinels.open)}(\\d+)${escapeRegex(sentinels.close)}`,
-    "g",
-  );
-  return source.replace(pattern, (_, idx: string) => placeholders[Number(idx)] ?? "");
-}
-
-/**
- * Compute the unmatched-`[[` depth at each character offset of `source`.
- * Mirrors the lexer's `blockOpenerDepth`. Returns `Int32Array` of length
- * `source.length + 1`; `depths[k]` is the depth immediately before the
- * character at offset `k` is consumed.
- */
-export function computeBracketDepths(source: string): Int32Array {
-  const n = source.length;
-  const depths = new Int32Array(n + 1);
-  let depth = 0;
-  let i = 0;
-  while (i < n) {
-    depths[i] = depth;
-    const c = source.charCodeAt(i);
-    const c1 = i + 1 < n ? source.charCodeAt(i + 1) : -1;
-    const c2 = i + 2 < n ? source.charCodeAt(i + 2) : -1;
-
-    if (depth > 0 && c === 0x22 /* " */ && precededByEqualsAttr(source, i)) {
-      const end = findQuoteEnd(source, i + 1);
-      for (let k = i; k <= end; k++) depths[k] = depth;
-      i = end + 1;
-      continue;
-    }
-
-    if (c === 0x5b /* [ */ && c1 === 0x5b && c2 === 0x5b) {
-      const end = findTripleLinkEnd(source, i + 3);
-      for (let k = i; k <= end; k++) depths[k] = depth;
-      i = end + 1;
-      continue;
-    }
-
-    if (c === 0x5b && c1 === 0x5b) {
-      depth++;
-      depths[i + 1] = depth;
-      i += 2;
-      continue;
-    }
-
-    if (c === 0x5d /* ] */ && c1 === 0x5d) {
-      depth = Math.max(0, depth - 1);
-      depths[i + 1] = depth;
-      i += 2;
-      continue;
-    }
-
-    if (c === 0x0a /* \n */) {
-      // Block openers are single-line; reset depth at line boundaries so
-      // an unterminated `[[xxx` does not keep subsequent directives
-      // inside its (imaginary) opener context.
-      depth = 0;
-    }
-
-    i++;
-  }
-  depths[n] = depth;
-  return depths;
-}
-
-function precededByEqualsAttr(s: string, i: number): boolean {
-  let j = i - 1;
-  while (j >= 0) {
-    const ch = s.charCodeAt(j);
-    if (ch === 0x20 /* space */ || ch === 0x09 /* tab */) {
-      j--;
-      continue;
-    }
-    return ch === 0x3d; /* = */
-  }
-  return false;
-}
-
-function findQuoteEnd(s: string, from: number): number {
-  for (let i = from; i < s.length; i++) {
-    const ch = s.charCodeAt(i);
-    if (ch === 0x22 /* " */ || ch === 0x0a /* \n */) return i;
-  }
-  return s.length - 1;
-}
-
-function findTripleLinkEnd(s: string, from: number): number {
-  for (let i = from; i < s.length; i++) {
-    if (
-      s.charCodeAt(i) === 0x5d &&
-      i + 2 < s.length &&
-      s.charCodeAt(i + 1) === 0x5d &&
-      s.charCodeAt(i + 2) === 0x5d
-    ) {
-      return i + 2;
-    }
-    if (s.charCodeAt(i) === 0x0a && i + 1 < s.length && s.charCodeAt(i + 1) === 0x0a) {
-      return i;
-    }
-  }
-  return s.length - 1;
-}

package/src/parser/preprocess/whitespace.ts

@@ -1,111 +0,0 @@
-/**
- *
- * Whitespace normalization preprocessing for Wikidot markup.
- *
- * This module ensures the lexer and parser receive input with consistent
- * whitespace conventions. It handles platform differences (DOS/Mac newlines),
- * normalizes exotic whitespace characters that users may paste from external
- * sources, and applies Wikidot-specific behaviors like backslash line continuation.
- *
- * Substitutions are applied in a deliberate order:
- * 1. Newline normalization (DOS `\r\n` and legacy Mac `\r` to Unix `\n`)
- * 2. Non-standard leading whitespace replacement (nbsp, figure space to regular space)
- * 3. Whitespace-only line stripping (collapse to empty lines)
- * 4. Backslash line continuation (`\\\n` to line-break marker U+E000)
- * 5. Tab expansion (tab to four spaces)
- * 6. Null character replacement (NUL to space)
- * 7. Leading/trailing newline removal
- *
- * @module
- */
-
-/**
- * Matches non-standard whitespace characters (non-breaking space U+00A0,
- * figure space U+2007) at the start of lines. These are replaced with
- * regular ASCII spaces so the parser's indentation logic works correctly.
- */
-const LEADING_NONSTANDARD_WHITESPACE = /^[\u00a0\u2007]+/gm;
-
-/** Matches lines containing only whitespace (collapsed to empty lines). */
-const WHITESPACE_ONLY_LINE = /^\s+$/gm;
-
-/** Matches one or more newlines at the very start of the text. */
-const LEADING_NEWLINES = /^\n+/;
-
-/** Matches one or more newlines at the very end of the text. */
-const TRAILING_NEWLINES = /\n+$/;
-
-/** Matches DOS (`\r\n`) and legacy Mac (`\r`) line endings. */
-const DOS_MAC_NEWLINES = /\r\n?/g;
-
-/**
- * Matches a backslash immediately followed by a newline.
- * In Wikidot, `\` at end of line acts as an explicit line break (`<br />`).
- */
-const CONCAT_LINES = /\\\n/g;
-
-/** Matches tab characters (expanded to four spaces). */
-const TABS = /\t/g;
-
-/** Matches null (NUL) characters (replaced with spaces). */
-const NULL_CHARS = /\0/g;
-
-/**
- * Replace non-standard whitespace characters at the start of each line
- * with the same number of regular ASCII spaces.
- *
- * This ensures indentation-sensitive constructs (like nested lists) work
- * correctly regardless of whether the user typed regular spaces, non-breaking
- * spaces, or figure spaces.
- *
- * @param text - Input text with potentially non-standard leading whitespace
- * @returns Text with leading non-standard whitespace replaced by ASCII spaces
- */
-function replaceLeadingSpaces(text: string): string {
-  return text.replace(LEADING_NONSTANDARD_WHITESPACE, (match) => {
-    return " ".repeat(match.length);
-  });
-}
-
-/**
- * Apply all whitespace normalization substitutions to the given text.
- *
- * Substitutions are applied in a specific order that avoids interference
- * between steps (e.g., DOS newlines must be normalized before backslash
- * continuation can be detected).
- *
- * The backslash continuation step converts `\\\n` to the Private Use Area
- * character U+E000, which the lexer later recognizes as an explicit line break.
- * This approach avoids ambiguity with other uses of the backslash character.
- *
- * @param text - Raw input text
- * @returns Text with normalized whitespace, ready for typography preprocessing
- */
-export function substitute(text: string): string {
-  let result = text;
-
-  // Replace DOS and Mac newlines
-  result = result.replace(DOS_MAC_NEWLINES, "\n");
-
-  // Replace leading non-standard spaces with regular spaces
-  result = replaceLeadingSpaces(result);
-
-  // Strip lines with only whitespace
-  result = result.replace(WHITESPACE_ONLY_LINE, "");
-
-  // Backslash at end of line → line break marker (U+E000)
-  // Wikidot treats \ at end of line as <br />
-  result = result.replace(CONCAT_LINES, String.fromCharCode(0xe000));
-
-  // Tabs to spaces
-  result = result.replace(TABS, "    ");
-
-  // Null characters to spaces
-  result = result.replace(NULL_CHARS, " ");
-
-  // Remove leading and trailing newlines
-  result = result.replace(LEADING_NEWLINES, "");
-  result = result.replace(TRAILING_NEWLINES, "");
-
-  return result;
-}