@wdprlib/parser 3.2.0 → 4.0.0

package/src/parser/preprocess/expr/parse.ts

@@ -0,0 +1,103 @@
+import { isWhitespace } from "./chars";
+import { matchDirectiveKind } from "./kind";
+import type { DirectiveKind, DirectiveMatch } from "./types";
+
+/**
+ * Try to parse a single `[[#kind ...]]` directive starting at `start`.
+ * Returns `null` when the directive is malformed (no closing `]]`) or
+ * when its body contains another `[[#kind]]` of the same family
+ * (so the caller should keep descending). The substrings are returned
+ * raw; callers decide how to evaluate them.
+ */
+export function tryParseInnermostDirective(
+  source: string,
+  start: number,
+  kind: DirectiveKind,
+): DirectiveMatch | null {
+  const keywordLen = kind === "ifexpr" ? 6 : kind === "expr" ? 4 : 2;
+  let pos = start + 3 + keywordLen;
+  while (pos < source.length && isWhitespace(source[pos])) pos++;
+
+  const headStart = pos;
+  let blockDepth = 0;
+  let linkDepth = 0;
+  const pipes: number[] = [];
+  let closeStart = -1;
+
+  while (pos < source.length) {
+    if (matchDirectiveKind(source, pos) !== null) {
+      return null;
+    }
+    if (source.startsWith("[[[", pos)) {
+      linkDepth++;
+      pos += 3;
+      continue;
+    }
+    if (linkDepth > 0 && source.startsWith("]]]", pos)) {
+      linkDepth--;
+      pos += 3;
+      continue;
+    }
+    if (linkDepth > 0) {
+      pos++;
+      continue;
+    }
+    if (source.startsWith("[[", pos)) {
+      blockDepth++;
+      pos += 2;
+      continue;
+    }
+    if (source.startsWith("]]", pos)) {
+      if (blockDepth === 0) {
+        closeStart = pos;
+        break;
+      }
+      blockDepth--;
+      pos += 2;
+      continue;
+    }
+    if (source[pos] === "|" && blockDepth === 0 && linkDepth === 0) {
+      pipes.push(pos);
+    }
+    pos++;
+  }
+
+  if (closeStart === -1) return null;
+  const hasPipe = pipes.length > 0;
+  if (!hasPipe && (kind === "if" || kind === "ifexpr")) return null;
+
+  return buildDirectiveMatch(source, headStart, closeStart, pipes, hasPipe);
+}
+
+function buildDirectiveMatch(
+  source: string,
+  headStart: number,
+  closeStart: number,
+  pipes: number[],
+  hasPipe: boolean,
+): DirectiveMatch {
+  if (!hasPipe) {
+    return {
+      end: closeStart + 2,
+      head: source.slice(headStart, closeStart).trim(),
+      thenText: "",
+      elseText: "",
+      hasPipe,
+    };
+  }
+
+  const head = source.slice(headStart, pipes[0]!).trim();
+  const thenText =
+    pipes.length >= 2
+      ? source.slice(pipes[0]! + 1, pipes[1]!).trim()
+      : source.slice(pipes[0]! + 1, closeStart).trim();
+  const elseText = pipes.length >= 2 ? source.slice(pipes[1]! + 1, closeStart).trim() : "";
+
+  return {
+    end: closeStart + 2,
+    head,
+    thenText,
+    elseText,
+    hasPipe,
+  };
+}

package/src/parser/preprocess/expr/scan.ts

@@ -0,0 +1,34 @@
+import { computeBracketDepths } from "../utils";
+import { evaluateDirective } from "./evaluate";
+import { matchDirectiveKind } from "./kind";
+import { tryParseInnermostDirective } from "./parse";
+
+/**
+ * Walk `source`, locate every innermost `[[#if]]` / `[[#ifexpr]]` /
+ * `[[#expr]]` directive that sits inside an unclosed `[[`, and replace
+ * it with its evaluated string. Returns the source unchanged when no
+ * replacements were made.
+ */
+export function expandInnermost(source: string): string {
+  const depths = computeBracketDepths(source);
+  let result = "";
+  let i = 0;
+  let replaced = false;
+
+  while (i < source.length) {
+    const kind = matchDirectiveKind(source, i);
+    if (kind !== null && depths[i]! > 0) {
+      const match = tryParseInnermostDirective(source, i, kind);
+      if (match !== null) {
+        result += evaluateDirective(kind, match);
+        i = match.end;
+        replaced = true;
+        continue;
+      }
+    }
+    result += source[i];
+    i++;
+  }
+
+  return replaced ? result : source;
+}

package/src/parser/preprocess/expr/types.ts

@@ -0,0 +1,14 @@
+export type DirectiveKind = "if" | "ifexpr" | "expr";
+
+export interface DirectiveMatch {
+  /** Position just past the closing `]]`. */
+  end: number;
+  /** Raw condition / expression. */
+  head: string;
+  /** Raw `then` branch. */
+  thenText: string;
+  /** Raw `else` branch. */
+  elseText: string;
+  /** Whether the directive supplied a `|` at all. */
+  hasPipe: boolean;
+}

package/src/parser/preprocess/typography.ts

@@ -32,6 +32,61 @@ const LOW_DOUBLE_QUOTE = "\u201e"; // „
 /** Unicode horizontal ellipsis (U+2026) */
 const ELLIPSIS = "\u2026"; // …
 
+function replaceExactEllipsisPattern(text: string, pattern: string): string {
+  let searchFrom = 0;
+  let result = "";
+  let lastCopied = 0;
+  const patternLength = pattern.length;
+
+  while (searchFrom < text.length) {
+    const index = text.indexOf(pattern, searchFrom);
+    if (index === -1) break;
+
+    const prev = index > 0 ? text[index - 1] : "";
+    const next = index + patternLength < text.length ? text[index + patternLength] : "";
+    if (prev !== "." && next !== ".") {
+      result += text.slice(lastCopied, index) + ELLIPSIS;
+      lastCopied = index + patternLength;
+      searchFrom = lastCopied;
+    } else {
+      searchFrom = index + 1;
+    }
+  }
+
+  return lastCopied === 0 ? text : result + text.slice(lastCopied);
+}
+
+function replaceDelimitedTypography(
+  text: string,
+  opener: string,
+  closer: string,
+  leftQuote: string,
+  rightQuote: string,
+): string {
+  let searchFrom = 0;
+  let result = "";
+  let lastCopied = 0;
+
+  while (searchFrom < text.length) {
+    const openIndex = text.indexOf(opener, searchFrom);
+    if (openIndex === -1) break;
+
+    const contentStart = openIndex + opener.length;
+    const closeIndex = text.indexOf(closer, contentStart);
+    if (closeIndex === -1) break;
+
+    result += text.slice(lastCopied, openIndex);
+    result += leftQuote;
+    result += text.slice(contentStart, closeIndex);
+    result += rightQuote;
+
+    lastCopied = closeIndex + closer.length;
+    searchFrom = lastCopied;
+  }
+
+  return lastCopied === 0 ? text : result + text.slice(lastCopied);
+}
+
 /**
  * Apply all typographic substitutions to the given text.
  *
@@ -47,21 +102,31 @@ export function substitute(text: string): string {
   let result = text;
 
   // Double quotes: ``...'' -> "..."
-  result = result.replace(/``(.*?)''/g, `${LEFT_DOUBLE_QUOTE}$1${RIGHT_DOUBLE_QUOTE}`);
+  if (result.includes("``") && result.includes("''")) {
+    result = replaceDelimitedTypography(result, "``", "''", LEFT_DOUBLE_QUOTE, RIGHT_DOUBLE_QUOTE);
+  }
 
   // Low double quotes: ,,..'' -> „..."
-  result = result.replace(/,,(.*?)''/g, `${LOW_DOUBLE_QUOTE}$1${RIGHT_DOUBLE_QUOTE}`);
+  if (result.includes(",,") && result.includes("''")) {
+    result = replaceDelimitedTypography(result, ",,", "''", LOW_DOUBLE_QUOTE, RIGHT_DOUBLE_QUOTE);
+  }
 
   // Single quotes: `...' -> '...'
-  result = result.replace(/`(.*?)'/g, `${LEFT_SINGLE_QUOTE}$1${RIGHT_SINGLE_QUOTE}`);
+  if (result.includes("`") && result.includes("'")) {
+    result = replaceDelimitedTypography(result, "`", "'", LEFT_SINGLE_QUOTE, RIGHT_SINGLE_QUOTE);
+  }
 
   // Ellipsis: ... or . . . -> …
   // Must be exactly 3 dots, not preceded or followed by more dots
   // Handle continuous dots: ...
-  result = result.replace(/(?<![.])\.\.\.(?![.])/g, ELLIPSIS);
+  if (result.includes("...")) {
+    result = replaceExactEllipsisPattern(result, "...");
+  }
 
   // Handle spaced dots: . . .
-  result = result.replace(/(?<![.])\. \. \.(?![.])/g, ELLIPSIS);
+  if (result.includes(". . .")) {
+    result = replaceExactEllipsisPattern(result, ". . .");
+  }
 
   return result;
 }

package/src/parser/preprocess/utils/bracket-depths.ts

@@ -0,0 +1,98 @@
+/**
+ * Compute the unmatched-`[[` depth at each character offset of `source`.
+ * Mirrors the lexer's `blockOpenerDepth`. Returns `Int32Array` of length
+ * `source.length + 1`; `depths[k]` is the depth immediately before the
+ * character at offset `k` is consumed.
+ */
+export function computeBracketDepths(source: string): Int32Array {
+  const n = source.length;
+  const depths = new Int32Array(n + 1);
+  let depth = 0;
+  let i = 0;
+
+  while (i < n) {
+    depths[i] = depth;
+    const c = source.charCodeAt(i);
+    const c1 = i + 1 < n ? source.charCodeAt(i + 1) : -1;
+    const c2 = i + 2 < n ? source.charCodeAt(i + 2) : -1;
+
+    if (depth > 0 && c === 0x22 /* " */ && precededByEqualsAttr(source, i)) {
+      const end = findQuoteEnd(source, i + 1);
+      for (let k = i; k <= end; k++) depths[k] = depth;
+      i = end + 1;
+      continue;
+    }
+
+    if (c === 0x5b /* [ */ && c1 === 0x5b && c2 === 0x5b) {
+      const end = findTripleLinkEnd(source, i + 3);
+      for (let k = i; k <= end; k++) depths[k] = depth;
+      i = end + 1;
+      continue;
+    }
+
+    if (c === 0x5b && c1 === 0x5b) {
+      depth++;
+      depths[i + 1] = depth;
+      i += 2;
+      continue;
+    }
+
+    if (c === 0x5d /* ] */ && c1 === 0x5d) {
+      depth = Math.max(0, depth - 1);
+      depths[i + 1] = depth;
+      i += 2;
+      continue;
+    }
+
+    if (c === 0x0a /* \n */) {
+      depth = 0;
+    }
+
+    i++;
+  }
+
+  depths[n] = depth;
+  return depths;
+}
+
+function precededByEqualsAttr(source: string, pos: number): boolean {
+  let j = pos - 1;
+  while (j >= 0) {
+    const ch = source.charCodeAt(j);
+    if (ch === 0x20 /* space */ || ch === 0x09 /* tab */) {
+      j--;
+      continue;
+    }
+    return ch === 0x3d; /* = */
+  }
+  return false;
+}
+
+function findQuoteEnd(source: string, from: number): number {
+  for (let i = from; i < source.length; i++) {
+    const ch = source.charCodeAt(i);
+    if (ch === 0x22 /* " */ || ch === 0x0a /* \n */) return i;
+  }
+  return source.length - 1;
+}
+
+function findTripleLinkEnd(source: string, from: number): number {
+  for (let i = from; i < source.length; i++) {
+    if (
+      source.charCodeAt(i) === 0x5d &&
+      i + 2 < source.length &&
+      source.charCodeAt(i + 1) === 0x5d &&
+      source.charCodeAt(i + 2) === 0x5d
+    ) {
+      return i + 2;
+    }
+    if (
+      source.charCodeAt(i) === 0x0a &&
+      i + 1 < source.length &&
+      source.charCodeAt(i + 1) === 0x0a
+    ) {
+      return i;
+    }
+  }
+  return source.length - 1;
+}

package/src/parser/preprocess/utils/index.ts

@@ -0,0 +1,13 @@
+/**
+ *
+ * Shared helper facade for source-level preprocess passes.
+ *
+ * Raw-region masking and bracket-depth tracking are implemented in separate
+ * files so preprocess passes can depend on a small stable import surface.
+ *
+ * @module
+ */
+
+export type { Sentinels } from "./raw-regions";
+export { makeUniqueSentinels, maskRawRegions, restorePlaceholders } from "./raw-regions";
+export { computeBracketDepths } from "./bracket-depths";

package/src/parser/preprocess/utils/raw-regions.ts

@@ -0,0 +1,153 @@
+const BASE_PLACEHOLDER_OPEN = "\uE000";
+const BASE_PLACEHOLDER_CLOSE = "\uE001";
+
+const RAW_BLOCK_OPEN_PATTERN = /\[\[\s*(code|html)\b[^\]]*\]\]/iy;
+
+/** Unique sentinel characters used to wrap raw-region placeholders. */
+export interface Sentinels {
+  open: string;
+  close: string;
+}
+
+/**
+ * Choose sentinel strings that are guaranteed not to appear in `source`.
+ * The placeholders we splice into the masked source have the form
+ * `<open><digits><close>`, so the restore pass must not confuse them
+ * with content. Extends both sentinel characters until neither appears.
+ */
+export function makeUniqueSentinels(source: string): Sentinels {
+  let open = BASE_PLACEHOLDER_OPEN;
+  let close = BASE_PLACEHOLDER_CLOSE;
+  while (source.includes(open) || source.includes(close)) {
+    open += BASE_PLACEHOLDER_OPEN;
+    close += BASE_PLACEHOLDER_CLOSE;
+  }
+  return { open, close };
+}
+
+/**
+ * Walk `source` and replace each raw region with a placeholder token so
+ * downstream passes do not transform their bodies.
+ */
+export function maskRawRegions(
+  source: string,
+  sentinels: Sentinels,
+): { masked: string; placeholders: string[] } {
+  const placeholders: string[] = [];
+  let masked = "";
+  let i = 0;
+
+  while (i < source.length) {
+    const rawBlock = tryMaskRawBlock(source, i, placeholders, sentinels);
+    if (rawBlock) {
+      masked += rawBlock.placeholder;
+      i = rawBlock.end;
+      continue;
+    }
+
+    const rawInline = tryMaskRawInline(source, i, placeholders, sentinels);
+    if (rawInline) {
+      masked += rawInline.placeholder;
+      i = rawInline.end;
+      continue;
+    }
+
+    masked += source[i];
+    i++;
+  }
+
+  return { masked, placeholders };
+}
+
+/** Inverse of {@link maskRawRegions}: replace placeholders with originals. */
+export function restorePlaceholders(
+  source: string,
+  placeholders: string[],
+  sentinels: Sentinels,
+): string {
+  const pattern = new RegExp(
+    `${escapeRegex(sentinels.open)}(\\d+)${escapeRegex(sentinels.close)}`,
+    "g",
+  );
+  return source.replace(pattern, (_, idx: string) => placeholders[Number(idx)] ?? "");
+}
+
+function tryMaskRawBlock(
+  source: string,
+  pos: number,
+  placeholders: string[],
+  sentinels: Sentinels,
+): { placeholder: string; end: number } | null {
+  if (source[pos] !== "[" || source[pos + 1] !== "[") return null;
+
+  RAW_BLOCK_OPEN_PATTERN.lastIndex = pos;
+  const openMatch = RAW_BLOCK_OPEN_PATTERN.exec(source);
+  if (!openMatch) return null;
+
+  const name = openMatch[1]!.toLowerCase();
+  const openLen = openMatch[0].length;
+  const closePattern = new RegExp(`\\[\\[\\/\\s*${name}\\s*\\]\\]`, "ig");
+  closePattern.lastIndex = pos + openLen;
+  const closeMatch = closePattern.exec(source);
+
+  if (closeMatch) {
+    const end = closeMatch.index + closeMatch[0].length;
+    return {
+      placeholder: pushPlaceholder(placeholders, source.slice(pos, end), sentinels),
+      end,
+    };
+  }
+
+  if (name !== "code") return null;
+
+  return {
+    placeholder: pushPlaceholder(placeholders, source.slice(pos), sentinels),
+    end: source.length,
+  };
+}
+
+function tryMaskRawInline(
+  source: string,
+  pos: number,
+  placeholders: string[],
+  sentinels: Sentinels,
+): { placeholder: string; end: number } | null {
+  if (source[pos] === "@" && source[pos + 1] === "<") {
+    return tryMaskSingleLineRaw(source, pos, 2, ">@", placeholders, sentinels);
+  }
+
+  if (source[pos] === "@" && source[pos + 1] === "@") {
+    return tryMaskSingleLineRaw(source, pos, 2, "@@", placeholders, sentinels);
+  }
+
+  return null;
+}
+
+function tryMaskSingleLineRaw(
+  source: string,
+  pos: number,
+  openerLength: number,
+  close: string,
+  placeholders: string[],
+  sentinels: Sentinels,
+): { placeholder: string; end: number } | null {
+  const closePos = source.indexOf(close, pos + openerLength);
+  const newline = source.indexOf("\n", pos + openerLength);
+  if (closePos === -1 || (newline !== -1 && newline < closePos)) return null;
+
+  const end = closePos + close.length;
+  return {
+    placeholder: pushPlaceholder(placeholders, source.slice(pos, end), sentinels),
+    end,
+  };
+}
+
+function pushPlaceholder(placeholders: string[], text: string, sentinels: Sentinels): string {
+  const idx = placeholders.length;
+  placeholders.push(text);
+  return `${sentinels.open}${idx}${sentinels.close}`;
+}
+
+function escapeRegex(str: string): string {
+  return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}

package/src/parser/preprocess/whitespace/detection.ts

@@ -0,0 +1,39 @@
+export function needsWhitespaceSubstitution(text: string): boolean {
+  if (text.length === 0) return false;
+
+  if (text[0] === "\n" || text[0] === " " || text[text.length - 1] === "\n") {
+    return true;
+  }
+
+  return (
+    text.indexOf("\r") !== -1 ||
+    text.indexOf("\t") !== -1 ||
+    text.indexOf("\0") !== -1 ||
+    text.indexOf("\u00a0") !== -1 ||
+    text.indexOf("\u2007") !== -1 ||
+    text.indexOf("\\\n") !== -1 ||
+    text.indexOf("\n\n\n") !== -1 ||
+    text.indexOf("\n ") !== -1
+  );
+}
+
+export function mayContainWhitespaceOnlyLine(text: string): boolean {
+  const first = text[0];
+  if (
+    first === " " ||
+    first === "\t" ||
+    first === "\n" ||
+    first === "\u00a0" ||
+    first === "\u2007"
+  ) {
+    return true;
+  }
+
+  return (
+    text.indexOf("\n ") !== -1 ||
+    text.indexOf("\n\n") !== -1 ||
+    text.indexOf("\n\t") !== -1 ||
+    text.indexOf("\n\u00a0") !== -1 ||
+    text.indexOf("\n\u2007") !== -1
+  );
+}

package/src/parser/preprocess/whitespace/index.ts

@@ -0,0 +1,79 @@
+/**
+ *
+ * Whitespace normalization preprocessing for Wikidot markup.
+ *
+ * This module ensures the lexer and parser receive input with consistent
+ * whitespace conventions. It handles platform differences (DOS/Mac newlines),
+ * normalizes exotic whitespace characters that users may paste from external
+ * sources, and applies Wikidot-specific behaviors like backslash line continuation.
+ *
+ * @module
+ */
+
+import { needsWhitespaceSubstitution, mayContainWhitespaceOnlyLine } from "./detection";
+import { replaceLeadingSpaces } from "./leading-spaces";
+import { CONCAT_LINES, DOS_MAC_NEWLINES, NULL_CHARS, TABS, WHITESPACE_ONLY_LINE } from "./patterns";
+
+/**
+ * Apply all whitespace normalization substitutions to the given text.
+ *
+ * Substitutions are applied in a specific order that avoids interference
+ * between steps (e.g., DOS newlines must be normalized before backslash
+ * continuation can be detected).
+ */
+export function substitute(text: string): string {
+  if (!needsWhitespaceSubstitution(text)) {
+    return text;
+  }
+
+  let result = text;
+
+  if (result.indexOf("\r") !== -1) {
+    result = result.replace(DOS_MAC_NEWLINES, "\n");
+  }
+
+  if (result.indexOf("\u00a0") !== -1 || result.indexOf("\u2007") !== -1) {
+    result = replaceLeadingSpaces(result);
+  }
+
+  if (mayContainWhitespaceOnlyLine(result)) {
+    result = result.replace(WHITESPACE_ONLY_LINE, "");
+  }
+
+  if (result.indexOf("\\\n") !== -1) {
+    result = result.replace(CONCAT_LINES, String.fromCharCode(0xe000));
+  }
+
+  if (result.indexOf("\t") !== -1) {
+    result = result.replace(TABS, "    ");
+  }
+
+  if (result.indexOf("\0") !== -1) {
+    result = result.replace(NULL_CHARS, " ");
+  }
+
+  if (result[0] === "\n") {
+    result = trimLeadingNewlines(result);
+  }
+  if (result[result.length - 1] === "\n") {
+    result = trimTrailingNewlines(result);
+  }
+
+  return result;
+}
+
+function trimLeadingNewlines(text: string): string {
+  let index = 0;
+  while (text[index] === "\n") {
+    index++;
+  }
+  return index === 0 ? text : text.slice(index);
+}
+
+function trimTrailingNewlines(text: string): string {
+  let end = text.length;
+  while (end > 0 && text[end - 1] === "\n") {
+    end--;
+  }
+  return end === text.length ? text : text.slice(0, end);
+}

package/src/parser/preprocess/whitespace/leading-spaces.ts

@@ -0,0 +1,11 @@
+import { LEADING_NONSTANDARD_WHITESPACE } from "./patterns";
+
+/**
+ * Replace non-standard whitespace characters at the start of each line
+ * with the same number of regular ASCII spaces.
+ */
+export function replaceLeadingSpaces(text: string): string {
+  return text.replace(LEADING_NONSTANDARD_WHITESPACE, (match) => {
+    return " ".repeat(match.length);
+  });
+}

package/src/parser/preprocess/whitespace/patterns.ts

@@ -0,0 +1,23 @@
+/**
+ * Matches non-standard whitespace characters (non-breaking space U+00A0,
+ * figure space U+2007) at the start of lines.
+ */
+export const LEADING_NONSTANDARD_WHITESPACE: RegExp = /^[\u00a0\u2007]+/gm;
+
+/** Matches lines containing only whitespace (collapsed to empty lines). */
+export const WHITESPACE_ONLY_LINE: RegExp = /^\s+$/gm;
+
+/** Matches DOS (`\r\n`) and legacy Mac (`\r`) line endings. */
+export const DOS_MAC_NEWLINES: RegExp = /\r\n?/g;
+
+/**
+ * Matches a backslash immediately followed by a newline.
+ * In Wikidot, `\` at end of line acts as an explicit line break (`<br />`).
+ */
+export const CONCAT_LINES: RegExp = /\\\n/g;
+
+/** Matches tab characters (expanded to four spaces). */
+export const TABS: RegExp = /\t/g;
+
+/** Matches null (NUL) characters (replaced with spaces). */
+export const NULL_CHARS: RegExp = new RegExp(String.fromCharCode(0), "g");