@wdprlib/parser 3.1.2 → 4.0.0

package/src/parser/preprocess/utils/bracket-depths.ts

@@ -0,0 +1,98 @@
+/**
+ * Compute the unmatched-`[[` depth at each character offset of `source`.
+ * Mirrors the lexer's `blockOpenerDepth`. Returns `Int32Array` of length
+ * `source.length + 1`; `depths[k]` is the depth immediately before the
+ * character at offset `k` is consumed.
+ */
+export function computeBracketDepths(source: string): Int32Array {
+  const n = source.length;
+  const depths = new Int32Array(n + 1);
+  let depth = 0;
+  let i = 0;
+
+  while (i < n) {
+    depths[i] = depth;
+    const c = source.charCodeAt(i);
+    const c1 = i + 1 < n ? source.charCodeAt(i + 1) : -1;
+    const c2 = i + 2 < n ? source.charCodeAt(i + 2) : -1;
+
+    if (depth > 0 && c === 0x22 /* " */ && precededByEqualsAttr(source, i)) {
+      const end = findQuoteEnd(source, i + 1);
+      for (let k = i; k <= end; k++) depths[k] = depth;
+      i = end + 1;
+      continue;
+    }
+
+    if (c === 0x5b /* [ */ && c1 === 0x5b && c2 === 0x5b) {
+      const end = findTripleLinkEnd(source, i + 3);
+      for (let k = i; k <= end; k++) depths[k] = depth;
+      i = end + 1;
+      continue;
+    }
+
+    if (c === 0x5b && c1 === 0x5b) {
+      depth++;
+      depths[i + 1] = depth;
+      i += 2;
+      continue;
+    }
+
+    if (c === 0x5d /* ] */ && c1 === 0x5d) {
+      depth = Math.max(0, depth - 1);
+      depths[i + 1] = depth;
+      i += 2;
+      continue;
+    }
+
+    if (c === 0x0a /* \n */) {
+      depth = 0;
+    }
+
+    i++;
+  }
+
+  depths[n] = depth;
+  return depths;
+}
+
+function precededByEqualsAttr(source: string, pos: number): boolean {
+  let j = pos - 1;
+  while (j >= 0) {
+    const ch = source.charCodeAt(j);
+    if (ch === 0x20 /* space */ || ch === 0x09 /* tab */) {
+      j--;
+      continue;
+    }
+    return ch === 0x3d; /* = */
+  }
+  return false;
+}
+
+function findQuoteEnd(source: string, from: number): number {
+  for (let i = from; i < source.length; i++) {
+    const ch = source.charCodeAt(i);
+    if (ch === 0x22 /* " */ || ch === 0x0a /* \n */) return i;
+  }
+  return source.length - 1;
+}
+
+function findTripleLinkEnd(source: string, from: number): number {
+  for (let i = from; i < source.length; i++) {
+    if (
+      source.charCodeAt(i) === 0x5d &&
+      i + 2 < source.length &&
+      source.charCodeAt(i + 1) === 0x5d &&
+      source.charCodeAt(i + 2) === 0x5d
+    ) {
+      return i + 2;
+    }
+    if (
+      source.charCodeAt(i) === 0x0a &&
+      i + 1 < source.length &&
+      source.charCodeAt(i + 1) === 0x0a
+    ) {
+      return i;
+    }
+  }
+  return source.length - 1;
+}

package/src/parser/preprocess/utils/index.ts

@@ -0,0 +1,13 @@
+/**
+ *
+ * Shared helper facade for source-level preprocess passes.
+ *
+ * Raw-region masking and bracket-depth tracking are implemented in separate
+ * files so preprocess passes can depend on a small stable import surface.
+ *
+ * @module
+ */
+
+export type { Sentinels } from "./raw-regions";
+export { makeUniqueSentinels, maskRawRegions, restorePlaceholders } from "./raw-regions";
+export { computeBracketDepths } from "./bracket-depths";

package/src/parser/preprocess/utils/raw-regions.ts

@@ -0,0 +1,153 @@
+const BASE_PLACEHOLDER_OPEN = "\uE000";
+const BASE_PLACEHOLDER_CLOSE = "\uE001";
+
+const RAW_BLOCK_OPEN_PATTERN = /\[\[\s*(code|html)\b[^\]]*\]\]/iy;
+
+/** Unique sentinel characters used to wrap raw-region placeholders. */
+export interface Sentinels {
+  open: string;
+  close: string;
+}
+
+/**
+ * Choose sentinel strings that are guaranteed not to appear in `source`.
+ * The placeholders we splice into the masked source have the form
+ * `<open><digits><close>`, so the restore pass must not confuse them
+ * with content. Extends both sentinel characters until neither appears.
+ */
+export function makeUniqueSentinels(source: string): Sentinels {
+  let open = BASE_PLACEHOLDER_OPEN;
+  let close = BASE_PLACEHOLDER_CLOSE;
+  while (source.includes(open) || source.includes(close)) {
+    open += BASE_PLACEHOLDER_OPEN;
+    close += BASE_PLACEHOLDER_CLOSE;
+  }
+  return { open, close };
+}
+
+/**
+ * Walk `source` and replace each raw region with a placeholder token so
+ * downstream passes do not transform their bodies.
+ */
+export function maskRawRegions(
+  source: string,
+  sentinels: Sentinels,
+): { masked: string; placeholders: string[] } {
+  const placeholders: string[] = [];
+  let masked = "";
+  let i = 0;
+
+  while (i < source.length) {
+    const rawBlock = tryMaskRawBlock(source, i, placeholders, sentinels);
+    if (rawBlock) {
+      masked += rawBlock.placeholder;
+      i = rawBlock.end;
+      continue;
+    }
+
+    const rawInline = tryMaskRawInline(source, i, placeholders, sentinels);
+    if (rawInline) {
+      masked += rawInline.placeholder;
+      i = rawInline.end;
+      continue;
+    }
+
+    masked += source[i];
+    i++;
+  }
+
+  return { masked, placeholders };
+}
+
+/** Inverse of {@link maskRawRegions}: replace placeholders with originals. */
+export function restorePlaceholders(
+  source: string,
+  placeholders: string[],
+  sentinels: Sentinels,
+): string {
+  const pattern = new RegExp(
+    `${escapeRegex(sentinels.open)}(\\d+)${escapeRegex(sentinels.close)}`,
+    "g",
+  );
+  return source.replace(pattern, (_, idx: string) => placeholders[Number(idx)] ?? "");
+}
+
+function tryMaskRawBlock(
+  source: string,
+  pos: number,
+  placeholders: string[],
+  sentinels: Sentinels,
+): { placeholder: string; end: number } | null {
+  if (source[pos] !== "[" || source[pos + 1] !== "[") return null;
+
+  RAW_BLOCK_OPEN_PATTERN.lastIndex = pos;
+  const openMatch = RAW_BLOCK_OPEN_PATTERN.exec(source);
+  if (!openMatch) return null;
+
+  const name = openMatch[1]!.toLowerCase();
+  const openLen = openMatch[0].length;
+  const closePattern = new RegExp(`\\[\\[\\/\\s*${name}\\s*\\]\\]`, "ig");
+  closePattern.lastIndex = pos + openLen;
+  const closeMatch = closePattern.exec(source);
+
+  if (closeMatch) {
+    const end = closeMatch.index + closeMatch[0].length;
+    return {
+      placeholder: pushPlaceholder(placeholders, source.slice(pos, end), sentinels),
+      end,
+    };
+  }
+
+  if (name !== "code") return null;
+
+  return {
+    placeholder: pushPlaceholder(placeholders, source.slice(pos), sentinels),
+    end: source.length,
+  };
+}
+
+function tryMaskRawInline(
+  source: string,
+  pos: number,
+  placeholders: string[],
+  sentinels: Sentinels,
+): { placeholder: string; end: number } | null {
+  if (source[pos] === "@" && source[pos + 1] === "<") {
+    return tryMaskSingleLineRaw(source, pos, 2, ">@", placeholders, sentinels);
+  }
+
+  if (source[pos] === "@" && source[pos + 1] === "@") {
+    return tryMaskSingleLineRaw(source, pos, 2, "@@", placeholders, sentinels);
+  }
+
+  return null;
+}
+
+function tryMaskSingleLineRaw(
+  source: string,
+  pos: number,
+  openerLength: number,
+  close: string,
+  placeholders: string[],
+  sentinels: Sentinels,
+): { placeholder: string; end: number } | null {
+  const closePos = source.indexOf(close, pos + openerLength);
+  const newline = source.indexOf("\n", pos + openerLength);
+  if (closePos === -1 || (newline !== -1 && newline < closePos)) return null;
+
+  const end = closePos + close.length;
+  return {
+    placeholder: pushPlaceholder(placeholders, source.slice(pos, end), sentinels),
+    end,
+  };
+}
+
+function pushPlaceholder(placeholders: string[], text: string, sentinels: Sentinels): string {
+  const idx = placeholders.length;
+  placeholders.push(text);
+  return `${sentinels.open}${idx}${sentinels.close}`;
+}
+
+function escapeRegex(str: string): string {
+  return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}

package/src/parser/preprocess/whitespace/detection.ts

@@ -0,0 +1,39 @@
+export function needsWhitespaceSubstitution(text: string): boolean {
+  if (text.length === 0) return false;
+
+  if (text[0] === "\n" || text[0] === " " || text[text.length - 1] === "\n") {
+    return true;
+  }
+
+  return (
+    text.indexOf("\r") !== -1 ||
+    text.indexOf("\t") !== -1 ||
+    text.indexOf("\0") !== -1 ||
+    text.indexOf("\u00a0") !== -1 ||
+    text.indexOf("\u2007") !== -1 ||
+    text.indexOf("\\\n") !== -1 ||
+    text.indexOf("\n\n\n") !== -1 ||
+    text.indexOf("\n ") !== -1
+  );
+}
+
+export function mayContainWhitespaceOnlyLine(text: string): boolean {
+  const first = text[0];
+  if (
+    first === " " ||
+    first === "\t" ||
+    first === "\n" ||
+    first === "\u00a0" ||
+    first === "\u2007"
+  ) {
+    return true;
+  }
+
+  return (
+    text.indexOf("\n ") !== -1 ||
+    text.indexOf("\n\n") !== -1 ||
+    text.indexOf("\n\t") !== -1 ||
+    text.indexOf("\n\u00a0") !== -1 ||
+    text.indexOf("\n\u2007") !== -1
+  );
+}

package/src/parser/preprocess/whitespace/index.ts

@@ -0,0 +1,79 @@
+/**
+ *
+ * Whitespace normalization preprocessing for Wikidot markup.
+ *
+ * This module ensures the lexer and parser receive input with consistent
+ * whitespace conventions. It handles platform differences (DOS/Mac newlines),
+ * normalizes exotic whitespace characters that users may paste from external
+ * sources, and applies Wikidot-specific behaviors like backslash line continuation.
+ *
+ * @module
+ */
+
+import { needsWhitespaceSubstitution, mayContainWhitespaceOnlyLine } from "./detection";
+import { replaceLeadingSpaces } from "./leading-spaces";
+import { CONCAT_LINES, DOS_MAC_NEWLINES, NULL_CHARS, TABS, WHITESPACE_ONLY_LINE } from "./patterns";
+
+/**
+ * Apply all whitespace normalization substitutions to the given text.
+ *
+ * Substitutions are applied in a specific order that avoids interference
+ * between steps (e.g., DOS newlines must be normalized before backslash
+ * continuation can be detected).
+ */
+export function substitute(text: string): string {
+  if (!needsWhitespaceSubstitution(text)) {
+    return text;
+  }
+
+  let result = text;
+
+  if (result.indexOf("\r") !== -1) {
+    result = result.replace(DOS_MAC_NEWLINES, "\n");
+  }
+
+  if (result.indexOf("\u00a0") !== -1 || result.indexOf("\u2007") !== -1) {
+    result = replaceLeadingSpaces(result);
+  }
+
+  if (mayContainWhitespaceOnlyLine(result)) {
+    result = result.replace(WHITESPACE_ONLY_LINE, "");
+  }
+
+  if (result.indexOf("\\\n") !== -1) {
+    result = result.replace(CONCAT_LINES, String.fromCharCode(0xe000));
+  }
+
+  if (result.indexOf("\t") !== -1) {
+    result = result.replace(TABS, "    ");
+  }
+
+  if (result.indexOf("\0") !== -1) {
+    result = result.replace(NULL_CHARS, " ");
+  }
+
+  if (result[0] === "\n") {
+    result = trimLeadingNewlines(result);
+  }
+  if (result[result.length - 1] === "\n") {
+    result = trimTrailingNewlines(result);
+  }
+
+  return result;
+}
+
+function trimLeadingNewlines(text: string): string {
+  let index = 0;
+  while (text[index] === "\n") {
+    index++;
+  }
+  return index === 0 ? text : text.slice(index);
+}
+
+function trimTrailingNewlines(text: string): string {
+  let end = text.length;
+  while (end > 0 && text[end - 1] === "\n") {
+    end--;
+  }
+  return end === text.length ? text : text.slice(0, end);
+}

package/src/parser/preprocess/whitespace/leading-spaces.ts

@@ -0,0 +1,11 @@
+import { LEADING_NONSTANDARD_WHITESPACE } from "./patterns";
+
+/**
+ * Replace non-standard whitespace characters at the start of each line
+ * with the same number of regular ASCII spaces.
+ */
+export function replaceLeadingSpaces(text: string): string {
+  return text.replace(LEADING_NONSTANDARD_WHITESPACE, (match) => {
+    return " ".repeat(match.length);
+  });
+}

package/src/parser/preprocess/whitespace/patterns.ts

@@ -0,0 +1,23 @@
+/**
+ * Matches non-standard whitespace characters (non-breaking space U+00A0,
+ * figure space U+2007) at the start of lines.
+ */
+export const LEADING_NONSTANDARD_WHITESPACE: RegExp = /^[\u00a0\u2007]+/gm;
+
+/** Matches lines containing only whitespace (collapsed to empty lines). */
+export const WHITESPACE_ONLY_LINE: RegExp = /^\s+$/gm;
+
+/** Matches DOS (`\r\n`) and legacy Mac (`\r`) line endings. */
+export const DOS_MAC_NEWLINES: RegExp = /\r\n?/g;
+
+/**
+ * Matches a backslash immediately followed by a newline.
+ * In Wikidot, `\` at end of line acts as an explicit line break (`<br />`).
+ */
+export const CONCAT_LINES: RegExp = /\\\n/g;
+
+/** Matches tab characters (expanded to four spaces). */
+export const TABS: RegExp = /\t/g;
+
+/** Matches null (NUL) characters (replaced with spaces). */
+export const NULL_CHARS: RegExp = new RegExp(String.fromCharCode(0), "g");

package/src/parser/rules/block/align/body.ts

@@ -0,0 +1,46 @@
+import type { Element } from "@wdprlib/ast";
+import type { ParseContext } from "../../types";
+import { parseBlocksUntil } from "../utils";
+import { type AlignDirection, parseAlignClose } from "./syntax";
+
+export interface AlignBodyResult {
+  elements: Element[];
+  consumed: number;
+  foundClose: boolean;
+}
+
+export function parseAlignBody(
+  ctx: ParseContext,
+  startPos: number,
+  direction: AlignDirection,
+): AlignBodyResult {
+  const bodyCtx: ParseContext = { ...ctx, pos: startPos };
+  const bodyResult = parseBlocksUntil(bodyCtx, (checkCtx) => {
+    return parseAlignClose(checkCtx, direction).match;
+  });
+
+  let pos = startPos + bodyResult.consumed;
+  let consumed = bodyResult.consumed;
+
+  const closeCheck = parseAlignClose({ ...ctx, pos }, direction);
+  if (!closeCheck.match) {
+    return {
+      elements: bodyResult.elements,
+      consumed,
+      foundClose: false,
+    };
+  }
+
+  consumed += closeCheck.consumed;
+  pos += closeCheck.consumed;
+
+  if (ctx.tokens[pos]?.type === "NEWLINE") {
+    consumed++;
+  }
+
+  return {
+    elements: bodyResult.elements,
+    consumed,
+    foundClose: true,
+  };
+}

package/src/parser/rules/block/align/element.ts

@@ -0,0 +1,13 @@
+import type { Element } from "@wdprlib/ast";
+import type { AlignDirection } from "./syntax";
+
+export function createAlignElement(direction: AlignDirection, elements: Element[]): Element {
+  return {
+    element: "container",
+    data: {
+      type: { align: direction },
+      attributes: {},
+      elements,
+    },
+  };
+}

package/src/parser/rules/block/align/index.ts

@@ -0,0 +1,90 @@
+/**
+ *
+ * Block rule for Wikidot alignment containers.
+ *
+ * Wikidot provides a shorthand bracket syntax for wrapping content in a
+ * directional alignment container:
+ *
+ * ```
+ * [[>]]        ... [[/>]]        right-aligned
+ * [[<]]        ... [[/<]]        left-aligned
+ * [[=]]        ... [[/=]]        center-aligned
+ * [[==]]       ... [[/==]]       justify-aligned
+ * ```
+ *
+ * Each pair acts as a block-level wrapper. The opening tag must appear at
+ * the start of a line and be followed by a newline. Body content is parsed
+ * recursively as block-level markup, and the matching closing tag terminates
+ * the container.
+ *
+ * @module
+ */
+import type { Element } from "@wdprlib/ast";
+import type { BlockRule, ParseContext, RuleResult } from "../../types";
+import { currentToken } from "../../types";
+import { parseAlignBody } from "./body";
+import { createAlignElement } from "./element";
+import { alignDirectionSymbol, parseAlignOpen } from "./syntax";
+
+/**
+ * Block rule that matches Wikidot directional alignment containers.
+ *
+ * `preservesPrecedingLineBreak` is `true` because, unlike most block
+ * constructs, an alignment block does not suppress a preceding `\n` from
+ * becoming a `<br />` in Wikidot's output.
+ */
+export const alignRule: BlockRule = {
+  name: "align",
+  startTokens: ["BLOCK_OPEN"],
+  requiresLineStart: true,
+  preservesPrecedingLineBreak: true,
+
+  isStartPattern(ctx: ParseContext, pos: number): boolean {
+    if (ctx.tokens[pos]?.type !== "BLOCK_OPEN") return false;
+    return parseAlignOpen(ctx, pos + 1) !== null;
+  },
+
+  parse(ctx: ParseContext): RuleResult<Element> {
+    const openToken = currentToken(ctx);
+    if (openToken.type !== "BLOCK_OPEN") {
+      return { success: false };
+    }
+
+    let pos = ctx.pos + 1;
+    let consumed = 1;
+
+    const alignResult = parseAlignOpen(ctx, pos);
+    if (!alignResult) {
+      return { success: false };
+    }
+
+    const { direction } = alignResult;
+    pos += alignResult.consumed;
+    consumed += alignResult.consumed;
+
+    if (ctx.tokens[pos]?.type !== "NEWLINE") {
+      return { success: false };
+    }
+    pos++;
+    consumed++;
+
+    const bodyResult = parseAlignBody(ctx, pos, direction);
+    consumed += bodyResult.consumed;
+
+    if (!bodyResult.foundClose) {
+      const directionSymbol = alignDirectionSymbol(direction);
+      ctx.diagnostics.push({
+        severity: "warning",
+        code: "unclosed-block",
+        message: `Missing closing tag [[/${directionSymbol}]] for [[${directionSymbol}]]`,
+        position: openToken.position,
+      });
+    }
+
+    return {
+      success: true,
+      elements: [createAlignElement(direction, bodyResult.elements)],
+      consumed,
+    };
+  },
+};

package/src/parser/rules/block/align/syntax.ts

@@ -0,0 +1,113 @@
+import type { ParseContext } from "../../types";
+
+/** The four text-alignment directions Wikidot supports. */
+export type AlignDirection = "left" | "right" | "center" | "justify";
+
+/**
+ * Attempts to parse the interior of an align opening tag starting after
+ * the BLOCK_OPEN (`[[`) token.
+ */
+export function parseAlignOpen(
+  ctx: ParseContext,
+  pos: number,
+): { direction: AlignDirection; consumed: number } | null {
+  const tokens = ctx.tokens;
+  const firstToken = tokens[pos];
+  if (!firstToken) return null;
+
+  if (
+    firstToken.type === "BLOCKQUOTE_MARKER" &&
+    firstToken.value === ">" &&
+    tokens[pos + 1]?.type === "BLOCK_CLOSE"
+  ) {
+    return { direction: "right", consumed: 2 };
+  }
+
+  if (
+    firstToken.type === "TEXT" &&
+    firstToken.value === ">" &&
+    tokens[pos + 1]?.type === "BLOCK_CLOSE"
+  ) {
+    return { direction: "right", consumed: 2 };
+  }
+
+  if (
+    firstToken.type === "TEXT" &&
+    firstToken.value === "<" &&
+    tokens[pos + 1]?.type === "BLOCK_CLOSE"
+  ) {
+    return { direction: "left", consumed: 2 };
+  }
+
+  if (firstToken.type === "EQUALS" && tokens[pos + 1]?.type === "BLOCK_CLOSE") {
+    return { direction: "center", consumed: 2 };
+  }
+
+  if (
+    firstToken.type === "EQUALS" &&
+    tokens[pos + 1]?.type === "EQUALS" &&
+    tokens[pos + 2]?.type === "BLOCK_CLOSE"
+  ) {
+    return { direction: "justify", consumed: 3 };
+  }
+
+  return null;
+}
+
+/**
+ * Tests whether the tokens at the current position form a closing align tag.
+ */
+export function parseAlignClose(
+  ctx: ParseContext,
+  direction: AlignDirection,
+): { match: boolean; consumed: number } {
+  const tokens = ctx.tokens;
+  let pos = ctx.pos;
+
+  if (tokens[pos]?.type !== "BLOCK_END_OPEN") {
+    return { match: false, consumed: 0 };
+  }
+  pos++;
+
+  if (direction === "right") {
+    if (
+      (tokens[pos]?.type === "BLOCKQUOTE_MARKER" || tokens[pos]?.type === "TEXT") &&
+      tokens[pos]?.value === ">" &&
+      tokens[pos + 1]?.type === "BLOCK_CLOSE"
+    ) {
+      return { match: true, consumed: 3 };
+    }
+  }
+
+  if (direction === "left") {
+    if (
+      tokens[pos]?.type === "TEXT" &&
+      tokens[pos]?.value === "<" &&
+      tokens[pos + 1]?.type === "BLOCK_CLOSE"
+    ) {
+      return { match: true, consumed: 3 };
+    }
+  }
+
+  if (direction === "center") {
+    if (tokens[pos]?.type === "EQUALS" && tokens[pos + 1]?.type === "BLOCK_CLOSE") {
+      return { match: true, consumed: 3 };
+    }
+  }
+
+  if (direction === "justify") {
+    if (
+      tokens[pos]?.type === "EQUALS" &&
+      tokens[pos + 1]?.type === "EQUALS" &&
+      tokens[pos + 2]?.type === "BLOCK_CLOSE"
+    ) {
+      return { match: true, consumed: 4 };
+    }
+  }
+
+  return { match: false, consumed: 0 };
+}
+
+export function alignDirectionSymbol(direction: AlignDirection): string {
+  return { left: "<", right: ">", center: "=", justify: "==" }[direction];
+}