@wdprlib/parser 3.1.2 → 4.0.0

package/src/parser/parse/plain-non-ascii.ts

@@ -0,0 +1,129 @@
+import type { Element, ParseResult } from "@wdprlib/ast";
+
+const LARGE_PLAIN_TEXT_MIN_LENGTH = 100_000;
+
+export function parsePlainNonAsciiDocument(source: string): ParseResult | null {
+  if (!isPlainNonAsciiDocument(source)) {
+    return null;
+  }
+
+  const text = trimNewlineEdges(source);
+  const elements = text === "" ? [] : buildPlainParagraphs(text);
+
+  elements.push({
+    element: "footnote-block",
+    data: { title: null, hide: false },
+  });
+
+  return { ast: { elements }, diagnostics: [] };
+}
+
+function isPlainNonAsciiDocument(source: string): boolean {
+  for (let i = 0; i < source.length; i++) {
+    const code = source.charCodeAt(i);
+    if ((code >= 0x00 && code <= 0x09) || (code >= 0x0b && code <= 0x7f) || code === 0xe000) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+export function parseLargePlainTextDocument(source: string): ParseResult | null {
+  if (source.length < LARGE_PLAIN_TEXT_MIN_LENGTH || hasWikitextSyntaxCandidate(source)) {
+    return null;
+  }
+
+  const text = trimNewlineEdges(source);
+  const elements = text === "" ? [] : buildPlainParagraphs(text);
+
+  elements.push({
+    element: "footnote-block",
+    data: { title: null, hide: false },
+  });
+
+  return { ast: { elements }, diagnostics: [] };
+}
+
+function hasWikitextSyntaxCandidate(source: string): boolean {
+  for (let i = 0; i < source.length; i++) {
+    switch (source.charCodeAt(i)) {
+      case 42: // *
+      case 43: // +
+      case 47: // /
+      case 60: // <
+      case 61: // =
+      case 62: // >
+      case 64: // @
+      case 91: // [
+      case 92: // \
+      case 93: // ]
+      case 94: // ^
+      case 95: // _
+      case 124: // |
+      case 126: // ~
+      case 0xe000:
+        return true;
+      case 35: // #
+        if (isLineStart(source, i)) return true;
+        break;
+      case 45: // -
+        if (source.charCodeAt(i + 1) === 45) return true;
+        break;
+      case 44: // ,
+        if (source.charCodeAt(i + 1) === 44) return true;
+        break;
+      case 123: // {
+        if (source.charCodeAt(i + 1) === 123) return true;
+        break;
+      case 125: // }
+        if (source.charCodeAt(i + 1) === 125) return true;
+        break;
+    }
+  }
+
+  return false;
+}
+
+function isLineStart(source: string, index: number): boolean {
+  if (index === 0) return true;
+  const prev = source.charCodeAt(index - 1);
+  return prev === 10 || prev === 13;
+}
+
+function trimNewlineEdges(source: string): string {
+  let start = 0;
+  let end = source.length;
+  while (start < end && source.charCodeAt(start) === 10) start++;
+  while (end > start && source.charCodeAt(end - 1) === 10) end--;
+  return start === 0 && end === source.length ? source : source.slice(start, end);
+}
+
+function buildPlainParagraphs(text: string): Element[] {
+  const paragraphs =
+    text.indexOf("\n\n") === -1 ? [text] : text.split(/\n{2,}/).filter((part) => part !== "");
+
+  return paragraphs.map((paragraphText) => ({
+    element: "container",
+    data: {
+      type: "paragraph",
+      attributes: {},
+      elements: buildPlainParagraphElements(paragraphText),
+    },
+  }));
+}
+
+function buildPlainParagraphElements(text: string): Element[] {
+  const lines = text.split("\n");
+  const elements: Element[] = Array.from({ length: lines.length * 2 - 1 });
+  let out = 0;
+
+  for (let i = 0; i < lines.length; i++) {
+    elements[out++] = { element: "text", data: lines[i]! };
+    if (i !== lines.length - 1) {
+      elements[out++] = { element: "line-break" };
+    }
+  }
+
+  return elements;
+}

package/src/parser/parse/result.ts

@@ -0,0 +1,57 @@
+import type { Element, ParseResult, SyntaxTree } from "@wdprlib/ast";
+import type { ParseContext } from "../rules";
+import {
+  cleanInternalFlags,
+  mergeSpanStripParagraphs,
+  suppressDivAdjacentParagraphs,
+} from "../postprocess";
+import { buildTableOfContents } from "../toc";
+import { containsFootnoteBlock } from "./footnotes";
+
+export function finalizeParseResult(ctx: ParseContext, children: Element[]): ParseResult {
+  const cleanedChildren = postprocessChildren(children);
+  appendImplicitFootnoteBlock(cleanedChildren);
+
+  return {
+    ast: buildSyntaxTree(ctx, cleanedChildren),
+    diagnostics: ctx.diagnostics,
+  };
+}
+
+function postprocessChildren(children: Element[]): Element[] {
+  const mergedChildren = mergeSpanStripParagraphs(children);
+  const divProcessed = suppressDivAdjacentParagraphs(mergedChildren);
+  return cleanInternalFlags(divProcessed);
+}
+
+function appendImplicitFootnoteBlock(elements: Element[]): void {
+  if (containsFootnoteBlock(elements)) return;
+
+  elements.push({
+    element: "footnote-block",
+    data: { title: null, hide: false },
+  });
+}
+
+function buildSyntaxTree(ctx: ParseContext, elements: Element[]): SyntaxTree {
+  const tableOfContents = buildTableOfContents(ctx.tocEntries);
+  const result: SyntaxTree = { elements };
+
+  if (tableOfContents.length > 0) {
+    result["table-of-contents"] = tableOfContents;
+  }
+
+  if (ctx.footnotes.length > 0) {
+    result.footnotes = ctx.footnotes;
+  }
+
+  if (ctx.codeBlocks.length > 0) {
+    result["code-blocks"] = ctx.codeBlocks;
+  }
+
+  if (ctx.htmlBlocks.length > 0) {
+    result["html-blocks"] = ctx.htmlBlocks;
+  }
+
+  return result;
+}

package/src/parser/parse/source.ts

@@ -0,0 +1,11 @@
+import { preprocess } from "../preprocess";
+import { preprocessExpr } from "../preprocess/expr";
+import { preprocessIftags } from "../rules/block/module/iftags/preprocess";
+import type { ParserOptions } from "./options";
+
+export function prepareSourceForParse(source: string, options: ParserOptions | undefined): string {
+  const ifProcessed = preprocessExpr(source);
+  const iftagsProcessed =
+    options?.pageTags !== undefined ? preprocessIftags(ifProcessed, options.pageTags) : ifProcessed;
+  return preprocess(iftagsProcessed);
+}

package/src/parser/postprocess/divAdjacentParagraph.ts

@@ -0,0 +1,76 @@
+/**
+ *
+ * Post-processing pass: suppress paragraph wrapping adjacent to div containers.
+ *
+ * In Wikidot, when a paragraph is a direct sibling of a `<div>` block (no other
+ * block elements between them), the `<p>` wrapping is removed and the inner
+ * elements are promoted to the parent level.
+ *
+ * When the unwrapped paragraph follows a div, a line-break element is prepended
+ * to represent the newline between the closing `</div>` and the bare text.
+ *
+ * Examples:
+ *   `[[div]]inline[[/div]]\n[[div]]\n[[/div]]`  → no `<p>` (adjacent to div)
+ *   `[[div]]inline[[/div]]\n> a\n[[div]]\n[[/div]]` → has `<p>` (blockquote between)
+ *
+ * @module
+ */
+import type { Element, ContainerData } from "@wdprlib/ast";
+
+function isParagraphContainer(el: Element | undefined): boolean {
+  if (!el || el.element !== "container") return false;
+  return (el.data as ContainerData).type === "paragraph";
+}
+
+function isDivContainer(el: Element | undefined): boolean {
+  if (!el || el.element !== "container") return false;
+  return (el.data as ContainerData).type === "div";
+}
+
+/**
+ * At a single nesting level, unwrap paragraph containers that are directly
+ * adjacent to div containers. A line-break is prepended when the paragraph
+ * follows a div.
+ */
+function suppressAtLevel(elements: Element[]): Element[] {
+  if (elements.length <= 1) return elements;
+
+  const unwrap = Array.from({ length: elements.length }, () => false);
+
+  for (let i = 0; i < elements.length; i++) {
+    if (!isParagraphContainer(elements[i])) continue;
+    const prevIsDiv = i > 0 && isDivContainer(elements[i - 1]);
+    const nextIsDiv = i < elements.length - 1 && isDivContainer(elements[i + 1]);
+    if (prevIsDiv || nextIsDiv) {
+      unwrap[i] = true;
+    }
+  }
+
+  const result: Element[] = [];
+  for (let i = 0; i < elements.length; i++) {
+    const el = elements[i];
+    if (!el) continue;
+
+    if (unwrap[i] && el.element === "container") {
+      const inner = (el.data as ContainerData).elements;
+      if (i > 0 && isDivContainer(elements[i - 1])) {
+        result.push({ element: "line-break" });
+      }
+      result.push(...inner);
+    } else {
+      result.push(el);
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Suppress paragraph wrapping adjacent to div containers.
+ *
+ * Applied only at the top level. Inside div containers, paragraphs adjacent
+ * to nested divs retain their `<p>` wrapping (matching Wikidot behavior).
+ */
+export function suppressDivAdjacentParagraphs(elements: Element[]): Element[] {
+  return suppressAtLevel(elements);
+}

package/src/parser/postprocess/index.ts

@@ -0,0 +1,15 @@
+/**
+ *
+ * Post-processing pipeline for the parsed AST.
+ *
+ * After the main parser produces an initial AST, post-processing transforms handle
+ * behaviors that cannot be resolved during single-pass parsing. This includes merging
+ * paragraphs around `span_` (paragraph-strip) elements, splitting paragraphs at
+ * empty `[[#expr ]]` elements, and cleaning internal flags that were used as
+ * inter-pass communication markers.
+ *
+ * @module
+ */
+
+export { mergeSpanStripParagraphs, cleanInternalFlags } from "./spanStrip";
+export { suppressDivAdjacentParagraphs } from "./divAdjacentParagraph";

package/src/parser/postprocess/spanStrip/clean-element.ts

@@ -0,0 +1,168 @@
+import type { DefinitionListItem, Element, ListItem } from "@wdprlib/ast";
+import type { InternalContainerData } from "./predicates";
+
+type CleanElements = (elements: Element[]) => Element[];
+
+export function cleanElement(el: Element, cleanElements: CleanElements): Element {
+  if (el.element === "line-break") {
+    return cleanLineBreak(el);
+  }
+
+  if (el.element === "container") {
+    return cleanContainer(el, cleanElements);
+  }
+
+  if (el.element === "collapsible") {
+    const elements = cleanElements(el.data.elements);
+    if (elements === el.data.elements) {
+      return el;
+    }
+    return {
+      element: "collapsible",
+      data: {
+        ...el.data,
+        elements,
+      },
+    };
+  }
+
+  if (el.element === "color") {
+    const elements = cleanElements(el.data.elements);
+    if (elements === el.data.elements) {
+      return el;
+    }
+    return {
+      element: "color",
+      data: {
+        ...el.data,
+        elements,
+      },
+    };
+  }
+
+  if (el.element === "list") {
+    let items: typeof el.data.items | null = null;
+    for (let i = 0; i < el.data.items.length; i++) {
+      const item = el.data.items[i]!;
+      const cleaned = cleanListItem(item, cleanElements);
+      if (items !== null) {
+        items.push(cleaned);
+      } else if (cleaned !== item) {
+        items = el.data.items.slice(0, i);
+        items.push(cleaned);
+      }
+    }
+
+    if (items === null) {
+      return el;
+    }
+
+    return {
+      element: "list",
+      data: {
+        ...el.data,
+        items,
+      },
+    };
+  }
+
+  if (el.element === "definition-list") {
+    let items: typeof el.data | null = null;
+    for (let i = 0; i < el.data.length; i++) {
+      const item = el.data[i]!;
+      const cleaned = cleanDefinitionListItem(item, cleanElements);
+      if (items !== null) {
+        items.push(cleaned);
+      } else if (cleaned !== item) {
+        items = el.data.slice(0, i);
+        items.push(cleaned);
+      }
+    }
+
+    if (items === null) {
+      return el;
+    }
+
+    return {
+      element: "definition-list",
+      data: items,
+    };
+  }
+
+  return el;
+}
+
+function cleanLineBreak(el: Element): Element {
+  for (const key in el) {
+    if (key !== "element") {
+      return { element: "line-break" };
+    }
+  }
+  return el;
+}
+
+function cleanContainer(
+  el: Extract<Element, { element: "container" }>,
+  cleanElements: CleanElements,
+): Element {
+  const data = el.data as InternalContainerData;
+  const elements = cleanElements(data.elements);
+  const hasInternalFlags =
+    data._paragraphStrip !== undefined ||
+    data._emptyParagraphStrip !== undefined ||
+    data._escapedFromParagraph !== undefined ||
+    data._splitByBlankLine !== undefined;
+
+  if (!hasInternalFlags && elements === data.elements) {
+    return el;
+  }
+
+  return {
+    element: "container",
+    data: {
+      type: data.type,
+      attributes: data.attributes,
+      elements,
+    },
+  };
+}
+
+function cleanListItem(item: ListItem, cleanElements: CleanElements): ListItem {
+  if (item["item-type"] === "elements") {
+    const elements = cleanElements(item.elements);
+    return elements === item.elements
+      ? item
+      : {
+          ...item,
+          elements,
+        };
+  }
+
+  if (item["item-type"] === "sub-list") {
+    const cleanedList = cleanElement({ element: "list", data: item.data }, cleanElements);
+    return cleanedList.element === "list" && cleanedList.data !== item.data
+      ? {
+          "item-type": "sub-list",
+          element: "list",
+          data: cleanedList.data,
+        }
+      : item;
+  }
+
+  return item;
+}
+
+function cleanDefinitionListItem(
+  item: DefinitionListItem,
+  cleanElements: CleanElements,
+): DefinitionListItem {
+  const key = cleanElements(item.key);
+  const value = cleanElements(item.value);
+  return key === item.key && value === item.value
+    ? item
+    : {
+        ...item,
+        key,
+        value,
+      };
+}

package/src/parser/postprocess/spanStrip/cleanup.ts

@@ -0,0 +1,25 @@
+import type { Element } from "@wdprlib/ast";
+import { cleanElement } from "./clean-element";
+import { removeEmptySpansAndAdjacentWhitespace } from "./empty-spans";
+
+/**
+ * Recursively remove internal flags from AST elements and clean up empty spans.
+ */
+export function cleanInternalFlags(elements: Element[]): Element[] {
+  let cleaned: Element[] | null = null;
+
+  for (let i = 0; i < elements.length; i++) {
+    const el = elements[i];
+    if (!el) continue;
+
+    const next = cleanElement(el, cleanInternalFlags);
+    if (cleaned !== null) {
+      cleaned.push(next);
+    } else if (next !== el) {
+      cleaned = elements.slice(0, i);
+      cleaned.push(next);
+    }
+  }
+
+  return removeEmptySpansAndAdjacentWhitespace(cleaned ?? elements);
+}

package/src/parser/postprocess/spanStrip/empty-spans.ts

@@ -0,0 +1,36 @@
+import type { Element } from "@wdprlib/ast";
+import { isEmptySpan, isWhitespaceText } from "./predicates";
+
+export function removeEmptySpansAndAdjacentWhitespace(elements: Element[]): Element[] {
+  let hasEmptySpan = false;
+  for (const el of elements) {
+    if (el && isEmptySpan(el)) {
+      hasEmptySpan = true;
+      break;
+    }
+  }
+  if (!hasEmptySpan) {
+    return elements;
+  }
+
+  const result: Element[] = [];
+
+  for (let i = 0; i < elements.length; i++) {
+    const el = elements[i];
+    if (!el) continue;
+
+    if (isEmptySpan(el)) {
+      if (result.length > 0 && isWhitespaceText(result[result.length - 1]!)) {
+        result.pop();
+      }
+      while (i + 1 < elements.length && elements[i + 1] && isWhitespaceText(elements[i + 1]!)) {
+        i++;
+      }
+      continue;
+    }
+
+    result.push(el);
+  }
+
+  return result;
+}

package/src/parser/postprocess/spanStrip/escaped.ts

@@ -0,0 +1,78 @@
+import type { Element } from "@wdprlib/ast";
+import { spanElement } from "./factory";
+import {
+  getContainerData,
+  isEscapedSpan,
+  isSpanStripMarker,
+  type InternalContainerData,
+} from "./predicates";
+
+export function extractEscapedSpans(children: Element[]): Element[] {
+  const escaped: Element[] = [];
+
+  let firstEscapedIndex = -1;
+  for (let i = 0; i < children.length; i++) {
+    const child = children[i];
+    if (child && isEscapedSpan(child)) {
+      firstEscapedIndex = i;
+      break;
+    }
+  }
+
+  if (firstEscapedIndex === -1) {
+    return escaped;
+  }
+
+  let currentSpanChildren: Element[] = [];
+
+  for (let i = firstEscapedIndex; i < children.length; i++) {
+    const child = children[i];
+    if (!child) continue;
+
+    if (isEscapedSpan(child)) {
+      if (currentSpanChildren.length > 0) {
+        escaped.push(spanElement(currentSpanChildren));
+        currentSpanChildren = [];
+      }
+      escaped.push(child);
+    } else if (isSpanStripMarker(child)) {
+      const childData = getContainerData(child);
+      if (childData) {
+        escaped.push(spanElement(childData.elements, childData.attributes));
+      }
+    } else {
+      currentSpanChildren.push(child);
+    }
+  }
+
+  if (currentSpanChildren.length > 0) {
+    escaped.push(spanElement(currentSpanChildren));
+  }
+
+  children.splice(firstEscapedIndex);
+
+  return escaped;
+}
+
+export function removeLineBreaksAroundSpanStrip(children: Element[]): void {
+  for (let i = children.length - 1; i >= 0; i--) {
+    const child = children[i];
+    if (!child) continue;
+
+    if (child.element === "line-break") {
+      const prev = children[i - 1];
+      const next = children[i + 1];
+      if (isSpanStripMarker(prev) || isSpanStripMarker(next)) {
+        children.splice(i, 1);
+        continue;
+      }
+    }
+
+    if (child.element === "container") {
+      const data = child.data as InternalContainerData;
+      if (data.type === "span" && data._emptyParagraphStrip) {
+        children.splice(i, 1);
+      }
+    }
+  }
+}

package/src/parser/postprocess/spanStrip/factory.ts

@@ -0,0 +1,23 @@
+import type { Element } from "@wdprlib/ast";
+
+export function paragraphElement(elements: Element[]): Element {
+  return {
+    element: "container",
+    data: {
+      type: "paragraph",
+      attributes: {},
+      elements,
+    },
+  };
+}
+
+export function spanElement(elements: Element[], attributes: Record<string, string> = {}): Element {
+  return {
+    element: "container",
+    data: {
+      type: "span",
+      attributes,
+      elements,
+    },
+  };
+}

package/src/parser/postprocess/spanStrip/index.ts

@@ -0,0 +1,8 @@
+/**
+ * Public facade for span-strip post-processing.
+ *
+ * @module
+ */
+
+export { mergeSpanStripParagraphs } from "./merge";
+export { cleanInternalFlags } from "./cleanup";