@wdprlib/parser 3.2.0 → 4.0.0

package/src/parser/depth/index.ts

@@ -0,0 +1,111 @@
+/**
+ *
+ * Depth processing module for converting flat lists into nested tree structures.
+ *
+ * This is a TypeScript port of Wikidot's `depth.rs`. It handles the conversion
+ * of flat depth-annotated items (such as bullet/numbered list entries at various
+ * indentation levels) into properly nested tree structures. The algorithm uses an
+ * internal stack to track open nesting levels and collapses them as depth decreases.
+ *
+ * Used primarily by the list parser and the table-of-contents builder to transform
+ * flat sequences of items with depth annotations into hierarchical AST structures.
+ *
+ * @module
+ */
+import { DepthStack } from "./stack";
+
+/**
+ * Represents a single node in a depth tree.
+ *
+ * A node is either a leaf item containing a value, or a nested list containing
+ * children. This recursive type allows arbitrarily deep nesting.
+ *
+ * @typeParam L - The list type discriminator (e.g., "bullet" vs "number" for lists,
+ *               or `null` when list type distinction is not needed)
+ * @typeParam T - The type of leaf item values
+ */
+export type DepthItem<L, T> =
+  | { kind: "item"; value: T }
+  | { kind: "list"; ltype: L; children: DepthList<L, T> };
+
+/**
+ * An ordered collection of depth tree nodes at the same level.
+ *
+ * @typeParam L - The list type discriminator
+ * @typeParam T - The type of leaf item values
+ */
+export type DepthList<L, T> = DepthItem<L, T>[];
+
+/**
+ * Process a flat list of depth-annotated items into nested tree structures.
+ *
+ * This is the main entry point for the depth module. It takes a sequence of items,
+ * each annotated with a nesting depth and a list type, and produces one or more
+ * nested trees. Multiple trees are produced when the list type changes at the
+ * root level (depth 0).
+ *
+ * The algorithm iterates through items sequentially, using a stack to track
+ * open nesting levels. When depth increases, new levels are pushed; when depth
+ * decreases, levels are popped and collapsed into their parent. When the list
+ * type changes at the same depth, the current list is finalized and a new one begins.
+ *
+ * @typeParam L - The list type discriminator
+ * @typeParam T - The type of leaf item values
+ * @param topLtype - The default list type for the root level
+ * @param items - Flat sequence of depth-annotated items, where each item has:
+ *   - `depth`: the 0-based nesting level
+ *   - `ltype`: the list type for grouping (e.g., "bullet" vs "number")
+ *   - `value`: the actual item content
+ * @param ltypeEquals - Equality comparator for list types (defaults to `===`)
+ * @returns Array of finished trees, each with an `ltype` and a `list` of nested items.
+ *          Multiple trees are returned when the list type changes at depth 0.
+ */
+export function processDepths<L, T>(
+  topLtype: L,
+  items: Array<{ depth: number; ltype: L; value: T }>,
+  ltypeEquals: (a: L, b: L) => boolean = (a, b) => a === b,
+): Array<{ ltype: L; list: DepthList<L, T> }> {
+  const stack = new DepthStack<L, T>(topLtype);
+
+  // The depth value for the previous item
+  let previous = 0;
+
+  // Iterate through each of the items
+  for (const { depth, ltype, value } of items) {
+    // Add or remove new depth levels as appropriate,
+    // based on what our new depth value is compared
+    // to the value in the previous iteration.
+    //
+    // If previous == depth, then neither of these for loops will run
+    // If previous < depth, then only the first will run
+    // If previous > depth, then only the second will run
+
+    // Open new levels
+    for (let i = previous; i < depth; i++) {
+      stack.increaseDepth(ltype);
+    }
+
+    // Close existing levels
+    for (let i = depth; i < previous; i++) {
+      stack.decreaseDepth();
+    }
+
+    // Create new level if the type doesn't match
+    //
+    // Here we decrease and increase the depth to close
+    // the current layer, then make a new one with the
+    // type this item has.
+    //
+    // We'll keep appending to this remade layer until
+    // we hit a different depth or a different type.
+    if (!ltypeEquals(stack.lastType(), ltype)) {
+      stack.newList(ltype);
+    }
+
+    // Push element and update state
+    stack.pushItem(value);
+    previous = depth;
+  }
+
+  return stack.intoTrees();
+}

package/src/parser/depth/stack.ts

@@ -0,0 +1,82 @@
+import type { DepthItem, DepthList } from "../depth";
+
+/**
+ * Internal stack-based builder for constructing depth trees incrementally.
+ */
+export class DepthStack<L, T> {
+  private finished: Array<{ ltype: L; list: DepthList<L, T> }> = [];
+  private stack: Array<{ ltype: L; items: DepthItem<L, T>[] }>;
+
+  constructor(topLtype: L) {
+    this.stack = [{ ltype: topLtype, items: [] }];
+  }
+
+  private get last(): { ltype: L; items: DepthItem<L, T>[] } {
+    return this.stack[this.stack.length - 1]!;
+  }
+
+  private get first(): { ltype: L; items: DepthItem<L, T>[] } {
+    return this.stack[0]!;
+  }
+
+  private isSingle(): boolean {
+    return this.stack.length === 1;
+  }
+
+  increaseDepth(ltype: L): void {
+    this.stack.push({ ltype, items: [] });
+  }
+
+  decreaseDepth(): void {
+    const popped = this.stack.pop();
+    if (!popped) {
+      throw new Error("No depth to pop off!");
+    }
+    this.push({ kind: "list", ltype: popped.ltype, children: popped.items });
+  }
+
+  newList(ltype: L): void {
+    if (this.isSingle()) {
+      this.finishDepthList(ltype);
+      return;
+    }
+
+    this.decreaseDepth();
+    this.increaseDepth(ltype);
+  }
+
+  pushItem(item: T): void {
+    this.push({ kind: "item", value: item });
+  }
+
+  lastType(): L {
+    return this.last.ltype;
+  }
+
+  intoTrees(): Array<{ ltype: L; list: DepthList<L, T> }> {
+    this.finishDepthList(null);
+    return this.finished;
+  }
+
+  private push(item: DepthItem<L, T>): void {
+    this.last.items.push(item);
+  }
+
+  private finishDepthList(newLtype: L | null): void {
+    while (this.stack.length > 1) {
+      this.decreaseDepth();
+    }
+
+    const first = this.first;
+    const ltype = first.ltype;
+    const list = first.items;
+    const actualNewLtype = newLtype ?? ltype;
+
+    first.ltype = actualNewLtype;
+    first.items = [];
+
+    if (list.length > 0) {
+      this.finished.push({ ltype, list });
+    }
+  }
+}

package/src/parser/parse/block.ts

@@ -0,0 +1,42 @@
+import type { Element } from "@wdprlib/ast";
+import type { ParseContext } from "../rules";
+import { getCandidateBlockRules } from "../rules/block/utils";
+
+export function parseNextBlock(
+  ctx: ParseContext,
+  skipWhitespace: () => void,
+  isAtEnd: () => boolean,
+): Element[] {
+  skipWhitespace();
+
+  if (isAtEnd()) {
+    return [];
+  }
+
+  const token = ctx.tokens[ctx.pos];
+  if (!token) {
+    return [];
+  }
+
+  if (token.type === "NEWLINE") {
+    ctx.pos++;
+    return [];
+  }
+
+  for (const rule of getCandidateBlockRules(ctx.blockRules, token)) {
+    const result = rule.parse(ctx);
+    if (result.success) {
+      ctx.pos += result.consumed;
+      return result.elements;
+    }
+  }
+
+  const result = ctx.blockFallbackRule.parse(ctx);
+  if (result.success && result.elements.length > 0) {
+    ctx.pos += result.consumed;
+    return result.elements;
+  }
+
+  ctx.pos++;
+  return [];
+}

package/src/parser/parse/context.ts

@@ -0,0 +1,26 @@
+import { DEFAULT_SETTINGS } from "@wdprlib/ast";
+import type { Token } from "../../lexer";
+import { blockFallbackRule, blockRules, inlineRules, type ParseContext } from "../rules";
+import type { ParserOptions } from "./options";
+
+export function createParseContext(tokens: Token[], options: ParserOptions = {}): ParseContext {
+  return {
+    tokens,
+    pos: 0,
+    version: options.version ?? "wikidot",
+    trackPositions: options.trackPositions ?? true,
+    settings: options.settings ?? DEFAULT_SETTINGS,
+    footnotes: [],
+    tocEntries: [],
+    codeBlocks: [],
+    htmlBlocks: [],
+    bibcites: [],
+    diagnostics: [],
+    blockRules,
+    blockFallbackRule,
+    inlineRules,
+    scope: {
+      footnoteBlockParsed: false,
+    },
+  };
+}

package/src/parser/parse/footnotes.ts

@@ -0,0 +1,25 @@
+import type { Element } from "@wdprlib/ast";
+import { walkElements } from "../rules/block/module/walk";
+
+/**
+ * Return `true` when an explicit `footnote-block` element exists anywhere
+ * in the tree.
+ *
+ * Reuses the parser's general-purpose {@link walkElements} so descent
+ * into containers, collapsibles, list items, table cells, tab panels,
+ * definition lists, etc. matches the rest of the parser exactly.
+ *
+ * Note on `[[iftags]]`: the walker also descends into iftags bodies, so
+ * a `[[footnoteblock]]` that only renders conditionally still suppresses
+ * the auto-append. This matches the previous behaviour for unresolved
+ * iftags but means the implicit block does not reappear if the iftags
+ * condition evaluates to false during `resolveModules`. Tracked as a
+ * known limitation; iftags preprocessing would fix it.
+ */
+export function containsFootnoteBlock(elements: Element[]): boolean {
+  let found = false;
+  walkElements(elements, (element) => {
+    if (element.element === "footnote-block") found = true;
+  });
+  return found;
+}

package/src/parser/parse/index.ts

@@ -0,0 +1,42 @@
+import type { ParseResult } from "@wdprlib/ast";
+import { tokenize } from "../../lexer";
+import { Parser } from "./parser";
+import { parseLargePlainTextDocument, parsePlainNonAsciiDocument } from "./plain-non-ascii";
+import { prepareSourceForParse } from "./source";
+import type { ParserOptions } from "./options";
+
+const COMPACT_TEXT_RUN_SOURCE_LENGTH = 100_000;
+
+export type { ParserOptions } from "./options";
+export { Parser } from "./parser";
+
+/**
+ * Parse a Wikidot markup string into an AST with diagnostics.
+ *
+ * @example
+ * ```ts
+ * import { parse } from "@wdprlib/parser";
+ *
+ * const { ast, diagnostics } = parse("**bold** and //italic//");
+ * ```
+ *
+ * @since 2.0.0
+ */
+export function parse(source: string, options?: ParserOptions): ParseResult {
+  const plainResult = parsePlainNonAsciiDocument(source);
+  if (plainResult) {
+    return plainResult;
+  }
+
+  const preprocessed = prepareSourceForParse(source, options);
+  const largePlainResult = parseLargePlainTextDocument(preprocessed);
+  if (largePlainResult) {
+    return largePlainResult;
+  }
+
+  const tokens = tokenize(preprocessed, {
+    trackPositions: options?.trackPositions,
+    compactTextRuns: preprocessed.length >= COMPACT_TEXT_RUN_SOURCE_LENGTH,
+  });
+  return new Parser(tokens, options).parse();
+}

package/src/parser/parse/options.ts

@@ -0,0 +1,34 @@
+import type { WikitextSettings } from "@wdprlib/ast";
+
+/**
+ * Configuration for the {@link Parser} and the {@link parse} function.
+ *
+ * All fields are optional; sensible defaults are applied when omitted.
+ *
+ * @group Parser
+ */
+export interface ParserOptions {
+  /** Markup dialect. Currently only `"wikidot"` is supported. */
+  version?: "wikidot";
+  /**
+   * Propagate source-position data into every AST node.
+   * Defaults to `true`. Set to `false` for smaller output when positions
+   * are not needed.
+   */
+  trackPositions?: boolean;
+  /**
+   * Context-dependent feature flags (page vs. forum-post, etc.).
+   * Defaults to {@link DEFAULT_SETTINGS} (full page mode).
+   */
+  settings?: WikitextSettings;
+  /**
+   * Page tags consulted when expanding `[[iftags]]` directives that are
+   * embedded inside another block's opener.
+   *
+   * Values:
+   * - omitted / `undefined`: no preprocess pass.
+   * - `null`: opener-embedded iftags only, evaluated as if the page has no tags.
+   * - `string[]`: every iftags block is evaluated against the given tags eagerly.
+   */
+  pageTags?: string[] | null;
+}

package/src/parser/parse/parser.ts

@@ -0,0 +1,79 @@
+import type { Element, ParseResult } from "@wdprlib/ast";
+import type { Token } from "../../lexer";
+import type { ParseContext } from "../rules";
+import type { ParserOptions } from "./options";
+import { parseNextBlock } from "./block";
+import { createParseContext } from "./context";
+import { finalizeParseResult } from "./result";
+
+/**
+ * Converts a token stream into a Wikidot {@link SyntaxTree}.
+ *
+ * The parser consumes tokens produced by the `Lexer` and emits a tree of
+ * {@link Element} nodes. Block-level rules are tried in priority order; when
+ * none match, the fallback paragraph rule collects inline tokens until the
+ * next blank line.
+ *
+ * For most use-cases the standalone {@link parse} function is simpler than
+ * constructing a `Parser` directly.
+ *
+ * @group Parser
+ */
+export class Parser {
+  private ctx: ParseContext;
+
+  constructor(tokens: Token[], options: ParserOptions = {}) {
+    this.ctx = createParseContext(tokens, options);
+  }
+
+  /**
+   * Parse tokens into a {@link ParseResult} containing the AST and
+   * any diagnostics emitted during parsing.
+   *
+   * @since 2.0.0
+   */
+  parse(): ParseResult {
+    const children: Element[] = [];
+
+    while (!this.isAtEnd()) {
+      const blocks = this.parseBlock();
+      children.push(...blocks);
+    }
+
+    return finalizeParseResult(this.ctx, children);
+  }
+
+  private isAtEnd(): boolean {
+    return this.ctx.pos >= this.ctx.tokens.length || this.currentToken().type === "EOF";
+  }
+
+  private currentToken(): Token {
+    return this.ctx.tokens[this.ctx.pos] ?? this.eofToken();
+  }
+
+  private eofToken(): Token {
+    return {
+      type: "EOF",
+      value: "",
+      position: {
+        start: { line: 0, column: 0, offset: 0 },
+        end: { line: 0, column: 0, offset: 0 },
+      },
+      lineStart: false,
+    };
+  }
+
+  private skipWhitespace(): void {
+    while (this.currentToken().type === "WHITESPACE") {
+      this.ctx.pos++;
+    }
+  }
+
+  private parseBlock(): Element[] {
+    return parseNextBlock(
+      this.ctx,
+      () => this.skipWhitespace(),
+      () => this.isAtEnd(),
+    );
+  }
+}

package/src/parser/parse/plain-non-ascii.ts

@@ -0,0 +1,129 @@
+import type { Element, ParseResult } from "@wdprlib/ast";
+
+const LARGE_PLAIN_TEXT_MIN_LENGTH = 100_000;
+
+export function parsePlainNonAsciiDocument(source: string): ParseResult | null {
+  if (!isPlainNonAsciiDocument(source)) {
+    return null;
+  }
+
+  const text = trimNewlineEdges(source);
+  const elements = text === "" ? [] : buildPlainParagraphs(text);
+
+  elements.push({
+    element: "footnote-block",
+    data: { title: null, hide: false },
+  });
+
+  return { ast: { elements }, diagnostics: [] };
+}
+
+function isPlainNonAsciiDocument(source: string): boolean {
+  for (let i = 0; i < source.length; i++) {
+    const code = source.charCodeAt(i);
+    if ((code >= 0x00 && code <= 0x09) || (code >= 0x0b && code <= 0x7f) || code === 0xe000) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+export function parseLargePlainTextDocument(source: string): ParseResult | null {
+  if (source.length < LARGE_PLAIN_TEXT_MIN_LENGTH || hasWikitextSyntaxCandidate(source)) {
+    return null;
+  }
+
+  const text = trimNewlineEdges(source);
+  const elements = text === "" ? [] : buildPlainParagraphs(text);
+
+  elements.push({
+    element: "footnote-block",
+    data: { title: null, hide: false },
+  });
+
+  return { ast: { elements }, diagnostics: [] };
+}
+
+function hasWikitextSyntaxCandidate(source: string): boolean {
+  for (let i = 0; i < source.length; i++) {
+    switch (source.charCodeAt(i)) {
+      case 42: // *
+      case 43: // +
+      case 47: // /
+      case 60: // <
+      case 61: // =
+      case 62: // >
+      case 64: // @
+      case 91: // [
+      case 92: // \
+      case 93: // ]
+      case 94: // ^
+      case 95: // _
+      case 124: // |
+      case 126: // ~
+      case 0xe000:
+        return true;
+      case 35: // #
+        if (isLineStart(source, i)) return true;
+        break;
+      case 45: // -
+        if (source.charCodeAt(i + 1) === 45) return true;
+        break;
+      case 44: // ,
+        if (source.charCodeAt(i + 1) === 44) return true;
+        break;
+      case 123: // {
+        if (source.charCodeAt(i + 1) === 123) return true;
+        break;
+      case 125: // }
+        if (source.charCodeAt(i + 1) === 125) return true;
+        break;
+    }
+  }
+
+  return false;
+}
+
+function isLineStart(source: string, index: number): boolean {
+  if (index === 0) return true;
+  const prev = source.charCodeAt(index - 1);
+  return prev === 10 || prev === 13;
+}
+
+function trimNewlineEdges(source: string): string {
+  let start = 0;
+  let end = source.length;
+  while (start < end && source.charCodeAt(start) === 10) start++;
+  while (end > start && source.charCodeAt(end - 1) === 10) end--;
+  return start === 0 && end === source.length ? source : source.slice(start, end);
+}
+
+function buildPlainParagraphs(text: string): Element[] {
+  const paragraphs =
+    text.indexOf("\n\n") === -1 ? [text] : text.split(/\n{2,}/).filter((part) => part !== "");
+
+  return paragraphs.map((paragraphText) => ({
+    element: "container",
+    data: {
+      type: "paragraph",
+      attributes: {},
+      elements: buildPlainParagraphElements(paragraphText),
+    },
+  }));
+}
+
+function buildPlainParagraphElements(text: string): Element[] {
+  const lines = text.split("\n");
+  const elements: Element[] = Array.from({ length: lines.length * 2 - 1 });
+  let out = 0;
+
+  for (let i = 0; i < lines.length; i++) {
+    elements[out++] = { element: "text", data: lines[i]! };
+    if (i !== lines.length - 1) {
+      elements[out++] = { element: "line-break" };
+    }
+  }
+
+  return elements;
+}

package/src/parser/parse/result.ts

@@ -0,0 +1,57 @@
+import type { Element, ParseResult, SyntaxTree } from "@wdprlib/ast";
+import type { ParseContext } from "../rules";
+import {
+  cleanInternalFlags,
+  mergeSpanStripParagraphs,
+  suppressDivAdjacentParagraphs,
+} from "../postprocess";
+import { buildTableOfContents } from "../toc";
+import { containsFootnoteBlock } from "./footnotes";
+
+export function finalizeParseResult(ctx: ParseContext, children: Element[]): ParseResult {
+  const cleanedChildren = postprocessChildren(children);
+  appendImplicitFootnoteBlock(cleanedChildren);
+
+  return {
+    ast: buildSyntaxTree(ctx, cleanedChildren),
+    diagnostics: ctx.diagnostics,
+  };
+}
+
+function postprocessChildren(children: Element[]): Element[] {
+  const mergedChildren = mergeSpanStripParagraphs(children);
+  const divProcessed = suppressDivAdjacentParagraphs(mergedChildren);
+  return cleanInternalFlags(divProcessed);
+}
+
+function appendImplicitFootnoteBlock(elements: Element[]): void {
+  if (containsFootnoteBlock(elements)) return;
+
+  elements.push({
+    element: "footnote-block",
+    data: { title: null, hide: false },
+  });
+}
+
+function buildSyntaxTree(ctx: ParseContext, elements: Element[]): SyntaxTree {
+  const tableOfContents = buildTableOfContents(ctx.tocEntries);
+  const result: SyntaxTree = { elements };
+
+  if (tableOfContents.length > 0) {
+    result["table-of-contents"] = tableOfContents;
+  }
+
+  if (ctx.footnotes.length > 0) {
+    result.footnotes = ctx.footnotes;
+  }
+
+  if (ctx.codeBlocks.length > 0) {
+    result["code-blocks"] = ctx.codeBlocks;
+  }
+
+  if (ctx.htmlBlocks.length > 0) {
+    result["html-blocks"] = ctx.htmlBlocks;
+  }
+
+  return result;
+}

package/src/parser/parse/source.ts

@@ -0,0 +1,11 @@
+import { preprocess } from "../preprocess";
+import { preprocessExpr } from "../preprocess/expr";
+import { preprocessIftags } from "../rules/block/module/iftags/preprocess";
+import type { ParserOptions } from "./options";
+
+export function prepareSourceForParse(source: string, options: ParserOptions | undefined): string {
+  const ifProcessed = preprocessExpr(source);
+  const iftagsProcessed =
+    options?.pageTags !== undefined ? preprocessIftags(ifProcessed, options.pageTags) : ifProcessed;
+  return preprocess(iftagsProcessed);
+}

package/src/parser/postprocess/divAdjacentParagraph.ts

@@ -35,7 +35,7 @@ function isDivContainer(el: Element | undefined): boolean {
 function suppressAtLevel(elements: Element[]): Element[] {
   if (elements.length <= 1) return elements;
 
-  const unwrap = new Array<boolean>(elements.length).fill(false);
+  const unwrap = Array.from({ length: elements.length }, () => false);
 
   for (let i = 0; i < elements.length; i++) {
     if (!isParagraphContainer(elements[i])) continue;