@wdprlib/parser 3.1.2 → 4.0.0

package/src/parser/constants.ts

@@ -0,0 +1,175 @@
+/**
+ *
+ * Parser constants that define structural boundaries in Wikidot markup.
+ *
+ * These constants are used by the paragraph rule to determine when a new
+ * block-level construct begins, which terminates the current paragraph.
+ * When any of these token types appear at the start of a line, the parser
+ * stops collecting inline content for the current paragraph and begins
+ * processing the new block element.
+ *
+ * @module
+ */
+
+import type { TokenType } from "../lexer";
+
+/**
+ * Token types that signal the start of a block-level construct in Wikidot markup.
+ *
+ * When the parser encounters any of these tokens at the beginning of a line while
+ * building a paragraph, it stops the paragraph and delegates to the appropriate
+ * block rule. Each token maps to a specific Wikidot syntax element (documented
+ * inline with comments).
+ */
+export const BLOCK_START_TOKENS: TokenType[] = [
+  "BLOCKQUOTE_MARKER",
+  "LIST_BULLET",
+  "LIST_NUMBER",
+  "HEADING_MARKER",
+  "HR_MARKER",
+  "TABLE_MARKER",
+  "COLON", // Definition list
+  "BLOCK_OPEN", // [[footnoteblock]], [[div]], etc.
+  "BLOCK_END_OPEN", // [[/div]], [[/collapsible]], etc.
+  "EQUALS", // Center align (= text) or content separator (====)
+  "CLEAR_FLOAT", // ~~~~
+  "CLEAR_FLOAT_LEFT", // ~~~~<
+  "CLEAR_FLOAT_RIGHT", // ~~~~>
+];
+
+export const BLOCK_START_TOKEN_SET: ReadonlySet<TokenType> = new Set(BLOCK_START_TOKENS);
+
+/**
+ * Set of block names recognized by the parser at `[[name]]` / `[[/name]]`.
+ *
+ * Used by inline-parser logic to distinguish real block boundaries from
+ * unknown tokens like `[[foo]]`, which Wikidot treats as inline text
+ * rather than as a paragraph-breaking block.
+ *
+ * Keep in sync with the set of block rules registered in
+ * `packages/parser/src/parser/rules/block/index.ts`. Align-style markers
+ * (`<`, `>`, `=`, `==`) are intentionally included because `[[<]]` etc.
+ * open `alignRule`.
+ */
+export const KNOWN_BLOCK_NAMES: ReadonlySet<string> = new Set<string>([
+  // structural containers
+  "collapsible",
+  "div",
+  "div_",
+  "code",
+  // list blocks
+  "ul",
+  "ol",
+  "li",
+  // table blocks
+  "table",
+  "row",
+  "cell",
+  "hcell",
+  // tabview / module
+  "tabview",
+  "tabs",
+  "module",
+  "module654",
+  // misc named blocks
+  "bibliography",
+  "footnoteblock",
+  "toc",
+  "iframe",
+  "math",
+  "html",
+  "iftags",
+  "include",
+  "f", // float TOC prefix: `[[f<toc]]`, `[[f>toc]]` (see toc rule)
+  // embed family
+  "embed",
+  "embedvideo",
+  "embedaudio",
+  // align markers
+  "<",
+  ">",
+  "=",
+  "==",
+  // inline-level constructs that use BLOCK_OPEN tokens; recognized here so
+  // that the paragraph parser keeps existing block-boundary behavior for
+  // `[[span]]`, `[[user ...]]`, `[[$ ... $]]`, etc. when they appear at
+  // the start of a line.
+  "span",
+  "span_",
+  "user",
+  "a",
+  "anchor",
+  "size",
+  "footnote",
+  "eref",
+  "$",
+  "image",
+  "gallery",
+  "file",
+]);
+
+/**
+ * Block names whose rule sets `requiresLineStart: false`, i.e. they can
+ * legitimately start a block even when the `[[...]]` opener is preceded
+ * by leading whitespace on its line.
+ *
+ * Used by the inline parser to decide whether a `\n<indent>[[name]]`
+ * sequence ends the current paragraph. Without this list, the inline
+ * parser would either:
+ *   - keep `lineStart` strict and miss legitimately-indented container
+ *     blocks (Wikidot accepts e.g. `\n     [[div_]]`); the inner block
+ *     gets absorbed into the parent paragraph as literal text, or
+ *   - drop the `lineStart` check entirely and prematurely break out of
+ *     paragraphs for `\n  [[toc]]` — a rule with `requiresLineStart: true`
+ *     would refuse the indented token, leaving the paragraph split but
+ *     the block unconsumed (literal `[[toc]]` text in a new paragraph).
+ *
+ * Each entry corresponds to a name handled by a block rule whose
+ * `requiresLineStart` is `false`. Keep this list in sync when adding or
+ * changing such rules; the inline-level constructs that happen to share
+ * `BLOCK_OPEN` (`[[span]]`, `[[image]]`, `[[user]]`, etc.) are
+ * intentionally excluded — they remain inline and should not split
+ * paragraphs based on indentation alone.
+ *
+ * Sources (block rule → handled names):
+ * - `bibliographyRule` → bibliography
+ * - `blockListRule` → ul, ol, li
+ * - `codeRule` → code
+ * - `collapsibleRule` → collapsible
+ * - `divRule` → div, div_
+ * - `embedBlockRule` → embed, embedvideo, embedaudio
+ * - `htmlRule` → html
+ * - `iframeRule` → iframe
+ * - `iftagsRule` → iftags
+ * - `mathRule` → math
+ * - `moduleRule` → module, module654
+ * - `orphanLiRule` → li (also under blockListRule)
+ * - `tableBlockRule` → table (row, cell, hcell are private to the in-table
+ *   parser, never accepted by the top-level dispatcher)
+ * - `tabviewRule` → tabview, tabs (tab is private to the in-tabview parser)
+ *
+ * `includeRule` is omitted because `[[include ...]]` is expanded as a
+ * text-level macro by `resolveIncludes` before the parser sees it.
+ */
+export const INDENT_ACCEPTING_BLOCK_NAMES: ReadonlySet<string> = new Set<string>([
+  "bibliography",
+  "ul",
+  "ol",
+  "li",
+  "code",
+  "collapsible",
+  "div",
+  "div_",
+  "embed",
+  "embedvideo",
+  "embedaudio",
+  "html",
+  "iframe",
+  "iftags",
+  "math",
+  "module",
+  "module654",
+  "table",
+  "tabview",
+  "tabs",
+]);

package/src/parser/depth/index.ts

@@ -0,0 +1,111 @@
+/**
+ *
+ * Depth processing module for converting flat lists into nested tree structures.
+ *
+ * This is a TypeScript port of Wikidot's `depth.rs`. It handles the conversion
+ * of flat depth-annotated items (such as bullet/numbered list entries at various
+ * indentation levels) into properly nested tree structures. The algorithm uses an
+ * internal stack to track open nesting levels and collapses them as depth decreases.
+ *
+ * Used primarily by the list parser and the table-of-contents builder to transform
+ * flat sequences of items with depth annotations into hierarchical AST structures.
+ *
+ * @module
+ */
+import { DepthStack } from "./stack";
+
+/**
+ * Represents a single node in a depth tree.
+ *
+ * A node is either a leaf item containing a value, or a nested list containing
+ * children. This recursive type allows arbitrarily deep nesting.
+ *
+ * @typeParam L - The list type discriminator (e.g., "bullet" vs "number" for lists,
+ *               or `null` when list type distinction is not needed)
+ * @typeParam T - The type of leaf item values
+ */
+export type DepthItem<L, T> =
+  | { kind: "item"; value: T }
+  | { kind: "list"; ltype: L; children: DepthList<L, T> };
+
+/**
+ * An ordered collection of depth tree nodes at the same level.
+ *
+ * @typeParam L - The list type discriminator
+ * @typeParam T - The type of leaf item values
+ */
+export type DepthList<L, T> = DepthItem<L, T>[];
+
+/**
+ * Process a flat list of depth-annotated items into nested tree structures.
+ *
+ * This is the main entry point for the depth module. It takes a sequence of items,
+ * each annotated with a nesting depth and a list type, and produces one or more
+ * nested trees. Multiple trees are produced when the list type changes at the
+ * root level (depth 0).
+ *
+ * The algorithm iterates through items sequentially, using a stack to track
+ * open nesting levels. When depth increases, new levels are pushed; when depth
+ * decreases, levels are popped and collapsed into their parent. When the list
+ * type changes at the same depth, the current list is finalized and a new one begins.
+ *
+ * @typeParam L - The list type discriminator
+ * @typeParam T - The type of leaf item values
+ * @param topLtype - The default list type for the root level
+ * @param items - Flat sequence of depth-annotated items, where each item has:
+ *   - `depth`: the 0-based nesting level
+ *   - `ltype`: the list type for grouping (e.g., "bullet" vs "number")
+ *   - `value`: the actual item content
+ * @param ltypeEquals - Equality comparator for list types (defaults to `===`)
+ * @returns Array of finished trees, each with an `ltype` and a `list` of nested items.
+ *          Multiple trees are returned when the list type changes at depth 0.
+ */
+export function processDepths<L, T>(
+  topLtype: L,
+  items: Array<{ depth: number; ltype: L; value: T }>,
+  ltypeEquals: (a: L, b: L) => boolean = (a, b) => a === b,
+): Array<{ ltype: L; list: DepthList<L, T> }> {
+  const stack = new DepthStack<L, T>(topLtype);
+
+  // The depth value for the previous item
+  let previous = 0;
+
+  // Iterate through each of the items
+  for (const { depth, ltype, value } of items) {
+    // Add or remove new depth levels as appropriate,
+    // based on what our new depth value is compared
+    // to the value in the previous iteration.
+    //
+    // If previous == depth, then neither of these for loops will run
+    // If previous < depth, then only the first will run
+    // If previous > depth, then only the second will run
+
+    // Open new levels
+    for (let i = previous; i < depth; i++) {
+      stack.increaseDepth(ltype);
+    }
+
+    // Close existing levels
+    for (let i = depth; i < previous; i++) {
+      stack.decreaseDepth();
+    }
+
+    // Create new level if the type doesn't match
+    //
+    // Here we decrease and increase the depth to close
+    // the current layer, then make a new one with the
+    // type this item has.
+    //
+    // We'll keep appending to this remade layer until
+    // we hit a different depth or a different type.
+    if (!ltypeEquals(stack.lastType(), ltype)) {
+      stack.newList(ltype);
+    }
+
+    // Push element and update state
+    stack.pushItem(value);
+    previous = depth;
+  }
+
+  return stack.intoTrees();
+}

package/src/parser/depth/stack.ts

@@ -0,0 +1,82 @@
+import type { DepthItem, DepthList } from "../depth";
+
+/**
+ * Internal stack-based builder for constructing depth trees incrementally.
+ */
+export class DepthStack<L, T> {
+  private finished: Array<{ ltype: L; list: DepthList<L, T> }> = [];
+  private stack: Array<{ ltype: L; items: DepthItem<L, T>[] }>;
+
+  constructor(topLtype: L) {
+    this.stack = [{ ltype: topLtype, items: [] }];
+  }
+
+  private get last(): { ltype: L; items: DepthItem<L, T>[] } {
+    return this.stack[this.stack.length - 1]!;
+  }
+
+  private get first(): { ltype: L; items: DepthItem<L, T>[] } {
+    return this.stack[0]!;
+  }
+
+  private isSingle(): boolean {
+    return this.stack.length === 1;
+  }
+
+  increaseDepth(ltype: L): void {
+    this.stack.push({ ltype, items: [] });
+  }
+
+  decreaseDepth(): void {
+    const popped = this.stack.pop();
+    if (!popped) {
+      throw new Error("No depth to pop off!");
+    }
+    this.push({ kind: "list", ltype: popped.ltype, children: popped.items });
+  }
+
+  newList(ltype: L): void {
+    if (this.isSingle()) {
+      this.finishDepthList(ltype);
+      return;
+    }
+
+    this.decreaseDepth();
+    this.increaseDepth(ltype);
+  }
+
+  pushItem(item: T): void {
+    this.push({ kind: "item", value: item });
+  }
+
+  lastType(): L {
+    return this.last.ltype;
+  }
+
+  intoTrees(): Array<{ ltype: L; list: DepthList<L, T> }> {
+    this.finishDepthList(null);
+    return this.finished;
+  }
+
+  private push(item: DepthItem<L, T>): void {
+    this.last.items.push(item);
+  }
+
+  private finishDepthList(newLtype: L | null): void {
+    while (this.stack.length > 1) {
+      this.decreaseDepth();
+    }
+
+    const first = this.first;
+    const ltype = first.ltype;
+    const list = first.items;
+    const actualNewLtype = newLtype ?? ltype;
+
+    first.ltype = actualNewLtype;
+    first.items = [];
+
+    if (list.length > 0) {
+      this.finished.push({ ltype, list });
+    }
+  }
+}

package/src/parser/index.ts

@@ -0,0 +1,18 @@
+/**
+ *
+ * Main parser for Wikidot markup.
+ *
+ * The parser consumes a token stream from the lexer and produces an AST
+ * (Abstract Syntax Tree) conforming to the `@wdprlib/ast` package types.
+ * It applies block rules and inline rules in a recursive-descent fashion,
+ * followed by post-processing passes for paragraph merging and cleanup.
+ *
+ * The main entry points are:
+ * - `parse()` - convenience function that parses a string in one call
+ * - `Parser` class - for more control over parsing options
+ *
+ * @module
+ */
+
+export type { ParserOptions } from "./parse";
+export { Parser, parse } from "./parse";

package/src/parser/parse/block.ts

@@ -0,0 +1,42 @@
+import type { Element } from "@wdprlib/ast";
+import type { ParseContext } from "../rules";
+import { getCandidateBlockRules } from "../rules/block/utils";
+
+export function parseNextBlock(
+  ctx: ParseContext,
+  skipWhitespace: () => void,
+  isAtEnd: () => boolean,
+): Element[] {
+  skipWhitespace();
+
+  if (isAtEnd()) {
+    return [];
+  }
+
+  const token = ctx.tokens[ctx.pos];
+  if (!token) {
+    return [];
+  }
+
+  if (token.type === "NEWLINE") {
+    ctx.pos++;
+    return [];
+  }
+
+  for (const rule of getCandidateBlockRules(ctx.blockRules, token)) {
+    const result = rule.parse(ctx);
+    if (result.success) {
+      ctx.pos += result.consumed;
+      return result.elements;
+    }
+  }
+
+  const result = ctx.blockFallbackRule.parse(ctx);
+  if (result.success && result.elements.length > 0) {
+    ctx.pos += result.consumed;
+    return result.elements;
+  }
+
+  ctx.pos++;
+  return [];
+}

package/src/parser/parse/context.ts

@@ -0,0 +1,26 @@
+import { DEFAULT_SETTINGS } from "@wdprlib/ast";
+import type { Token } from "../../lexer";
+import { blockFallbackRule, blockRules, inlineRules, type ParseContext } from "../rules";
+import type { ParserOptions } from "./options";
+
+export function createParseContext(tokens: Token[], options: ParserOptions = {}): ParseContext {
+  return {
+    tokens,
+    pos: 0,
+    version: options.version ?? "wikidot",
+    trackPositions: options.trackPositions ?? true,
+    settings: options.settings ?? DEFAULT_SETTINGS,
+    footnotes: [],
+    tocEntries: [],
+    codeBlocks: [],
+    htmlBlocks: [],
+    bibcites: [],
+    diagnostics: [],
+    blockRules,
+    blockFallbackRule,
+    inlineRules,
+    scope: {
+      footnoteBlockParsed: false,
+    },
+  };
+}

package/src/parser/parse/footnotes.ts

@@ -0,0 +1,25 @@
+import type { Element } from "@wdprlib/ast";
+import { walkElements } from "../rules/block/module/walk";
+
+/**
+ * Return `true` when an explicit `footnote-block` element exists anywhere
+ * in the tree.
+ *
+ * Reuses the parser's general-purpose {@link walkElements} so descent
+ * into containers, collapsibles, list items, table cells, tab panels,
+ * definition lists, etc. matches the rest of the parser exactly.
+ *
+ * Note on `[[iftags]]`: the walker also descends into iftags bodies, so
+ * a `[[footnoteblock]]` that only renders conditionally still suppresses
+ * the auto-append. This matches the previous behaviour for unresolved
+ * iftags but means the implicit block does not reappear if the iftags
+ * condition evaluates to false during `resolveModules`. Tracked as a
+ * known limitation; iftags preprocessing would fix it.
+ */
+export function containsFootnoteBlock(elements: Element[]): boolean {
+  let found = false;
+  walkElements(elements, (element) => {
+    if (element.element === "footnote-block") found = true;
+  });
+  return found;
+}

package/src/parser/parse/index.ts

@@ -0,0 +1,42 @@
+import type { ParseResult } from "@wdprlib/ast";
+import { tokenize } from "../../lexer";
+import { Parser } from "./parser";
+import { parseLargePlainTextDocument, parsePlainNonAsciiDocument } from "./plain-non-ascii";
+import { prepareSourceForParse } from "./source";
+import type { ParserOptions } from "./options";
+
+const COMPACT_TEXT_RUN_SOURCE_LENGTH = 100_000;
+
+export type { ParserOptions } from "./options";
+export { Parser } from "./parser";
+
+/**
+ * Parse a Wikidot markup string into an AST with diagnostics.
+ *
+ * @example
+ * ```ts
+ * import { parse } from "@wdprlib/parser";
+ *
+ * const { ast, diagnostics } = parse("**bold** and //italic//");
+ * ```
+ *
+ * @since 2.0.0
+ */
+export function parse(source: string, options?: ParserOptions): ParseResult {
+  const plainResult = parsePlainNonAsciiDocument(source);
+  if (plainResult) {
+    return plainResult;
+  }
+
+  const preprocessed = prepareSourceForParse(source, options);
+  const largePlainResult = parseLargePlainTextDocument(preprocessed);
+  if (largePlainResult) {
+    return largePlainResult;
+  }
+
+  const tokens = tokenize(preprocessed, {
+    trackPositions: options?.trackPositions,
+    compactTextRuns: preprocessed.length >= COMPACT_TEXT_RUN_SOURCE_LENGTH,
+  });
+  return new Parser(tokens, options).parse();
+}

package/src/parser/parse/options.ts

@@ -0,0 +1,34 @@
+import type { WikitextSettings } from "@wdprlib/ast";
+
+/**
+ * Configuration for the {@link Parser} and the {@link parse} function.
+ *
+ * All fields are optional; sensible defaults are applied when omitted.
+ *
+ * @group Parser
+ */
+export interface ParserOptions {
+  /** Markup dialect. Currently only `"wikidot"` is supported. */
+  version?: "wikidot";
+  /**
+   * Propagate source-position data into every AST node.
+   * Defaults to `true`. Set to `false` for smaller output when positions
+   * are not needed.
+   */
+  trackPositions?: boolean;
+  /**
+   * Context-dependent feature flags (page vs. forum-post, etc.).
+   * Defaults to {@link DEFAULT_SETTINGS} (full page mode).
+   */
+  settings?: WikitextSettings;
+  /**
+   * Page tags consulted when expanding `[[iftags]]` directives that are
+   * embedded inside another block's opener.
+   *
+   * Values:
+   * - omitted / `undefined`: no preprocess pass.
+   * - `null`: opener-embedded iftags only, evaluated as if the page has no tags.
+   * - `string[]`: every iftags block is evaluated against the given tags eagerly.
+   */
+  pageTags?: string[] | null;
+}

package/src/parser/parse/parser.ts

@@ -0,0 +1,79 @@
+import type { Element, ParseResult } from "@wdprlib/ast";
+import type { Token } from "../../lexer";
+import type { ParseContext } from "../rules";
+import type { ParserOptions } from "./options";
+import { parseNextBlock } from "./block";
+import { createParseContext } from "./context";
+import { finalizeParseResult } from "./result";
+
+/**
+ * Converts a token stream into a Wikidot {@link SyntaxTree}.
+ *
+ * The parser consumes tokens produced by the `Lexer` and emits a tree of
+ * {@link Element} nodes. Block-level rules are tried in priority order; when
+ * none match, the fallback paragraph rule collects inline tokens until the
+ * next blank line.
+ *
+ * For most use-cases the standalone {@link parse} function is simpler than
+ * constructing a `Parser` directly.
+ *
+ * @group Parser
+ */
+export class Parser {
+  private ctx: ParseContext;
+
+  constructor(tokens: Token[], options: ParserOptions = {}) {
+    this.ctx = createParseContext(tokens, options);
+  }
+
+  /**
+   * Parse tokens into a {@link ParseResult} containing the AST and
+   * any diagnostics emitted during parsing.
+   *
+   * @since 2.0.0
+   */
+  parse(): ParseResult {
+    const children: Element[] = [];
+
+    while (!this.isAtEnd()) {
+      const blocks = this.parseBlock();
+      children.push(...blocks);
+    }
+
+    return finalizeParseResult(this.ctx, children);
+  }
+
+  private isAtEnd(): boolean {
+    return this.ctx.pos >= this.ctx.tokens.length || this.currentToken().type === "EOF";
+  }
+
+  private currentToken(): Token {
+    return this.ctx.tokens[this.ctx.pos] ?? this.eofToken();
+  }
+
+  private eofToken(): Token {
+    return {
+      type: "EOF",
+      value: "",
+      position: {
+        start: { line: 0, column: 0, offset: 0 },
+        end: { line: 0, column: 0, offset: 0 },
+      },
+      lineStart: false,
+    };
+  }
+
+  private skipWhitespace(): void {
+    while (this.currentToken().type === "WHITESPACE") {
+      this.ctx.pos++;
+    }
+  }
+
+  private parseBlock(): Element[] {
+    return parseNextBlock(
+      this.ctx,
+      () => this.skipWhitespace(),
+      () => this.isAtEnd(),
+    );
+  }
+}