@wdprlib/parser 3.1.1 → 3.2.0

package/src/lexer/tokens.ts

@@ -0,0 +1,141 @@
+import type { Position } from "@wdprlib/ast";
+
+/**
+ * Every distinct lexeme the Wikidot lexer can produce.
+ *
+ * Each value corresponds to a fixed character sequence (or class of
+ * sequences) in Wikidot markup. The inline comments show the literal
+ * text that produces each token type.
+ *
+ * @group Lexer
+ */
+export type TokenType =
+  // Special
+  | "EOF"
+  | "TEXT"
+  | "IDENTIFIER" // alphanumeric word
+  | "NEWLINE"
+  | "WHITESPACE"
+
+  // Block delimiters
+  | "BLOCK_OPEN" // [[
+  | "BLOCK_CLOSE" // ]]
+  | "BLOCK_END_OPEN" // [[/
+
+  // Inline formatting
+  | "BOLD_MARKER" // **
+  | "ITALIC_MARKER" // //
+  | "UNDERLINE_MARKER" // __
+  | "STRIKE_MARKER" // --
+  | "SUPER_MARKER" // ^^
+  | "SUB_MARKER" // ,,
+  | "MONO_MARKER" // {{
+  | "MONO_CLOSE" // }}
+
+  // Special syntax
+  | "HEADING_MARKER" // + (at line start)
+  | "HR_MARKER" // ---- (at line start)
+  | "LIST_BULLET" // * (at line start)
+  | "LIST_NUMBER" // # (at line start)
+  | "BLOCKQUOTE_MARKER" // > (at line start)
+  | "TABLE_MARKER" // || (at line start)
+  | "TABLE_HEADER" // ||~ (header cell)
+  | "TABLE_LEFT" // ||< (left align)
+  | "TABLE_CENTER" // ||= (center align)
+  | "TABLE_RIGHT" // ||> (right align)
+
+  // Code blocks
+  | "CODE_OPEN" // [[code]]
+  | "CODE_CLOSE" // [[/code]]
+
+  // Links
+  | "LINK_OPEN" // [[[
+  | "LINK_CLOSE" // ]]]
+  | "BRACKET_OPEN" // [
+  | "BRACKET_CLOSE" // ]
+  | "BRACKET_ANCHOR" // [#
+  | "BRACKET_STAR" // [*
+
+  // Special characters
+  | "PIPE" // |
+  | "EQUALS" // =
+  | "COLON" // :
+  | "SLASH" // /
+  | "STAR" // *
+  | "HASH" // #
+  | "AT" // @
+  | "AMPERSAND" // &
+  | "BACKSLASH" // \
+  | "QUOTED_STRING" // "..."
+
+  // Raw/Escape
+  | "RAW_OPEN" // @@
+  | "RAW_CLOSE" // @@
+  | "RAW_BLOCK_OPEN" // @<
+  | "RAW_BLOCK_CLOSE" // >@
+
+  // Color
+  | "COLOR_MARKER" // ##
+
+  // Other
+  | "UNDERSCORE" // _ (single underscore, for line break)
+  | "BACKSLASH_BREAK" // U+E000 (preproc marker for \ at end of line)
+
+  // Comment
+  | "COMMENT_OPEN" // [!--
+  | "COMMENT_CLOSE" // --]
+
+  // Clear float
+  | "CLEAR_FLOAT" // ~~~
+  | "CLEAR_FLOAT_LEFT" // ~~~<
+  | "CLEAR_FLOAT_RIGHT" // ~~~>
+
+  // Double angle (guillemet)
+  | "LEFT_DOUBLE_ANGLE" // <<
+  | "RIGHT_DOUBLE_ANGLE"; // >> (non-line-start)
+
+/**
+ * A single lexical token produced by the `Lexer`.
+ *
+ * Tokens are the input to the parser stage. Each token carries its
+ * literal text (`value`), source location (`position`), and a flag
+ * indicating whether it appeared at the beginning of a line — which
+ * matters because several Wikidot constructs (headings, lists,
+ * blockquotes, horizontal rules) are only valid at line start.
+ *
+ * @group Lexer
+ */
+export interface Token {
+  /** The lexeme category */
+  type: TokenType;
+  /** The literal source text that produced this token */
+  value: string;
+  /** Start/end location in the original source string */
+  position: Position;
+  /**
+   * `true` when this token is the first non-whitespace token on its
+   * line. Block-level rules (headings, lists, blockquotes) check this
+   * flag before attempting to match.
+   */
+  lineStart: boolean;
+}
+
+/**
+ * Construct a {@link Token} value.
+ *
+ * @param type - The lexeme category
+ * @param value - Literal source text
+ * @param position - Source location range
+ * @param lineStart - Whether the token starts a new line
+ * @returns A new token object
+ *
+ * @group Lexer
+ */
+export function createToken(
+  type: TokenType,
+  value: string,
+  position: Position,
+  lineStart = false,
+): Token {
+  return { type, value, position, lineStart };
+}

package/src/parser/constants.ts

@@ -0,0 +1,173 @@
+/**
+ *
+ * Parser constants that define structural boundaries in Wikidot markup.
+ *
+ * These constants are used by the paragraph rule to determine when a new
+ * block-level construct begins, which terminates the current paragraph.
+ * When any of these token types appear at the start of a line, the parser
+ * stops collecting inline content for the current paragraph and begins
+ * processing the new block element.
+ *
+ * @module
+ */
+
+import type { TokenType } from "../lexer";
+
+/**
+ * Token types that signal the start of a block-level construct in Wikidot markup.
+ *
+ * When the parser encounters any of these tokens at the beginning of a line while
+ * building a paragraph, it stops the paragraph and delegates to the appropriate
+ * block rule. Each token maps to a specific Wikidot syntax element (documented
+ * inline with comments).
+ */
+export const BLOCK_START_TOKENS: TokenType[] = [
+  "BLOCKQUOTE_MARKER",
+  "LIST_BULLET",
+  "LIST_NUMBER",
+  "HEADING_MARKER",
+  "HR_MARKER",
+  "TABLE_MARKER",
+  "COLON", // Definition list
+  "BLOCK_OPEN", // [[footnoteblock]], [[div]], etc.
+  "BLOCK_END_OPEN", // [[/div]], [[/collapsible]], etc.
+  "EQUALS", // Center align (= text) or content separator (====)
+  "CLEAR_FLOAT", // ~~~~
+  "CLEAR_FLOAT_LEFT", // ~~~~<
+  "CLEAR_FLOAT_RIGHT", // ~~~~>
+];
+
+/**
+ * Set of block names recognized by the parser at `[[name]]` / `[[/name]]`.
+ *
+ * Used by inline-parser logic to distinguish real block boundaries from
+ * unknown tokens like `[[foo]]`, which Wikidot treats as inline text
+ * rather than as a paragraph-breaking block.
+ *
+ * Keep in sync with the set of block rules registered in
+ * `packages/parser/src/parser/rules/block/index.ts`. Align-style markers
+ * (`<`, `>`, `=`, `==`) are intentionally included because `[[<]]` etc.
+ * open `alignRule`.
+ */
+export const KNOWN_BLOCK_NAMES: ReadonlySet<string> = new Set<string>([
+  // structural containers
+  "collapsible",
+  "div",
+  "div_",
+  "code",
+  // list blocks
+  "ul",
+  "ol",
+  "li",
+  // table blocks
+  "table",
+  "row",
+  "cell",
+  "hcell",
+  // tabview / module
+  "tabview",
+  "tabs",
+  "module",
+  "module654",
+  // misc named blocks
+  "bibliography",
+  "footnoteblock",
+  "toc",
+  "iframe",
+  "math",
+  "html",
+  "iftags",
+  "include",
+  "f", // float TOC prefix: `[[f<toc]]`, `[[f>toc]]` (see toc rule)
+  // embed family
+  "embed",
+  "embedvideo",
+  "embedaudio",
+  // align markers
+  "<",
+  ">",
+  "=",
+  "==",
+  // inline-level constructs that use BLOCK_OPEN tokens; recognized here so
+  // that the paragraph parser keeps existing block-boundary behavior for
+  // `[[span]]`, `[[user ...]]`, `[[$ ... $]]`, etc. when they appear at
+  // the start of a line.
+  "span",
+  "span_",
+  "user",
+  "a",
+  "anchor",
+  "size",
+  "footnote",
+  "eref",
+  "$",
+  "image",
+  "gallery",
+  "file",
+]);
+
+/**
+ * Block names whose rule sets `requiresLineStart: false`, i.e. they can
+ * legitimately start a block even when the `[[...]]` opener is preceded
+ * by leading whitespace on its line.
+ *
+ * Used by the inline parser to decide whether a `\n<indent>[[name]]`
+ * sequence ends the current paragraph. Without this list, the inline
+ * parser would either:
+ *   - keep `lineStart` strict and miss legitimately-indented container
+ *     blocks (Wikidot accepts e.g. `\n     [[div_]]`); the inner block
+ *     gets absorbed into the parent paragraph as literal text, or
+ *   - drop the `lineStart` check entirely and prematurely break out of
+ *     paragraphs for `\n  [[toc]]` — a rule with `requiresLineStart: true`
+ *     would refuse the indented token, leaving the paragraph split but
+ *     the block unconsumed (literal `[[toc]]` text in a new paragraph).
+ *
+ * Each entry corresponds to a name handled by a block rule whose
+ * `requiresLineStart` is `false`. Keep this list in sync when adding or
+ * changing such rules; the inline-level constructs that happen to share
+ * `BLOCK_OPEN` (`[[span]]`, `[[image]]`, `[[user]]`, etc.) are
+ * intentionally excluded — they remain inline and should not split
+ * paragraphs based on indentation alone.
+ *
+ * Sources (block rule → handled names):
+ * - `bibliographyRule` → bibliography
+ * - `blockListRule` → ul, ol, li
+ * - `codeRule` → code
+ * - `collapsibleRule` → collapsible
+ * - `divRule` → div, div_
+ * - `embedBlockRule` → embed, embedvideo, embedaudio
+ * - `htmlRule` → html
+ * - `iframeRule` → iframe
+ * - `iftagsRule` → iftags
+ * - `mathRule` → math
+ * - `moduleRule` → module, module654
+ * - `orphanLiRule` → li (also under blockListRule)
+ * - `tableBlockRule` → table (row, cell, hcell are private to the in-table
+ *   parser, never accepted by the top-level dispatcher)
+ * - `tabviewRule` → tabview, tabs (tab is private to the in-tabview parser)
+ *
+ * `includeRule` is omitted because `[[include ...]]` is expanded as a
+ * text-level macro by `resolveIncludes` before the parser sees it.
+ */
+export const INDENT_ACCEPTING_BLOCK_NAMES: ReadonlySet<string> = new Set<string>([
+  "bibliography",
+  "ul",
+  "ol",
+  "li",
+  "code",
+  "collapsible",
+  "div",
+  "div_",
+  "embed",
+  "embedvideo",
+  "embedaudio",
+  "html",
+  "iframe",
+  "iftags",
+  "math",
+  "module",
+  "module654",
+  "table",
+  "tabview",
+  "tabs",
+]);

package/src/parser/depth.ts

@@ -0,0 +1,251 @@
+/**
+ *
+ * Depth processing module for converting flat lists into nested tree structures.
+ *
+ * This is a TypeScript port of Wikidot's `depth.rs`. It handles the conversion
+ * of flat depth-annotated items (such as bullet/numbered list entries at various
+ * indentation levels) into properly nested tree structures. The algorithm uses an
+ * internal stack to track open nesting levels and collapses them as depth decreases.
+ *
+ * Used primarily by the list parser and the table-of-contents builder to transform
+ * flat sequences of items with depth annotations into hierarchical AST structures.
+ *
+ * @module
+ */
+
+/**
+ * Represents a single node in a depth tree.
+ *
+ * A node is either a leaf item containing a value, or a nested list containing
+ * children. This recursive type allows arbitrarily deep nesting.
+ *
+ * @typeParam L - The list type discriminator (e.g., "bullet" vs "number" for lists,
+ *               or `null` when list type distinction is not needed)
+ * @typeParam T - The type of leaf item values
+ */
+export type DepthItem<L, T> =
+  | { kind: "item"; value: T }
+  | { kind: "list"; ltype: L; children: DepthList<L, T> };
+
+/**
+ * An ordered collection of depth tree nodes at the same level.
+ *
+ * @typeParam L - The list type discriminator
+ * @typeParam T - The type of leaf item values
+ */
+export type DepthList<L, T> = DepthItem<L, T>[];
+
+/**
+ * Internal stack-based builder for constructing depth trees incrementally.
+ *
+ * The stack tracks open nesting levels. As items are added at increasing depths,
+ * new levels are pushed. When depth decreases, levels are popped and collapsed
+ * into their parent as nested list nodes. When the list type changes at the same
+ * depth, the current list is finalized and a new one begins.
+ *
+ * @typeParam L - The list type discriminator
+ * @typeParam T - The type of leaf item values
+ */
+class DepthStack<L, T> {
+  private finished: Array<{ ltype: L; list: DepthList<L, T> }> = [];
+  private stack: Array<{ ltype: L; items: DepthItem<L, T>[] }>;
+
+  /**
+   * @param topLtype - The list type for the initial (top-level) nesting layer
+   */
+  constructor(topLtype: L) {
+    this.stack = [{ ltype: topLtype, items: [] }];
+  }
+
+  /** Returns the topmost (deepest nesting) layer on the stack. */
+  private get last(): { ltype: L; items: DepthItem<L, T>[] } {
+    return this.stack[this.stack.length - 1]!;
+  }
+
+  /** Returns the bottommost (root) layer on the stack. */
+  private get first(): { ltype: L; items: DepthItem<L, T>[] } {
+    return this.stack[0]!;
+  }
+
+  /** Returns true if only the root layer remains on the stack. */
+  private isSingle(): boolean {
+    return this.stack.length === 1;
+  }
+
+  /**
+   * Push a new nesting level onto the stack.
+   * @param ltype - The list type for the new level
+   */
+  increaseDepth(ltype: L): void {
+    this.stack.push({ ltype, items: [] });
+  }
+
+  /**
+   * Pop the topmost nesting level and collapse it into a list node on its parent.
+   * @throws Error if there is no level to pop (stack is empty)
+   */
+  decreaseDepth(): void {
+    const popped = this.stack.pop();
+    if (!popped) {
+      throw new Error("No depth to pop off!");
+    }
+    this.push({ kind: "list", ltype: popped.ltype, children: popped.items });
+  }
+
+  /**
+   * Start a new list at the current depth with a different list type.
+   *
+   * When the list type changes (e.g., from bullet to numbered) at the same depth,
+   * this method finalizes the current list and begins a new one. At the root layer
+   * the entire tree is finalized; at deeper layers a pop/push cycle suffices.
+   *
+   * @param ltype - The list type for the new list
+   */
+  newList(ltype: L): void {
+    if (this.isSingle()) {
+      // This is the last layer, so the pop/push trick doesn't work.
+      // Instead, output this entire thing as a finished list tree,
+      // then create a new one for the process to continue.
+      this.finishDepthList(ltype);
+    } else {
+      // We can just decrease and increase to make a new list
+      this.decreaseDepth();
+      this.increaseDepth(ltype);
+    }
+  }
+
+  /** Append a depth item to the current (topmost) layer. */
+  private push(item: DepthItem<L, T>): void {
+    this.last.items.push(item);
+  }
+
+  /**
+   * Add a leaf item to the current nesting level.
+   * @param item - The value to wrap in a leaf node
+   */
+  pushItem(item: T): void {
+    this.push({ kind: "item", value: item });
+  }
+
+  /** Returns the list type of the topmost nesting level. */
+  lastType(): L {
+    return this.last.ltype;
+  }
+
+  /**
+   * Finalize the current tree by collapsing all open layers into a single
+   * finished tree, then reset the stack for continued processing.
+   *
+   * @param newLtype - The list type for the next tree, or null to reuse the current type.
+   *                   Null is used by {@link intoTrees} since no further items will be added.
+   */
+  private finishDepthList(newLtype: L | null): void {
+    // Wrap all opened layers
+    // Start at 1 since we always have at least one layer
+    while (this.stack.length > 1) {
+      this.decreaseDepth();
+    }
+
+    // Return top-level layer
+    const first = this.first;
+    const ltype = first.ltype;
+    const list = first.items;
+
+    // For intoTrees(), we don't care what the new ltype is,
+    // so we just reuse the last one.
+    // But for newList() we do, we want a new list layer.
+    const actualNewLtype = newLtype ?? ltype;
+
+    // Reset the first layer
+    first.ltype = actualNewLtype;
+    first.items = [];
+
+    // Only push if the list has elements
+    if (list.length > 0) {
+      this.finished.push({ ltype, list });
+    }
+  }
+
+  /**
+   * Finalize all remaining layers and return the completed trees.
+   * @returns Array of finished trees, each with its list type and items
+   */
+  intoTrees(): Array<{ ltype: L; list: DepthList<L, T> }> {
+    this.finishDepthList(null);
+    return this.finished;
+  }
+}
+
+/**
+ * Process a flat list of depth-annotated items into nested tree structures.
+ *
+ * This is the main entry point for the depth module. It takes a sequence of items,
+ * each annotated with a nesting depth and a list type, and produces one or more
+ * nested trees. Multiple trees are produced when the list type changes at the
+ * root level (depth 0).
+ *
+ * The algorithm iterates through items sequentially, using a stack to track
+ * open nesting levels. When depth increases, new levels are pushed; when depth
+ * decreases, levels are popped and collapsed into their parent. When the list
+ * type changes at the same depth, the current list is finalized and a new one begins.
+ *
+ * @typeParam L - The list type discriminator
+ * @typeParam T - The type of leaf item values
+ * @param topLtype - The default list type for the root level
+ * @param items - Flat sequence of depth-annotated items, where each item has:
+ *   - `depth`: the 0-based nesting level
+ *   - `ltype`: the list type for grouping (e.g., "bullet" vs "number")
+ *   - `value`: the actual item content
+ * @param ltypeEquals - Equality comparator for list types (defaults to `===`)
+ * @returns Array of finished trees, each with an `ltype` and a `list` of nested items.
+ *          Multiple trees are returned when the list type changes at depth 0.
+ */
+export function processDepths<L, T>(
+  topLtype: L,
+  items: Array<{ depth: number; ltype: L; value: T }>,
+  ltypeEquals: (a: L, b: L) => boolean = (a, b) => a === b,
+): Array<{ ltype: L; list: DepthList<L, T> }> {
+  const stack = new DepthStack<L, T>(topLtype);
+
+  // The depth value for the previous item
+  let previous = 0;
+
+  // Iterate through each of the items
+  for (const { depth, ltype, value } of items) {
+    // Add or remove new depth levels as appropriate,
+    // based on what our new depth value is compared
+    // to the value in the previous iteration.
+    //
+    // If previous == depth, then neither of these for loops will run
+    // If previous < depth, then only the first will run
+    // If previous > depth, then only the second will run
+
+    // Open new levels
+    for (let i = previous; i < depth; i++) {
+      stack.increaseDepth(ltype);
+    }
+
+    // Close existing levels
+    for (let i = depth; i < previous; i++) {
+      stack.decreaseDepth();
+    }
+
+    // Create new level if the type doesn't match
+    //
+    // Here we decrease and increase the depth to close
+    // the current layer, then make a new one with the
+    // type this item has.
+    //
+    // We'll keep appending to this remade layer until
+    // we hit a different depth or a different type.
+    if (!ltypeEquals(stack.lastType(), ltype)) {
+      stack.newList(ltype);
+    }
+
+    // Push element and update state
+    stack.pushItem(value);
+    previous = depth;
+  }
+
+  return stack.intoTrees();
+}

package/src/parser/index.ts

@@ -0,0 +1,18 @@
+/**
+ *
+ * Main parser for Wikidot markup.
+ *
+ * The parser consumes a token stream from the lexer and produces an AST
+ * (Abstract Syntax Tree) conforming to the `@wdprlib/ast` package types.
+ * It applies block rules and inline rules in a recursive-descent fashion,
+ * followed by post-processing passes for paragraph merging and cleanup.
+ *
+ * The main entry points are:
+ * - `parse()` - convenience function that parses a string in one call
+ * - `Parser` class - for more control over parsing options
+ *
+ * @module
+ */
+
+export type { ParserOptions } from "./parse";
+export { Parser, parse } from "./parse";