@wdprlib/parser 3.2.0 → 4.0.0

package/src/parser/rules/types.ts

@@ -1,252 +1,9 @@
-import type { Token, TokenType } from "../../lexer";
-import type { Version, WikitextSettings, Diagnostic } from "@wdprlib/ast";
-import type { Element, CodeBlockData, TocEntry } from "@wdprlib/ast";
-
-/**
- * Per-scope state propagated by spread + override semantics.
- *
- * Every field is `readonly` so a rule cannot accidentally mutate the
- * parent scope by writing through a shared reference. Updates must be
- * expressed as a replacement: `ctx.scope = { ...ctx.scope, X: ... }`
- * (or, more commonly, by constructing a new child context with the
- * desired scope override).
- *
- * The motivation is to keep speculative parse rollback safe: when a
- * block rule fails, any scope it built up is discarded with the failed
- * context. A shared-state design that mutates fields in place does not
- * survive rollback — grouping per-scope fields here and forbidding
- * nested mutation makes the semantics explicit at the type level.
- */
-export interface ScopeContext {
-  /**
-   * Close condition for the current block. The paragraph parser calls
-   * it to decide when to stop collecting inline content.
-   */
-  readonly blockCloseCondition?: (ctx: ParseContext) => boolean;
-  /**
-   * Block names excluded from paragraph-boundary detection. When a
-   * BLOCK_OPEN/BLOCK_END_OPEN for an excluded name appears at line
-   * start, the inline parser does NOT treat it as a paragraph break.
-   * Used by `[[collapsible]]` to prevent nested `[[collapsible]]` from
-   * splitting paragraphs.
-   */
-  readonly excludedBlockNames?: ReadonlySet<string>;
-  /**
-   * Budget for div nesting: tracks how many more nested divs can open.
-   * When 0, the div rule fails (innermost excess opens become text).
-   * `undefined` means "not yet calculated" (top-level or non-div context).
-   */
-  readonly divClosesBudget?: number;
-  /**
-   * Used by the footnote-block rule to reject duplicate occurrences.
-   *
-   * **Scope is per spread copy of `ParseContext`, not document-global.**
-   * `parseBlocksUntil` creates a fresh `{ ...ctx, pos, ... }` on every
-   * iteration, so the flag does not propagate between sibling rules in
-   * a body, between sibling bodies, or up to the top-level parser.
-   *
-   * Practical effect today:
-   * - Two `[[footnoteblock]]` at the top level: the second one fails
-   *   (the top-level dispatch hands the parser's own `ctx` to rules,
-   *   so mutations are visible to the next top-level iteration).
-   * - Two `[[footnoteblock]]` inside the same body, or across nested
-   *   bodies: both currently succeed, even though Wikidot's
-   *   "first-only" rule should reject the duplicate.
-   *
-   * Fixing the cross-scope case requires either an AST-level dedup pass
-   * after parsing (similar to the auto-append walk) or a shared-state
-   * design with proper rollback for speculative parses. Tracked
-   * separately; this flag intentionally keeps the original primitive
-   * semantics to avoid regressing the top-level duplicate-rejection
-   * test fixtures.
-   *
-   * The auto-append decision in `Parser.parse` deliberately ignores
-   * this flag and walks the final AST instead — see `containsFootnoteBlock`.
-   */
-  readonly footnoteBlockParsed: boolean;
-}
-
-/**
- * Parser context passed to rules.
- *
- * Fields are grouped by lifecycle:
- * - Static config (`tokens`, `version`, `trackPositions`, `settings`,
- *   rule arrays): constructor-fixed.
- * - `pos`: per-scope cursor; kept top-level for ergonomics because
- *   every rule spread overrides it.
- * - Accumulators (`footnotes`, `tocEntries`, …, `diagnostics`):
- *   reference-shared via array identity across spreads.
- * - `scope`: per-scope state explicitly grouped; see {@link ScopeContext}.
- */
-export interface ParseContext {
-  tokens: Token[];
-  pos: number;
-  version: Version;
-  trackPositions: boolean;
-  settings: WikitextSettings;
-  // Collections for SyntaxTree output
-  footnotes: Element[][];
-  tocEntries: TocEntry[];
-  codeBlocks: CodeBlockData[];
-  htmlBlocks: string[];
-  // Bibliography citation labels collected during parsing
-  bibcites: string[];
-  // Rules (injected to avoid circular dependency)
-  blockRules: BlockRule[];
-  blockFallbackRule: BlockRule;
-  inlineRules: InlineRule[];
-  // Diagnostics collected during parsing
-  diagnostics: Diagnostic[];
-  // Per-scope state (readonly fields, immutable-replace semantics).
-  scope: ScopeContext;
-}
-
-/**
- * Result of a rule attempt
- * Returns elements array None/Single/Multiple
- *
- * During migration: T can be either internal AST node or Element
- */
-export type RuleResult<T> = { success: true; elements: T[]; consumed: number } | { success: false };
-
-/**
- * Block rule interface
- */
-export interface BlockRule {
-  /** Rule name for debugging */
-  name: string;
-  /** Token types that can start this rule */
-  startTokens: TokenType[];
-  /** Whether this rule requires line start */
-  requiresLineStart: boolean;
-  /** Try to parse this block */
-  parse(ctx: ParseContext): RuleResult<Element>;
-  /**
-   * Check if tokens at the given position match this rule's start pattern.
-   * Used by inline parser to determine behavior before a block boundary
-   * (e.g. whether to generate a trailing line-break).
-   */
-  isStartPattern?(ctx: ParseContext, pos: number): boolean;
-  /**
-   * When true, a single newline before this block becomes a line-break.
-   * Wikidot's Divalign expands content inline, so \n before nested blocks
-   * becomes <br />. Other blocks (Code, Div, etc.) suppress this.
-   */
-  preservesPrecedingLineBreak?: boolean;
-}
-
-/**
- * Inline rule interface
- */
-export interface InlineRule {
-  /** Rule name for debugging */
-  name: string;
-  /** Token types that can start this rule */
-  startTokens: TokenType[];
-  /** Try to parse this inline element */
-  parse(ctx: ParseContext): RuleResult<Element>;
-}
-
-/**
- * Helper to get current token
- */
-export function currentToken(ctx: ParseContext): Token {
-  return ctx.tokens[ctx.pos] ?? eofToken();
-}
-
-/**
- * Helper to peek ahead
- */
-export function peekToken(ctx: ParseContext, n = 1): Token {
-  return ctx.tokens[ctx.pos + n] ?? eofToken();
-}
-
-/**
- * Helper to check token type
- */
-export function checkToken(ctx: ParseContext, type: TokenType): boolean {
-  return currentToken(ctx).type === type;
-}
-
-/**
- * Helper to check if at end
- */
-export function isAtEnd(ctx: ParseContext): boolean {
-  return ctx.pos >= ctx.tokens.length || currentToken(ctx).type === "EOF";
-}
-
-/**
- * Create EOF token
- */
-function eofToken(): Token {
-  return {
-    type: "EOF",
-    value: "",
-    position: { start: { line: 0, column: 0, offset: 0 }, end: { line: 0, column: 0, offset: 0 } },
-    lineStart: false,
-  };
-}
-
-/**
- * Check if closing marker exists before newline
- * If markerValue is provided, also check that the token value matches
- */
-export function hasClosingMarkerBeforeNewline(
-  ctx: ParseContext,
-  markerType: TokenType,
-  markerValue?: string,
-): boolean {
-  let pos = ctx.pos;
-  while (pos < ctx.tokens.length) {
-    const token = ctx.tokens[pos];
-    if (!token || token.type === "NEWLINE" || token.type === "EOF") {
-      return false;
-    }
-    if (token.type === markerType) {
-      if (markerValue === undefined || token.value === markerValue) {
-        return true;
-      }
-    }
-    pos++;
-  }
-  return false;
-}
-
-/**
- * Check if closing marker exists before paragraph break (double newline)
- * Allows inline formatting to span multiple lines within a paragraph
- */
-export function hasClosingMarkerBeforeParagraphBreak(
-  ctx: ParseContext,
-  markerType: TokenType,
-  markerValue?: string,
-): boolean {
-  let pos = ctx.pos;
-  while (pos < ctx.tokens.length) {
-    const token = ctx.tokens[pos];
-    if (!token || token.type === "EOF") {
-      return false;
-    }
-    // Check for paragraph break (NEWLINE followed by NEWLINE after optional whitespace)
-    if (token.type === "NEWLINE") {
-      let lookAhead = 1;
-      while (ctx.tokens[pos + lookAhead]?.type === "WHITESPACE") {
-        lookAhead++;
-      }
-      if (
-        ctx.tokens[pos + lookAhead]?.type === "NEWLINE" ||
-        ctx.tokens[pos + lookAhead]?.type === "EOF" ||
-        !ctx.tokens[pos + lookAhead]
-      ) {
-        return false; // Paragraph break - stop
-      }
-    }
-    if (token.type === markerType) {
-      if (markerValue === undefined || token.value === markerValue) {
-        return true;
-      }
-    }
-    pos++;
-  }
-  return false;
-}
+export type { BlockRule, InlineRule, ParseContext, RuleResult, ScopeContext } from "./contracts";
+export {
+  checkToken,
+  currentToken,
+  hasClosingMarkerBeforeNewline,
+  hasClosingMarkerBeforeParagraphBreak,
+  isAtEnd,
+  peekToken,
+} from "./tokens";

package/src/parser/depth.ts

@@ -1,251 +0,0 @@
-/**
- *
- * Depth processing module for converting flat lists into nested tree structures.
- *
- * This is a TypeScript port of Wikidot's `depth.rs`. It handles the conversion
- * of flat depth-annotated items (such as bullet/numbered list entries at various
- * indentation levels) into properly nested tree structures. The algorithm uses an
- * internal stack to track open nesting levels and collapses them as depth decreases.
- *
- * Used primarily by the list parser and the table-of-contents builder to transform
- * flat sequences of items with depth annotations into hierarchical AST structures.
- *
- * @module
- */
-
-/**
- * Represents a single node in a depth tree.
- *
- * A node is either a leaf item containing a value, or a nested list containing
- * children. This recursive type allows arbitrarily deep nesting.
- *
- * @typeParam L - The list type discriminator (e.g., "bullet" vs "number" for lists,
- *               or `null` when list type distinction is not needed)
- * @typeParam T - The type of leaf item values
- */
-export type DepthItem<L, T> =
-  | { kind: "item"; value: T }
-  | { kind: "list"; ltype: L; children: DepthList<L, T> };
-
-/**
- * An ordered collection of depth tree nodes at the same level.
- *
- * @typeParam L - The list type discriminator
- * @typeParam T - The type of leaf item values
- */
-export type DepthList<L, T> = DepthItem<L, T>[];
-
-/**
- * Internal stack-based builder for constructing depth trees incrementally.
- *
- * The stack tracks open nesting levels. As items are added at increasing depths,
- * new levels are pushed. When depth decreases, levels are popped and collapsed
- * into their parent as nested list nodes. When the list type changes at the same
- * depth, the current list is finalized and a new one begins.
- *
- * @typeParam L - The list type discriminator
- * @typeParam T - The type of leaf item values
- */
-class DepthStack<L, T> {
-  private finished: Array<{ ltype: L; list: DepthList<L, T> }> = [];
-  private stack: Array<{ ltype: L; items: DepthItem<L, T>[] }>;
-
-  /**
-   * @param topLtype - The list type for the initial (top-level) nesting layer
-   */
-  constructor(topLtype: L) {
-    this.stack = [{ ltype: topLtype, items: [] }];
-  }
-
-  /** Returns the topmost (deepest nesting) layer on the stack. */
-  private get last(): { ltype: L; items: DepthItem<L, T>[] } {
-    return this.stack[this.stack.length - 1]!;
-  }
-
-  /** Returns the bottommost (root) layer on the stack. */
-  private get first(): { ltype: L; items: DepthItem<L, T>[] } {
-    return this.stack[0]!;
-  }
-
-  /** Returns true if only the root layer remains on the stack. */
-  private isSingle(): boolean {
-    return this.stack.length === 1;
-  }
-
-  /**
-   * Push a new nesting level onto the stack.
-   * @param ltype - The list type for the new level
-   */
-  increaseDepth(ltype: L): void {
-    this.stack.push({ ltype, items: [] });
-  }
-
-  /**
-   * Pop the topmost nesting level and collapse it into a list node on its parent.
-   * @throws Error if there is no level to pop (stack is empty)
-   */
-  decreaseDepth(): void {
-    const popped = this.stack.pop();
-    if (!popped) {
-      throw new Error("No depth to pop off!");
-    }
-    this.push({ kind: "list", ltype: popped.ltype, children: popped.items });
-  }
-
-  /**
-   * Start a new list at the current depth with a different list type.
-   *
-   * When the list type changes (e.g., from bullet to numbered) at the same depth,
-   * this method finalizes the current list and begins a new one. At the root layer
-   * the entire tree is finalized; at deeper layers a pop/push cycle suffices.
-   *
-   * @param ltype - The list type for the new list
-   */
-  newList(ltype: L): void {
-    if (this.isSingle()) {
-      // This is the last layer, so the pop/push trick doesn't work.
-      // Instead, output this entire thing as a finished list tree,
-      // then create a new one for the process to continue.
-      this.finishDepthList(ltype);
-    } else {
-      // We can just decrease and increase to make a new list
-      this.decreaseDepth();
-      this.increaseDepth(ltype);
-    }
-  }
-
-  /** Append a depth item to the current (topmost) layer. */
-  private push(item: DepthItem<L, T>): void {
-    this.last.items.push(item);
-  }
-
-  /**
-   * Add a leaf item to the current nesting level.
-   * @param item - The value to wrap in a leaf node
-   */
-  pushItem(item: T): void {
-    this.push({ kind: "item", value: item });
-  }
-
-  /** Returns the list type of the topmost nesting level. */
-  lastType(): L {
-    return this.last.ltype;
-  }
-
-  /**
-   * Finalize the current tree by collapsing all open layers into a single
-   * finished tree, then reset the stack for continued processing.
-   *
-   * @param newLtype - The list type for the next tree, or null to reuse the current type.
-   *                   Null is used by {@link intoTrees} since no further items will be added.
-   */
-  private finishDepthList(newLtype: L | null): void {
-    // Wrap all opened layers
-    // Start at 1 since we always have at least one layer
-    while (this.stack.length > 1) {
-      this.decreaseDepth();
-    }
-
-    // Return top-level layer
-    const first = this.first;
-    const ltype = first.ltype;
-    const list = first.items;
-
-    // For intoTrees(), we don't care what the new ltype is,
-    // so we just reuse the last one.
-    // But for newList() we do, we want a new list layer.
-    const actualNewLtype = newLtype ?? ltype;
-
-    // Reset the first layer
-    first.ltype = actualNewLtype;
-    first.items = [];
-
-    // Only push if the list has elements
-    if (list.length > 0) {
-      this.finished.push({ ltype, list });
-    }
-  }
-
-  /**
-   * Finalize all remaining layers and return the completed trees.
-   * @returns Array of finished trees, each with its list type and items
-   */
-  intoTrees(): Array<{ ltype: L; list: DepthList<L, T> }> {
-    this.finishDepthList(null);
-    return this.finished;
-  }
-}
-
-/**
- * Process a flat list of depth-annotated items into nested tree structures.
- *
- * This is the main entry point for the depth module. It takes a sequence of items,
- * each annotated with a nesting depth and a list type, and produces one or more
- * nested trees. Multiple trees are produced when the list type changes at the
- * root level (depth 0).
- *
- * The algorithm iterates through items sequentially, using a stack to track
- * open nesting levels. When depth increases, new levels are pushed; when depth
- * decreases, levels are popped and collapsed into their parent. When the list
- * type changes at the same depth, the current list is finalized and a new one begins.
- *
- * @typeParam L - The list type discriminator
- * @typeParam T - The type of leaf item values
- * @param topLtype - The default list type for the root level
- * @param items - Flat sequence of depth-annotated items, where each item has:
- *   - `depth`: the 0-based nesting level
- *   - `ltype`: the list type for grouping (e.g., "bullet" vs "number")
- *   - `value`: the actual item content
- * @param ltypeEquals - Equality comparator for list types (defaults to `===`)
- * @returns Array of finished trees, each with an `ltype` and a `list` of nested items.
- *          Multiple trees are returned when the list type changes at depth 0.
- */
-export function processDepths<L, T>(
-  topLtype: L,
-  items: Array<{ depth: number; ltype: L; value: T }>,
-  ltypeEquals: (a: L, b: L) => boolean = (a, b) => a === b,
-): Array<{ ltype: L; list: DepthList<L, T> }> {
-  const stack = new DepthStack<L, T>(topLtype);
-
-  // The depth value for the previous item
-  let previous = 0;
-
-  // Iterate through each of the items
-  for (const { depth, ltype, value } of items) {
-    // Add or remove new depth levels as appropriate,
-    // based on what our new depth value is compared
-    // to the value in the previous iteration.
-    //
-    // If previous == depth, then neither of these for loops will run
-    // If previous < depth, then only the first will run
-    // If previous > depth, then only the second will run
-
-    // Open new levels
-    for (let i = previous; i < depth; i++) {
-      stack.increaseDepth(ltype);
-    }
-
-    // Close existing levels
-    for (let i = depth; i < previous; i++) {
-      stack.decreaseDepth();
-    }
-
-    // Create new level if the type doesn't match
-    //
-    // Here we decrease and increase the depth to close
-    // the current layer, then make a new one with the
-    // type this item has.
-    //
-    // We'll keep appending to this remade layer until
-    // we hit a different depth or a different type.
-    if (!ltypeEquals(stack.lastType(), ltype)) {
-      stack.newList(ltype);
-    }
-
-    // Push element and update state
-    stack.pushItem(value);
-    previous = depth;
-  }
-
-  return stack.intoTrees();
-}