@wdprlib/parser 3.1.1 → 3.2.0

package/src/lexer/index.ts

@@ -0,0 +1,20 @@
+/**
+ *
+ * Lexer (tokenizer) for Wikidot markup.
+ *
+ * The lexer converts preprocessed wikitext into a flat sequence of tokens
+ * that the parser consumes. Each token has a type (e.g., `HEADING_MARKER`,
+ * `BOLD`, `TEXT`) and a string value. The lexer is context-free and does
+ * not build any tree structure; that is the parser's responsibility.
+ *
+ * The main entry points are:
+ * - `tokenize()` - convenience function that tokenizes a string in one call
+ * - `Lexer` class - for more control over tokenization options
+ *
+ * @module
+ */
+
+export type { TokenType, Token } from "./tokens";
+export { createToken } from "./tokens";
+export type { LexerOptions } from "./lexer";
+export { Lexer, tokenize } from "./lexer";

package/src/lexer/lexer.ts

@@ -0,0 +1,687 @@
+import { createPoint, createPosition } from "@wdprlib/ast";
+import { createToken, type Token, type TokenType } from "./tokens";
+
+/**
+ * Configuration for the {@link Lexer}.
+ *
+ * @group Lexer
+ */
+export interface LexerOptions {
+  /**
+   * When `true` (default), every token carries accurate line/column/offset
+   * data. Set to `false` to skip position tracking for faster tokenisation
+   * when source-map information is not needed.
+   */
+  trackPositions?: boolean;
+}
+
+/**
+ * Internal mutable state carried through a single tokenisation pass.
+ */
+interface LexerState {
+  source: string;
+  pos: number;
+  line: number;
+  column: number;
+  lineStart: boolean;
+  tokens: Token[];
+}
+
+/**
+ * Converts a Wikidot markup source string into a flat array of {@link Token}s.
+ *
+ * The lexer is single-pass and greedy: it tries the longest-matching
+ * multi-character pattern first (e.g. `[[[` before `[[`, `**` before `*`).
+ * Context-sensitive constructs (line-start headings, blockquote markers)
+ * are disambiguated via the `lineStart` state flag.
+ *
+ * For convenience, use the standalone {@link tokenize} function instead
+ * of constructing a `Lexer` directly.
+ *
+ * @group Lexer
+ */
+export class Lexer {
+  private state: LexerState;
+  private options: Required<LexerOptions>;
+  // Positions where ]] should be split into ] + ] (for invalid anchor names)
+  private splitBlockClosePositions: Set<number> = new Set();
+  /**
+   * Nesting depth of block-opener context (between `[[` / `[[/` and the
+   * matching `]]`). Used to scope `QUOTED_STRING` recognition so that
+   * `"` after `=` only becomes a quoted attribute value while we are
+   * actually parsing block attributes — otherwise inline `=` followed by
+   * `"` (e.g. inside `[[footnote]]="[[/footnote]]`) would erroneously
+   * consume content up to the next `"` or newline.
+   */
+  private blockOpenerDepth = 0;
+
+  constructor(source: string, options: LexerOptions = {}) {
+    this.options = {
+      trackPositions: options.trackPositions ?? true,
+    };
+    this.state = {
+      source,
+      pos: 0,
+      line: 1,
+      column: 1,
+      lineStart: true,
+      tokens: [],
+    };
+  }
+
+  /**
+   * Tokenize the entire source
+   */
+  tokenize(): Token[] {
+    while (!this.isAtEnd()) {
+      this.scanToken();
+    }
+
+    this.addToken("EOF", "");
+    return this.state.tokens;
+  }
+
+  /**
+   * Check if at end of source
+   */
+  private isAtEnd(): boolean {
+    return this.state.pos >= this.state.source.length;
+  }
+
+  /**
+   * Get current character
+   */
+  private current(): string {
+    return this.state.source[this.state.pos] ?? "";
+  }
+
+  /**
+   * Check if [[# is followed by an invalid anchor name that closes with ]].
+   * Valid: [[# valid-name]] where name matches [-_A-Za-z0-9.%]+
+   * Invalid: [[# name with spaces]] or [[# name$special]]
+   * When invalid, returns the position of the closing ]] so the lexer can
+   * emit tokens that allow the inner [# text] to be parsed as a described link.
+   */
+  private findInvalidAnchorNameEnd(): number | null {
+    const src = this.state.source;
+    const pos = this.state.pos;
+
+    // Must start with [[#
+    if (src[pos] !== "[" || src[pos + 1] !== "[" || src[pos + 2] !== "#") {
+      return null;
+    }
+
+    // Must have space after #
+    if (src[pos + 3] !== " ") {
+      return null;
+    }
+
+    // Skip spaces after #
+    let i = pos + 4;
+    while (i < src.length && src[i] === " ") {
+      i++;
+    }
+
+    // Scan for invalid characters
+    let foundInvalid = false;
+    while (i < src.length) {
+      const ch = src[i]!;
+      if (ch === "\n") return null;
+      if (ch === "]" && src[i + 1] === "]") {
+        // Reached ]] - if we found invalid chars, this is an invalid anchor name
+        return foundInvalid ? i : null;
+      }
+      const code = ch.charCodeAt(0);
+      const isValid =
+        (code >= 48 && code <= 57) || // 0-9
+        (code >= 65 && code <= 90) || // A-Z
+        (code >= 97 && code <= 122) || // a-z
+        code === 45 || // -
+        code === 95 || // _
+        code === 46 || // .
+        code === 37; // %
+      if (!isValid) {
+        foundInvalid = true;
+      }
+      i++;
+    }
+
+    return null;
+  }
+
+  /**
+   * Check if source matches pattern at current position
+   */
+  private match(pattern: string): boolean {
+    for (let i = 0; i < pattern.length; i++) {
+      if (this.state.source[this.state.pos + i] !== pattern[i]) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  /**
+   * Advance position by n characters
+   */
+  private advance(n = 1): string {
+    let result = "";
+    for (let i = 0; i < n && !this.isAtEnd(); i++) {
+      const char = this.current();
+      result += char;
+      this.state.pos++;
+
+      if (char === "\n") {
+        this.state.line++;
+        this.state.column = 1;
+        this.state.lineStart = true;
+      } else {
+        this.state.column++;
+        if (char !== " " && char !== "\t") {
+          this.state.lineStart = false;
+        }
+      }
+    }
+    return result;
+  }
+
+  /**
+   * Returns the type of the last non-whitespace token, or null if none.
+   */
+  private lastNonWhitespaceTokenType(): TokenType | null {
+    for (let i = this.state.tokens.length - 1; i >= 0; i--) {
+      const t = this.state.tokens[i]!;
+      if (t.type !== "WHITESPACE") return t.type;
+    }
+    return null;
+  }
+
+  /**
+   * Add token
+   */
+  private addToken(type: TokenType, value: string): void {
+    const startPos = createPoint(
+      this.state.line,
+      this.state.column - value.length,
+      this.state.pos - value.length,
+    );
+    const endPos = createPoint(this.state.line, this.state.column, this.state.pos);
+    const position = this.options.trackPositions
+      ? createPosition(startPos, endPos)
+      : createPosition(createPoint(0, 0, 0), createPoint(0, 0, 0));
+
+    const lineStart =
+      this.state.tokens.length === 0 ||
+      this.state.tokens[this.state.tokens.length - 1]?.type === "NEWLINE";
+
+    this.state.tokens.push(createToken(type, value, position, lineStart));
+
+    // Track block-opener nesting so `"` after `=` is only recognised as a
+    // quoted attribute value while we are actually inside `[[ ... ]]`.
+    if (type === "BLOCK_OPEN" || type === "BLOCK_END_OPEN") {
+      this.blockOpenerDepth++;
+    } else if (type === "BLOCK_CLOSE" && this.blockOpenerDepth > 0) {
+      this.blockOpenerDepth--;
+    }
+  }
+
+  /**
+   * Scan a single token
+   */
+  private scanToken(): void {
+    const char = this.current();
+    const isLineStart = this.state.lineStart;
+
+    // Newline
+    if (char === "\n") {
+      this.advance();
+      this.addToken("NEWLINE", "\n");
+      return;
+    }
+
+    // Whitespace (non-newline)
+    if (char === " " || char === "\t") {
+      let ws = "";
+      while (!this.isAtEnd() && (this.current() === " " || this.current() === "\t")) {
+        ws += this.advance();
+      }
+      this.addToken("WHITESPACE", ws);
+      return;
+    }
+
+    // Comment open [!-- (must check before [[[)
+    if (this.match("[!--")) {
+      this.advance(4);
+      this.addToken("COMMENT_OPEN", "[!--");
+      return;
+    }
+
+    // Link open [[[ (must check before [[)
+    if (this.match("[[[")) {
+      this.advance(3);
+      this.addToken("LINK_OPEN", "[[[");
+      return;
+    }
+
+    // Block end open [[/
+    if (this.match("[[/")) {
+      this.advance(3);
+      this.addToken("BLOCK_END_OPEN", "[[/");
+      return;
+    }
+
+    // Block open [[
+    if (this.match("[[")) {
+      // Check for invalid anchor name pattern: [[# name-with-spaces]]
+      // Wikidot's Anchor regex requires [-_A-Za-z0-9.%] only after [[# .
+      // If [[# is followed by invalid anchor name, decompose into
+      // TEXT "[" so the inner [# text] is parsed as a described anchor link.
+      // The closing ]] will also be split: ] (BRACKET_CLOSE) + ] (TEXT).
+      const invalidEnd = this.findInvalidAnchorNameEnd();
+      if (invalidEnd !== null) {
+        this.splitBlockClosePositions.add(invalidEnd);
+        this.advance(1);
+        this.addToken("TEXT", "[");
+        return;
+      }
+      this.advance(2);
+      this.addToken("BLOCK_OPEN", "[[");
+      return;
+    }
+
+    // Link close ]]] (must check before ]])
+    if (this.match("]]]")) {
+      this.advance(3);
+      this.addToken("LINK_CLOSE", "]]]");
+      return;
+    }
+
+    // Block close ]]
+    if (this.match("]]")) {
+      // For invalid anchor names, split ]] into ] (BRACKET_CLOSE) + ] (TEXT)
+      if (this.splitBlockClosePositions.has(this.state.pos)) {
+        this.splitBlockClosePositions.delete(this.state.pos);
+        this.advance(1);
+        this.addToken("BRACKET_CLOSE", "]");
+        this.advance(1);
+        this.addToken("TEXT", "]");
+        return;
+      }
+      this.advance(2);
+      this.addToken("BLOCK_CLOSE", "]]");
+      return;
+    }
+
+    // Raw/escape @@
+    if (this.match("@@")) {
+      this.advance(2);
+      this.addToken("RAW_OPEN", "@@");
+      return;
+    }
+
+    // Raw block @<
+    if (this.match("@<")) {
+      this.advance(2);
+      this.addToken("RAW_BLOCK_OPEN", "@<");
+      return;
+    }
+
+    // Raw block close >@
+    if (this.match(">@")) {
+      this.advance(2);
+      this.addToken("RAW_BLOCK_CLOSE", ">@");
+      return;
+    }
+
+    // Monospace open {{
+    if (this.match("{{")) {
+      this.advance(2);
+      this.addToken("MONO_MARKER", "{{");
+      return;
+    }
+
+    // Monospace close }}
+    if (this.match("}}")) {
+      this.advance(2);
+      this.addToken("MONO_CLOSE", "}}");
+      return;
+    }
+
+    // Bold **
+    if (this.match("**")) {
+      this.advance(2);
+      this.addToken("BOLD_MARKER", "**");
+      return;
+    }
+
+    // Horizontal rule ---- or more (4+ hyphens, check before --)
+    if (isLineStart && this.match("----")) {
+      let dashes = "";
+      while (this.current() === "-") {
+        dashes += this.advance();
+      }
+      this.addToken("HR_MARKER", dashes);
+      return;
+    }
+
+    // Comment close --] (must check before --)
+    if (this.match("--]")) {
+      this.advance(3);
+      this.addToken("COMMENT_CLOSE", "--]");
+      return;
+    }
+
+    // Strikethrough -- (Wikidot only uses --)
+    if (this.match("--")) {
+      this.advance(2);
+      this.addToken("STRIKE_MARKER", "--");
+      return;
+    }
+
+    // Left double angle << (guillemet)
+    if (this.match("<<")) {
+      this.advance(2);
+      this.addToken("LEFT_DOUBLE_ANGLE", "<<");
+      return;
+    }
+
+    // Clear float ~~~~ or more (at line start only, Wikidot requires 4+)
+    if (isLineStart && this.match("~~~~")) {
+      let tildes = "";
+      while (this.current() === "~") {
+        tildes += this.advance();
+      }
+      // Check for directional clear float
+      if (this.current() === "<") {
+        this.advance();
+        this.addToken("CLEAR_FLOAT_LEFT", `${tildes}<`);
+        return;
+      }
+      if (this.current() === ">") {
+        this.advance();
+        this.addToken("CLEAR_FLOAT_RIGHT", `${tildes}>`);
+        return;
+      }
+      this.addToken("CLEAR_FLOAT", `${tildes}`);
+      return;
+    }
+
+    // Single hyphen (not part of --)
+    if (char === "-") {
+      this.advance();
+      this.addToken("TEXT", "-");
+      return;
+    }
+
+    // Underline __ (check before single _)
+    if (this.match("__")) {
+      this.advance(2);
+      this.addToken("UNDERLINE_MARKER", "__");
+      return;
+    }
+
+    // Single underscore _ (for line break)
+    if (char === "_") {
+      this.advance();
+      this.addToken("UNDERSCORE", "_");
+      return;
+    }
+
+    // Superscript ^^
+    if (this.match("^^")) {
+      this.advance(2);
+      this.addToken("SUPER_MARKER", "^^");
+      return;
+    }
+
+    // Subscript ,,
+    if (this.match(",,")) {
+      this.advance(2);
+      this.addToken("SUB_MARKER", ",,");
+      return;
+    }
+
+    // Italic //
+    if (this.match("//")) {
+      this.advance(2);
+      this.addToken("ITALIC_MARKER", "//");
+      return;
+    }
+
+    // Table markers
+    // ||~ (header), ||< (left), ||= (center), ||> (right), || (normal)
+    if (this.match("||~")) {
+      this.advance(3);
+      this.addToken("TABLE_HEADER", "||~");
+      return;
+    }
+    if (this.match("||<")) {
+      this.advance(3);
+      this.addToken("TABLE_LEFT", "||<");
+      return;
+    }
+    if (this.match("||=")) {
+      this.advance(3);
+      this.addToken("TABLE_CENTER", "||=");
+      return;
+    }
+    if (this.match("||>")) {
+      this.advance(3);
+      this.addToken("TABLE_RIGHT", "||>");
+      return;
+    }
+    if (this.match("||")) {
+      this.advance(2);
+      this.addToken("TABLE_MARKER", "||");
+      return;
+    }
+
+    // Heading + (at line start)
+    if (isLineStart && char === "+") {
+      let plusCount = 0;
+      while (this.current() === "+") {
+        plusCount++;
+        this.advance();
+      }
+      this.addToken("HEADING_MARKER", "+".repeat(plusCount));
+      return;
+    }
+
+    // List bullet * (at line start)
+    if (isLineStart && char === "*") {
+      this.advance();
+      this.addToken("LIST_BULLET", "*");
+      return;
+    }
+
+    // Color marker ## (check before LIST_NUMBER)
+    if (this.match("##")) {
+      this.advance(2);
+      this.addToken("COLOR_MARKER", "##");
+      return;
+    }
+
+    // List number # (at line start)
+    if (isLineStart && char === "#") {
+      this.advance();
+      this.addToken("LIST_NUMBER", "#");
+      return;
+    }
+
+    // Blockquote > or >>> (at line start only for blockquote)
+    if (char === ">") {
+      if (isLineStart) {
+        // At line start: consume all consecutive > as a single blockquote marker
+        let depth = "";
+        while (this.current() === ">") {
+          depth += this.advance();
+        }
+        this.addToken("BLOCKQUOTE_MARKER", depth);
+        return;
+      }
+      // Not at line start
+      if (this.match(">>")) {
+        // >> not at line start - guillemet
+        this.advance(2);
+        this.addToken("RIGHT_DOUBLE_ANGLE", ">>");
+        return;
+      }
+      // Single > not at line start - just text
+      this.advance();
+      this.addToken("TEXT", ">");
+      return;
+    }
+
+    // Bracket anchor [#
+    if (this.match("[#")) {
+      this.advance(2);
+      this.addToken("BRACKET_ANCHOR", "[#");
+      return;
+    }
+
+    // Bracket star [* (for new tab links)
+    if (this.match("[*")) {
+      this.advance(2);
+      this.addToken("BRACKET_STAR", "[*");
+      return;
+    }
+
+    // Single characters
+    if (char === "[") {
+      this.advance();
+      this.addToken("BRACKET_OPEN", "[");
+      return;
+    }
+
+    if (char === "]") {
+      this.advance();
+      this.addToken("BRACKET_CLOSE", "]");
+      return;
+    }
+
+    if (char === "|") {
+      this.advance();
+      this.addToken("PIPE", "|");
+      return;
+    }
+
+    if (char === "=") {
+      this.advance();
+      this.addToken("EQUALS", "=");
+      return;
+    }
+
+    // Quoted string (only after EQUALS for block attribute values)
+    // In inline context (outside of a `[[...]]` opener), `"` is just a
+    // text character (typographic quote). Without the depth gate, an
+    // inline `=` followed by `"` (e.g. `[[footnote]]="[[/footnote]]`)
+    // would otherwise eat the closing tag.
+    if (char === '"') {
+      const lastNonWs = this.lastNonWhitespaceTokenType();
+      if (this.blockOpenerDepth > 0 && lastNonWs === "EQUALS") {
+        let quoted = this.advance(); // opening "
+        while (!this.isAtEnd() && this.current() !== '"' && this.current() !== "\n") {
+          quoted += this.advance();
+        }
+        if (this.current() === '"') {
+          quoted += this.advance(); // closing "
+        }
+        this.addToken("QUOTED_STRING", quoted);
+        return;
+      }
+      this.advance();
+      this.addToken("TEXT", '"');
+      return;
+    }
+
+    if (char === ":") {
+      this.advance();
+      this.addToken("COLON", ":");
+      return;
+    }
+
+    if (char === "/") {
+      this.advance();
+      this.addToken("SLASH", "/");
+      return;
+    }
+
+    if (char === "*") {
+      this.advance();
+      this.addToken("STAR", "*");
+      return;
+    }
+
+    if (char === "#") {
+      this.advance();
+      this.addToken("HASH", "#");
+      return;
+    }
+
+    if (char === "@") {
+      this.advance();
+      this.addToken("AT", "@");
+      return;
+    }
+
+    if (char === "&") {
+      this.advance();
+      this.addToken("AMPERSAND", "&");
+      return;
+    }
+
+    if (char === "\\") {
+      this.advance();
+      this.addToken("BACKSLASH", "\\");
+      return;
+    }
+
+    // Backslash line break marker (U+E000, inserted by preproc)
+    if (char.charCodeAt(0) === 0xe000) {
+      this.advance();
+      this.addToken("BACKSLASH_BREAK", char);
+      return;
+    }
+
+    // Identifier: alphanumeric sequence
+    if (this.isAlphanumeric(char)) {
+      let ident = "";
+      while (!this.isAtEnd() && this.isAlphanumeric(this.current())) {
+        ident += this.advance();
+      }
+      this.addToken("IDENTIFIER", ident);
+      return;
+    }
+
+    // Default: single character as text
+    const text = this.advance();
+    this.addToken("TEXT", text);
+  }
+
+  /**
+   * Check if character is alphanumeric (for identifier tokens)
+   */
+  private isAlphanumeric(char: string): boolean {
+    const code = char.charCodeAt(0);
+    return (
+      (code >= 48 && code <= 57) || // 0-9
+      (code >= 65 && code <= 90) || // A-Z
+      (code >= 97 && code <= 122) // a-z
+    );
+  }
+}
+
+/**
+ * Tokenise a Wikidot markup source string in one call.
+ *
+ * Shorthand for `new Lexer(source, options).tokenize()`.
+ *
+ * @param source - Raw Wikidot markup
+ * @param options - Optional lexer configuration
+ * @returns A flat array of tokens, ending with an `EOF` token
+ *
+ * @group Lexer
+ */
+export function tokenize(source: string, options?: LexerOptions): Token[] {
+  return new Lexer(source, options).tokenize();
+}