@wdprlib/parser 3.1.2 → 4.0.0

package/src/lexer/state.ts

@@ -0,0 +1,103 @@
+import type { Token } from "./tokens";
+import type { TokenType } from "./tokens";
+
+/**
+ * Internal mutable state carried through a single tokenisation pass.
+ */
+export interface LexerState {
+  source: string;
+  pos: number;
+  line: number;
+  column: number;
+  lineStart: boolean;
+  tokens: Token[];
+}
+
+export function createInitialLexerState(source: string): LexerState {
+  return {
+    source,
+    pos: 0,
+    line: 1,
+    column: 1,
+    lineStart: true,
+    tokens: [],
+  };
+}
+
+export function isAtEnd(state: LexerState): boolean {
+  return state.pos >= state.source.length;
+}
+
+export function current(state: LexerState): string {
+  return state.source[state.pos] ?? "";
+}
+
+export function advance(state: LexerState, n = 1): string {
+  const start = state.pos;
+  const end = Math.min(state.pos + n, state.source.length);
+  const value = state.source.slice(start, end);
+  updatePosition(state, start, end);
+  return value;
+}
+
+export function advanceBy(state: LexerState, n = 1): void {
+  const start = state.pos;
+  const end = Math.min(state.pos + n, state.source.length);
+  updatePosition(state, start, end);
+}
+
+export function advanceByToken(state: LexerState, type: TokenType, length: number): void {
+  state.pos += length;
+
+  if (type === "NEWLINE") {
+    state.line++;
+    state.column = 1;
+    state.lineStart = true;
+    return;
+  }
+
+  state.column += length;
+  if (type !== "WHITESPACE") {
+    state.lineStart = false;
+  }
+}
+
+function updatePosition(state: LexerState, start: number, end: number): void {
+  state.pos = end;
+  updatePositionFromValue(state, state.source.slice(start, end));
+}
+
+function updatePositionFromValue(state: LexerState, value: string): void {
+  const firstNewline = value.indexOf("\n");
+  if (firstNewline === -1) {
+    state.column += value.length;
+    if (state.lineStart && hasNonLineStartSpacing(value, 0)) {
+      state.lineStart = false;
+    }
+    return;
+  }
+
+  const lastNewline = value.lastIndexOf("\n");
+  let newlineCount = 1;
+  let searchFrom = firstNewline + 1;
+  while (searchFrom <= lastNewline) {
+    const nextNewline = value.indexOf("\n", searchFrom);
+    if (nextNewline === -1) break;
+    newlineCount++;
+    searchFrom = nextNewline + 1;
+  }
+
+  state.line += newlineCount;
+  state.column = value.length - lastNewline;
+  state.lineStart = !hasNonLineStartSpacing(value, lastNewline + 1);
+}
+
+function hasNonLineStartSpacing(value: string, start: number): boolean {
+  for (let i = start; i < value.length; i++) {
+    const char = value[i];
+    if (char !== " " && char !== "\t") {
+      return true;
+    }
+  }
+  return false;
+}

package/src/lexer/syntax-actions.ts

@@ -0,0 +1,80 @@
+import type { TokenAction } from "./token-actions";
+import { findRepeatedCharRunEnd } from "./runs";
+import type { TokenType } from "./tokens";
+
+export function scanSimpleSyntaxToken(
+  src: string,
+  pos: number,
+  isLineStart: boolean,
+): TokenAction | null {
+  switch (src[pos]) {
+    case "{":
+      return pairedToken(src, pos, "{", "MONO_MARKER", "{{");
+    case "}":
+      return pairedToken(src, pos, "}", "MONO_CLOSE", "}}");
+    case "*":
+      return scanStarToken(src, pos, isLineStart);
+    case "<":
+      return pairedToken(src, pos, "<", "LEFT_DOUBLE_ANGLE", "<<");
+    case "_":
+      return pairedToken(src, pos, "_", "UNDERLINE_MARKER", "__") ?? token("UNDERSCORE", "_");
+    case "^":
+      return pairedToken(src, pos, "^", "SUPER_MARKER", "^^");
+    case ",":
+      return pairedToken(src, pos, ",", "SUB_MARKER", ",,");
+    case "/":
+      return pairedToken(src, pos, "/", "ITALIC_MARKER", "//") ?? token("SLASH", "/");
+    case "+":
+      return scanHeadingToken(src, pos, isLineStart);
+    case "#":
+      return scanHashToken(src, pos, isLineStart);
+    case "=":
+      return token("EQUALS", "=");
+    case ":":
+      return token("COLON", ":");
+    case "&":
+      return token("AMPERSAND", "&");
+    case "\\":
+      return token("BACKSLASH", "\\");
+    default:
+      return null;
+  }
+}
+
+function scanStarToken(src: string, pos: number, isLineStart: boolean): TokenAction {
+  if (src[pos + 1] === "*") {
+    return token("BOLD_MARKER", "**");
+  }
+  return isLineStart ? token("LIST_BULLET", "*") : token("STAR", "*");
+}
+
+function scanHeadingToken(src: string, pos: number, isLineStart: boolean): TokenAction | null {
+  return isLineStart
+    ? runToken(src, pos, findRepeatedCharRunEnd(src, pos, "+"), "HEADING_MARKER")
+    : null;
+}
+
+function scanHashToken(src: string, pos: number, isLineStart: boolean): TokenAction {
+  if (src[pos + 1] === "#") {
+    return token("COLOR_MARKER", "##");
+  }
+  return isLineStart ? token("LIST_NUMBER", "#") : token("HASH", "#");
+}
+
+function pairedToken(
+  src: string,
+  pos: number,
+  secondChar: string,
+  type: TokenType,
+  value: string,
+): TokenAction | null {
+  return src[pos + 1] === secondChar ? token(type, value) : null;
+}
+
+function token(type: TokenType, value: string): TokenAction {
+  return { type, value, length: value.length };
+}
+
+function runToken(src: string, pos: number, end: number, type: TokenType): TokenAction {
+  return { type, value: src.slice(pos, end), length: end - pos };
+}

package/src/lexer/text-actions.ts

@@ -0,0 +1,41 @@
+import {
+  findAsciiIdentifierEnd,
+  findCompactPlainTextRunEnd,
+  findLongPlainTextRunEnd,
+  isAsciiAlphanumericCode,
+} from "./runs";
+import type { TokenAction } from "./token-actions";
+
+export function scanTextToken(src: string, pos: number): TokenAction {
+  const char = src[pos] ?? "";
+
+  // Backslash line break marker (U+E000, inserted by preproc)
+  if (char.charCodeAt(0) === 0xe000) {
+    return token("BACKSLASH_BREAK", char);
+  }
+
+  const plainTextRunEnd = findLongPlainTextRunEnd(src, pos);
+  if (plainTextRunEnd !== null) {
+    return runToken(src, pos, plainTextRunEnd, "TEXT");
+  }
+
+  const code = char.charCodeAt(0);
+  if (isAsciiAlphanumericCode(code)) {
+    return runToken(src, pos, findAsciiIdentifierEnd(src, pos), "IDENTIFIER");
+  }
+
+  return token("TEXT", char);
+}
+
+export function scanCompactTextToken(src: string, pos: number): TokenAction | null {
+  const end = findCompactPlainTextRunEnd(src, pos);
+  return end > pos ? runToken(src, pos, end, "TEXT") : null;
+}
+
+function token(type: TokenAction["type"], value: string): TokenAction {
+  return { type, value, length: value.length };
+}
+
+function runToken(src: string, pos: number, end: number, type: TokenAction["type"]): TokenAction {
+  return { type, value: src.slice(pos, end), length: end - pos };
+}

package/src/lexer/token-actions.ts

@@ -0,0 +1,136 @@
+import type { TokenType } from "./tokens";
+import { findRepeatedCharRunEnd } from "./runs";
+
+export interface TokenAction {
+  type: TokenType;
+  value: string;
+  length: number;
+  splitBlockCloseAt?: number;
+}
+
+export function scanOpeningBracketToken(
+  src: string,
+  pos: number,
+  invalidAnchorEnd: number | null,
+): TokenAction {
+  if (src[pos + 1] === "!" && src[pos + 2] === "-" && src[pos + 3] === "-") {
+    return token("COMMENT_OPEN", "[!--");
+  }
+  if (src[pos + 1] === "[" && src[pos + 2] === "[") {
+    return token("LINK_OPEN", "[[[");
+  }
+  if (src[pos + 1] === "[" && src[pos + 2] === "/") {
+    return token("BLOCK_END_OPEN", "[[/");
+  }
+  if (src[pos + 1] === "[") {
+    if (invalidAnchorEnd !== null) {
+      return { ...token("TEXT", "["), splitBlockCloseAt: invalidAnchorEnd };
+    }
+    return token("BLOCK_OPEN", "[[");
+  }
+  if (src[pos + 1] === "#") {
+    return token("BRACKET_ANCHOR", "[#");
+  }
+  if (src[pos + 1] === "*") {
+    return token("BRACKET_STAR", "[*");
+  }
+  return token("BRACKET_OPEN", "[");
+}
+
+export function scanClosingBracketToken(
+  src: string,
+  pos: number,
+  splitBlockClose: boolean,
+): TokenAction | TokenAction[] {
+  if (src[pos + 1] === "]" && src[pos + 2] === "]") {
+    return token("LINK_CLOSE", "]]]");
+  }
+  if (src[pos + 1] === "]") {
+    if (splitBlockClose) {
+      return [token("BRACKET_CLOSE", "]"), token("TEXT", "]")];
+    }
+    return token("BLOCK_CLOSE", "]]");
+  }
+  return token("BRACKET_CLOSE", "]");
+}
+
+export function scanAtToken(src: string, pos: number): TokenAction {
+  if (src[pos + 1] === "@") {
+    return token("RAW_OPEN", "@@");
+  }
+  if (src[pos + 1] === "<") {
+    return token("RAW_BLOCK_OPEN", "@<");
+  }
+  return token("AT", "@");
+}
+
+export function scanGreaterToken(src: string, pos: number, isLineStart: boolean): TokenAction {
+  if (src[pos + 1] === "@") {
+    return token("RAW_BLOCK_CLOSE", ">@");
+  }
+  if (isLineStart) {
+    return runToken(src, pos, findRepeatedCharRunEnd(src, pos, ">"), "BLOCKQUOTE_MARKER");
+  }
+  if (src[pos + 1] === ">") {
+    return token("RIGHT_DOUBLE_ANGLE", ">>");
+  }
+  return token("TEXT", ">");
+}
+
+export function scanDashToken(src: string, pos: number, isLineStart: boolean): TokenAction {
+  if (isLineStart && src[pos + 1] === "-" && src[pos + 2] === "-" && src[pos + 3] === "-") {
+    return runToken(src, pos, findRepeatedCharRunEnd(src, pos, "-"), "HR_MARKER");
+  }
+  if (src[pos + 1] === "-" && src[pos + 2] === "]") {
+    return token("COMMENT_CLOSE", "--]");
+  }
+  if (src[pos + 1] === "-") {
+    return token("STRIKE_MARKER", "--");
+  }
+  return token("TEXT", "-");
+}
+
+export function scanTildeToken(src: string, pos: number, isLineStart: boolean): TokenAction | null {
+  if (!isLineStart || src[pos + 1] !== "~" || src[pos + 2] !== "~" || src[pos + 3] !== "~") {
+    return null;
+  }
+
+  const end = findRepeatedCharRunEnd(src, pos, "~");
+  const next = src[end];
+  if (next === "<") {
+    return runToken(src, pos, end + 1, "CLEAR_FLOAT_LEFT");
+  }
+  if (next === ">") {
+    return runToken(src, pos, end + 1, "CLEAR_FLOAT_RIGHT");
+  }
+  return runToken(src, pos, end, "CLEAR_FLOAT");
+}
+
+export function scanPipeToken(src: string, pos: number): TokenAction {
+  if (src[pos + 1] !== "|") {
+    return token("PIPE", "|");
+  }
+
+  const third = src[pos + 2];
+  if (third === "~") {
+    return token("TABLE_HEADER", "||~");
+  }
+  if (third === "<") {
+    return token("TABLE_LEFT", "||<");
+  }
+  if (third === "=") {
+    return token("TABLE_CENTER", "||=");
+  }
+  if (third === ">") {
+    return token("TABLE_RIGHT", "||>");
+  }
+  return token("TABLE_MARKER", "||");
+}
+
+function token(type: TokenType, value: string): TokenAction {
+  return { type, value, length: value.length };
+}
+
+function runToken(src: string, pos: number, end: number, type: TokenType): TokenAction {
+  return { type, value: src.slice(pos, end), length: end - pos };
+}

package/src/lexer/token-factory.ts

@@ -0,0 +1,62 @@
+import type { Position } from "@wdprlib/ast";
+import type { LexerState } from "./state";
+import type { Token, TokenType } from "./tokens";
+
+const ZERO_POSITION: Position = {
+  start: { line: 0, column: 0, offset: 0 },
+  end: { line: 0, column: 0, offset: 0 },
+};
+
+export function createLexerToken(
+  state: LexerState,
+  type: TokenType,
+  value: string,
+  trackPositions: boolean,
+): Token {
+  return {
+    type,
+    value,
+    position: trackPositions ? currentTokenPosition(state, value) : ZERO_POSITION,
+    lineStart: isTokenAtLineStart(state),
+  };
+}
+
+export function updateLastNonWhitespaceType(
+  current: TokenType | null,
+  type: TokenType,
+): TokenType | null {
+  return type === "WHITESPACE" ? current : type;
+}
+
+/**
+ * Track block-opener nesting so `"` after `=` is only recognised as a quoted
+ * attribute value while inside `[[ ... ]]`.
+ */
+export function nextBlockOpenerDepth(current: number, type: TokenType): number {
+  if (type === "BLOCK_OPEN" || type === "BLOCK_END_OPEN") {
+    return current + 1;
+  }
+  if (type === "BLOCK_CLOSE" && current > 0) {
+    return current - 1;
+  }
+  return current;
+}
+
+function currentTokenPosition(state: LexerState, value: string): Position {
+  return {
+    start: {
+      line: state.line,
+      column: state.column - value.length,
+      offset: state.pos - value.length,
+    },
+    end: {
+      line: state.line,
+      column: state.column,
+      offset: state.pos,
+    },
+  };
+}
+
+function isTokenAtLineStart(state: LexerState): boolean {
+  return state.tokens.length === 0 || state.tokens[state.tokens.length - 1]?.type === "NEWLINE";
+}

package/src/lexer/tokenize.ts

@@ -0,0 +1,18 @@
+import type { Token } from "./tokens";
+import type { LexerOptions } from "./options";
+import { Lexer } from "./lexer";
+
+/**
+ * Tokenise a Wikidot markup source string in one call.
+ *
+ * Shorthand for `new Lexer(source, options).tokenize()`.
+ *
+ * @param source - Raw Wikidot markup
+ * @param options - Optional lexer configuration
+ * @returns A flat array of tokens, ending with an `EOF` token
+ *
+ * @group Lexer
+ */
+export function tokenize(source: string, options?: LexerOptions): Token[] {
+  return new Lexer(source, options).tokenize();
+}

package/src/lexer/tokens.ts

@@ -0,0 +1,141 @@
+import type { Position } from "@wdprlib/ast";
+
+/**
+ * Every distinct lexeme the Wikidot lexer can produce.
+ *
+ * Each value corresponds to a fixed character sequence (or class of
+ * sequences) in Wikidot markup. The inline comments show the literal
+ * text that produces each token type.
+ *
+ * @group Lexer
+ */
+export type TokenType =
+  // Special
+  | "EOF"
+  | "TEXT"
+  | "IDENTIFIER" // alphanumeric word
+  | "NEWLINE"
+  | "WHITESPACE"
+
+  // Block delimiters
+  | "BLOCK_OPEN" // [[
+  | "BLOCK_CLOSE" // ]]
+  | "BLOCK_END_OPEN" // [[/
+
+  // Inline formatting
+  | "BOLD_MARKER" // **
+  | "ITALIC_MARKER" // //
+  | "UNDERLINE_MARKER" // __
+  | "STRIKE_MARKER" // --
+  | "SUPER_MARKER" // ^^
+  | "SUB_MARKER" // ,,
+  | "MONO_MARKER" // {{
+  | "MONO_CLOSE" // }}
+
+  // Special syntax
+  | "HEADING_MARKER" // + (at line start)
+  | "HR_MARKER" // ---- (at line start)
+  | "LIST_BULLET" // * (at line start)
+  | "LIST_NUMBER" // # (at line start)
+  | "BLOCKQUOTE_MARKER" // > (at line start)
+  | "TABLE_MARKER" // || (at line start)
+  | "TABLE_HEADER" // ||~ (header cell)
+  | "TABLE_LEFT" // ||< (left align)
+  | "TABLE_CENTER" // ||= (center align)
+  | "TABLE_RIGHT" // ||> (right align)
+
+  // Code blocks
+  | "CODE_OPEN" // [[code]]
+  | "CODE_CLOSE" // [[/code]]
+
+  // Links
+  | "LINK_OPEN" // [[[
+  | "LINK_CLOSE" // ]]]
+  | "BRACKET_OPEN" // [
+  | "BRACKET_CLOSE" // ]
+  | "BRACKET_ANCHOR" // [#
+  | "BRACKET_STAR" // [*
+
+  // Special characters
+  | "PIPE" // |
+  | "EQUALS" // =
+  | "COLON" // :
+  | "SLASH" // /
+  | "STAR" // *
+  | "HASH" // #
+  | "AT" // @
+  | "AMPERSAND" // &
+  | "BACKSLASH" // \
+  | "QUOTED_STRING" // "..."
+
+  // Raw/Escape
+  | "RAW_OPEN" // @@
+  | "RAW_CLOSE" // @@
+  | "RAW_BLOCK_OPEN" // @<
+  | "RAW_BLOCK_CLOSE" // >@
+
+  // Color
+  | "COLOR_MARKER" // ##
+
+  // Other
+  | "UNDERSCORE" // _ (single underscore, for line break)
+  | "BACKSLASH_BREAK" // U+E000 (preproc marker for \ at end of line)
+
+  // Comment
+  | "COMMENT_OPEN" // [!--
+  | "COMMENT_CLOSE" // --]
+
+  // Clear float
+  | "CLEAR_FLOAT" // ~~~
+  | "CLEAR_FLOAT_LEFT" // ~~~<
+  | "CLEAR_FLOAT_RIGHT" // ~~~>
+
+  // Double angle (guillemet)
+  | "LEFT_DOUBLE_ANGLE" // <<
+  | "RIGHT_DOUBLE_ANGLE"; // >> (non-line-start)
+
+/**
+ * A single lexical token produced by the `Lexer`.
+ *
+ * Tokens are the input to the parser stage. Each token carries its
+ * literal text (`value`), source location (`position`), and a flag
+ * indicating whether it appeared at the beginning of a line — which
+ * matters because several Wikidot constructs (headings, lists,
+ * blockquotes, horizontal rules) are only valid at line start.
+ *
+ * @group Lexer
+ */
+export interface Token {
+  /** The lexeme category */
+  type: TokenType;
+  /** The literal source text that produced this token */
+  value: string;
+  /** Start/end location in the original source string */
+  position: Position;
+  /**
+   * `true` when this token is the first non-whitespace token on its
+   * line. Block-level rules (headings, lists, blockquotes) check this
+   * flag before attempting to match.
+   */
+  lineStart: boolean;
+}
+
+/**
+ * Construct a {@link Token} value.
+ *
+ * @param type - The lexeme category
+ * @param value - Literal source text
+ * @param position - Source location range
+ * @param lineStart - Whether the token starts a new line
+ * @returns A new token object
+ *
+ * @group Lexer
+ */
+export function createToken(
+  type: TokenType,
+  value: string,
+  position: Position,
+  lineStart = false,
+): Token {
+  return { type, value, position, lineStart };
+}