@wdprlib/parser 3.2.0 → 4.0.0

package/src/lexer/options.ts

@@ -0,0 +1,19 @@
+/**
+ * Configuration for the {@link Lexer}.
+ *
+ * @group Lexer
+ */
+export interface LexerOptions {
+  /**
+   * When `true` (default), every token carries accurate line/column/offset
+   * data. Set to `false` to skip position tracking for faster tokenisation
+   * when source-map information is not needed.
+   */
+  trackPositions?: boolean;
+  /**
+   * Coalesce ordinary text outside `[[...]]` openers into larger TEXT tokens.
+   * This keeps block names and attributes tokenized normally while reducing
+   * token volume for large documents.
+   */
+  compactTextRuns?: boolean;
+}

package/src/lexer/punctuation.ts

@@ -0,0 +1,70 @@
+import {
+  scanAtToken,
+  scanClosingBracketToken,
+  scanDashToken,
+  scanGreaterToken,
+  scanOpeningBracketToken,
+  scanPipeToken,
+  scanTildeToken,
+  type TokenAction,
+} from "./token-actions";
+
+export interface PunctuationScanInput {
+  char: string;
+  source: string;
+  pos: number;
+  lineStart: boolean;
+  splitBlockClose: boolean;
+  findInvalidAnchorNameEnd: () => number | null;
+}
+
+export type PunctuationScanResult =
+  | { handled: false }
+  | {
+      handled: true;
+      actions: TokenAction | TokenAction[];
+      splitBlockCloseAt?: number;
+      clearSplitBlockCloseAt?: number;
+    };
+
+export function scanPunctuationToken(input: PunctuationScanInput): PunctuationScanResult {
+  const { char, source, pos, lineStart } = input;
+
+  switch (char) {
+    case "[": {
+      const action = scanOpeningBracketToken(source, pos, input.findInvalidAnchorNameEnd());
+      return {
+        handled: true,
+        actions: action,
+        splitBlockCloseAt: action.splitBlockCloseAt,
+      };
+    }
+
+    case "]":
+      return {
+        handled: true,
+        actions: scanClosingBracketToken(source, pos, input.splitBlockClose),
+        clearSplitBlockCloseAt: input.splitBlockClose ? pos : undefined,
+      };
+
+    case "@":
+      return { handled: true, actions: scanAtToken(source, pos) };
+
+    case ">":
+      return { handled: true, actions: scanGreaterToken(source, pos, lineStart) };
+
+    case "-":
+      return { handled: true, actions: scanDashToken(source, pos, lineStart) };
+
+    case "~": {
+      const action = scanTildeToken(source, pos, lineStart);
+      return action ? { handled: true, actions: action } : { handled: false };
+    }
+
+    case "|":
+      return { handled: true, actions: scanPipeToken(source, pos) };
+
+    default:
+      return { handled: false };
+  }
+}

package/src/lexer/quoted-string.ts

@@ -0,0 +1,16 @@
+import { advance, current, isAtEnd, type LexerState } from "./state";
+
+/**
+ * Scan a quoted block-attribute value, including the opening quote and optional
+ * closing quote. Newline terminates the token without being consumed.
+ */
+export function scanQuotedString(state: LexerState): string {
+  let quoted = advance(state);
+  while (!isAtEnd(state) && current(state) !== '"' && current(state) !== "\n") {
+    quoted += advance(state);
+  }
+  if (current(state) === '"') {
+    quoted += advance(state);
+  }
+  return quoted;
+}

package/src/lexer/runs.ts

@@ -0,0 +1,85 @@
+const MIN_PLAIN_TEXT_RUN_LENGTH = 32;
+
+export function findWhitespaceRunEnd(src: string, pos: number): number {
+  let end = pos + 1;
+  while (end < src.length && (src[end] === " " || src[end] === "\t")) {
+    end++;
+  }
+  return end;
+}
+
+export function findRepeatedCharRunEnd(src: string, pos: number, char: string): number {
+  let end = pos + 1;
+  while (end < src.length && src[end] === char) {
+    end++;
+  }
+  return end;
+}
+
+export function findLongPlainTextRunEnd(src: string, pos: number): number | null {
+  let end = pos;
+  while (end < src.length) {
+    const code = src.charCodeAt(end);
+    if (code <= 0x7f || code === 0xe000) {
+      break;
+    }
+    end++;
+  }
+
+  return end - pos >= MIN_PLAIN_TEXT_RUN_LENGTH ? end : null;
+}
+
+export function findAsciiIdentifierEnd(src: string, pos: number): number {
+  let end = pos + 1;
+  while (end < src.length && isAsciiAlphanumericCode(src.charCodeAt(end))) {
+    end++;
+  }
+  return end;
+}
+
+export function isAsciiAlphanumericCode(code: number): boolean {
+  return (code >= 48 && code <= 57) || (code >= 65 && code <= 90) || (code >= 97 && code <= 122);
+}
+
+export function findCompactPlainTextRunEnd(src: string, pos: number): number {
+  let end = pos;
+  while (end < src.length) {
+    const code = src.charCodeAt(end);
+    if (isCompactPlainTextBoundary(code)) {
+      break;
+    }
+    end++;
+  }
+  return end;
+}
+
+function isCompactPlainTextBoundary(code: number): boolean {
+  switch (code) {
+    case 0x0a: // \n
+    case 0xe000: // preprocessed backslash break marker
+    case 0x5b: // [
+    case 0x5d: // ]
+    case 0x40: // @
+    case 0x3e: // >
+    case 0x2d: // -
+    case 0x7e: // ~
+    case 0x7c: // |
+    case 0x7b: // {
+    case 0x7d: // }
+    case 0x2a: // *
+    case 0x3c: // <
+    case 0x5f: // _
+    case 0x5e: // ^
+    case 0x2c: // ,
+    case 0x2f: // /
+    case 0x2b: // +
+    case 0x23: // #
+    case 0x3d: // =
+    case 0x3a: // :
+    case 0x26: // &
+    case 0x5c: // \
+      return true;
+    default:
+      return false;
+  }
+}

package/src/lexer/spacing-actions.ts

@@ -0,0 +1,24 @@
+import { findWhitespaceRunEnd } from "./runs";
+import type { TokenAction } from "./token-actions";
+
+export function scanSpacingToken(src: string, pos: number): TokenAction | null {
+  const char = src[pos];
+
+  if (char === "\n") {
+    return token("NEWLINE", "\n");
+  }
+
+  if (char === " " || char === "\t") {
+    return runToken(src, pos, findWhitespaceRunEnd(src, pos), "WHITESPACE");
+  }
+
+  return null;
+}
+
+function token(type: TokenAction["type"], value: string): TokenAction {
+  return { type, value, length: value.length };
+}
+
+function runToken(src: string, pos: number, end: number, type: TokenAction["type"]): TokenAction {
+  return { type, value: src.slice(pos, end), length: end - pos };
+}

package/src/lexer/state.ts

@@ -0,0 +1,103 @@
+import type { Token } from "./tokens";
+import type { TokenType } from "./tokens";
+
+/**
+ * Internal mutable state carried through a single tokenisation pass.
+ */
+export interface LexerState {
+  source: string;
+  pos: number;
+  line: number;
+  column: number;
+  lineStart: boolean;
+  tokens: Token[];
+}
+
+export function createInitialLexerState(source: string): LexerState {
+  return {
+    source,
+    pos: 0,
+    line: 1,
+    column: 1,
+    lineStart: true,
+    tokens: [],
+  };
+}
+
+export function isAtEnd(state: LexerState): boolean {
+  return state.pos >= state.source.length;
+}
+
+export function current(state: LexerState): string {
+  return state.source[state.pos] ?? "";
+}
+
+export function advance(state: LexerState, n = 1): string {
+  const start = state.pos;
+  const end = Math.min(state.pos + n, state.source.length);
+  const value = state.source.slice(start, end);
+  updatePosition(state, start, end);
+  return value;
+}
+
+export function advanceBy(state: LexerState, n = 1): void {
+  const start = state.pos;
+  const end = Math.min(state.pos + n, state.source.length);
+  updatePosition(state, start, end);
+}
+
+export function advanceByToken(state: LexerState, type: TokenType, length: number): void {
+  state.pos += length;
+
+  if (type === "NEWLINE") {
+    state.line++;
+    state.column = 1;
+    state.lineStart = true;
+    return;
+  }
+
+  state.column += length;
+  if (type !== "WHITESPACE") {
+    state.lineStart = false;
+  }
+}
+
+function updatePosition(state: LexerState, start: number, end: number): void {
+  state.pos = end;
+  updatePositionFromValue(state, state.source.slice(start, end));
+}
+
+function updatePositionFromValue(state: LexerState, value: string): void {
+  const firstNewline = value.indexOf("\n");
+  if (firstNewline === -1) {
+    state.column += value.length;
+    if (state.lineStart && hasNonLineStartSpacing(value, 0)) {
+      state.lineStart = false;
+    }
+    return;
+  }
+
+  const lastNewline = value.lastIndexOf("\n");
+  let newlineCount = 1;
+  let searchFrom = firstNewline + 1;
+  while (searchFrom <= lastNewline) {
+    const nextNewline = value.indexOf("\n", searchFrom);
+    if (nextNewline === -1) break;
+    newlineCount++;
+    searchFrom = nextNewline + 1;
+  }
+
+  state.line += newlineCount;
+  state.column = value.length - lastNewline;
+  state.lineStart = !hasNonLineStartSpacing(value, lastNewline + 1);
+}
+
+function hasNonLineStartSpacing(value: string, start: number): boolean {
+  for (let i = start; i < value.length; i++) {
+    const char = value[i];
+    if (char !== " " && char !== "\t") {
+      return true;
+    }
+  }
+  return false;
+}

package/src/lexer/syntax-actions.ts

@@ -0,0 +1,80 @@
+import type { TokenAction } from "./token-actions";
+import { findRepeatedCharRunEnd } from "./runs";
+import type { TokenType } from "./tokens";
+
+export function scanSimpleSyntaxToken(
+  src: string,
+  pos: number,
+  isLineStart: boolean,
+): TokenAction | null {
+  switch (src[pos]) {
+    case "{":
+      return pairedToken(src, pos, "{", "MONO_MARKER", "{{");
+    case "}":
+      return pairedToken(src, pos, "}", "MONO_CLOSE", "}}");
+    case "*":
+      return scanStarToken(src, pos, isLineStart);
+    case "<":
+      return pairedToken(src, pos, "<", "LEFT_DOUBLE_ANGLE", "<<");
+    case "_":
+      return pairedToken(src, pos, "_", "UNDERLINE_MARKER", "__") ?? token("UNDERSCORE", "_");
+    case "^":
+      return pairedToken(src, pos, "^", "SUPER_MARKER", "^^");
+    case ",":
+      return pairedToken(src, pos, ",", "SUB_MARKER", ",,");
+    case "/":
+      return pairedToken(src, pos, "/", "ITALIC_MARKER", "//") ?? token("SLASH", "/");
+    case "+":
+      return scanHeadingToken(src, pos, isLineStart);
+    case "#":
+      return scanHashToken(src, pos, isLineStart);
+    case "=":
+      return token("EQUALS", "=");
+    case ":":
+      return token("COLON", ":");
+    case "&":
+      return token("AMPERSAND", "&");
+    case "\\":
+      return token("BACKSLASH", "\\");
+    default:
+      return null;
+  }
+}
+
+function scanStarToken(src: string, pos: number, isLineStart: boolean): TokenAction {
+  if (src[pos + 1] === "*") {
+    return token("BOLD_MARKER", "**");
+  }
+  return isLineStart ? token("LIST_BULLET", "*") : token("STAR", "*");
+}
+
+function scanHeadingToken(src: string, pos: number, isLineStart: boolean): TokenAction | null {
+  return isLineStart
+    ? runToken(src, pos, findRepeatedCharRunEnd(src, pos, "+"), "HEADING_MARKER")
+    : null;
+}
+
+function scanHashToken(src: string, pos: number, isLineStart: boolean): TokenAction {
+  if (src[pos + 1] === "#") {
+    return token("COLOR_MARKER", "##");
+  }
+  return isLineStart ? token("LIST_NUMBER", "#") : token("HASH", "#");
+}
+
+function pairedToken(
+  src: string,
+  pos: number,
+  secondChar: string,
+  type: TokenType,
+  value: string,
+): TokenAction | null {
+  return src[pos + 1] === secondChar ? token(type, value) : null;
+}
+
+function token(type: TokenType, value: string): TokenAction {
+  return { type, value, length: value.length };
+}
+
+function runToken(src: string, pos: number, end: number, type: TokenType): TokenAction {
+  return { type, value: src.slice(pos, end), length: end - pos };
+}

package/src/lexer/text-actions.ts

@@ -0,0 +1,41 @@
+import {
+  findAsciiIdentifierEnd,
+  findCompactPlainTextRunEnd,
+  findLongPlainTextRunEnd,
+  isAsciiAlphanumericCode,
+} from "./runs";
+import type { TokenAction } from "./token-actions";
+
+export function scanTextToken(src: string, pos: number): TokenAction {
+  const char = src[pos] ?? "";
+
+  // Backslash line break marker (U+E000, inserted by preproc)
+  if (char.charCodeAt(0) === 0xe000) {
+    return token("BACKSLASH_BREAK", char);
+  }
+
+  const plainTextRunEnd = findLongPlainTextRunEnd(src, pos);
+  if (plainTextRunEnd !== null) {
+    return runToken(src, pos, plainTextRunEnd, "TEXT");
+  }
+
+  const code = char.charCodeAt(0);
+  if (isAsciiAlphanumericCode(code)) {
+    return runToken(src, pos, findAsciiIdentifierEnd(src, pos), "IDENTIFIER");
+  }
+
+  return token("TEXT", char);
+}
+
+export function scanCompactTextToken(src: string, pos: number): TokenAction | null {
+  const end = findCompactPlainTextRunEnd(src, pos);
+  return end > pos ? runToken(src, pos, end, "TEXT") : null;
+}
+
+function token(type: TokenAction["type"], value: string): TokenAction {
+  return { type, value, length: value.length };
+}
+
+function runToken(src: string, pos: number, end: number, type: TokenAction["type"]): TokenAction {
+  return { type, value: src.slice(pos, end), length: end - pos };
+}

package/src/lexer/token-actions.ts

@@ -0,0 +1,136 @@
+import type { TokenType } from "./tokens";
+import { findRepeatedCharRunEnd } from "./runs";
+
+export interface TokenAction {
+  type: TokenType;
+  value: string;
+  length: number;
+  splitBlockCloseAt?: number;
+}
+
+export function scanOpeningBracketToken(
+  src: string,
+  pos: number,
+  invalidAnchorEnd: number | null,
+): TokenAction {
+  if (src[pos + 1] === "!" && src[pos + 2] === "-" && src[pos + 3] === "-") {
+    return token("COMMENT_OPEN", "[!--");
+  }
+  if (src[pos + 1] === "[" && src[pos + 2] === "[") {
+    return token("LINK_OPEN", "[[[");
+  }
+  if (src[pos + 1] === "[" && src[pos + 2] === "/") {
+    return token("BLOCK_END_OPEN", "[[/");
+  }
+  if (src[pos + 1] === "[") {
+    if (invalidAnchorEnd !== null) {
+      return { ...token("TEXT", "["), splitBlockCloseAt: invalidAnchorEnd };
+    }
+    return token("BLOCK_OPEN", "[[");
+  }
+  if (src[pos + 1] === "#") {
+    return token("BRACKET_ANCHOR", "[#");
+  }
+  if (src[pos + 1] === "*") {
+    return token("BRACKET_STAR", "[*");
+  }
+  return token("BRACKET_OPEN", "[");
+}
+
+export function scanClosingBracketToken(
+  src: string,
+  pos: number,
+  splitBlockClose: boolean,
+): TokenAction | TokenAction[] {
+  if (src[pos + 1] === "]" && src[pos + 2] === "]") {
+    return token("LINK_CLOSE", "]]]");
+  }
+  if (src[pos + 1] === "]") {
+    if (splitBlockClose) {
+      return [token("BRACKET_CLOSE", "]"), token("TEXT", "]")];
+    }
+    return token("BLOCK_CLOSE", "]]");
+  }
+  return token("BRACKET_CLOSE", "]");
+}
+
+export function scanAtToken(src: string, pos: number): TokenAction {
+  if (src[pos + 1] === "@") {
+    return token("RAW_OPEN", "@@");
+  }
+  if (src[pos + 1] === "<") {
+    return token("RAW_BLOCK_OPEN", "@<");
+  }
+  return token("AT", "@");
+}
+
+export function scanGreaterToken(src: string, pos: number, isLineStart: boolean): TokenAction {
+  if (src[pos + 1] === "@") {
+    return token("RAW_BLOCK_CLOSE", ">@");
+  }
+  if (isLineStart) {
+    return runToken(src, pos, findRepeatedCharRunEnd(src, pos, ">"), "BLOCKQUOTE_MARKER");
+  }
+  if (src[pos + 1] === ">") {
+    return token("RIGHT_DOUBLE_ANGLE", ">>");
+  }
+  return token("TEXT", ">");
+}
+
+export function scanDashToken(src: string, pos: number, isLineStart: boolean): TokenAction {
+  if (isLineStart && src[pos + 1] === "-" && src[pos + 2] === "-" && src[pos + 3] === "-") {
+    return runToken(src, pos, findRepeatedCharRunEnd(src, pos, "-"), "HR_MARKER");
+  }
+  if (src[pos + 1] === "-" && src[pos + 2] === "]") {
+    return token("COMMENT_CLOSE", "--]");
+  }
+  if (src[pos + 1] === "-") {
+    return token("STRIKE_MARKER", "--");
+  }
+  return token("TEXT", "-");
+}
+
+export function scanTildeToken(src: string, pos: number, isLineStart: boolean): TokenAction | null {
+  if (!isLineStart || src[pos + 1] !== "~" || src[pos + 2] !== "~" || src[pos + 3] !== "~") {
+    return null;
+  }
+
+  const end = findRepeatedCharRunEnd(src, pos, "~");
+  const next = src[end];
+  if (next === "<") {
+    return runToken(src, pos, end + 1, "CLEAR_FLOAT_LEFT");
+  }
+  if (next === ">") {
+    return runToken(src, pos, end + 1, "CLEAR_FLOAT_RIGHT");
+  }
+  return runToken(src, pos, end, "CLEAR_FLOAT");
+}
+
+export function scanPipeToken(src: string, pos: number): TokenAction {
+  if (src[pos + 1] !== "|") {
+    return token("PIPE", "|");
+  }
+
+  const third = src[pos + 2];
+  if (third === "~") {
+    return token("TABLE_HEADER", "||~");
+  }
+  if (third === "<") {
+    return token("TABLE_LEFT", "||<");
+  }
+  if (third === "=") {
+    return token("TABLE_CENTER", "||=");
+  }
+  if (third === ">") {
+    return token("TABLE_RIGHT", "||>");
+  }
+  return token("TABLE_MARKER", "||");
+}
+
+function token(type: TokenType, value: string): TokenAction {
+  return { type, value, length: value.length };
+}
+
+function runToken(src: string, pos: number, end: number, type: TokenType): TokenAction {
+  return { type, value: src.slice(pos, end), length: end - pos };
+}

package/src/lexer/token-factory.ts

@@ -0,0 +1,62 @@
+import type { Position } from "@wdprlib/ast";
+import type { LexerState } from "./state";
+import type { Token, TokenType } from "./tokens";
+
+const ZERO_POSITION: Position = {
+  start: { line: 0, column: 0, offset: 0 },
+  end: { line: 0, column: 0, offset: 0 },
+};
+
+export function createLexerToken(
+  state: LexerState,
+  type: TokenType,
+  value: string,
+  trackPositions: boolean,
+): Token {
+  return {
+    type,
+    value,
+    position: trackPositions ? currentTokenPosition(state, value) : ZERO_POSITION,
+    lineStart: isTokenAtLineStart(state),
+  };
+}
+
+export function updateLastNonWhitespaceType(
+  current: TokenType | null,
+  type: TokenType,
+): TokenType | null {
+  return type === "WHITESPACE" ? current : type;
+}
+
+/**
+ * Track block-opener nesting so `"` after `=` is only recognised as a quoted
+ * attribute value while inside `[[ ... ]]`.
+ */
+export function nextBlockOpenerDepth(current: number, type: TokenType): number {
+  if (type === "BLOCK_OPEN" || type === "BLOCK_END_OPEN") {
+    return current + 1;
+  }
+  if (type === "BLOCK_CLOSE" && current > 0) {
+    return current - 1;
+  }
+  return current;
+}
+
+function currentTokenPosition(state: LexerState, value: string): Position {
+  return {
+    start: {
+      line: state.line,
+      column: state.column - value.length,
+      offset: state.pos - value.length,
+    },
+    end: {
+      line: state.line,
+      column: state.column,
+      offset: state.pos,
+    },
+  };
+}
+
+function isTokenAtLineStart(state: LexerState): boolean {
+  return state.tokens.length === 0 || state.tokens[state.tokens.length - 1]?.type === "NEWLINE";
+}

package/src/lexer/tokenize.ts

@@ -0,0 +1,18 @@
+import type { Token } from "./tokens";
+import type { LexerOptions } from "./options";
+import { Lexer } from "./lexer";
+
+/**
+ * Tokenise a Wikidot markup source string in one call.
+ *
+ * Shorthand for `new Lexer(source, options).tokenize()`.
+ *
+ * @param source - Raw Wikidot markup
+ * @param options - Optional lexer configuration
+ * @returns A flat array of tokens, ending with an `EOF` token
+ *
+ * @group Lexer
+ */
+export function tokenize(source: string, options?: LexerOptions): Token[] {
+  return new Lexer(source, options).tokenize();
+}

package/src/parser/constants.ts

@@ -37,6 +37,8 @@ export const BLOCK_START_TOKENS: TokenType[] = [
   "CLEAR_FLOAT_RIGHT", // ~~~~>
 ];
 
+export const BLOCK_START_TOKEN_SET: ReadonlySet<TokenType> = new Set(BLOCK_START_TOKENS);
+
 /**
  * Set of block names recognized by the parser at `[[name]]` / `[[/name]]`.
  *