@wdprlib/parser 3.1.2 → 4.0.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wdprlib/parser",
-  "version": "3.1.2",
+  "version": "4.0.0",
   "description": "Parser for Wikidot markup",
   "keywords": [
     "ast",
@@ -15,7 +15,8 @@
     "directory": "packages/parser"
   },
   "files": [
-    "dist"
+    "dist",
+    "src"
   ],
   "type": "module",
   "sideEffects": false,
@@ -24,6 +25,7 @@
   "types": "./dist/index.d.ts",
   "exports": {
     ".": {
+      "bun": "./src/index.ts",
       "import": {
         "types": "./dist/index.d.ts",
         "default": "./dist/index.js"
@@ -39,6 +41,6 @@
   },
   "dependencies": {
     "@braintree/sanitize-url": "^7.1.1",
-    "@wdprlib/ast": "2.0.0"
+    "@wdprlib/ast": "2.1.0"
   }
 }

package/src/index.ts

@@ -0,0 +1,170 @@
+/**
+ * Wikidot markup parser.
+ *
+ * This package converts Wikidot wikitext source into an abstract syntax
+ * tree (AST) defined by `@wdprlib/ast`. It also provides module-resolution
+ * utilities for dynamic constructs such as `[[module ListPages]]`,
+ * `[[module ListUsers]]`, `[[include]]`, and `[[iftags]]`.
+ *
+ * Typical usage:
+ *
+ * ```ts
+ * import { parse } from "@wdprlib/parser";
+ *
+ * const { ast, diagnostics } = parse("**bold** and //italic//");
+ * ```
+ *
+ * For server-side module resolution, see {@link extractDataRequirements},
+ * {@link resolveModules}, and {@link resolveIncludes}.
+ *
+ * @packageDocumentation
+ */
+
+// Re-export AST types and utilities from @wdprlib/ast
+export type {
+  Position,
+  Point,
+  Version,
+  Element,
+  SyntaxTree,
+  ContainerType,
+  ContainerData,
+  AttributeMap,
+  VariableMap,
+  Alignment,
+  LinkType,
+  LinkLocation,
+  LinkLabel,
+  PageRef,
+  ImageSource,
+  FloatAlignment,
+  ListType,
+  ListItem,
+  ListData,
+  CodeBlockData,
+  TabData,
+  TableCell,
+  TableRow,
+  TableData,
+  DefinitionListItem,
+  Module,
+  CollapsibleData,
+  ClearFloat,
+  AnchorTarget,
+  HeaderType,
+  AlignType,
+  HeadingLevel,
+  Heading,
+  DateItem,
+  Embed,
+  TocEntry,
+  // Diagnostics
+  Diagnostic,
+  DiagnosticSeverity,
+  ParseResult,
+} from "@wdprlib/ast";
+export {
+  createPoint,
+  createPosition,
+  text,
+  container,
+  paragraph,
+  bold,
+  italics,
+  heading,
+  lineBreak,
+  horizontalRule,
+  link,
+  list,
+  listItemElements,
+  listItemSubList,
+} from "@wdprlib/ast";
+
+// Wikitext settings (re-exported from @wdprlib/ast)
+export type { WikitextMode, WikitextSettings } from "@wdprlib/ast";
+export { createSettings, DEFAULT_SETTINGS } from "@wdprlib/ast";
+
+// Lexer
+export type { TokenType, Token, LexerOptions } from "./lexer";
+export { Lexer, tokenize, createToken } from "./lexer";
+
+// Parser
+export type { ParserOptions } from "./parser";
+export { Parser, parse } from "./parser";
+
+// Modules (ListPages, ListUsers, IfTags, Include, etc.)
+export type {
+  // ListPages query types
+  ListPagesQuery,
+  ListPagesVariable,
+  // Data requirement types
+  ListPagesDataRequirement,
+  DataRequirements,
+  // External data types
+  UserInfo,
+  PageData,
+  SiteContext,
+  ListPagesExternalData,
+  // Callback types
+  ListPagesDataFetcher,
+  DataProvider,
+  // Template types
+  VariableContext,
+  CompiledTemplate,
+  // Extraction types
+  ExtractionResult,
+  // Resolution types
+  ParseFunction,
+  ModuleSourceTransform,
+  ResolveOptions,
+  // Include resolution
+  IncludeFetcher,
+  AsyncIncludeFetcher,
+  ResolveIncludesOptions,
+  IncludeReference,
+  IncludeDependency,
+  IncludeIterationTrace,
+  ResolveIncludesTraceResult,
+  // ListUsers types
+  ListUsersVariable,
+  ListUsersUserData,
+  ListUsersDataRequirement,
+  ListUsersExternalData,
+  ListUsersDataFetcher,
+  ListUsersVariableContext,
+  ListUsersCompiledTemplate,
+  // Normalized query types
+  NormalizedListPagesQuery,
+  NormalizedTags,
+  NormalizedCategory,
+  NormalizedOrder,
+  NormalizedParent,
+  NormalizedDateSelector,
+  NormalizedNumericSelector,
+} from "./parser/rules/block/module/index";
+export {
+  extractDataRequirements,
+  resolveModules,
+  STYLE_SLOT_PREFIX,
+  compileTemplate,
+  // Include resolution
+  extractIncludeReferences,
+  resolveIncludes,
+  resolveIncludesAsync,
+  resolveIncludesWithTrace,
+  // IfTags source-level preprocessing (run between include expansion and parse)
+  preprocessIftags,
+  // Query normalization (for advanced use cases)
+  normalizeQuery,
+  parseTags,
+  parseCategory,
+  parseOrder,
+  parseParent,
+  parseDateSelector,
+  parseNumericSelector,
+  // ListUsers
+  extractListUsersVariables,
+  compileListUsersTemplate,
+  isListUsersModule,
+  resolveListUsers,
+} from "./parser/rules/block/module/index";

package/src/lexer/anchor.ts

@@ -0,0 +1,48 @@
+/**
+ * Check if `[[#` is followed by an invalid anchor name that closes with `]]`.
+ *
+ * Valid anchor names match `[-_A-Za-z0-9.%]+`; invalid names are decomposed so
+ * the parser can handle the inner `[# text]` as a described anchor link.
+ */
+export function findInvalidAnchorNameEnd(src: string, pos: number): number | null {
+  if (src[pos] !== "[" || src[pos + 1] !== "[" || src[pos + 2] !== "#") {
+    return null;
+  }
+  if (src[pos + 3] !== " ") {
+    return null;
+  }
+
+  let i = pos + 4;
+  while (i < src.length && src[i] === " ") {
+    i++;
+  }
+
+  let foundInvalid = false;
+  while (i < src.length) {
+    const ch = src[i]!;
+    if (ch === "\n") {
+      return null;
+    }
+    if (ch === "]" && src[i + 1] === "]") {
+      return foundInvalid ? i : null;
+    }
+    if (!isValidAnchorNameChar(ch.charCodeAt(0))) {
+      foundInvalid = true;
+    }
+    i++;
+  }
+
+  return null;
+}
+
+function isValidAnchorNameChar(code: number): boolean {
+  return (
+    (code >= 48 && code <= 57) ||
+    (code >= 65 && code <= 90) ||
+    (code >= 97 && code <= 122) ||
+    code === 45 ||
+    code === 95 ||
+    code === 46 ||
+    code === 37
+  );
+}

package/src/lexer/index.ts

@@ -0,0 +1,21 @@
+/**
+ *
+ * Lexer (tokenizer) for Wikidot markup.
+ *
+ * The lexer converts preprocessed wikitext into a flat sequence of tokens
+ * that the parser consumes. Each token has a type (e.g., `HEADING_MARKER`,
+ * `BOLD`, `TEXT`) and a string value. The lexer is context-free and does
+ * not build any tree structure; that is the parser's responsibility.
+ *
+ * The main entry points are:
+ * - `tokenize()` - convenience function that tokenizes a string in one call
+ * - `Lexer` class - for more control over tokenization options
+ *
+ * @module
+ */
+
+export type { TokenType, Token } from "./tokens";
+export { createToken } from "./tokens";
+export type { LexerOptions } from "./options";
+export { Lexer } from "./lexer";
+export { tokenize } from "./tokenize";

package/src/lexer/lexer.ts

@@ -0,0 +1,201 @@
+import type { Token, TokenType } from "./tokens";
+import type { LexerOptions } from "./options";
+import {
+  createLexerToken,
+  nextBlockOpenerDepth,
+  updateLastNonWhitespaceType,
+} from "./token-factory";
+import {
+  advance,
+  advanceByToken,
+  createInitialLexerState,
+  current,
+  isAtEnd,
+  type LexerState,
+} from "./state";
+import { findInvalidAnchorNameEnd } from "./anchor";
+import { scanQuotedString } from "./quoted-string";
+import { scanSimpleSyntaxToken } from "./syntax-actions";
+import type { TokenAction } from "./token-actions";
+import { scanPunctuationToken } from "./punctuation";
+import { scanCompactTextToken, scanTextToken } from "./text-actions";
+import { scanSpacingToken } from "./spacing-actions";
+
+/**
+ * Converts a Wikidot markup source string into a flat array of {@link Token}s.
+ *
+ * The lexer is single-pass and greedy: it tries the longest-matching
+ * multi-character pattern first (e.g. `[[[` before `[[`, `**` before `*`).
+ * Context-sensitive constructs (line-start headings, blockquote markers)
+ * are disambiguated via the `lineStart` state flag.
+ *
+ * For convenience, use the standalone {@link tokenize} function instead
+ * of constructing a `Lexer` directly.
+ *
+ * @group Lexer
+ */
+export class Lexer {
+  private state: LexerState;
+  private options: Required<LexerOptions>;
+  // Positions where ]] should be split into ] + ] (for invalid anchor names)
+  private splitBlockClosePositions: Set<number> = new Set();
+  private lastNonWhitespaceType: TokenType | null = null;
+  /**
+   * Nesting depth of block-opener context (between `[[` / `[[/` and the
+   * matching `]]`). Used to scope `QUOTED_STRING` recognition so that
+   * `"` after `=` only becomes a quoted attribute value while we are
+   * actually parsing block attributes — otherwise inline `=` followed by
+   * `"` (e.g. inside `[[footnote]]="[[/footnote]]`) would erroneously
+   * consume content up to the next `"` or newline.
+   */
+  private blockOpenerDepth = 0;
+
+  constructor(source: string, options: LexerOptions = {}) {
+    this.options = {
+      trackPositions: options.trackPositions ?? true,
+      compactTextRuns: options.compactTextRuns ?? false,
+    };
+    this.state = createInitialLexerState(source);
+  }
+
+  /**
+   * Tokenize the entire source
+   */
+  tokenize(): Token[] {
+    while (!this.isAtEnd()) {
+      this.scanToken();
+    }
+
+    this.addToken("EOF", "");
+    return this.state.tokens;
+  }
+
+  /**
+   * Check if at end of source
+   */
+  private isAtEnd(): boolean {
+    return isAtEnd(this.state);
+  }
+
+  /**
+   * Get current character
+   */
+  private current(): string {
+    return current(this.state);
+  }
+
+  /**
+   * Check if [[# is followed by an invalid anchor name that closes with ]].
+   * Valid: [[# valid-name]] where name matches [-_A-Za-z0-9.%]+
+   * Invalid: [[# name with spaces]] or [[# name$special]]
+   * When invalid, returns the position of the closing ]] so the lexer can
+   * emit tokens that allow the inner [# text] to be parsed as a described link.
+   */
+  private findInvalidAnchorNameEnd(): number | null {
+    return findInvalidAnchorNameEnd(this.state.source, this.state.pos);
+  }
+
+  /**
+   * Advance position by n characters
+   */
+  private advance(n = 1): string {
+    return advance(this.state, n);
+  }
+
+  /**
+   * Returns the type of the last non-whitespace token, or null if none.
+   */
+  private lastNonWhitespaceTokenType(): TokenType | null {
+    return this.lastNonWhitespaceType;
+  }
+
+  /**
+   * Add token
+   */
+  private addToken(type: TokenType, value: string): void {
+    this.state.tokens.push(createLexerToken(this.state, type, value, this.options.trackPositions));
+    this.lastNonWhitespaceType = updateLastNonWhitespaceType(this.lastNonWhitespaceType, type);
+    this.blockOpenerDepth = nextBlockOpenerDepth(this.blockOpenerDepth, type);
+  }
+
+  private emitTokenAction(action: TokenAction): void {
+    advanceByToken(this.state, action.type, action.length);
+    this.addToken(action.type, action.value);
+  }
+
+  private emitTokenActions(actions: TokenAction | TokenAction[]): void {
+    if (Array.isArray(actions)) {
+      for (const action of actions) {
+        this.emitTokenAction(action);
+      }
+      return;
+    }
+    this.emitTokenAction(actions);
+  }
+
+  /**
+   * Scan a single token
+   */
+  private scanToken(): void {
+    const char = this.current();
+    const isLineStart = this.state.lineStart;
+    const src = this.state.source;
+
+    const spacingAction = scanSpacingToken(src, this.state.pos);
+    if (spacingAction) {
+      this.emitTokenAction(spacingAction);
+      return;
+    }
+
+    const punctuation = scanPunctuationToken({
+      char,
+      source: src,
+      pos: this.state.pos,
+      lineStart: isLineStart,
+      splitBlockClose: this.splitBlockClosePositions.has(this.state.pos),
+      findInvalidAnchorNameEnd: () => this.findInvalidAnchorNameEnd(),
+    });
+    if (punctuation.handled) {
+      if (punctuation.clearSplitBlockCloseAt !== undefined) {
+        this.splitBlockClosePositions.delete(punctuation.clearSplitBlockCloseAt);
+      }
+      if (punctuation.splitBlockCloseAt !== undefined) {
+        this.splitBlockClosePositions.add(punctuation.splitBlockCloseAt);
+      }
+      this.emitTokenActions(punctuation.actions);
+      return;
+    }
+
+    // Quoted string (only after EQUALS for block attribute values)
+    // In inline context (outside of a `[[...]]` opener), `"` is just a
+    // text character (typographic quote). Without the depth gate, an
+    // inline `=` followed by `"` (e.g. `[[footnote]]="[[/footnote]]`)
+    // would otherwise eat the closing tag.
+    if (char === '"') {
+      const lastNonWs = this.lastNonWhitespaceTokenType();
+      if (this.blockOpenerDepth > 0 && lastNonWs === "EQUALS") {
+        this.addToken("QUOTED_STRING", scanQuotedString(this.state));
+        return;
+      }
+      this.advance();
+      this.addToken("TEXT", '"');
+      return;
+    }
+
+    const simpleAction = scanSimpleSyntaxToken(src, this.state.pos, isLineStart);
+    if (simpleAction) {
+      this.emitTokenAction(simpleAction);
+      return;
+    }
+
+    if (this.options.compactTextRuns && this.blockOpenerDepth === 0) {
+      const compactTextAction = scanCompactTextToken(src, this.state.pos);
+      if (compactTextAction) {
+        this.emitTokenAction(compactTextAction);
+        return;
+      }
+    }
+
+    this.emitTokenAction(scanTextToken(src, this.state.pos));
+  }
+}

package/src/lexer/options.ts

@@ -0,0 +1,19 @@
+/**
+ * Configuration for the {@link Lexer}.
+ *
+ * @group Lexer
+ */
+export interface LexerOptions {
+  /**
+   * When `true` (default), every token carries accurate line/column/offset
+   * data. Set to `false` to skip position tracking for faster tokenisation
+   * when source-map information is not needed.
+   */
+  trackPositions?: boolean;
+  /**
+   * Coalesce ordinary text outside `[[...]]` openers into larger TEXT tokens.
+   * This keeps block names and attributes tokenized normally while reducing
+   * token volume for large documents.
+   */
+  compactTextRuns?: boolean;
+}

package/src/lexer/punctuation.ts

@@ -0,0 +1,70 @@
+import {
+  scanAtToken,
+  scanClosingBracketToken,
+  scanDashToken,
+  scanGreaterToken,
+  scanOpeningBracketToken,
+  scanPipeToken,
+  scanTildeToken,
+  type TokenAction,
+} from "./token-actions";
+
+export interface PunctuationScanInput {
+  char: string;
+  source: string;
+  pos: number;
+  lineStart: boolean;
+  splitBlockClose: boolean;
+  findInvalidAnchorNameEnd: () => number | null;
+}
+
+export type PunctuationScanResult =
+  | { handled: false }
+  | {
+      handled: true;
+      actions: TokenAction | TokenAction[];
+      splitBlockCloseAt?: number;
+      clearSplitBlockCloseAt?: number;
+    };
+
+export function scanPunctuationToken(input: PunctuationScanInput): PunctuationScanResult {
+  const { char, source, pos, lineStart } = input;
+
+  switch (char) {
+    case "[": {
+      const action = scanOpeningBracketToken(source, pos, input.findInvalidAnchorNameEnd());
+      return {
+        handled: true,
+        actions: action,
+        splitBlockCloseAt: action.splitBlockCloseAt,
+      };
+    }
+
+    case "]":
+      return {
+        handled: true,
+        actions: scanClosingBracketToken(source, pos, input.splitBlockClose),
+        clearSplitBlockCloseAt: input.splitBlockClose ? pos : undefined,
+      };
+
+    case "@":
+      return { handled: true, actions: scanAtToken(source, pos) };
+
+    case ">":
+      return { handled: true, actions: scanGreaterToken(source, pos, lineStart) };
+
+    case "-":
+      return { handled: true, actions: scanDashToken(source, pos, lineStart) };
+
+    case "~": {
+      const action = scanTildeToken(source, pos, lineStart);
+      return action ? { handled: true, actions: action } : { handled: false };
+    }
+
+    case "|":
+      return { handled: true, actions: scanPipeToken(source, pos) };
+
+    default:
+      return { handled: false };
+  }
+}

package/src/lexer/quoted-string.ts

@@ -0,0 +1,16 @@
+import { advance, current, isAtEnd, type LexerState } from "./state";
+
+/**
+ * Scan a quoted block-attribute value, including the opening quote and optional
+ * closing quote. Newline terminates the token without being consumed.
+ */
+export function scanQuotedString(state: LexerState): string {
+  let quoted = advance(state);
+  while (!isAtEnd(state) && current(state) !== '"' && current(state) !== "\n") {
+    quoted += advance(state);
+  }
+  if (current(state) === '"') {
+    quoted += advance(state);
+  }
+  return quoted;
+}

package/src/lexer/runs.ts

@@ -0,0 +1,85 @@
+const MIN_PLAIN_TEXT_RUN_LENGTH = 32;
+
+export function findWhitespaceRunEnd(src: string, pos: number): number {
+  let end = pos + 1;
+  while (end < src.length && (src[end] === " " || src[end] === "\t")) {
+    end++;
+  }
+  return end;
+}
+
+export function findRepeatedCharRunEnd(src: string, pos: number, char: string): number {
+  let end = pos + 1;
+  while (end < src.length && src[end] === char) {
+    end++;
+  }
+  return end;
+}
+
+export function findLongPlainTextRunEnd(src: string, pos: number): number | null {
+  let end = pos;
+  while (end < src.length) {
+    const code = src.charCodeAt(end);
+    if (code <= 0x7f || code === 0xe000) {
+      break;
+    }
+    end++;
+  }
+
+  return end - pos >= MIN_PLAIN_TEXT_RUN_LENGTH ? end : null;
+}
+
+export function findAsciiIdentifierEnd(src: string, pos: number): number {
+  let end = pos + 1;
+  while (end < src.length && isAsciiAlphanumericCode(src.charCodeAt(end))) {
+    end++;
+  }
+  return end;
+}
+
+export function isAsciiAlphanumericCode(code: number): boolean {
+  return (code >= 48 && code <= 57) || (code >= 65 && code <= 90) || (code >= 97 && code <= 122);
+}
+
+export function findCompactPlainTextRunEnd(src: string, pos: number): number {
+  let end = pos;
+  while (end < src.length) {
+    const code = src.charCodeAt(end);
+    if (isCompactPlainTextBoundary(code)) {
+      break;
+    }
+    end++;
+  }
+  return end;
+}
+
+function isCompactPlainTextBoundary(code: number): boolean {
+  switch (code) {
+    case 0x0a: // \n
+    case 0xe000: // preprocessed backslash break marker
+    case 0x5b: // [
+    case 0x5d: // ]
+    case 0x40: // @
+    case 0x3e: // >
+    case 0x2d: // -
+    case 0x7e: // ~
+    case 0x7c: // |
+    case 0x7b: // {
+    case 0x7d: // }
+    case 0x2a: // *
+    case 0x3c: // <
+    case 0x5f: // _
+    case 0x5e: // ^
+    case 0x2c: // ,
+    case 0x2f: // /
+    case 0x2b: // +
+    case 0x23: // #
+    case 0x3d: // =
+    case 0x3a: // :
+    case 0x26: // &
+    case 0x5c: // \
+      return true;
+    default:
+      return false;
+  }
+}

package/src/lexer/spacing-actions.ts

@@ -0,0 +1,24 @@
+import { findWhitespaceRunEnd } from "./runs";
+import type { TokenAction } from "./token-actions";
+
+export function scanSpacingToken(src: string, pos: number): TokenAction | null {
+  const char = src[pos];
+
+  if (char === "\n") {
+    return token("NEWLINE", "\n");
+  }
+
+  if (char === " " || char === "\t") {
+    return runToken(src, pos, findWhitespaceRunEnd(src, pos), "WHITESPACE");
+  }
+
+  return null;
+}
+
+function token(type: TokenAction["type"], value: string): TokenAction {
+  return { type, value, length: value.length };
+}
+
+function runToken(src: string, pos: number, end: number, type: TokenAction["type"]): TokenAction {
+  return { type, value: src.slice(pos, end), length: end - pos };
+}