@wdprlib/parser 5.1.6 → 5.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wdprlib/parser",
-  "version": "5.1.6",
+  "version": "5.3.0",
   "description": "Parser for Wikidot markup",
   "keywords": [
     "ast",
@@ -44,6 +44,6 @@
   },
   "dependencies": {
     "@braintree/sanitize-url": "^7.1.1",
-    "@wdprlib/ast": "4.0.2"
+    "@wdprlib/ast": "4.2.0"
   }
 }

package/src/build-info.generated.ts

@@ -0,0 +1,11 @@
+// Generated by bun run build:info. Do not edit.
+/** Build-time package metadata; unavailable Git information is null. */
+export const buildInfo: Readonly<{
+  version: string;
+  sha: string | null;
+  dirty: boolean | null;
+}> = Object.freeze({
+  version: "5.3.0",
+  sha: "b71b84c705e05717f5ea12fd89684a91b669dbff",
+  dirty: false,
+});

package/src/index.ts

@@ -20,6 +20,8 @@
  * @packageDocumentation
  */
 
+export { buildInfo } from "./build-info.generated";
+
 // Re-export AST types and utilities from @wdprlib/ast
 export type {
   Position,

package/src/lexer/lexer.ts

@@ -11,6 +11,7 @@ import {
   createInitialLexerState,
   current,
   isAtEnd,
+  isSyntaxLineStart,
   type LexerState,
 } from "./state";
 import { findInvalidAnchorNameEnd } from "./anchor";
@@ -19,7 +20,7 @@ import { scanSimpleSyntaxToken } from "./syntax-actions";
 import type { TokenAction } from "./token-actions";
 import { scanPunctuationToken } from "./punctuation";
 import { scanCompactTextToken, scanTextToken } from "./text-actions";
-import { scanSpacingToken } from "./spacing-actions";
+import { limitBlockquotePrefixSpace, scanSpacingToken } from "./spacing-actions";
 
 /**
  * Converts a Wikidot markup source string into a flat array of {@link Token}s.
@@ -119,7 +120,7 @@ export class Lexer {
   }
 
   private emitTokenAction(action: TokenAction): void {
-    advanceByToken(this.state, action.type, action.length);
+    advanceByToken(this.state, action.type, action.length, action.value);
     this.addToken(action.type, action.value);
   }
 
@@ -138,12 +139,12 @@ export class Lexer {
    */
   private scanToken(): void {
     const char = this.current();
-    const isLineStart = this.state.lineStart;
+    const isLineStart = isSyntaxLineStart(this.state);
     const src = this.state.source;
 
     const spacingAction = scanSpacingToken(src, this.state.pos);
     if (spacingAction) {
-      this.emitTokenAction(spacingAction);
+      this.emitTokenAction(limitBlockquotePrefixSpace(spacingAction, this.state.tokens.at(-1)));
       return;
     }
 
@@ -152,6 +153,7 @@ export class Lexer {
       source: src,
       pos: this.state.pos,
       lineStart: isLineStart,
+      physicalLineStart: this.state.lineStart,
       splitBlockClose: this.splitBlockClosePositions.has(this.state.pos),
       findInvalidAnchorNameEnd: () => this.findInvalidAnchorNameEnd(),
     });

package/src/lexer/punctuation.ts

@@ -14,6 +14,7 @@ export interface PunctuationScanInput {
   source: string;
   pos: number;
   lineStart: boolean;
+  physicalLineStart: boolean;
   splitBlockClose: boolean;
   findInvalidAnchorNameEnd: () => number | null;
 }
@@ -51,7 +52,7 @@ export function scanPunctuationToken(input: PunctuationScanInput): PunctuationSc
       return { handled: true, actions: scanAtToken(source, pos) };
 
     case ">":
-      return { handled: true, actions: scanGreaterToken(source, pos, lineStart) };
+      return { handled: true, actions: scanGreaterToken(source, pos, input.physicalLineStart) };
 
     case "-":
       return { handled: true, actions: scanDashToken(source, pos, lineStart) };

package/src/lexer/spacing-actions.ts

@@ -1,5 +1,20 @@
 import { findWhitespaceRunEnd } from "./runs";
+import { isLineStartQuoteMarker } from "./state";
 import type { TokenAction } from "./token-actions";
+import type { Token } from "./tokens";
+
+export function limitBlockquotePrefixSpace(
+  action: TokenAction,
+  previous: Token | undefined,
+): TokenAction {
+  if (!isLineStartQuoteMarker(previous) || action.type !== "WHITESPACE") {
+    return action;
+  }
+  if (action.length <= 1 || !action.value.startsWith(" ")) {
+    return action;
+  }
+  return { type: "WHITESPACE", value: " ", length: 1 };
+}
 
 export function scanSpacingToken(src: string, pos: number): TokenAction | null {
   const char = src[pos];

package/src/lexer/state.ts

@@ -10,6 +10,7 @@ export interface LexerState {
   line: number;
   column: number;
   lineStart: boolean;
+  quoteContentStart: boolean;
   tokens: Token[];
 }
 
@@ -20,10 +21,20 @@ export function createInitialLexerState(source: string): LexerState {
     line: 1,
     column: 1,
     lineStart: true,
+    quoteContentStart: false,
     tokens: [],
   };
 }
 
+export function isSyntaxLineStart(state: LexerState): boolean {
+  return state.lineStart || state.quoteContentStart;
+}
+
+/** An indented `>` is not a blockquote, so its content keeps the enclosing line. */
+export function isLineStartQuoteMarker(token: Token | undefined): boolean {
+  return token?.type === "BLOCKQUOTE_MARKER" && token.lineStart;
+}
+
 export function isAtEnd(state: LexerState): boolean {
   return state.pos >= state.source.length;
 }
@@ -46,20 +57,33 @@ export function advanceBy(state: LexerState, n = 1): void {
   updatePosition(state, start, end);
 }
 
-export function advanceByToken(state: LexerState, type: TokenType, length: number): void {
+export function advanceByToken(
+  state: LexerState,
+  type: TokenType,
+  length: number,
+  value = "",
+): void {
+  const afterQuoteMarker = isLineStartQuoteMarker(state.tokens[state.tokens.length - 1]);
   state.pos += length;
 
   if (type === "NEWLINE") {
     state.line++;
     state.column = 1;
     state.lineStart = true;
+    state.quoteContentStart = false;
     return;
   }
 
   state.column += length;
-  if (type !== "WHITESPACE") {
-    state.lineStart = false;
+  if (type === "WHITESPACE") {
+    if (afterQuoteMarker && value === " ") {
+      state.quoteContentStart = true;
+    }
+    return;
   }
+
+  state.lineStart = false;
+  state.quoteContentStart = false;
 }
 
 function updatePosition(state: LexerState, start: number, end: number): void {
@@ -71,8 +95,9 @@ function updatePositionFromValue(state: LexerState, value: string): void {
   const firstNewline = value.indexOf("\n");
   if (firstNewline === -1) {
     state.column += value.length;
-    if (state.lineStart && hasNonLineStartSpacing(value, 0)) {
+    if (hasNonLineStartSpacing(value, 0)) {
       state.lineStart = false;
+      state.quoteContentStart = false;
     }
     return;
   }
@@ -90,6 +115,7 @@ function updatePositionFromValue(state: LexerState, value: string): void {
   state.line += newlineCount;
   state.column = value.length - lastNewline;
   state.lineStart = !hasNonLineStartSpacing(value, lastNewline + 1);
+  state.quoteContentStart = false;
 }
 
 function hasNonLineStartSpacing(value: string, start: number): boolean {

package/src/lexer/token-factory.ts

@@ -1,5 +1,5 @@
 import type { Position } from "@wdprlib/ast";
-import type { LexerState } from "./state";
+import { isLineStartQuoteMarker, type LexerState } from "./state";
 import type { Token, TokenType } from "./tokens";
 
 const ZERO_POSITION: Position = {
@@ -58,5 +58,18 @@ function currentTokenPosition(state: LexerState, value: string): Position {
 }
 
 function isTokenAtLineStart(state: LexerState): boolean {
-  return state.tokens.length === 0 || state.tokens[state.tokens.length - 1]?.type === "NEWLINE";
+  if (state.tokens.length === 0) {
+    return true;
+  }
+
+  const last = state.tokens[state.tokens.length - 1];
+  if (last?.type === "NEWLINE") {
+    return true;
+  }
+
+  return (
+    last?.type === "WHITESPACE" &&
+    last.value === " " &&
+    isLineStartQuoteMarker(state.tokens[state.tokens.length - 2])
+  );
 }

package/src/lexer/tokens.ts

@@ -113,9 +113,10 @@ export interface Token {
   /** Start/end location in the original source string */
   position: Position;
   /**
-   * `true` when this token is the first non-whitespace token on its
-   * line. Block-level rules (headings, lists, blockquotes) check this
-   * flag before attempting to match.
+   * `true` when this token opens a logical line: the first non-whitespace
+   * token on a source line, or the first token after a blockquote prefix.
+   * Block-level rules (headings, lists, blockquotes) check this flag before
+   * attempting to match.
    */
   lineStart: boolean;
 }

package/src/parser/rules/block/blockquote/build.ts

@@ -1,55 +1,67 @@
 import type { Element } from "@wdprlib/ast";
+import { createToken, type Token } from "../../../../lexer";
 import { processDepths, type DepthList } from "../../../depth";
+import type { ParseContext } from "../../types";
+import { parseBlocksUntil } from "../utils";
 import type { BlockquoteLine, ParsedBlockquoteLine } from "./lines";
 
-export function buildBlockquoteElements(lines: ParsedBlockquoteLine[]): Element[] {
+/**
+ * Blocks Wikidot protects before parsing, by matching an opening tag at the
+ * start of a line. A `>` prefix defeats that match, so inside a blockquote
+ * only their tags stay literal while the body parses as usual.
+ */
+const EXCLUDED_BLOCK_NAMES: ReadonlySet<string> = new Set([
+  "bibliography",
+  "code",
+  "html",
+  "include",
+  "math",
+  "module",
+]);
+
+const NEVER_CLOSES = () => false;
+
+export function buildBlockquoteElements(
+  ctx: ParseContext,
+  lines: ParsedBlockquoteLine[],
+): Element[] {
   const depthTrees = processDepths<null, BlockquoteLine>(null, lines);
-  return depthTrees.map(({ list }) => buildBlockquoteElement(list));
+  return depthTrees
+    .map(({ list }) => buildBlockquoteElement(ctx, list))
+    .filter((element): element is Element => element !== null);
 }
 
-function buildBlockquoteElement(list: DepthList<null, BlockquoteLine>): Element {
+function buildBlockquoteElement(
+  ctx: ParseContext,
+  list: DepthList<null, BlockquoteLine>,
+): Element | null {
   const children: Element[] = [];
-  let currentParagraphChildren: Element[] = [];
-
-  function flushParagraph() {
-    if (currentParagraphChildren.length > 0) {
-      while (
-        currentParagraphChildren.length > 0 &&
-        currentParagraphChildren[currentParagraphChildren.length - 1]?.element === "line-break"
-      ) {
-        currentParagraphChildren.pop();
-      }
-      if (currentParagraphChildren.length > 0) {
-        children.push({
-          element: "container",
-          data: {
-            type: "paragraph",
-            attributes: {},
-            elements: currentParagraphChildren,
-          },
-        });
-      }
-      currentParagraphChildren = [];
-    }
+  let pending: BlockquoteLine[] = [];
+
+  function flushPending() {
+    if (pending.length === 0) return;
+    children.push(...parseLines(ctx, pending));
+    pending = [];
   }
 
   for (const item of list) {
     if (item.kind === "item") {
-      if (item.value.elements.length === 0) {
-        flushParagraph();
-        continue;
-      }
-      currentParagraphChildren.push(...item.value.elements);
-      if (item.value.hasLineBreak) {
-        currentParagraphChildren.push({ element: "line-break" });
-      }
-    } else {
-      flushParagraph();
-      children.push(buildBlockquoteElement(item.children));
+      pending.push(item.value);
+      continue;
+    }
+
+    flushPending();
+    const nested = buildBlockquoteElement(ctx, item.children);
+    if (nested) {
+      children.push(nested);
     }
   }
 
-  flushParagraph();
+  flushPending();
+
+  if (children.length === 0) {
+    return null;
+  }
 
   return {
     element: "container",
@@ -60,3 +72,35 @@ function buildBlockquoteElement(list: DepthList<null, BlockquoteLine>): Element
     },
   };
 }
+
+function parseLines(ctx: ParseContext, lines: BlockquoteLine[]): Element[] {
+  const tokens = sliceLineTokens(ctx, lines);
+  const lineCtx: ParseContext = { ...ctx, tokens, pos: 0 };
+
+  return parseBlocksUntil(lineCtx, NEVER_CLOSES, {
+    excludedBlockNames: EXCLUDED_BLOCK_NAMES,
+  }).elements;
+}
+
+function sliceLineTokens(ctx: ParseContext, lines: BlockquoteLine[]): Token[] {
+  const tokens: Token[] = [];
+
+  for (const { start, end } of lines) {
+    for (let pos = start; pos < end; pos++) {
+      const token = ctx.tokens[pos];
+      if (token) {
+        tokens.push(token);
+      }
+    }
+  }
+
+  const last = tokens[tokens.length - 1];
+  tokens.push(createToken("EOF", "", last?.position ?? ZERO_POSITION));
+
+  return tokens;
+}
+
+const ZERO_POSITION = {
+  start: { line: 0, column: 0, offset: 0 },
+  end: { line: 0, column: 0, offset: 0 },
+};

package/src/parser/rules/block/blockquote/index.ts

@@ -35,11 +35,11 @@ import { buildBlockquoteElements, collectBlockquoteLines } from "./lines";
  * Parsing strategy:
  * 1. Collect consecutive lines that begin with BLOCKQUOTE_MARKER at line start.
  * 2. For each line, record the depth (number of `>` chars, zero-indexed)
- *    and parse the inline content after the mandatory space.
+ *    and the token range of the content after the mandatory space.
  * 3. Lines missing the required space are consumed but produce no output.
  * 4. Feed the flat depth list into {@link processDepths} to build a nested tree.
- * 5. Recursively convert the tree into nested blockquote container elements
- *    via `buildBlockquoteElement()`.
+ * 5. Recursively convert the tree into nested blockquote container elements,
+ *    re-parsing each run of content tokens as blocks.
  */
 export const blockquoteRule: BlockRule = {
   name: "blockquote",
@@ -64,11 +64,10 @@ export const blockquoteRule: BlockRule = {
       return { success: false };
     }
 
-    const blockquotes = buildBlockquoteElements(blockquoteLines.lines);
+    const blockquotes = buildBlockquoteElements(ctx, blockquoteLines.lines);
 
-    // Return first blockquote (should usually be only one)
     if (blockquotes.length === 0) {
-      return { success: false };
+      return { success: true, elements: [], consumed: blockquoteLines.consumed };
     }
 
     return {

package/src/parser/rules/block/blockquote/line.ts

@@ -1,5 +1,4 @@
 import type { ParseContext } from "../../types";
-import { parseInlineUntil } from "../../inline/utils";
 import type { ParsedBlockquoteLine } from "./lines";
 
 /**
@@ -35,19 +34,21 @@ export function parseBlockquoteLine(
     return { kind: "skipped", consumed: consumeLineRemainder(ctx, pos) + consumed };
   }
 
-  while (ctx.tokens[pos]?.type === "WHITESPACE") {
+  pos++;
+  consumed++;
+
+  const contentStart = pos;
+  while (
+    pos < ctx.tokens.length &&
+    ctx.tokens[pos]?.type !== "NEWLINE" &&
+    ctx.tokens[pos]?.type !== "EOF"
+  ) {
     pos++;
     consumed++;
   }
 
-  const inlineCtx: ParseContext = { ...ctx, pos };
-  const inlineResult = parseInlineUntil(inlineCtx, "NEWLINE");
-  consumed += inlineResult.consumed;
-  pos += inlineResult.consumed;
-
-  let hasLineBreak = false;
   if (ctx.tokens[pos]?.type === "NEWLINE") {
-    hasLineBreak = true;
+    pos++;
     consumed++;
   }
 
@@ -56,7 +57,7 @@ export function parseBlockquoteLine(
     line: {
       depth: depth - 1,
       ltype: null,
-      value: { elements: inlineResult.elements, hasLineBreak },
+      value: { start: contentStart, end: pos },
     },
     consumed,
   };

package/src/parser/rules/block/blockquote/lines.ts

@@ -1,11 +1,11 @@
-import type { Element } from "@wdprlib/ast";
 import type { ParseContext } from "../../types";
 import { parseBlockquoteLine } from "./line";
 export { buildBlockquoteElements } from "./build";
 
+/** Half-open range of the line's content tokens in the enclosing token stream. */
 export interface BlockquoteLine {
-  elements: Element[];
-  hasLineBreak: boolean;
+  start: number;
+  end: number;
 }
 
 export interface ParsedBlockquoteLine {
@@ -26,7 +26,14 @@ export function collectBlockquoteLines(ctx: ParseContext): ParsedBlockquoteLines
 
   while (pos < ctx.tokens.length) {
     const result = parseBlockquoteLine(ctx, pos);
-    if (result.kind === "stop") break;
+    if (result.kind === "stop") {
+      const resumed = extendToCommentClose(ctx, lines, pos);
+      if (resumed === pos) break;
+
+      consumed += resumed - pos;
+      pos = resumed;
+      continue;
+    }
 
     pos += result.consumed;
     consumed += result.consumed;
@@ -35,5 +42,96 @@ export function collectBlockquoteLines(ctx: ParseContext): ParsedBlockquoteLines
     }
   }
 
+  blankCommentOnlyLines(ctx, lines);
+
   return { lines, consumed };
 }
+
+/**
+ * Comments are stripped before the quoted block is split, so one opened on a
+ * quoted line closes wherever its `--]` is, quoted or not. The rest of the
+ * closing line joins the quoted line the comment started on.
+ */
+function extendToCommentClose(
+  ctx: ParseContext,
+  lines: ParsedBlockquoteLine[],
+  stopPos: number,
+): number {
+  const last = lines[lines.length - 1];
+  if (!last || !endsInsideComment(ctx, lines)) {
+    return stopPos;
+  }
+
+  let pos = stopPos;
+  while (pos < ctx.tokens.length && ctx.tokens[pos]?.type !== "COMMENT_CLOSE") {
+    if (ctx.tokens[pos]?.type === "EOF") return stopPos;
+    pos++;
+  }
+
+  while (pos < ctx.tokens.length && ctx.tokens[pos]?.type !== "EOF") {
+    pos++;
+    if (ctx.tokens[pos - 1]?.type === "NEWLINE") break;
+  }
+
+  last.value.end = pos;
+  return pos;
+}
+
+function endsInsideComment(ctx: ParseContext, lines: ParsedBlockquoteLine[]): boolean {
+  let open = false;
+
+  for (const { value } of lines) {
+    for (let pos = value.start; pos < value.end; pos++) {
+      const type = ctx.tokens[pos]?.type;
+      if (type === "COMMENT_OPEN") open = true;
+      else if (type === "COMMENT_CLOSE") open = false;
+    }
+  }
+
+  return open;
+}
+
+/**
+ * Wikidot strips comments before parsing the quoted content, so a line left
+ * with nothing becomes blank and separates paragraphs. Unterminated comments
+ * stay literal, hence the scan only covers closed ones.
+ */
+function blankCommentOnlyLines(ctx: ParseContext, lines: ParsedBlockquoteLine[]): void {
+  const positions: { line: number; pos: number }[] = [];
+  lines.forEach((line, index) => {
+    for (let pos = line.value.start; pos < line.value.end; pos++) {
+      if (ctx.tokens[pos]?.type !== "NEWLINE") {
+        positions.push({ line: index, pos });
+      }
+    }
+  });
+
+  const commented: boolean[] = Array.from({ length: positions.length }, () => false);
+  for (let open = 0; open < positions.length; open++) {
+    if (ctx.tokens[positions[open]!.pos]?.type !== "COMMENT_OPEN") continue;
+
+    let close = open + 1;
+    while (
+      close < positions.length &&
+      ctx.tokens[positions[close]!.pos]?.type !== "COMMENT_CLOSE"
+    ) {
+      close++;
+    }
+    if (close === positions.length) break;
+
+    commented.fill(true, open, close + 1);
+    open = close;
+  }
+
+  const hasContent: boolean[] = Array.from({ length: lines.length }, () => false);
+  positions.forEach(({ line, pos }, index) => {
+    if (commented[index] || ctx.tokens[pos]?.type === "WHITESPACE") return;
+    hasContent[line] = true;
+  });
+
+  lines.forEach((line, index) => {
+    if (hasContent[index]) return;
+    const { end } = line.value;
+    line.value.start = ctx.tokens[end - 1]?.type === "NEWLINE" ? end - 1 : end;
+  });
+}

package/src/parser/rules/block/code/content.ts

@@ -49,9 +49,6 @@ function consumeCodeClose(ctx: ParseContext, startPos: number, closeNameConsumed
     pos++;
     consumed++;
   }
-  if (ctx.tokens[pos]?.type === "NEWLINE") {
-    consumed++;
-  }
 
   return consumed;
 }

package/src/parser/rules/block/code/index.ts

@@ -15,6 +15,9 @@ import { currentToken } from "../../types";
 import { parseBlockName } from "../utils";
 import { parseAttributesRaw } from "../utils";
 import { repairSwallowedCodeClose } from "./attributes";
+import { parseInlineUntil } from "../../inline/utils";
+import { getParagraphNewlineBoundary } from "../../inline/parsing/paragraph-boundary";
+import { normalizeParagraphElements } from "../paragraph/normalize";
 import { collectCodeContent } from "./content";
 
 /**
@@ -86,15 +89,16 @@ export const codeBlockRule: BlockRule = {
     };
     ctx.codeBlocks.push(codeBlockData);
 
-    return {
-      success: true,
-      elements: [
-        {
-          element: "code",
-          data: codeBlockData,
-        },
-      ],
-      consumed,
-    };
+    const elements: Element[] = [{ element: "code", data: codeBlockData }];
+    if (
+      ctx.tokens[pos]?.type === "NEWLINE" &&
+      !getParagraphNewlineBoundary(ctx, pos, false).shouldBreak &&
+      !ctx.scope.blockCloseCondition?.({ ...ctx, pos: pos + 1 })
+    ) {
+      const after = parseInlineUntil({ ...ctx, pos: pos + 1 }, "PARAGRAPH_BREAK");
+      elements.push({ element: "line-break" }, ...normalizeParagraphElements(after.elements));
+      consumed += 1 + after.consumed;
+    }
+    return { success: true, elements, consumed };
   },
 };

package/src/parser/rules/block/definition-list/index.ts

@@ -14,10 +14,8 @@
  * Key parsing details:
  * - Whitespace after the first colon is required (`": key"` not `":key"`).
  * - The key portion supports inline markup (bold, links, etc.).
- * - The value continues until a double newline, a new definition entry, or
- *   the end of the document.
- * - A single newline within the value does NOT break the entry -- parsing
- *   continues on the next line.
+ * - The value ends at the next newline or the end of the document.
+ * - Explicit line continuations (` _`) can extend a value across source lines.
  *
  * @module
  */
@@ -49,14 +47,10 @@ export const definitionListRule: BlockRule = {
       return { success: false };
     }
 
+    const items = toDefinitionListItems(result.items.filter((item) => item.value.length > 0));
     return {
       success: true,
-      elements: [
-        {
-          element: "definition-list",
-          data: toDefinitionListItems(result.items),
-        },
-      ],
+      elements: items.length > 0 ? [{ element: "definition-list", data: items }] : [],
       consumed: result.consumed,
     };
   },