@wdprlib/parser 5.1.5 → 5.2.0

package/src/lexer/state.ts

@@ -10,6 +10,7 @@ export interface LexerState {
   line: number;
   column: number;
   lineStart: boolean;
+  quoteContentStart: boolean;
   tokens: Token[];
 }
 
@@ -20,10 +21,20 @@ export function createInitialLexerState(source: string): LexerState {
     line: 1,
     column: 1,
     lineStart: true,
+    quoteContentStart: false,
     tokens: [],
   };
 }
 
+export function isSyntaxLineStart(state: LexerState): boolean {
+  return state.lineStart || state.quoteContentStart;
+}
+
+/** An indented `>` is not a blockquote, so its content keeps the enclosing line. */
+export function isLineStartQuoteMarker(token: Token | undefined): boolean {
+  return token?.type === "BLOCKQUOTE_MARKER" && token.lineStart;
+}
+
 export function isAtEnd(state: LexerState): boolean {
   return state.pos >= state.source.length;
 }
@@ -46,20 +57,33 @@ export function advanceBy(state: LexerState, n = 1): void {
   updatePosition(state, start, end);
 }
 
-export function advanceByToken(state: LexerState, type: TokenType, length: number): void {
+export function advanceByToken(
+  state: LexerState,
+  type: TokenType,
+  length: number,
+  value = "",
+): void {
+  const afterQuoteMarker = isLineStartQuoteMarker(state.tokens[state.tokens.length - 1]);
   state.pos += length;
 
   if (type === "NEWLINE") {
     state.line++;
     state.column = 1;
     state.lineStart = true;
+    state.quoteContentStart = false;
     return;
   }
 
   state.column += length;
-  if (type !== "WHITESPACE") {
-    state.lineStart = false;
+  if (type === "WHITESPACE") {
+    if (afterQuoteMarker && value === " ") {
+      state.quoteContentStart = true;
+    }
+    return;
   }
+
+  state.lineStart = false;
+  state.quoteContentStart = false;
 }
 
 function updatePosition(state: LexerState, start: number, end: number): void {
@@ -71,8 +95,9 @@ function updatePositionFromValue(state: LexerState, value: string): void {
   const firstNewline = value.indexOf("\n");
   if (firstNewline === -1) {
     state.column += value.length;
-    if (state.lineStart && hasNonLineStartSpacing(value, 0)) {
+    if (hasNonLineStartSpacing(value, 0)) {
       state.lineStart = false;
+      state.quoteContentStart = false;
     }
     return;
   }
@@ -90,6 +115,7 @@ function updatePositionFromValue(state: LexerState, value: string): void {
   state.line += newlineCount;
   state.column = value.length - lastNewline;
   state.lineStart = !hasNonLineStartSpacing(value, lastNewline + 1);
+  state.quoteContentStart = false;
 }
 
 function hasNonLineStartSpacing(value: string, start: number): boolean {

package/src/lexer/token-factory.ts

@@ -1,5 +1,5 @@
 import type { Position } from "@wdprlib/ast";
-import type { LexerState } from "./state";
+import { isLineStartQuoteMarker, type LexerState } from "./state";
 import type { Token, TokenType } from "./tokens";
 
 const ZERO_POSITION: Position = {
@@ -58,5 +58,18 @@ function currentTokenPosition(state: LexerState, value: string): Position {
 }
 
 function isTokenAtLineStart(state: LexerState): boolean {
-  return state.tokens.length === 0 || state.tokens[state.tokens.length - 1]?.type === "NEWLINE";
+  if (state.tokens.length === 0) {
+    return true;
+  }
+
+  const last = state.tokens[state.tokens.length - 1];
+  if (last?.type === "NEWLINE") {
+    return true;
+  }
+
+  return (
+    last?.type === "WHITESPACE" &&
+    last.value === " " &&
+    isLineStartQuoteMarker(state.tokens[state.tokens.length - 2])
+  );
 }

package/src/lexer/tokens.ts

@@ -113,9 +113,10 @@ export interface Token {
   /** Start/end location in the original source string */
   position: Position;
   /**
-   * `true` when this token is the first non-whitespace token on its
-   * line. Block-level rules (headings, lists, blockquotes) check this
-   * flag before attempting to match.
+   * `true` when this token opens a logical line: the first non-whitespace
+   * token on a source line, or the first token after a blockquote prefix.
+   * Block-level rules (headings, lists, blockquotes) check this flag before
+   * attempting to match.
    */
   lineStart: boolean;
 }

package/src/parser/rules/block/blockquote/build.ts

@@ -1,55 +1,67 @@
 import type { Element } from "@wdprlib/ast";
+import { createToken, type Token } from "../../../../lexer";
 import { processDepths, type DepthList } from "../../../depth";
+import type { ParseContext } from "../../types";
+import { parseBlocksUntil } from "../utils";
 import type { BlockquoteLine, ParsedBlockquoteLine } from "./lines";
 
-export function buildBlockquoteElements(lines: ParsedBlockquoteLine[]): Element[] {
+/**
+ * Blocks Wikidot protects before parsing, by matching an opening tag at the
+ * start of a line. A `>` prefix defeats that match, so inside a blockquote
+ * only their tags stay literal while the body parses as usual.
+ */
+const EXCLUDED_BLOCK_NAMES: ReadonlySet<string> = new Set([
+  "bibliography",
+  "code",
+  "html",
+  "include",
+  "math",
+  "module",
+]);
+
+const NEVER_CLOSES = () => false;
+
+export function buildBlockquoteElements(
+  ctx: ParseContext,
+  lines: ParsedBlockquoteLine[],
+): Element[] {
   const depthTrees = processDepths<null, BlockquoteLine>(null, lines);
-  return depthTrees.map(({ list }) => buildBlockquoteElement(list));
+  return depthTrees
+    .map(({ list }) => buildBlockquoteElement(ctx, list))
+    .filter((element): element is Element => element !== null);
 }
 
-function buildBlockquoteElement(list: DepthList<null, BlockquoteLine>): Element {
+function buildBlockquoteElement(
+  ctx: ParseContext,
+  list: DepthList<null, BlockquoteLine>,
+): Element | null {
   const children: Element[] = [];
-  let currentParagraphChildren: Element[] = [];
-
-  function flushParagraph() {
-    if (currentParagraphChildren.length > 0) {
-      while (
-        currentParagraphChildren.length > 0 &&
-        currentParagraphChildren[currentParagraphChildren.length - 1]?.element === "line-break"
-      ) {
-        currentParagraphChildren.pop();
-      }
-      if (currentParagraphChildren.length > 0) {
-        children.push({
-          element: "container",
-          data: {
-            type: "paragraph",
-            attributes: {},
-            elements: currentParagraphChildren,
-          },
-        });
-      }
-      currentParagraphChildren = [];
-    }
+  let pending: BlockquoteLine[] = [];
+
+  function flushPending() {
+    if (pending.length === 0) return;
+    children.push(...parseLines(ctx, pending));
+    pending = [];
   }
 
   for (const item of list) {
     if (item.kind === "item") {
-      if (item.value.elements.length === 0) {
-        flushParagraph();
-        continue;
-      }
-      currentParagraphChildren.push(...item.value.elements);
-      if (item.value.hasLineBreak) {
-        currentParagraphChildren.push({ element: "line-break" });
-      }
-    } else {
-      flushParagraph();
-      children.push(buildBlockquoteElement(item.children));
+      pending.push(item.value);
+      continue;
+    }
+
+    flushPending();
+    const nested = buildBlockquoteElement(ctx, item.children);
+    if (nested) {
+      children.push(nested);
     }
   }
 
-  flushParagraph();
+  flushPending();
+
+  if (children.length === 0) {
+    return null;
+  }
 
   return {
     element: "container",
@@ -60,3 +72,35 @@ function buildBlockquoteElement(list: DepthList<null, BlockquoteLine>): Element
     },
   };
 }
+
+function parseLines(ctx: ParseContext, lines: BlockquoteLine[]): Element[] {
+  const tokens = sliceLineTokens(ctx, lines);
+  const lineCtx: ParseContext = { ...ctx, tokens, pos: 0 };
+
+  return parseBlocksUntil(lineCtx, NEVER_CLOSES, {
+    excludedBlockNames: EXCLUDED_BLOCK_NAMES,
+  }).elements;
+}
+
+function sliceLineTokens(ctx: ParseContext, lines: BlockquoteLine[]): Token[] {
+  const tokens: Token[] = [];
+
+  for (const { start, end } of lines) {
+    for (let pos = start; pos < end; pos++) {
+      const token = ctx.tokens[pos];
+      if (token) {
+        tokens.push(token);
+      }
+    }
+  }
+
+  const last = tokens[tokens.length - 1];
+  tokens.push(createToken("EOF", "", last?.position ?? ZERO_POSITION));
+
+  return tokens;
+}
+
+const ZERO_POSITION = {
+  start: { line: 0, column: 0, offset: 0 },
+  end: { line: 0, column: 0, offset: 0 },
+};

package/src/parser/rules/block/blockquote/index.ts

@@ -35,11 +35,11 @@ import { buildBlockquoteElements, collectBlockquoteLines } from "./lines";
  * Parsing strategy:
  * 1. Collect consecutive lines that begin with BLOCKQUOTE_MARKER at line start.
  * 2. For each line, record the depth (number of `>` chars, zero-indexed)
- *    and parse the inline content after the mandatory space.
+ *    and the token range of the content after the mandatory space.
  * 3. Lines missing the required space are consumed but produce no output.
  * 4. Feed the flat depth list into {@link processDepths} to build a nested tree.
- * 5. Recursively convert the tree into nested blockquote container elements
- *    via `buildBlockquoteElement()`.
+ * 5. Recursively convert the tree into nested blockquote container elements,
+ *    re-parsing each run of content tokens as blocks.
  */
 export const blockquoteRule: BlockRule = {
   name: "blockquote",
@@ -64,11 +64,10 @@ export const blockquoteRule: BlockRule = {
       return { success: false };
     }
 
-    const blockquotes = buildBlockquoteElements(blockquoteLines.lines);
+    const blockquotes = buildBlockquoteElements(ctx, blockquoteLines.lines);
 
-    // Return first blockquote (should usually be only one)
     if (blockquotes.length === 0) {
-      return { success: false };
+      return { success: true, elements: [], consumed: blockquoteLines.consumed };
     }
 
     return {

package/src/parser/rules/block/blockquote/line.ts

@@ -1,5 +1,4 @@
 import type { ParseContext } from "../../types";
-import { parseInlineUntil } from "../../inline/utils";
 import type { ParsedBlockquoteLine } from "./lines";
 
 /**
@@ -35,19 +34,21 @@ export function parseBlockquoteLine(
     return { kind: "skipped", consumed: consumeLineRemainder(ctx, pos) + consumed };
   }
 
-  while (ctx.tokens[pos]?.type === "WHITESPACE") {
+  pos++;
+  consumed++;
+
+  const contentStart = pos;
+  while (
+    pos < ctx.tokens.length &&
+    ctx.tokens[pos]?.type !== "NEWLINE" &&
+    ctx.tokens[pos]?.type !== "EOF"
+  ) {
     pos++;
     consumed++;
   }
 
-  const inlineCtx: ParseContext = { ...ctx, pos };
-  const inlineResult = parseInlineUntil(inlineCtx, "NEWLINE");
-  consumed += inlineResult.consumed;
-  pos += inlineResult.consumed;
-
-  let hasLineBreak = false;
   if (ctx.tokens[pos]?.type === "NEWLINE") {
-    hasLineBreak = true;
+    pos++;
     consumed++;
   }
 
@@ -56,7 +57,7 @@ export function parseBlockquoteLine(
     line: {
       depth: depth - 1,
       ltype: null,
-      value: { elements: inlineResult.elements, hasLineBreak },
+      value: { start: contentStart, end: pos },
     },
     consumed,
   };

package/src/parser/rules/block/blockquote/lines.ts

@@ -1,11 +1,11 @@
-import type { Element } from "@wdprlib/ast";
 import type { ParseContext } from "../../types";
 import { parseBlockquoteLine } from "./line";
 export { buildBlockquoteElements } from "./build";
 
+/** Half-open range of the line's content tokens in the enclosing token stream. */
 export interface BlockquoteLine {
-  elements: Element[];
-  hasLineBreak: boolean;
+  start: number;
+  end: number;
 }
 
 export interface ParsedBlockquoteLine {
@@ -26,7 +26,14 @@ export function collectBlockquoteLines(ctx: ParseContext): ParsedBlockquoteLines
 
   while (pos < ctx.tokens.length) {
     const result = parseBlockquoteLine(ctx, pos);
-    if (result.kind === "stop") break;
+    if (result.kind === "stop") {
+      const resumed = extendToCommentClose(ctx, lines, pos);
+      if (resumed === pos) break;
+
+      consumed += resumed - pos;
+      pos = resumed;
+      continue;
+    }
 
     pos += result.consumed;
     consumed += result.consumed;
@@ -35,5 +42,96 @@ export function collectBlockquoteLines(ctx: ParseContext): ParsedBlockquoteLines
     }
   }
 
+  blankCommentOnlyLines(ctx, lines);
+
   return { lines, consumed };
 }
+
+/**
+ * Comments are stripped before the quoted block is split, so one opened on a
+ * quoted line closes wherever its `--]` is, quoted or not. The rest of the
+ * closing line joins the quoted line the comment started on.
+ */
+function extendToCommentClose(
+  ctx: ParseContext,
+  lines: ParsedBlockquoteLine[],
+  stopPos: number,
+): number {
+  const last = lines[lines.length - 1];
+  if (!last || !endsInsideComment(ctx, lines)) {
+    return stopPos;
+  }
+
+  let pos = stopPos;
+  while (pos < ctx.tokens.length && ctx.tokens[pos]?.type !== "COMMENT_CLOSE") {
+    if (ctx.tokens[pos]?.type === "EOF") return stopPos;
+    pos++;
+  }
+
+  while (pos < ctx.tokens.length && ctx.tokens[pos]?.type !== "EOF") {
+    pos++;
+    if (ctx.tokens[pos - 1]?.type === "NEWLINE") break;
+  }
+
+  last.value.end = pos;
+  return pos;
+}
+
+function endsInsideComment(ctx: ParseContext, lines: ParsedBlockquoteLine[]): boolean {
+  let open = false;
+
+  for (const { value } of lines) {
+    for (let pos = value.start; pos < value.end; pos++) {
+      const type = ctx.tokens[pos]?.type;
+      if (type === "COMMENT_OPEN") open = true;
+      else if (type === "COMMENT_CLOSE") open = false;
+    }
+  }
+
+  return open;
+}
+
+/**
+ * Wikidot strips comments before parsing the quoted content, so a line left
+ * with nothing becomes blank and separates paragraphs. Unterminated comments
+ * stay literal, hence the scan only covers closed ones.
+ */
+function blankCommentOnlyLines(ctx: ParseContext, lines: ParsedBlockquoteLine[]): void {
+  const positions: { line: number; pos: number }[] = [];
+  lines.forEach((line, index) => {
+    for (let pos = line.value.start; pos < line.value.end; pos++) {
+      if (ctx.tokens[pos]?.type !== "NEWLINE") {
+        positions.push({ line: index, pos });
+      }
+    }
+  });
+
+  const commented: boolean[] = Array.from({ length: positions.length }, () => false);
+  for (let open = 0; open < positions.length; open++) {
+    if (ctx.tokens[positions[open]!.pos]?.type !== "COMMENT_OPEN") continue;
+
+    let close = open + 1;
+    while (
+      close < positions.length &&
+      ctx.tokens[positions[close]!.pos]?.type !== "COMMENT_CLOSE"
+    ) {
+      close++;
+    }
+    if (close === positions.length) break;
+
+    commented.fill(true, open, close + 1);
+    open = close;
+  }
+
+  const hasContent: boolean[] = Array.from({ length: lines.length }, () => false);
+  positions.forEach(({ line, pos }, index) => {
+    if (commented[index] || ctx.tokens[pos]?.type === "WHITESPACE") return;
+    hasContent[line] = true;
+  });
+
+  lines.forEach((line, index) => {
+    if (hasContent[index]) return;
+    const { end } = line.value;
+    line.value.start = ctx.tokens[end - 1]?.type === "NEWLINE" ? end - 1 : end;
+  });
+}

package/src/parser/rules/block/code/content.ts

@@ -49,9 +49,6 @@ function consumeCodeClose(ctx: ParseContext, startPos: number, closeNameConsumed
     pos++;
     consumed++;
   }
-  if (ctx.tokens[pos]?.type === "NEWLINE") {
-    consumed++;
-  }
 
   return consumed;
 }

package/src/parser/rules/block/code/index.ts

@@ -15,6 +15,9 @@ import { currentToken } from "../../types";
 import { parseBlockName } from "../utils";
 import { parseAttributesRaw } from "../utils";
 import { repairSwallowedCodeClose } from "./attributes";
+import { parseInlineUntil } from "../../inline/utils";
+import { getParagraphNewlineBoundary } from "../../inline/parsing/paragraph-boundary";
+import { normalizeParagraphElements } from "../paragraph/normalize";
 import { collectCodeContent } from "./content";
 
 /**
@@ -86,15 +89,16 @@ export const codeBlockRule: BlockRule = {
     };
     ctx.codeBlocks.push(codeBlockData);
 
-    return {
-      success: true,
-      elements: [
-        {
-          element: "code",
-          data: codeBlockData,
-        },
-      ],
-      consumed,
-    };
+    const elements: Element[] = [{ element: "code", data: codeBlockData }];
+    if (
+      ctx.tokens[pos]?.type === "NEWLINE" &&
+      !getParagraphNewlineBoundary(ctx, pos, false).shouldBreak &&
+      !ctx.scope.blockCloseCondition?.({ ...ctx, pos: pos + 1 })
+    ) {
+      const after = parseInlineUntil({ ...ctx, pos: pos + 1 }, "PARAGRAPH_BREAK");
+      elements.push({ element: "line-break" }, ...normalizeParagraphElements(after.elements));
+      consumed += 1 + after.consumed;
+    }
+    return { success: true, elements, consumed };
   },
 };

package/src/parser/rules/block/definition-list/index.ts

@@ -14,10 +14,8 @@
  * Key parsing details:
  * - Whitespace after the first colon is required (`": key"` not `":key"`).
  * - The key portion supports inline markup (bold, links, etc.).
- * - The value continues until a double newline, a new definition entry, or
- *   the end of the document.
- * - A single newline within the value does NOT break the entry -- parsing
- *   continues on the next line.
+ * - The value ends at the next newline or the end of the document.
+ * - Explicit line continuations (` _`) can extend a value across source lines.
  *
  * @module
  */
@@ -49,14 +47,10 @@ export const definitionListRule: BlockRule = {
       return { success: false };
     }
 
+    const items = toDefinitionListItems(result.items.filter((item) => item.value.length > 0));
     return {
       success: true,
-      elements: [
-        {
-          element: "definition-list",
-          data: toDefinitionListItems(result.items),
-        },
-      ],
+      elements: items.length > 0 ? [{ element: "definition-list", data: items }] : [],
       consumed: result.consumed,
     };
   },

package/src/parser/rules/block/definition-list/item-value.ts

@@ -27,17 +27,8 @@ export function parseDefinitionItemValue(
     }
 
     if (token.type === "NEWLINE") {
-      const nextToken = ctx.tokens[pos + 1];
-      if (nextToken?.type === "COLON" && nextToken.lineStart) {
-        pos++;
-        consumed++;
-        break;
-      }
-      if (nextToken?.type === "NEWLINE" || !nextToken || nextToken.type === "EOF") {
-        pos++;
-        consumed++;
-        break;
-      }
+      consumed++;
+      break;
     }
 
     const inlineCtx: ParseContext = { ...ctx, pos };

package/src/parser/rules/block/definition-list/items.ts

@@ -21,8 +21,7 @@ export interface ParsedDefinitionItem {
  *
  * The function expects `startPos` to point at a line-start COLON token.
  * It consumes the first colon, mandatory whitespace, key tokens up to
- * the second colon, then value tokens until a double newline, a new entry,
- * or end of input.
+ * the second colon, then value tokens until a newline or end of input.
  */
 export function parseDefinitionItem(
   ctx: ParseContext,

package/src/parser/rules/block/embed-block/index.ts

@@ -8,9 +8,6 @@
  * between the tags is stored verbatim as an `embed-block` element. Validation
  * and sanitisation are expected to happen at rendering time or on the server.
  *
- * The embed block is wrapped in a paragraph container in the AST, matching
- * Wikidot's rendering behaviour where embeds sit inside `<p>` tags.
- *
  * If no closing tag is found, the rule fails to prevent consuming the rest
  * of the document.
  *
@@ -18,6 +15,8 @@
  */
 import type { Element } from "@wdprlib/ast";
 import type { BlockRule, ParseContext, RuleResult } from "../../types";
+import { parseInlineUntil } from "../../inline/utils";
+import { normalizeParagraphElements } from "../paragraph/normalize";
 import { currentToken } from "../../types";
 import { collectEmbedContent, consumeEmbedClose } from "./content";
 import { parseEmbedBlockOpen } from "./open";
@@ -34,6 +33,8 @@ export const embedBlockRule: BlockRule = {
   name: "embed-block",
   startTokens: ["BLOCK_OPEN"],
   requiresLineStart: false,
+  preservesPrecedingLineBreak: true,
+  isStartPattern: (ctx, pos) => parseEmbedBlockOpen(ctx, pos) !== null,
 
   parse(ctx: ParseContext): RuleResult<Element> {
     const openToken = currentToken(ctx);
@@ -53,12 +54,17 @@ export const embedBlockRule: BlockRule = {
     consumed += contentResult.consumed;
 
     if (!contentResult.foundClose) {
-      ctx.diagnostics.push({
-        severity: "warning",
-        code: "unclosed-block",
-        message: `Missing closing tag [[/${openResult.blockName}]] for [[${openResult.blockName}]]`,
-        position: openToken.position,
-      });
+      if (
+        !ctx.diagnostics.some(
+          (d) => d.code === "unclosed-block" && d.position === openToken.position,
+        )
+      )
+        ctx.diagnostics.push({
+          severity: "warning",
+          code: "unclosed-block",
+          message: `Missing closing tag [[/${openResult.blockName}]] for [[${openResult.blockName}]]`,
+          position: openToken.position,
+        });
       return { success: false };
     }
 
@@ -66,26 +72,20 @@ export const embedBlockRule: BlockRule = {
     pos += closeConsumed;
     consumed += closeConsumed;
 
-    return {
-      success: true,
-      elements: [
-        {
-          element: "container",
-          data: {
-            type: "paragraph",
-            attributes: {},
-            elements: [
-              {
-                element: "embed-block",
-                data: {
-                  contents: contentResult.contents.trim(),
-                },
-              },
-            ],
-          },
-        },
-      ],
-      consumed,
-    };
+    const elements: Element[] = [
+      { element: "embed-block", data: { contents: contentResult.contents.trim() } },
+    ];
+    if (
+      ctx.scope.inlineEnd === undefined &&
+      ctx.tokens[pos]?.type !== "NEWLINE" &&
+      ctx.tokens[pos]?.type !== "EOF"
+    ) {
+      const after = parseInlineUntil({ ...ctx, pos }, "PARAGRAPH_BREAK");
+      const tail = normalizeParagraphElements(after.elements);
+      if (tail[0]?.element === "text") tail[0].data = tail[0].data.trimStart();
+      elements.push(...tail);
+      consumed += after.consumed;
+    }
+    return { success: true, elements, consumed };
   },
 };