@wdprlib/parser 5.2.0 → 5.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wdprlib/parser",
-  "version": "5.2.0",
+  "version": "5.4.0",
   "description": "Parser for Wikidot markup",
   "keywords": [
     "ast",
@@ -44,6 +44,7 @@
   },
   "dependencies": {
     "@braintree/sanitize-url": "^7.1.1",
-    "@wdprlib/ast": "4.1.0"
+    "@wdprlib/ast": "4.3.0",
+    "entities": "^7.0.1"
   }
 }

package/src/build-info.generated.ts

@@ -5,7 +5,7 @@ export const buildInfo: Readonly<{
   sha: string | null;
   dirty: boolean | null;
 }> = Object.freeze({
-  version: "5.2.0",
-  sha: "cc290b0d882de3ed9ea33ec060ddfc8a7b9da59a",
+  version: "5.4.0",
+  sha: "5e221ddeccc585e64dd1a2eb49b14817c3c1be5d",
   dirty: false,
 });

package/src/lexer/lexer.ts

@@ -50,6 +50,8 @@ export class Lexer {
    * consume content up to the next `"` or newline.
    */
   private blockOpenerDepth = 0;
+  private rawTagBounds: { source: string; close: number; outerDepth: number } | null = null;
+  private rawClosesExhausted = false;
 
   constructor(source: string, options: LexerOptions = {}) {
     this.options = {
@@ -92,8 +94,8 @@ export class Lexer {
    * When invalid, returns the position of the closing ]] so the lexer can
    * emit tokens that allow the inner [# text] to be parsed as a described link.
    */
-  private findInvalidAnchorNameEnd(): number | null {
-    return findInvalidAnchorNameEnd(this.state.source, this.state.pos);
+  private findInvalidAnchorNameEnd(source: string): number | null {
+    return findInvalidAnchorNameEnd(source, this.state.pos);
   }
 
   /**
@@ -117,6 +119,24 @@ export class Lexer {
     this.state.tokens.push(createLexerToken(this.state, type, value, this.options.trackPositions));
     this.lastNonWhitespaceType = updateLastNonWhitespaceType(this.lastNonWhitespaceType, type);
     this.blockOpenerDepth = nextBlockOpenerDepth(this.blockOpenerDepth, type);
+    if (
+      this.rawTagBounds === null &&
+      !this.rawClosesExhausted &&
+      (value.toLowerCase() === "button" || value.toLowerCase() === "social") &&
+      this.state.tokens.at(-2)?.type === "BLOCK_OPEN" &&
+      /\s/.test(this.current())
+    ) {
+      const close = this.state.source.indexOf("]]", this.state.pos);
+      if (close < 0) {
+        this.rawClosesExhausted = true;
+      } else {
+        this.rawTagBounds = {
+          source: this.state.source.slice(0, close),
+          close,
+          outerDepth: this.blockOpenerDepth - 1,
+        };
+      }
+    }
   }
 
   private emitTokenAction(action: TokenAction): void {
@@ -140,7 +160,16 @@ export class Lexer {
   private scanToken(): void {
     const char = this.current();
     const isLineStart = isSyntaxLineStart(this.state);
-    const src = this.state.source;
+    const bounds = this.rawTagBounds;
+    const src = bounds?.source ?? this.state.source;
+
+    // Scanners see a bounded source so no token can consume part of the close.
+    if (bounds && this.state.pos === bounds.close) {
+      this.emitTokenAction({ type: "BLOCK_CLOSE", value: "]]", length: 2 });
+      this.blockOpenerDepth = bounds.outerDepth;
+      this.rawTagBounds = null;
+      return;
+    }
 
     const spacingAction = scanSpacingToken(src, this.state.pos);
     if (spacingAction) {
@@ -155,7 +184,7 @@ export class Lexer {
       lineStart: isLineStart,
       physicalLineStart: this.state.lineStart,
       splitBlockClose: this.splitBlockClosePositions.has(this.state.pos),
-      findInvalidAnchorNameEnd: () => this.findInvalidAnchorNameEnd(),
+      findInvalidAnchorNameEnd: () => this.findInvalidAnchorNameEnd(src),
     });
     if (punctuation.handled) {
       if (punctuation.clearSplitBlockCloseAt !== undefined) {
@@ -176,7 +205,7 @@ export class Lexer {
     if (char === '"') {
       const lastNonWs = this.lastNonWhitespaceTokenType();
       if (this.blockOpenerDepth > 0 && lastNonWs === "EQUALS") {
-        this.addToken("QUOTED_STRING", scanQuotedString(this.state));
+        this.addToken("QUOTED_STRING", scanQuotedString(this.state, src.length));
         return;
       }
       this.advance();

package/src/lexer/quoted-string.ts

@@ -1,15 +1,15 @@
-import { advance, current, isAtEnd, type LexerState } from "./state";
+import { advance, current, type LexerState } from "./state";
 
 /**
  * Scan a quoted block-attribute value, including the opening quote and optional
  * closing quote. Newline terminates the token without being consumed.
  */
-export function scanQuotedString(state: LexerState): string {
+export function scanQuotedString(state: LexerState, end: number = state.source.length): string {
   let quoted = advance(state);
-  while (!isAtEnd(state) && current(state) !== '"' && current(state) !== "\n") {
+  while (state.pos < end && current(state) !== '"' && current(state) !== "\n") {
     quoted += advance(state);
   }
-  if (current(state) === '"') {
+  if (state.pos < end && current(state) === '"') {
     quoted += advance(state);
   }
   return quoted;

package/src/lexer/syntax-actions.ts

@@ -1,6 +1,7 @@
 import type { TokenAction } from "./token-actions";
 import { findRepeatedCharRunEnd } from "./runs";
 import type { TokenType } from "./tokens";
+import { startsWithUrl } from "./url-schemes";
 
 export function scanSimpleSyntaxToken(
   src: string,
@@ -42,7 +43,11 @@ export function scanSimpleSyntaxToken(
 }
 
 function scanStarToken(src: string, pos: number, isLineStart: boolean): TokenAction {
-  if (src[pos + 1] === "*") {
+  // `**http://x…` は太字にならない。Wikidotは2つ目の`*`をURLの新規タブプレフィックスとして
+  // 扱い、開きの`**`ペアが壊れる（`*` リテラル + `*http://x…` autolink + 末尾`**` リテラル）。
+  // 直後が有効なURLのときだけBOLD_MARKERに結合せず単一の`*`として切り出す。
+  // `**http://**`のようにURL本体が無い場合はautolinkが成立しないため太字のまま扱う
+  if (src[pos + 1] === "*" && !startsWithUrl(src, pos + 2)) {
     return token("BOLD_MARKER", "**");
   }
   return isLineStart ? token("LIST_BULLET", "*") : token("STAR", "*");

package/src/lexer/text-actions.ts

@@ -5,6 +5,7 @@ import {
   isAsciiAlphanumericCode,
 } from "./runs";
 import type { TokenAction } from "./token-actions";
+import { TRAILING_URL_SCHEME } from "./url-schemes";
 
 export function scanTextToken(src: string, pos: number): TokenAction {
   const char = src[pos] ?? "";
@@ -28,7 +29,20 @@ export function scanTextToken(src: string, pos: number): TokenAction {
 }
 
 export function scanCompactTextToken(src: string, pos: number): TokenAction | null {
-  const end = findCompactPlainTextRunEnd(src, pos);
+  let end = findCompactPlainTextRunEnd(src, pos);
+
+  // 生URLの自動リンク化はIDENTIFIERトークンのスキーム名から発火する。
+  // このcompactモード（大きなソースで連続する平文を1つのTEXTトークンにまとめる高速化）が
+  // `see http` のようにスキーム名までTEXTに取り込むとリンク化されなくなるため、
+  // 次の文字が`:`でトークンがスキーム名で終わる場合はスキーム名の手前で切り、
+  // スキーム名を通常のIDENTIFIERスキャンに委ねる
+  if (end > pos && src[end] === ":") {
+    const match = TRAILING_URL_SCHEME.exec(src.slice(pos, end));
+    if (match) {
+      end -= match[1]!.length;
+    }
+  }
+
   return end > pos ? runToken(src, pos, end, "TEXT") : null;
 }
 

package/src/lexer/url-schemes.ts

@@ -0,0 +1,78 @@
+/**
+ *
+ * URL schemes recognized for bare-URL auto-linking.
+ *
+ * Mirrors Text_Wiki's Url rule configuration (http://, https://, ftp://,
+ * gopher://, news://, mailto:, mms://). The scheme names are shared between
+ * the lexer (compact text-run rewinding) and the autolink inline rule.
+ *
+ * @module
+ */
+
+/** Scheme names (without `:`) that can start an auto-linked bare URL. */
+export const URL_SCHEME_NAMES: ReadonlySet<string> = new Set([
+  "http",
+  "https",
+  "ftp",
+  "gopher",
+  "news",
+  "mailto",
+  "mms",
+]);
+
+/**
+ * Matches a compact text run that ends with a URL scheme name.
+ *
+ * The boundary is `[^A-Za-z0-9]` (not Text_Wiki's `[^A-Za-z]`) so that runs
+ * like `123http` stay fused, matching the non-compact lexer which tokenizes
+ * `123http` as a single IDENTIFIER (such URLs are not auto-linked either way).
+ */
+export const TRAILING_URL_SCHEME: RegExp = /(?:^|[^A-Za-z0-9])(https?|ftp|gopher|news|mailto|mms)$/;
+
+/**
+ * Matches the exact scheme prefix the autolink rule consumes:
+ * `://` for http/https/ftp/gopher/news/mms and `:` for mailto. Must stay in
+ * sync with the autolink rule's URL pattern (a looser `scheme:` test would
+ * split `**` markers for inputs like `**http:foo**` that never autolink,
+ * turning them into literal `**` instead of bold).
+ */
+const URL_SCHEME_PREFIX = /^(?:(?:https?|ftp|gopher|news|mms):\/\/|mailto:)/;
+
+/**
+ * URL body pattern, ported from Text_Wiki's Url rule regex: scheme + zero or
+ * more slash-terminated segments + final segment + one terminating character
+ * from `[A-Za-z%0-9/?=&~_]`.
+ *
+ * Scheme matching is case-sensitive (lowercase only) on purpose: Text_Wiki's
+ * inline URL regex has no `i` flag and lists lowercase schemes, so Wikidot does
+ * not auto-link `HTTP://…`. Shared by the autolink rule and the lexer's bold
+ * marker split.
+ */
+export const URL_PATTERN: RegExp =
+  /^(?:(?:https?|ftp|gopher|news|mms):\/\/|mailto:)(?:[^ \\/"']*\/)*[^ \t\n\\/"']*[A-Za-z%0-9/?=&~_]/;
+
+/**
+ * Whether `src` begins a URL scheme prefix (e.g. `http://`) at `pos`.
+ *
+ * A prefix-only check, used for single-bracket links (`[ftp://x Label]`) whose
+ * URL portion is already delimited by the label, so the full URL body does not
+ * need to be validated here.
+ */
+export function startsWithUrlScheme(src: string, pos: number): boolean {
+  return URL_SCHEME_PREFIX.test(src.slice(pos, pos + 10));
+}
+
+/**
+ * Whether a complete autolinkable URL (not just a scheme prefix) begins at
+ * `pos` in `src`.
+ *
+ * Used by the lexer to decide whether `**` before a scheme is a bold marker or
+ * a literal `*` plus a new-tab URL prefix: only split the marker when a valid
+ * URL actually follows, so `**http://**` (no URL body) stays bold while
+ * `**http://x**` splits into `*` + autolink + `**`.
+ */
+export function startsWithUrl(src: string, pos: number): boolean {
+  // Scheme + a short body is enough to decide; cap the slice so a very long
+  // line does not get copied on every `**`.
+  return URL_PATTERN.test(src.slice(pos, pos + 2048));
+}

package/src/parser/constants.ts

@@ -28,6 +28,10 @@ export const BLOCK_START_TOKENS: TokenType[] = [
   "HEADING_MARKER",
   "HR_MARKER",
   "TABLE_MARKER",
+  "TABLE_HEADER",
+  "TABLE_LEFT",
+  "TABLE_CENTER",
+  "TABLE_RIGHT",
   "COLON", // Definition list
   "BLOCK_OPEN", // [[footnoteblock]], [[div]], etc.
   "BLOCK_END_OPEN", // [[/div]], [[/collapsible]], etc.

package/src/parser/parse/block.ts

@@ -32,7 +32,7 @@ export function parseNextBlock(
   }
 
   const result = ctx.blockFallbackRule.parse(ctx);
-  if (result.success && result.elements.length > 0) {
+  if (result.success) {
     ctx.pos += result.consumed;
     return result.elements;
   }

package/src/parser/postprocess/divAdjacentParagraph.ts

@@ -1,25 +1,27 @@
 /**
- *
- * Post-processing pass: suppress paragraph wrapping adjacent to div containers.
- *
- * In Wikidot, when a paragraph is a direct sibling of a `<div>` block (no other
- * block elements between them), the `<p>` wrapping is removed and the inner
- * elements are promoted to the parent level.
- *
- * When the unwrapped paragraph follows a div, a line-break element is prepended
- * to represent the newline between the closing `</div>` and the bare text.
- *
- * Examples:
- *   `[[div]]inline[[/div]]\n[[div]]\n[[/div]]`  → no `<p>` (adjacent to div)
- *   `[[div]]inline[[/div]]\n> a\n[[div]]\n[[/div]]` → has `<p>` (blockquote between)
- *
- * @module
+ * Preserve the bare rendering of unparsed div syntax beside a valid div.
+ * Ordinary paragraphs retain their wrappers, including beside div containers.
  */
 import type { Element, ContainerData } from "@wdprlib/ast";
 
-function isParagraphContainer(el: Element | undefined): boolean {
+// Track the syntax node rather than its paragraph so splitting a paragraph
+// cannot mark a separate fragment containing only ordinary text.
+const unparsedDivStarts = new WeakSet<Element>();
+
+export function markUnparsedDivStart(elements: Element[]): void {
+  const first = elements[0];
+  const text =
+    first?.element === "container" && first.data.type === "paragraph"
+      ? first.data.elements[0]
+      : first;
+  if (text?.element === "text") unparsedDivStarts.add(text);
+}
+
+function isUnparsedDivParagraph(el: Element | undefined): boolean {
   if (!el || el.element !== "container") return false;
-  return (el.data as ContainerData).type === "paragraph";
+  return (
+    el.data.type === "paragraph" && el.data.elements.some((child) => unparsedDivStarts.has(child))
+  );
 }
 
 function isDivContainer(el: Element | undefined): boolean {
@@ -28,8 +30,8 @@ function isDivContainer(el: Element | undefined): boolean {
 }
 
 /**
- * At a single nesting level, unwrap paragraph containers that are directly
- * adjacent to div containers. A line-break is prepended when the paragraph
+ * At a single nesting level, unwrap paragraphs containing unparsed div syntax
+ * that are directly adjacent to div containers. A line-break is prepended when the paragraph
  * follows a div.
  */
 function suppressAtLevel(elements: Element[]): Element[] {
@@ -38,7 +40,7 @@ function suppressAtLevel(elements: Element[]): Element[] {
   const unwrap = Array.from({ length: elements.length }, () => false);
 
   for (let i = 0; i < elements.length; i++) {
-    if (!isParagraphContainer(elements[i])) continue;
+    if (!isUnparsedDivParagraph(elements[i])) continue;
     const prevIsDiv = i > 0 && isDivContainer(elements[i - 1]);
     const nextIsDiv = i < elements.length - 1 && isDivContainer(elements[i + 1]);
     if (prevIsDiv || nextIsDiv) {
@@ -66,7 +68,7 @@ function suppressAtLevel(elements: Element[]): Element[] {
 }
 
 /**
- * Suppress paragraph wrapping adjacent to div containers.
+ * Suppress wrapping of unparsed div syntax adjacent to div containers.
  *
  * Applied only at the top level. Inside div containers, paragraphs adjacent
  * to nested divs retain their `<p>` wrapping (matching Wikidot behavior).

package/src/parser/postprocess/spanStrip/merge.ts

@@ -7,6 +7,8 @@ import {
   isContainer,
   isEmptyExpr,
   isSplitSpan,
+  isSpanStripMarker,
+  isWhitespaceText,
 } from "./predicates";
 import { splitParagraphAtBlankLineSpans, splitParagraphAtEmptyExpr } from "./split";
 
@@ -40,6 +42,10 @@ export function mergeSpanStripParagraphs(children: Element[]): Element[] {
       continue;
     }
 
+    const firstMeaningful = paraData.elements.find(
+      (child) => child.element !== "line-break" && !isWhitespaceText(child),
+    );
+    const keepParagraph = firstMeaningful && !isSpanStripMarker(firstMeaningful);
     const mergedChildren: Element[] = [...paraData.elements];
     i++;
 
@@ -55,7 +61,7 @@ export function mergeSpanStripParagraphs(children: Element[]): Element[] {
       }
 
       const hasSpanStrip = hasParagraphStripSpan(nextPara);
-      mergedChildren.push(...nextParaData.elements);
+      for (const element of nextParaData.elements) mergedChildren.push(element);
       i++;
 
       if (!hasSpanStrip) {
@@ -69,7 +75,7 @@ export function mergeSpanStripParagraphs(children: Element[]): Element[] {
     const escapedSpans = extractEscapedSpans(mergedChildren);
     removeLineBreaksAroundSpanStrip(mergedChildren);
 
-    if (escapedSpans.length > 0) {
+    if (escapedSpans.length > 0 || keepParagraph) {
       if (mergedChildren.length > 0) {
         result.push(paragraphElement(mergedChildren));
       }

package/src/parser/preprocess/typography.ts

@@ -19,6 +19,8 @@
  * @module
  */
 
+import { makeUniqueSentinels, maskRawRegions, restorePlaceholders } from "./utils";
+
 /** Unicode left single quotation mark (U+2018) */
 const LEFT_SINGLE_QUOTE = "\u2018"; // '
 /** Unicode right single quotation mark (U+2019) */
@@ -66,14 +68,24 @@ function replaceDelimitedTypography(
   let searchFrom = 0;
   let result = "";
   let lastCopied = 0;
+  let closeIndex = -1;
+  let newlineIndex = -1;
 
   while (searchFrom < text.length) {
     const openIndex = text.indexOf(opener, searchFrom);
     if (openIndex === -1) break;
 
     const contentStart = openIndex + opener.length;
-    const closeIndex = text.indexOf(closer, contentStart);
+    if (closeIndex < contentStart) closeIndex = text.indexOf(closer, contentStart);
     if (closeIndex === -1) break;
+    if (newlineIndex < contentStart) {
+      const nextNewline = text.indexOf("\n", contentStart);
+      newlineIndex = nextNewline === -1 ? text.length : nextNewline;
+    }
+    if (newlineIndex < closeIndex) {
+      searchFrom = newlineIndex + 1;
+      continue;
+    }
 
     result += text.slice(lastCopied, openIndex);
     result += leftQuote;
@@ -99,7 +111,17 @@ function replaceDelimitedTypography(
  * @returns Text with ASCII typography patterns replaced by Unicode equivalents
  */
 export function substitute(text: string): string {
-  let result = text;
+  if (
+    !text.includes("`") &&
+    !text.includes(",,") &&
+    !text.includes("...") &&
+    !text.includes(". . .")
+  ) {
+    return text;
+  }
+  const sentinels = makeUniqueSentinels(text);
+  const { masked, placeholders } = maskRawRegions(text, sentinels);
+  let result = masked;
 
   // Double quotes: ``...'' -> "..."
   if (result.includes("``") && result.includes("''")) {
@@ -128,5 +150,5 @@ export function substitute(text: string): string {
     result = replaceExactEllipsisPattern(result, ". . .");
   }
 
-  return result;
+  return restorePlaceholders(result, placeholders, sentinels);
 }

package/src/parser/preprocess/utils/raw-regions.ts

@@ -1,7 +1,11 @@
+import { tokenize, type Token } from "../../../lexer";
+import { findCodeOpen } from "../../rules/block/code/open";
+import { findCodeBodyBounds } from "../../rules/block/code/boundary";
+
 const BASE_PLACEHOLDER_OPEN = "\uE000";
 const BASE_PLACEHOLDER_CLOSE = "\uE001";
 
-const RAW_BLOCK_OPEN_PATTERN = /\[\[\s*(code|html)\b[^\]]*\]\]/iy;
+const RAW_BLOCK_OPEN_PATTERN = /\[\[html\b[^\]]*\]\]/iy;
 
 /** Unique sentinel characters used to wrap raw-region placeholders. */
 export interface Sentinels {
@@ -43,11 +47,23 @@ export function maskRawRegions(
   sentinels: Sentinels,
 ): { masked: string; placeholders: string[] } {
   const placeholders: string[] = [];
+  let tokens: Token[] | undefined;
+  const getTokens = () => (tokens ??= tokenize(source));
   let masked = "";
   let i = 0;
 
   while (i < source.length) {
-    const rawBlock = tryMaskRawBlock(source, i, placeholders, sentinels);
+    if (source.startsWith("[!--", i)) {
+      const close = source.indexOf("--]", i + 4);
+      if (close !== -1) {
+        const end = close + 3;
+        masked += source.slice(i, end);
+        i = end;
+        continue;
+      }
+    }
+
+    const rawBlock = tryMaskRawBlock(source, i, placeholders, sentinels, getTokens);
     if (rawBlock) {
       masked += rawBlock.placeholder;
       i = rawBlock.end;
@@ -86,16 +102,30 @@ function tryMaskRawBlock(
   pos: number,
   placeholders: string[],
   sentinels: Sentinels,
+  getTokens: () => Token[],
 ): { placeholder: string; end: number } | null {
   if (source[pos] !== "[" || source[pos + 1] !== "[") return null;
 
+  if (source.slice(pos, pos + 6).toLowerCase() === "[[code") {
+    const tokens = getTokens();
+    const start = tokenAtOffset(tokens, pos);
+    const open = findCodeOpen(tokens, start);
+    if (!open) return null;
+    const bounds = open.closingSwallowed ? null : findCodeBodyBounds(tokens, open.bodyStart);
+    const end = open.closingSwallowed
+      ? tokens[open.attributesEnd - 1]!.position.end.offset
+      : bounds!.foundClose
+        ? tokens[bounds!.end - 1]!.position.end.offset
+        : source.length;
+    return { placeholder: pushPlaceholder(placeholders, source.slice(pos, end), sentinels), end };
+  }
+
   RAW_BLOCK_OPEN_PATTERN.lastIndex = pos;
   const openMatch = RAW_BLOCK_OPEN_PATTERN.exec(source);
   if (!openMatch) return null;
 
-  const name = openMatch[1]!.toLowerCase();
   const openLen = openMatch[0].length;
-  const closePattern = new RegExp(`\\[\\[\\/\\s*${name}\\s*\\]\\]`, "ig");
+  const closePattern = /\[\[\/\s*html\s*\]\]/gi;
   closePattern.lastIndex = pos + openLen;
   const closeMatch = closePattern.exec(source);
 
@@ -107,12 +137,18 @@ function tryMaskRawBlock(
     };
   }
 
-  if (name !== "code") return null;
+  return null;
+}
 
-  return {
-    placeholder: pushPlaceholder(placeholders, source.slice(pos), sentinels),
-    end: source.length,
-  };
+function tokenAtOffset(tokens: readonly Token[], offset: number): number {
+  let low = 0;
+  let high = tokens.length;
+  while (low < high) {
+    const middle = Math.floor((low + high) / 2);
+    if (tokens[middle]!.position.start.offset < offset) low = middle + 1;
+    else high = middle;
+  }
+  return tokens[low]?.position.start.offset === offset ? low : tokens.length;
 }
 
 function tryMaskRawInline(
@@ -140,17 +176,26 @@ function tryMaskSingleLineRaw(
   placeholders: string[],
   sentinels: Sentinels,
 ): { placeholder: string; end: number } | null {
-  const closePos = source.indexOf(close, pos + openerLength);
-  const newline = source.indexOf("\n", pos + openerLength);
-  if (closePos === -1 || (newline !== -1 && newline < closePos)) return null;
-
-  const end = closePos + close.length;
+  const end = singleLineRawEnd(source, pos, openerLength, close);
+  if (end === pos) return null;
   return {
     placeholder: pushPlaceholder(placeholders, source.slice(pos, end), sentinels),
     end,
   };
 }
 
+function singleLineRawEnd(
+  source: string,
+  pos: number,
+  openerLength: number,
+  close: string,
+): number {
+  const closePos = source.indexOf(close, pos + openerLength);
+  const newline = source.indexOf("\n", pos + openerLength);
+  if (closePos === -1 || (newline !== -1 && newline < closePos)) return pos;
+  return closePos + close.length;
+}
+
 function pushPlaceholder(placeholders: string[], text: string, sentinels: Sentinels): string {
   const idx = placeholders.length;
   placeholders.push(text);

package/src/parser/preprocess/whitespace/index.ts

@@ -13,6 +13,7 @@
 import { needsWhitespaceSubstitution, mayContainWhitespaceOnlyLine } from "./detection";
 import { replaceLeadingSpaces } from "./leading-spaces";
 import { CONCAT_LINES, DOS_MAC_NEWLINES, NULL_CHARS, TABS, WHITESPACE_ONLY_LINE } from "./patterns";
+import { makeUniqueSentinels, maskRawRegions, restorePlaceholders } from "../utils";
 
 /**
  * Apply all whitespace normalization substitutions to the given text.
@@ -41,7 +42,13 @@ export function substitute(text: string): string {
   }
 
   if (result.indexOf("\\\n") !== -1) {
-    result = result.replace(CONCAT_LINES, String.fromCharCode(0xe000));
+    const sentinels = makeUniqueSentinels(result);
+    const { masked, placeholders } = maskRawRegions(result, sentinels);
+    result = restorePlaceholders(
+      masked.replace(CONCAT_LINES, String.fromCharCode(0xe000)),
+      placeholders,
+      sentinels,
+    );
   }
 
   if (result.indexOf("\t") !== -1) {

package/src/parser/rules/block/bibliography/entry-content.ts

@@ -60,7 +60,7 @@ export function parseBibliographyContent(
     const inlineCtx: ParseContext = { ...ctx, pos };
     const result = parseInlineUntil(inlineCtx, "NEWLINE");
     if (result.elements.length > 0) {
-      content.push(...result.elements);
+      for (const element of result.elements) content.push(element);
       pos += result.consumed;
       consumed += result.consumed;
     } else {

package/src/parser/rules/block/block-list/bare-content.ts

@@ -1,3 +1,4 @@
+import { stripAutomaticLineBreak } from "../../inline/parsing/automatic-line-break";
 import type { ListItem } from "@wdprlib/ast";
 import type { ParseContext } from "../../types";
 import { getCandidateInlineRules } from "../../inline/utils";
@@ -52,7 +53,7 @@ export function parseBareListContent(
       if (consecutiveNewlines >= 2) {
         flushBareParagraph(paragraphState);
       } else {
-        appendBareParagraphLineBreakIfNeeded(paragraphState);
+        appendBareParagraphLineBreakIfNeeded(paragraphState, token);
       }
       continue;
     }
@@ -66,6 +67,7 @@ export function parseBareListContent(
     for (const rule of getCandidateInlineRules(ctx.inlineRules, token.type)) {
       const result = rule.parse(inlineCtx);
       if (result.success) {
+        stripAutomaticLineBreak(paragraphState.current, result.stripLeadingLineBreak);
         appendBareParagraphElements(paragraphState, result.elements);
         consumed += result.consumed;
         pos += result.consumed;

package/src/parser/rules/block/block-list/bare-paragraph.ts

@@ -1,3 +1,5 @@
+import type { Token } from "../../../../lexer";
+import { createAutomaticLineBreak } from "../../inline/parsing/automatic-line-break";
 import type { Element } from "@wdprlib/ast";
 
 export interface BareParagraphState {
@@ -20,9 +22,12 @@ export function appendBareParagraphText(state: BareParagraphState, text: string)
   state.current.push({ element: "text", data: text });
 }
 
-export function appendBareParagraphLineBreakIfNeeded(state: BareParagraphState): void {
+export function appendBareParagraphLineBreakIfNeeded(
+  state: BareParagraphState,
+  token: Token,
+): void {
   if (state.current.length > 0) {
-    state.current.push({ element: "line-break" });
+    state.current.push(createAutomaticLineBreak(token));
   }
 }