@wdprlib/parser 5.3.0 → 5.4.0

package/src/parser/rules/block/block-list/item-content.ts

@@ -1,3 +1,5 @@
+import type { Token } from "../../../../lexer";
+import { stripAutomaticLineBreak } from "../../inline/parsing/automatic-line-break";
 import type { Element } from "@wdprlib/ast";
 import type { ParseContext } from "../../types";
 import { getCandidateInlineRules } from "../../inline/utils";
@@ -8,6 +10,7 @@ interface ListItemContentResult {
   matched: boolean;
   elements: Element[];
   consumed: number;
+  stripLeadingLineBreak?: Token;
 }
 
 const blockListExcludedRulesCache = new WeakMap<
@@ -26,7 +29,7 @@ export function parseListItemBlockContent(
   for (const rule of getCandidateBlockRules(filteredBlockRules, token)) {
     const result = rule.parse(blockCtx);
     if (result.success) {
-      return { matched: true, elements: result.elements, consumed: result.consumed };
+      return { matched: true, ...result };
     }
   }
 
@@ -56,7 +59,7 @@ export function parseListItemInlineContent(
   for (const rule of getCandidateInlineRules(ctx.inlineRules, tokenType)) {
     const result = rule.parse(inlineCtx);
     if (result.success) {
-      return { matched: true, elements: result.elements, consumed: result.consumed };
+      return { matched: true, ...result };
     }
   }
 
@@ -108,7 +111,8 @@ export function collectPostLiTrailingContent(
 
     const inlineResult = parseListItemInlineContent(ctx, pos, token.type);
     if (inlineResult.matched) {
-      elements.push(...inlineResult.elements);
+      stripAutomaticLineBreak(elements, inlineResult.stripLeadingLineBreak);
+      for (const element of inlineResult.elements) elements.push(element);
       consumed += inlineResult.consumed;
       pos += inlineResult.consumed;
       continue;

package/src/parser/rules/block/block-list/li-content.ts

@@ -1,3 +1,7 @@
+import {
+  createAutomaticLineBreak,
+  stripAutomaticLineBreak,
+} from "../../inline/parsing/automatic-line-break";
 import type { Element } from "@wdprlib/ast";
 import type { ParseContext } from "../../types";
 import { parseListItemBlockContent, parseListItemInlineContent } from "./item-content";
@@ -50,7 +54,7 @@ export function collectLiItemContent(
     if (token.type === "NEWLINE") {
       const newlineResult = consumeLiItemNewlines(ctx, pos, elements.length > 0);
       if (newlineResult.addLineBreak) {
-        elements.push({ element: "line-break" });
+        elements.push(createAutomaticLineBreak(token));
       }
       pos += newlineResult.consumed;
       consumed += newlineResult.consumed;
@@ -59,7 +63,7 @@ export function collectLiItemContent(
 
     const blockResult = parseListItemBlockContent(ctx, pos, token);
     if (blockResult.matched) {
-      elements.push(...blockResult.elements);
+      for (const element of blockResult.elements) elements.push(element);
       consumed += blockResult.consumed;
       pos += blockResult.consumed;
       continue;
@@ -67,7 +71,8 @@ export function collectLiItemContent(
 
     const inlineResult = parseListItemInlineContent(ctx, pos, token.type);
     if (inlineResult.matched) {
-      elements.push(...inlineResult.elements);
+      stripAutomaticLineBreak(elements, inlineResult.stripLeadingLineBreak);
+      for (const element of inlineResult.elements) elements.push(element);
       consumed += inlineResult.consumed;
       pos += inlineResult.consumed;
       continue;

package/src/parser/rules/block/block-list/li-item.ts

@@ -62,7 +62,7 @@ export function parseLiItem(
     pos += closeConsumed;
 
     const trailingResult = collectPostLiTrailingContent(ctx, pos, listType);
-    contentElements.push(...trailingResult.elements);
+    for (const element of trailingResult.elements) contentElements.push(element);
     consumed += trailingResult.consumed;
   }
 

package/src/parser/rules/block/blockquote/build.ts

@@ -88,7 +88,7 @@ function sliceLineTokens(ctx: ParseContext, lines: BlockquoteLine[]): Token[] {
   for (const { start, end } of lines) {
     for (let pos = start; pos < end; pos++) {
       const token = ctx.tokens[pos];
-      if (token) {
+      if (token && token.type !== "BACKSLASH_BREAK") {
         tokens.push(token);
       }
     }

package/src/parser/rules/block/code/boundary.ts

@@ -0,0 +1,76 @@
+import type { Token } from "../../../../lexer";
+import { rawRegionEnd } from "../../inline/raw/end";
+import { findCodeOpen } from "./open";
+
+interface CodeBodyBounds {
+  closeStart: number;
+  end: number;
+  foundClose: boolean;
+}
+
+const caches = new WeakMap<readonly Token[], Map<number, CodeBodyBounds>>();
+
+function codeCloseEnd(tokens: readonly Token[], start: number): number {
+  if (tokens[start]?.type !== "BLOCK_END_OPEN" || tokens[start + 1]?.value.toLowerCase() !== "code")
+    return start;
+  let end = start + 2;
+  while (tokens[end]?.type === "WHITESPACE") end++;
+  return tokens[end]?.type === "BLOCK_CLOSE" ? end + 1 : start;
+}
+
+function nestedCodeBodyStart(tokens: readonly Token[], start: number): number {
+  if (tokens[start]?.type !== "BLOCK_OPEN" || tokens[start + 1]?.value.toLowerCase() !== "code")
+    return start;
+  for (let pos = start + 2; pos < tokens.length; pos++) {
+    const type = tokens[pos]?.type;
+    if (type === "NEWLINE" || type === "EOF" || type === "BLOCK_OPEN") return start;
+    if (type === "BLOCK_CLOSE") {
+      const open = findCodeOpen(tokens, start);
+      return open && !open.repaired ? open.bodyStart : start;
+    }
+  }
+  return start;
+}
+
+export function findCodeBodyBounds(tokens: readonly Token[], start: number): CodeBodyBounds {
+  let cache = caches.get(tokens);
+  if (!cache) {
+    cache = new Map();
+    caches.set(tokens, cache);
+  }
+  const cached = cache.get(start);
+  if (cached) return cached;
+  const stack = [start];
+  let pos = start;
+  while (pos < tokens.length && tokens[pos]?.type !== "EOF") {
+    const rawEnd = rawRegionEnd(tokens, pos, tokens.length);
+    if (rawEnd > pos) {
+      pos = rawEnd;
+      continue;
+    }
+    const end = codeCloseEnd(tokens, pos);
+    if (end > pos) {
+      const result = { closeStart: pos, end, foundClose: true };
+      cache.set(stack.pop()!, result);
+      if (stack.length === 0) return result;
+      pos = end;
+      continue;
+    }
+    const bodyStart = nestedCodeBodyStart(tokens, pos);
+    if (bodyStart > pos) {
+      const child = cache.get(bodyStart);
+      if (child) {
+        pos = child.end;
+        if (!child.foundClose) break;
+      } else {
+        stack.push(bodyStart);
+        pos = bodyStart;
+      }
+      continue;
+    }
+    pos++;
+  }
+  const result = { closeStart: pos, end: pos, foundClose: false };
+  for (const bodyStart of stack) cache.set(bodyStart, result);
+  return result;
+}

package/src/parser/rules/block/code/content.ts

@@ -1,5 +1,5 @@
 import type { ParseContext } from "../../types";
-import { parseBlockName } from "../utils";
+import { findCodeBodyBounds } from "./boundary";
 
 export interface CodeContentResult {
   contents: string;
@@ -12,43 +12,14 @@ export function collectCodeContent(
   startPos: number,
   closingSwallowed: boolean,
 ): CodeContentResult {
-  const contentParts: string[] = [];
-  let pos = startPos;
-  let consumed = 0;
-  let foundClose = closingSwallowed;
-
-  while (!closingSwallowed && pos < ctx.tokens.length) {
-    const token = ctx.tokens[pos];
-    if (!token || token.type === "EOF") {
-      break;
-    }
-
-    if (token.type === "BLOCK_END_OPEN") {
-      const closeNameResult = parseBlockName(ctx, pos + 1);
-      if (closeNameResult?.name === "code") {
-        foundClose = true;
-        const closeConsumed = consumeCodeClose(ctx, pos, closeNameResult.consumed);
-        consumed += closeConsumed;
-        break;
-      }
-    }
-
-    contentParts.push(token.value);
-    pos++;
-    consumed++;
-  }
-
-  return { contents: contentParts.join(""), consumed, foundClose };
-}
-
-function consumeCodeClose(ctx: ParseContext, startPos: number, closeNameConsumed: number): number {
-  let pos = startPos + 1 + closeNameConsumed;
-  let consumed = 1 + closeNameConsumed;
-
-  if (ctx.tokens[pos]?.type === "BLOCK_CLOSE") {
-    pos++;
-    consumed++;
-  }
-
-  return consumed;
+  if (closingSwallowed) return { contents: "", consumed: 0, foundClose: true };
+  const bounds = findCodeBodyBounds(ctx.tokens, startPos);
+  return {
+    contents: ctx.tokens
+      .slice(startPos, bounds.closeStart)
+      .map((token) => token.value)
+      .join(""),
+    consumed: bounds.end - startPos,
+    foundClose: bounds.foundClose,
+  };
 }

package/src/parser/rules/block/code/index.ts

@@ -12,7 +12,7 @@
 import type { Element, CodeBlockData } from "@wdprlib/ast";
 import type { BlockRule, ParseContext, RuleResult } from "../../types";
 import { currentToken } from "../../types";
-import { parseBlockName } from "../utils";
+import { findCodeOpen } from "./open";
 import { parseAttributesRaw } from "../utils";
 import { repairSwallowedCodeClose } from "./attributes";
 import { parseInlineUntil } from "../../inline/utils";
@@ -38,37 +38,14 @@ export const codeBlockRule: BlockRule = {
       return { success: false };
     }
 
-    let pos = ctx.pos + 1;
-    let consumed = 1;
+    const open = findCodeOpen(ctx.tokens, ctx.pos);
+    if (!open) return { success: false };
+    const attrResult = parseAttributesRaw(ctx, ctx.pos + 2);
+    if (open.repaired) repairSwallowedCodeClose(ctx, open.attributesEnd, attrResult.attrs);
+    let pos = open.bodyStart;
+    let consumed = pos - ctx.pos;
 
-    const nameResult = parseBlockName(ctx, pos);
-    if (!nameResult || nameResult.name !== "code") {
-      return { success: false };
-    }
-
-    pos += nameResult.consumed;
-    consumed += nameResult.consumed;
-
-    const attrResult = parseAttributesRaw(ctx, pos);
-    pos += attrResult.consumed;
-    consumed += attrResult.consumed;
-
-    let closingSwallowed = false;
-    if (ctx.tokens[pos]?.type !== "BLOCK_CLOSE") {
-      const repaired = repairSwallowedCodeClose(ctx, pos, attrResult.attrs);
-      if (!repaired) return { success: false };
-      closingSwallowed = repaired.closingSwallowed;
-    } else {
-      pos++;
-      consumed++;
-    }
-
-    if (ctx.tokens[pos]?.type === "NEWLINE") {
-      pos++;
-      consumed++;
-    }
-
-    const contentResult = collectCodeContent(ctx, pos, closingSwallowed);
+    const contentResult = collectCodeContent(ctx, pos, open.closingSwallowed);
     let codeContent = contentResult.contents.replace(/\n$/, "");
     consumed += contentResult.consumed;
     pos += contentResult.consumed;

package/src/parser/rules/block/code/open.ts

@@ -0,0 +1,46 @@
+import type { Token } from "../../../../lexer";
+
+interface CodeOpenBounds {
+  attributesEnd: number;
+  bodyStart: number;
+  repaired: boolean;
+  closingSwallowed: boolean;
+}
+
+const attributeRanges = new WeakMap<readonly Token[], { start: number; end: number }>();
+
+function findAttributesEnd(tokens: readonly Token[], start: number): number {
+  const cached = attributeRanges.get(tokens);
+  if (cached && start >= cached.start && start <= cached.end) return cached.end;
+  let end = start;
+  while (end < tokens.length) {
+    const type = tokens[end]?.type;
+    if (type === "BLOCK_CLOSE" || type === "NEWLINE" || type === "EOF") break;
+    end++;
+  }
+  attributeRanges.set(tokens, { start, end });
+  return end;
+}
+
+export function findCodeOpen(tokens: readonly Token[], start: number): CodeOpenBounds | null {
+  const name = tokens[start + 1];
+  if (
+    tokens[start]?.type !== "BLOCK_OPEN" ||
+    (name?.type !== "IDENTIFIER" && name?.type !== "TEXT") ||
+    name.value.toLowerCase() !== "code" ||
+    tokens[start + 2]?.type === "UNDERSCORE"
+  )
+    return null;
+  const attributesEnd = findAttributesEnd(tokens, start + 2);
+  const repaired = tokens[attributesEnd]?.type !== "BLOCK_CLOSE";
+  const last = tokens[attributesEnd - 1];
+  if (repaired && (last?.type !== "QUOTED_STRING" || !last.value.includes("]]"))) return null;
+  let bodyStart = attributesEnd + (repaired ? 0 : 1);
+  if (tokens[bodyStart]?.type === "NEWLINE") bodyStart++;
+  return {
+    attributesEnd,
+    bodyStart,
+    repaired,
+    closingSwallowed: repaired && last!.value.includes("[[/code]]"),
+  };
+}

package/src/parser/rules/block/definition-list/item-key.ts

@@ -51,7 +51,7 @@ export function parseDefinitionItemKey(
     const inlineCtx: ParseContext = { ...ctx, pos };
     const result = parseInlineUntil(inlineCtx, "COLON");
     if (result.elements.length > 0) {
-      key.push(...result.elements);
+      for (const element of result.elements) key.push(element);
       for (let i = 0; i < result.consumed; i++) {
         const t = ctx.tokens[pos + i];
         if (t) keyTokens.push(t.value);

package/src/parser/rules/block/definition-list/item-value.ts

@@ -34,7 +34,7 @@ export function parseDefinitionItemValue(
     const inlineCtx: ParseContext = { ...ctx, pos };
     const result = parseInlineUntil(inlineCtx, "NEWLINE");
     if (result.elements.length > 0) {
-      value.push(...result.elements);
+      for (const element of result.elements) value.push(element);
       pos += result.consumed;
       consumed += result.consumed;
     } else {

package/src/parser/rules/block/div/failed.ts

@@ -1,3 +1,4 @@
+import { markUnparsedDivStart } from "../../../postprocess/divAdjacentParagraph";
 import type { Element } from "@wdprlib/ast";
 import type { ParseContext, RuleResult } from "../../types";
 import { parseAttributes, parseBlockName } from "../utils";
@@ -100,6 +101,7 @@ export function consumeFailedDiv(ctx: ParseContext): RuleResult<Element> {
     consumed++;
   }
 
+  markUnparsedDivStart(elements);
   return {
     success: true,
     elements: [

package/src/parser/rules/block/index.ts

@@ -27,6 +27,7 @@ import { blockListRule } from "./block-list";
 import { blockquoteRule } from "./blockquote";
 import { definitionListRule } from "./definition-list";
 import { paragraphRule } from "./paragraph";
+import { noteRule } from "./note";
 import { divRule } from "./div";
 import { codeBlockRule } from "./code";
 import { collapsibleRule } from "./collapsible";
@@ -58,6 +59,7 @@ export { blockListRule } from "./block-list";
 export { blockquoteRule } from "./blockquote";
 export { definitionListRule } from "./definition-list";
 export { paragraphRule } from "./paragraph";
+export { noteRule } from "./note";
 export { divRule } from "./div";
 export { codeBlockRule } from "./code";
 export { collapsibleRule } from "./collapsible";
@@ -123,6 +125,7 @@ export const blockRules: BlockRule[] = [
   iftagsRule,
   bibliographyRule,
   galleryRule,
+  noteRule,
   divRule,
   // paragraphRule is not included - used as fallback
 ];

package/src/parser/rules/block/list/line.ts

@@ -51,10 +51,13 @@ export function parseNativeListLine(ctx: ParseContext, startPos: number): ListLi
   pos++;
   consumed++;
 
-  if (ctx.tokens[pos]?.type === "WHITESPACE") {
-    pos++;
-    consumed++;
+  // Wikidotのリスト構文はマーカー直後の空白が必須（Text_Wiki List.php: `(\*|#) `）。
+  // 空白なしの行（例: 生URLの `*http://...`）はリストではなくインライン内容として扱う
+  if (ctx.tokens[pos]?.type !== "WHITESPACE") {
+    return { kind: "stop" };
   }
+  pos++;
+  consumed++;
 
   const inlineCtx: ParseContext = { ...ctx, pos };
   const inlineResult = parseInlineUntil(inlineCtx, "NEWLINE");

package/src/parser/rules/block/math/boundary.ts

@@ -0,0 +1,104 @@
+import type { Token } from "../../../../lexer";
+
+interface MathOpenBounds {
+  nameStart: number;
+  nameEnd: number;
+  bodyStart: number;
+}
+
+interface MathBodyBounds {
+  closeStart: number;
+  end: number;
+  foundClose: boolean;
+  hasContent: boolean;
+}
+
+const nameRanges = new WeakMap<readonly Token[], { start: number; end: number }>();
+const bodyCaches = new WeakMap<readonly Token[], Map<number, MathBodyBounds>>();
+
+function findNameEnd(tokens: readonly Token[], start: number): number {
+  const cached = nameRanges.get(tokens);
+  if (cached && start >= cached.start && start <= cached.end) return cached.end;
+  let end = start;
+  while (end < tokens.length) {
+    const type = tokens[end]?.type;
+    if (type === "BLOCK_CLOSE" || type === "WHITESPACE" || type === "NEWLINE" || type === "EOF")
+      break;
+    end++;
+  }
+  nameRanges.set(tokens, { start, end });
+  return end;
+}
+
+export function findMathOpen(tokens: readonly Token[], start: number): MathOpenBounds | null {
+  const name = tokens[start + 1];
+  if (
+    tokens[start]?.type !== "BLOCK_OPEN" ||
+    (name?.type !== "IDENTIFIER" && name?.type !== "TEXT") ||
+    name.value.toLowerCase() !== "math" ||
+    tokens[start + 2]?.type === "UNDERSCORE"
+  )
+    return null;
+  let pos = start + 2;
+  while (tokens[pos]?.type === "WHITESPACE") pos++;
+  const nameStart = pos;
+  if (tokens[pos]?.type === "IDENTIFIER" || tokens[pos]?.type === "TEXT") {
+    pos = findNameEnd(tokens, pos);
+  }
+  const nameEnd = pos;
+  while (tokens[pos]?.type === "WHITESPACE") pos++;
+  if (tokens[pos]?.type !== "BLOCK_CLOSE") return null;
+  pos++;
+  if (tokens[pos]?.type === "NEWLINE") pos++;
+  return { nameStart, nameEnd, bodyStart: pos };
+}
+
+function mathCloseEnd(tokens: readonly Token[], start: number): number {
+  if (
+    tokens[start]?.type !== "BLOCK_END_OPEN" ||
+    tokens[start + 1]?.value.toLowerCase() !== "math" ||
+    tokens[start + 2]?.type !== "BLOCK_CLOSE"
+  )
+    return start;
+  const end = start + 3;
+  return tokens[end]?.type === "NEWLINE" ? end + 1 : end;
+}
+
+export function findMathBodyBounds(tokens: readonly Token[], start: number): MathBodyBounds {
+  let cache = bodyCaches.get(tokens);
+  if (!cache) {
+    cache = new Map();
+    bodyCaches.set(tokens, cache);
+  }
+  const visited: number[] = [];
+  let result: MathBodyBounds = {
+    closeStart: tokens.length,
+    end: tokens.length,
+    foundClose: false,
+    hasContent: false,
+  };
+  for (let pos = start; pos < tokens.length; pos++) {
+    const cached = cache.get(pos);
+    if (cached) {
+      result = cached;
+      break;
+    }
+    const end = mathCloseEnd(tokens, pos);
+    if (end > pos) {
+      result = { closeStart: pos, end, foundClose: true, hasContent: false };
+      cache.set(pos, result);
+      break;
+    }
+    visited.push(pos);
+  }
+  // Empty-body detection must also be reusable without joining each candidate's suffix.
+  for (let i = visited.length - 1; i >= 0; i--) {
+    const pos = visited[i]!;
+    const token = tokens[pos]!;
+    if (!result.hasContent && (token.type === "BACKSLASH_BREAK" || token.value.trim() !== "")) {
+      result = { ...result, hasContent: true };
+    }
+    cache.set(pos, result);
+  }
+  return result;
+}

package/src/parser/rules/block/math/index.ts

@@ -11,9 +11,7 @@
 import type { Element } from "@wdprlib/ast";
 import type { BlockRule, ParseContext, RuleResult } from "../../types";
 import { currentToken } from "../../types";
-import { parseBlockName } from "../utils";
-import { collectMathContent, consumeMathClose } from "./content";
-import { parseMathName } from "./name";
+import { findMathOpen, findMathBodyBounds } from "./boundary";
 
 /**
  * Block rule for `[[math name]]...[[/math]]`.
@@ -28,79 +26,41 @@ export const mathBlockRule: BlockRule = {
 
   parse(ctx: ParseContext): RuleResult<Element> {
     const openToken = currentToken(ctx);
-    if (openToken.type !== "BLOCK_OPEN") {
-      return { success: false };
-    }
-
-    let pos = ctx.pos + 1;
-    let consumed = 1;
-
-    const nameResult = parseBlockName(ctx, pos);
-    if (!nameResult || nameResult.name !== "math") {
-      return { success: false };
-    }
-    pos += nameResult.consumed;
-    consumed += nameResult.consumed;
-
-    while (ctx.tokens[pos]?.type === "WHITESPACE") {
-      pos++;
-      consumed++;
-    }
-
-    const mathName = parseMathName(ctx, pos);
-    pos += mathName.consumed;
-    consumed += mathName.consumed;
-
-    while (ctx.tokens[pos]?.type === "WHITESPACE") {
-      pos++;
-      consumed++;
-    }
-
-    if (ctx.tokens[pos]?.type !== "BLOCK_CLOSE") {
-      return { success: false };
-    }
-    pos++;
-    consumed++;
-
-    if (ctx.tokens[pos]?.type === "NEWLINE") {
-      pos++;
-      consumed++;
-    }
-
-    const contentResult = collectMathContent(ctx, pos);
-    const latexSource = contentResult.latexSource.trim();
-    consumed += contentResult.consumed;
-    pos += contentResult.consumed;
-
-    if (!contentResult.foundClose) {
+    const open = findMathOpen(ctx.tokens, ctx.pos);
+    if (!open) return { success: false };
+    const bounds = findMathBodyBounds(ctx.tokens, open.bodyStart);
+    if (!bounds.foundClose) {
       ctx.diagnostics.push({
         severity: "warning",
         code: "unclosed-block",
         message: "Missing closing tag [[/math]] for [[math]]",
         position: openToken.position,
       });
-    } else {
-      const closeConsumed = consumeMathClose(ctx, pos);
-      pos += closeConsumed;
-      consumed += closeConsumed;
     }
-
-    if (!latexSource) {
+    if (!bounds.hasContent) {
       return { success: false };
     }
-
+    const latexSource = ctx.tokens
+      .slice(open.bodyStart, bounds.closeStart)
+      .map((token) => (token.type === "BACKSLASH_BREAK" ? "\\\n" : token.value))
+      .join("")
+      .trim();
+    const name = ctx.tokens
+      .slice(open.nameStart, open.nameEnd)
+      .map((token) => token.value)
+      .join("");
     return {
       success: true,
       elements: [
         {
           element: "math",
           data: {
-            name: mathName.name,
+            name: name || null,
             "latex-source": latexSource,
           },
         },
       ],
-      consumed,
+      consumed: bounds.end - ctx.pos,
     };
   },
 };