@wdprlib/parser 5.2.0 → 5.4.0

package/src/parser/rules/inline/parsing/preserved-line-break.ts

@@ -2,6 +2,7 @@ import type { Element } from "@wdprlib/ast";
 
 type PreservedLineBreak = Extract<Element, { element: "line-break" }> & {
   _preservedTrailingBreak?: boolean;
+  _preservedLeadingBreak?: boolean;
 };
 
 export function createPreservedTrailingLineBreak(): Element {
@@ -9,3 +10,15 @@ export function createPreservedTrailingLineBreak(): Element {
   lineBreak._preservedTrailingBreak = true;
   return lineBreak;
 }
+
+export function createPreservedLeadingLineBreak(): Element {
+  const lineBreak: PreservedLineBreak = { element: "line-break", _preservedLeadingBreak: true };
+  return lineBreak;
+}
+
+export function isPreservedLeadingLineBreak(element: Element | undefined): boolean {
+  return (
+    element?.element === "line-break" &&
+    (element as PreservedLineBreak)._preservedLeadingBreak === true
+  );
+}

package/src/parser/rules/inline/parsing/raw-tag.ts

@@ -0,0 +1,19 @@
+import type { Token } from "../../../../lexer";
+
+const unclosedRanges = new WeakMap<readonly Token[], { start: number; end: number }>();
+
+/** Raw directives end at the first ]], even inside a quoted value. */
+export function findRawTagClose(
+  tokens: readonly Token[],
+  start: number,
+  end: number,
+): number | null {
+  const unclosed = unclosedRanges.get(tokens);
+  if (unclosed && start >= unclosed.start && end <= unclosed.end) return null;
+  for (let close = start; close < end; close++) {
+    if (tokens[close]?.type === "BLOCK_CLOSE") return close;
+    if (!tokens[close] || tokens[close]?.type === "EOF") break;
+  }
+  unclosedRanges.set(tokens, { start, end });
+  return null;
+}

package/src/parser/rules/inline/raw/angle.ts

@@ -1,4 +1,5 @@
 import type { Element } from "@wdprlib/ast";
+import { decodeHTML } from "entities";
 import type { ParseContext, RuleResult } from "../../types";
 import { currentToken, hasClosingMarkerBeforeNewline } from "../../types";
 import { rawElement, textElement } from "./result";
@@ -36,5 +37,5 @@ export function parseAngleRaw(ctx: ParseContext): RuleResult<Element> {
     consumed++;
   }
 
-  return rawElement(value, consumed);
+  return rawElement(decodeHTML(value), consumed);
 }

package/src/parser/rules/inline/raw/end.ts

@@ -1,15 +1,35 @@
 import type { Token } from "../../../../lexer";
 
+// A failed angle-raw search also rules out later openers within the same immutable token range.
+const unclosedAngleRanges = new WeakMap<
+  readonly Token[],
+  { start: number; end: number; lineEnd: boolean }
+>();
+
 /** Exclusive end of a complete single-line raw region, or the original position. */
 export function rawRegionEnd(tokens: readonly Token[], start: number, end: number): number {
   const type = tokens[start]?.type;
   const close =
     type === "RAW_OPEN" ? "RAW_OPEN" : type === "RAW_BLOCK_OPEN" ? "RAW_BLOCK_CLOSE" : null;
   if (!close) return start;
+  const cached = type === "RAW_BLOCK_OPEN" ? unclosedAngleRanges.get(tokens) : undefined;
+  if (
+    cached &&
+    start >= cached.start &&
+    start < cached.end &&
+    (cached.lineEnd || end <= cached.end)
+  ) {
+    return start;
+  }
   for (let pos = start + 1; pos < end; pos++) {
-    if (tokens[pos]?.type === "NEWLINE" || tokens[pos]?.type === "EOF") break;
+    if (tokens[pos]?.type === "NEWLINE" || tokens[pos]?.type === "EOF") {
+      if (type === "RAW_BLOCK_OPEN")
+        unclosedAngleRanges.set(tokens, { start, end: pos, lineEnd: true });
+      return start;
+    }
     if (tokens[pos]?.type === close) return pos + 1;
   }
+  if (type === "RAW_BLOCK_OPEN") unclosedAngleRanges.set(tokens, { start, end, lineEnd: false });
   return start;
 }
 

package/src/parser/rules/inline/size/content.ts

@@ -32,7 +32,7 @@ export function parseSizeContent(ctx: ParseContext, startPos: number): SizeConte
     const inlineCtx: ParseContext = { ...ctx, pos };
     const inlineResult = parseInlineUntil(inlineCtx, "BLOCK_END_OPEN");
     if (inlineResult.elements.length > 0) {
-      children.push(...inlineResult.elements);
+      for (const element of inlineResult.elements) children.push(element);
       pos += inlineResult.consumed;
       consumed += inlineResult.consumed;
     } else {

package/src/parser/rules/inline/size/value.ts

@@ -1,6 +1,16 @@
 import type { ParseContext } from "../../types";
 
 const VALID_SIZE_UNITS = ["px", "em", "rem", "ex", "%", "cm", "mm", "in", "pc"];
+const VALID_SIZE_KEYWORDS = [
+  "smaller",
+  "larger",
+  "xx-small",
+  "x-small",
+  "small",
+  "large",
+  "x-large",
+  "xx-large",
+];
 
 export function parseSizeValue(
   ctx: ParseContext,
@@ -40,6 +50,7 @@ export function parseSizeValue(
 }
 
 function isValidSizeValue(size: string): boolean {
+  if (VALID_SIZE_KEYWORDS.includes(size)) return true;
   const unitPattern = VALID_SIZE_UNITS.join("|");
   return new RegExp(`^(\\d+(?:\\.\\d+)?)(${unitPattern})$`, "i").test(size);
 }

package/src/parser/rules/inline/social/index.ts

@@ -0,0 +1,17 @@
+import type { InlineRule } from "../../types";
+import { parseSocialSyntax } from "./syntax";
+
+export const socialRule: InlineRule = {
+  name: "social",
+  startTokens: ["BLOCK_OPEN"],
+  parse(ctx) {
+    const result = parseSocialSyntax(ctx, ctx.pos, ctx.scope.inlineEnd ?? ctx.tokens.length);
+    return result
+      ? {
+          success: true,
+          consumed: result.end - ctx.pos,
+          elements: [{ element: "social", data: result.data }],
+        }
+      : { success: false };
+  },
+};

package/src/parser/rules/inline/social/syntax.ts

@@ -0,0 +1,40 @@
+import type { SocialData } from "@wdprlib/ast";
+import type { Token } from "../../../../lexer";
+import type { ParseContext } from "../../types";
+import { findRawTagClose } from "../parsing/raw-tag";
+
+const invalidBodies = new WeakMap<readonly Token[], { start: number; bracket: number }>();
+
+export function parseSocialSyntax(
+  ctx: ParseContext,
+  start: number,
+  end: number,
+): { data: SocialData; end: number } | null {
+  const tokens = ctx.tokens;
+  if (tokens[start]?.type !== "BLOCK_OPEN" || tokens[start + 1]?.value.toLowerCase() !== "social")
+    return null;
+  const bodyStart = start + 2;
+  if (tokens[bodyStart]?.type !== "BLOCK_CLOSE" && !/^\s+$/.test(tokens[bodyStart]?.value ?? ""))
+    return null;
+  const invalid = invalidBodies.get(tokens);
+  if (invalid && bodyStart >= invalid.start && bodyStart <= invalid.bracket) return null;
+  const close = findRawTagClose(tokens, bodyStart, end);
+  if (close === null) return null;
+  for (let pos = bodyStart; pos < close; pos++) {
+    if (tokens[pos]!.value.includes("]")) {
+      invalidBodies.set(tokens, { start: bodyStart, bracket: pos });
+      return null;
+    }
+  }
+  const body = tokens
+    .slice(bodyStart, close)
+    .map((token) => token.value)
+    .join("");
+  const sites = body.trim()
+    ? body
+        .split(",")
+        .map((site) => site.trim().toLowerCase())
+        .filter(Boolean)
+    : null;
+  return { data: { sites }, end: close + 1 };
+}

package/src/parser/rules/inline/span/content.ts

@@ -1,3 +1,4 @@
+import { stripAutomaticLineBreak } from "../parsing/automatic-line-break";
 import { rawRegionEnd } from "../raw/end";
 import type { Element } from "@wdprlib/ast";
 import type { ParseContext } from "../../types";
@@ -118,7 +119,8 @@ function parseOneSpanChild(
   for (const rule of getCandidateInlineRules(inlineRules, token.type)) {
     const result = rule.parse(inlineCtx);
     if (result.success) {
-      targetChildren.push(...result.elements);
+      stripAutomaticLineBreak(targetChildren, result.stripLeadingLineBreak);
+      for (const element of result.elements) targetChildren.push(element);
       return { consumed: result.consumed };
     }
   }

package/src/parser/rules/inline/span/newline.ts

@@ -1,3 +1,4 @@
+import { createAutomaticLineBreak } from "../parsing/automatic-line-break";
 import type { Element } from "@wdprlib/ast";
 import type { ParseContext } from "../../types";
 
@@ -38,7 +39,7 @@ export function consumeSpanNewline(
   }
 
   const targetChildren = paragraphStrip && afterBlankLine ? escapedChildren : children;
-  targetChildren.push({ element: "line-break" });
+  targetChildren.push(createAutomaticLineBreak(ctx.tokens[pos]!));
   let consumed = 1;
   let nextPos = pos + 1;
   while (ctx.tokens[nextPos]?.type === "WHITESPACE" && ctx.tokens[nextPos]?.lineStart) {

package/src/parser/rules/opaque-probe.ts

@@ -0,0 +1,58 @@
+import type { BlockRule, InlineRule, ParseContext } from "./types";
+import { findCodeOpen } from "./block/code/open";
+import { findCodeBodyBounds } from "./block/code/boundary";
+import { findMathOpen, findMathBodyBounds } from "./block/math/boundary";
+
+/** Probe opaque syntax without registering its code, footnotes or diagnostics. */
+export function createOpaqueProbe(ctx: ParseContext): ParseContext {
+  return {
+    ...ctx,
+    diagnostics: [],
+    footnotes: [],
+    tocEntries: [],
+    codeBlocks: [],
+    htmlBlocks: [],
+    bibcites: [],
+    scope: {
+      ...ctx.scope,
+      inlineEnd: undefined,
+      tableFormatting: undefined,
+      blockCloseCondition: () => true,
+    },
+  };
+}
+
+export function opaqueRuleEnd(
+  probe: ParseContext,
+  pos: number,
+  rules: readonly (BlockRule | InlineRule)[],
+  requireClosed: boolean = false,
+): number {
+  const token = probe.tokens[pos];
+  if (!token) return pos;
+  probe.pos = pos;
+  for (const rule of rules) {
+    if (!rule.startTokens.includes(token.type)) continue;
+    if (rule.name === "code") {
+      const open = findCodeOpen(probe.tokens, pos);
+      if (!open) continue;
+      if (open.closingSwallowed) {
+        if (!requireClosed) return open.bodyStart;
+        continue;
+      }
+      const bounds = findCodeBodyBounds(probe.tokens, open.bodyStart);
+      if (!requireClosed || bounds.foundClose) return bounds.end;
+      continue;
+    }
+    if (rule.name === "math") {
+      const open = findMathOpen(probe.tokens, pos);
+      if (!open) continue;
+      const bounds = findMathBodyBounds(probe.tokens, open.bodyStart);
+      if (bounds.hasContent && (!requireClosed || bounds.foundClose)) return bounds.end;
+      continue;
+    }
+    const result = rule.parse(probe);
+    if (result.success) return pos + result.consumed;
+  }
+  return pos;
+}

package/src/pipeline/process.ts

@@ -82,6 +82,12 @@ export async function processWikitext<TPage extends WikitextPageContext>(
   const dataProvider = createModuleDataProvider(options, callbackContext);
   const parseFragment = parseSource;
   let ast = initial.ast;
+  const paginationState = { nextUnprefixed: 1 };
+  const requestedPath = options.page.urlPath;
+  const urlPath =
+    !requestedPath || /^\/(?:[?#]|$)/.test(requestedPath)
+      ? `/${options.page.fullName}${requestedPath?.slice(1) ?? ""}`
+      : requestedPath;
 
   for (let pass = 0; pass < DEFAULT_MODULE_MAX_PASSES; pass++) {
     const extraction = extractDataRequirements(ast);
@@ -94,7 +100,8 @@ export async function processWikitext<TPage extends WikitextPageContext>(
       compiledListPagesTemplates: extraction.compiledListPagesTemplates,
       compiledListUsersTemplates: extraction.compiledListUsersTemplates,
       requirements: extraction.requirements,
-      urlPath: options.page.urlPath,
+      urlPath,
+      paginationState,
       pageTags: options.page.tags,
     });
     ast = resolved.ast;

package/src/parser/rules/block/math/content.ts

@@ -1,54 +0,0 @@
-import type { ParseContext } from "../../types";
-import { parseBlockName } from "../utils";
-
-export interface MathContentResult {
-  latexSource: string;
-  consumed: number;
-  foundClose: boolean;
-}
-
-export function collectMathContent(ctx: ParseContext, startPos: number): MathContentResult {
-  let latexSource = "";
-  let pos = startPos;
-  let consumed = 0;
-  let foundClose = false;
-
-  while (pos < ctx.tokens.length) {
-    const token = ctx.tokens[pos];
-    if (!token) break;
-
-    if (token.type === "BLOCK_END_OPEN") {
-      const closeNameResult = parseBlockName(ctx, pos + 1);
-      if (closeNameResult?.name === "math") {
-        foundClose = true;
-        break;
-      }
-    }
-
-    latexSource += token.type === "BACKSLASH_BREAK" ? "\\\n" : token.value;
-    pos++;
-    consumed++;
-  }
-
-  return { latexSource, consumed, foundClose };
-}
-
-export function consumeMathClose(ctx: ParseContext, startPos: number): number {
-  let pos = startPos + 1;
-  let consumed = 1;
-
-  const closeNameResult = parseBlockName(ctx, pos);
-  if (closeNameResult) {
-    pos += closeNameResult.consumed;
-    consumed += closeNameResult.consumed;
-  }
-  if (ctx.tokens[pos]?.type === "BLOCK_CLOSE") {
-    pos++;
-    consumed++;
-  }
-  if (ctx.tokens[pos]?.type === "NEWLINE") {
-    consumed++;
-  }
-
-  return consumed;
-}

package/src/parser/rules/block/math/name.ts

@@ -1,35 +0,0 @@
-import type { ParseContext } from "../../types";
-
-export interface MathNameResult {
-  name: string | null;
-  consumed: number;
-}
-
-export function parseMathName(ctx: ParseContext, startPos: number): MathNameResult {
-  let pos = startPos;
-  let consumed = 0;
-  let name = "";
-
-  const first = ctx.tokens[pos];
-  if (first?.type !== "IDENTIFIER" && first?.type !== "TEXT") {
-    return { name: null, consumed: 0 };
-  }
-
-  while (pos < ctx.tokens.length) {
-    const token = ctx.tokens[pos];
-    if (
-      !token ||
-      token.type === "BLOCK_CLOSE" ||
-      token.type === "WHITESPACE" ||
-      token.type === "NEWLINE"
-    ) {
-      break;
-    }
-
-    name += token.value;
-    pos++;
-    consumed++;
-  }
-
-  return { name: name === "" ? null : name, consumed };
-}

package/src/parser/rules/inline/footnote/child.ts

@@ -1,22 +0,0 @@
-import type { Element } from "@wdprlib/ast";
-import type { ParseContext } from "../../types";
-import { parseInlineUntil } from "../utils";
-
-export interface FootnoteChildResult {
-  elements: Element[];
-  consumed: number;
-}
-
-export function parseFootnoteChild(ctx: ParseContext, pos: number): FootnoteChildResult {
-  const token = ctx.tokens[pos];
-  if (!token) {
-    return { elements: [], consumed: 0 };
-  }
-
-  const inlineResult = parseInlineUntil({ ...ctx, pos }, "BLOCK_END_OPEN");
-  if (inlineResult.elements.length > 0) {
-    return { elements: inlineResult.elements, consumed: inlineResult.consumed };
-  }
-
-  return { elements: [{ element: "text", data: token.value }], consumed: 1 };
-}

package/src/parser/rules/inline/footnote/newline.ts

@@ -1,27 +0,0 @@
-import type { ParseContext } from "../../types";
-
-export interface FootnoteNewlineResult {
-  consumed: number;
-  paragraphBreak: boolean;
-}
-
-export function consumeFootnoteNewline(ctx: ParseContext, pos: number): FootnoteNewlineResult {
-  let nextPos = pos + 1;
-  let consumed = 1;
-
-  while (ctx.tokens[nextPos]?.type === "WHITESPACE") {
-    nextPos++;
-    consumed++;
-  }
-
-  if (ctx.tokens[nextPos]?.type !== "NEWLINE") {
-    return { consumed, paragraphBreak: false };
-  }
-
-  while (ctx.tokens[nextPos]?.type === "NEWLINE") {
-    nextPos++;
-    consumed++;
-  }
-
-  return { consumed, paragraphBreak: true };
-}