@wdprlib/parser 5.3.0 → 6.0.0

package/src/parser/rules/inline/parsing/plain-text.ts

@@ -1,4 +1,6 @@
+import { getEmailCandidate } from "../email/candidates";
 import type { TokenType } from "../../../../lexer";
+import { URL_SCHEME_NAMES } from "../../../../lexer/url-schemes";
 import type { ParseContext } from "../../types";
 
 const MIN_INLINE_TEXT_RUN_LENGTH = 32;
@@ -18,6 +20,7 @@ export function collectLongPlainTextRun(
 ): PlainTextRun | null {
   const firstToken = ctx.tokens[startPos];
   if (
+    !getEmailCandidate(ctx.tokens, startPos) &&
     firstToken?.type === "TEXT" &&
     firstToken.value.length >= MIN_INLINE_TEXT_RUN_LENGTH &&
     firstToken.value !== "("
@@ -50,10 +53,12 @@ export function collectLongPlainTextRun(
 
 function isPlainTextRunToken(ctx: ParseContext, pos: number): boolean {
   const token = ctx.tokens[pos];
-  if (!token) return false;
+  if (!token || getEmailCandidate(ctx.tokens, pos)) return false;
 
   if (token.type === "IDENTIFIER") {
-    return true;
+    // 連続平文の一括テキスト化がURL先頭のスキーム名を取り込むと
+    // autolinkルールに到達しなくなるため、スキーム名の手前で止める
+    return !(URL_SCHEME_NAMES.has(token.value) && ctx.tokens[pos + 1]?.type === "COLON");
   }
 
   if (token.type === "WHITESPACE") {

package/src/parser/rules/inline/parsing/preserved-line-break.ts

@@ -2,6 +2,7 @@ import type { Element } from "@wdprlib/ast";
 
 type PreservedLineBreak = Extract<Element, { element: "line-break" }> & {
   _preservedTrailingBreak?: boolean;
+  _preservedLeadingBreak?: boolean;
 };
 
 export function createPreservedTrailingLineBreak(): Element {
@@ -9,3 +10,15 @@ export function createPreservedTrailingLineBreak(): Element {
   lineBreak._preservedTrailingBreak = true;
   return lineBreak;
 }
+
+export function createPreservedLeadingLineBreak(): Element {
+  const lineBreak: PreservedLineBreak = { element: "line-break", _preservedLeadingBreak: true };
+  return lineBreak;
+}
+
+export function isPreservedLeadingLineBreak(element: Element | undefined): boolean {
+  return (
+    element?.element === "line-break" &&
+    (element as PreservedLineBreak)._preservedLeadingBreak === true
+  );
+}

package/src/parser/rules/inline/parsing/raw-tag.ts

@@ -0,0 +1,19 @@
+import type { Token } from "../../../../lexer";
+
+const unclosedRanges = new WeakMap<readonly Token[], { start: number; end: number }>();
+
+/** Raw directives end at the first ]], even inside a quoted value. */
+export function findRawTagClose(
+  tokens: readonly Token[],
+  start: number,
+  end: number,
+): number | null {
+  const unclosed = unclosedRanges.get(tokens);
+  if (unclosed && start >= unclosed.start && end <= unclosed.end) return null;
+  for (let close = start; close < end; close++) {
+    if (tokens[close]?.type === "BLOCK_CLOSE") return close;
+    if (!tokens[close] || tokens[close]?.type === "EOF") break;
+  }
+  unclosedRanges.set(tokens, { start, end });
+  return null;
+}

package/src/parser/rules/inline/raw/angle.ts

@@ -1,4 +1,5 @@
 import type { Element } from "@wdprlib/ast";
+import { decodeHTML } from "entities";
 import type { ParseContext, RuleResult } from "../../types";
 import { currentToken, hasClosingMarkerBeforeNewline } from "../../types";
 import { rawElement, textElement } from "./result";
@@ -36,5 +37,5 @@ export function parseAngleRaw(ctx: ParseContext): RuleResult<Element> {
     consumed++;
   }
 
-  return rawElement(value, consumed);
+  return rawElement(decodeHTML(value), consumed);
 }

package/src/parser/rules/inline/raw/end.ts

@@ -1,15 +1,35 @@
 import type { Token } from "../../../../lexer";
 
+// A failed angle-raw search also rules out later openers within the same immutable token range.
+const unclosedAngleRanges = new WeakMap<
+  readonly Token[],
+  { start: number; end: number; lineEnd: boolean }
+>();
+
 /** Exclusive end of a complete single-line raw region, or the original position. */
 export function rawRegionEnd(tokens: readonly Token[], start: number, end: number): number {
   const type = tokens[start]?.type;
   const close =
     type === "RAW_OPEN" ? "RAW_OPEN" : type === "RAW_BLOCK_OPEN" ? "RAW_BLOCK_CLOSE" : null;
   if (!close) return start;
+  const cached = type === "RAW_BLOCK_OPEN" ? unclosedAngleRanges.get(tokens) : undefined;
+  if (
+    cached &&
+    start >= cached.start &&
+    start < cached.end &&
+    (cached.lineEnd || end <= cached.end)
+  ) {
+    return start;
+  }
   for (let pos = start + 1; pos < end; pos++) {
-    if (tokens[pos]?.type === "NEWLINE" || tokens[pos]?.type === "EOF") break;
+    if (tokens[pos]?.type === "NEWLINE" || tokens[pos]?.type === "EOF") {
+      if (type === "RAW_BLOCK_OPEN")
+        unclosedAngleRanges.set(tokens, { start, end: pos, lineEnd: true });
+      return start;
+    }
     if (tokens[pos]?.type === close) return pos + 1;
   }
+  if (type === "RAW_BLOCK_OPEN") unclosedAngleRanges.set(tokens, { start, end, lineEnd: false });
   return start;
 }
 

package/src/parser/rules/inline/size/content.ts

@@ -2,6 +2,8 @@ import type { Element } from "@wdprlib/ast";
 import type { ParseContext } from "../../types";
 import { parseBlockName } from "../../common";
 import { parseInlineUntil } from "../utils";
+import { protectedInlineRegionEnd } from "../raw/end";
+import { parseSizeOpen } from "./open";
 
 export interface SizeContentResult {
   children: Element[];
@@ -13,8 +15,9 @@ export function parseSizeContent(ctx: ParseContext, startPos: number): SizeConte
   const children: Element[] = [];
   let pos = startPos;
   let consumed = 0;
+  const inlineEnd = findSizeClose(ctx, startPos);
 
-  while (pos < ctx.tokens.length) {
+  while (pos < (ctx.scope.inlineEnd ?? ctx.tokens.length)) {
     const token = ctx.tokens[pos];
     if (!token || token.type === "EOF") {
       break;
@@ -29,10 +32,10 @@ export function parseSizeContent(ctx: ParseContext, startPos: number): SizeConte
       };
     }
 
-    const inlineCtx: ParseContext = { ...ctx, pos };
-    const inlineResult = parseInlineUntil(inlineCtx, "BLOCK_END_OPEN");
-    if (inlineResult.elements.length > 0) {
-      children.push(...inlineResult.elements);
+    const inlineCtx: ParseContext = { ...ctx, pos, scope: { ...ctx.scope, inlineEnd } };
+    const inlineResult = parseInlineUntil(inlineCtx, "EOF");
+    if (inlineResult.consumed > 0) {
+      for (const element of inlineResult.elements) children.push(element);
       pos += inlineResult.consumed;
       consumed += inlineResult.consumed;
     } else {
@@ -45,6 +48,30 @@ export function parseSizeContent(ctx: ParseContext, startPos: number): SizeConte
   return { children, consumed, foundClose: false };
 }
 
+function findSizeClose(ctx: ParseContext, startPos: number): number {
+  const end = ctx.scope.inlineEnd ?? ctx.tokens.length;
+  let depth = 0;
+  for (let pos = startPos; pos < end; pos++) {
+    const protectedEnd = protectedInlineRegionEnd(ctx.tokens, pos, end);
+    if (protectedEnd > pos) {
+      pos = protectedEnd - 1;
+      continue;
+    }
+    const open = ctx.tokens[pos]?.type === "BLOCK_OPEN" ? parseSizeOpen({ ...ctx, pos }) : null;
+    if (open) {
+      depth++;
+      pos = open.bodyStart - 1;
+      continue;
+    }
+    const close = tryConsumeSizeClose(ctx, pos);
+    if (!close) continue;
+    if (depth === 0) return pos;
+    depth--;
+    pos += close.consumed - 1;
+  }
+  return end;
+}
+
 function tryConsumeSizeClose(ctx: ParseContext, pos: number): { consumed: number } | null {
   if (ctx.tokens[pos]?.type !== "BLOCK_END_OPEN") {
     return null;

package/src/parser/rules/inline/size/value.ts

@@ -1,6 +1,16 @@
 import type { ParseContext } from "../../types";
 
 const VALID_SIZE_UNITS = ["px", "em", "rem", "ex", "%", "cm", "mm", "in", "pc"];
+const VALID_SIZE_KEYWORDS = [
+  "smaller",
+  "larger",
+  "xx-small",
+  "x-small",
+  "small",
+  "large",
+  "x-large",
+  "xx-large",
+];
 
 export function parseSizeValue(
   ctx: ParseContext,
@@ -40,6 +50,7 @@ export function parseSizeValue(
 }
 
 function isValidSizeValue(size: string): boolean {
+  if (VALID_SIZE_KEYWORDS.includes(size)) return true;
   const unitPattern = VALID_SIZE_UNITS.join("|");
   return new RegExp(`^(\\d+(?:\\.\\d+)?)(${unitPattern})$`, "i").test(size);
 }

package/src/parser/rules/inline/social/index.ts

@@ -0,0 +1,17 @@
+import type { InlineRule } from "../../types";
+import { parseSocialSyntax } from "./syntax";
+
+export const socialRule: InlineRule = {
+  name: "social",
+  startTokens: ["BLOCK_OPEN"],
+  parse(ctx) {
+    const result = parseSocialSyntax(ctx, ctx.pos, ctx.scope.inlineEnd ?? ctx.tokens.length);
+    return result
+      ? {
+          success: true,
+          consumed: result.end - ctx.pos,
+          elements: [{ element: "social", data: result.data }],
+        }
+      : { success: false };
+  },
+};

package/src/parser/rules/inline/social/syntax.ts

@@ -0,0 +1,40 @@
+import type { SocialData } from "@wdprlib/ast";
+import type { Token } from "../../../../lexer";
+import type { ParseContext } from "../../types";
+import { findRawTagClose } from "../parsing/raw-tag";
+
+const invalidBodies = new WeakMap<readonly Token[], { start: number; bracket: number }>();
+
+export function parseSocialSyntax(
+  ctx: ParseContext,
+  start: number,
+  end: number,
+): { data: SocialData; end: number } | null {
+  const tokens = ctx.tokens;
+  if (tokens[start]?.type !== "BLOCK_OPEN" || tokens[start + 1]?.value.toLowerCase() !== "social")
+    return null;
+  const bodyStart = start + 2;
+  if (tokens[bodyStart]?.type !== "BLOCK_CLOSE" && !/^\s+$/.test(tokens[bodyStart]?.value ?? ""))
+    return null;
+  const invalid = invalidBodies.get(tokens);
+  if (invalid && bodyStart >= invalid.start && bodyStart <= invalid.bracket) return null;
+  const close = findRawTagClose(tokens, bodyStart, end);
+  if (close === null) return null;
+  for (let pos = bodyStart; pos < close; pos++) {
+    if (tokens[pos]!.value.includes("]")) {
+      invalidBodies.set(tokens, { start: bodyStart, bracket: pos });
+      return null;
+    }
+  }
+  const body = tokens
+    .slice(bodyStart, close)
+    .map((token) => token.value)
+    .join("");
+  const sites = body.trim()
+    ? body
+        .split(",")
+        .map((site) => site.trim().toLowerCase())
+        .filter(Boolean)
+    : null;
+  return { data: { sites }, end: close + 1 };
+}

package/src/parser/rules/inline/span/content.ts

@@ -1,3 +1,4 @@
+import { stripAutomaticLineBreak } from "../parsing/automatic-line-break";
 import { rawRegionEnd } from "../raw/end";
 import type { Element } from "@wdprlib/ast";
 import type { ParseContext } from "../../types";
@@ -118,7 +119,8 @@ function parseOneSpanChild(
   for (const rule of getCandidateInlineRules(inlineRules, token.type)) {
     const result = rule.parse(inlineCtx);
     if (result.success) {
-      targetChildren.push(...result.elements);
+      stripAutomaticLineBreak(targetChildren, result.stripLeadingLineBreak);
+      for (const element of result.elements) targetChildren.push(element);
       return { consumed: result.consumed };
     }
   }

package/src/parser/rules/inline/span/newline.ts

@@ -1,3 +1,4 @@
+import { createAutomaticLineBreak } from "../parsing/automatic-line-break";
 import type { Element } from "@wdprlib/ast";
 import type { ParseContext } from "../../types";
 
@@ -38,7 +39,7 @@ export function consumeSpanNewline(
   }
 
   const targetChildren = paragraphStrip && afterBlankLine ? escapedChildren : children;
-  targetChildren.push({ element: "line-break" });
+  targetChildren.push(createAutomaticLineBreak(ctx.tokens[pos]!));
   let consumed = 1;
   let nextPos = pos + 1;
   while (ctx.tokens[nextPos]?.type === "WHITESPACE" && ctx.tokens[nextPos]?.lineStart) {

package/src/parser/rules/opaque-probe.ts

@@ -0,0 +1,58 @@
+import type { BlockRule, InlineRule, ParseContext } from "./types";
+import { findCodeOpen } from "./block/code/open";
+import { findCodeBodyBounds } from "./block/code/boundary";
+import { findMathOpen, findMathBodyBounds } from "./block/math/boundary";
+
+/** Probe opaque syntax without registering its code, footnotes or diagnostics. */
+export function createOpaqueProbe(ctx: ParseContext): ParseContext {
+  return {
+    ...ctx,
+    diagnostics: [],
+    footnotes: [],
+    tocEntries: [],
+    codeBlocks: [],
+    htmlBlocks: [],
+    bibcites: [],
+    scope: {
+      ...ctx.scope,
+      inlineEnd: undefined,
+      tableFormatting: undefined,
+      blockCloseCondition: () => true,
+    },
+  };
+}
+
+export function opaqueRuleEnd(
+  probe: ParseContext,
+  pos: number,
+  rules: readonly (BlockRule | InlineRule)[],
+  requireClosed: boolean = false,
+): number {
+  const token = probe.tokens[pos];
+  if (!token) return pos;
+  probe.pos = pos;
+  for (const rule of rules) {
+    if (!rule.startTokens.includes(token.type)) continue;
+    if (rule.name === "code") {
+      const open = findCodeOpen(probe.tokens, pos);
+      if (!open) continue;
+      if (open.closingSwallowed) {
+        if (!requireClosed) return open.bodyStart;
+        continue;
+      }
+      const bounds = findCodeBodyBounds(probe.tokens, open.bodyStart);
+      if (!requireClosed || bounds.foundClose) return bounds.end;
+      continue;
+    }
+    if (rule.name === "math") {
+      const open = findMathOpen(probe.tokens, pos);
+      if (!open) continue;
+      const bounds = findMathBodyBounds(probe.tokens, open.bodyStart);
+      if (bounds.hasContent && (!requireClosed || bounds.foundClose)) return bounds.end;
+      continue;
+    }
+    const result = rule.parse(probe);
+    if (result.success) return pos + result.consumed;
+  }
+  return pos;
+}

package/src/pipeline/process.ts

@@ -1,5 +1,8 @@
 import {
   DEFAULT_SETTINGS,
+  extractReadableText,
+  extractFirstParagraph,
+  countCharacters,
   type Diagnostic,
   type PageRef,
   type WikitextPageContext,
@@ -10,6 +13,7 @@ import type { AsyncIncludeFetcher, IncludeDependency } from "../parser/rules/blo
 import { resolveIncludesAsyncWithTraceSelective } from "../parser/rules/block/module/include/resolve";
 import type { DataProvider } from "../parser/rules/block/module/types-common";
 import { resolveModulesWithAsyncParse } from "../parser/rules/block/module/resolution/resolve-async";
+import { resolveRatings } from "../parser/rules/block/module/rate/resolve";
 import type {
   ProcessedWikitextDocument,
   ProcessWikitextCallbackContext,
@@ -117,8 +121,14 @@ export async function processWikitext<TPage extends WikitextPageContext>(
     );
   }
 
+  ast = await resolveRatings(ast, dataProvider.fetchRatings);
+  const readableText = extractReadableText(ast, options.readableText);
+
   return {
     ast,
+    readableText,
+    firstParagraph: extractFirstParagraph(ast, options.readableText),
+    characterCount: countCharacters(readableText),
     page: options.page,
     settings,
     diagnostics,
@@ -167,6 +177,9 @@ function createModuleDataProvider<TPage extends WikitextPageContext>(
 ): DataProvider {
   const provider = options.dataProvider;
   return {
+    fetchRatings: provider?.fetchRatings
+      ? (refs) => provider.fetchRatings!(refs, context)
+      : undefined,
     fetchListPages: provider?.fetchListPages
       ? (query, requirement) => provider.fetchListPages!(query, requirement, context)
       : undefined,

package/src/pipeline/types.ts

@@ -4,6 +4,9 @@ import type {
   SyntaxTree,
   WikitextPageContext,
   WikitextSettings,
+  RatingRef,
+  RatingState,
+  ReadableTextOptions,
 } from "@wdprlib/ast";
 import type { IncludeDependency } from "../parser/rules/block/module/include";
 import type {
@@ -26,6 +29,17 @@ export interface ProcessWikitextCallbackContext<TPage extends WikitextPageContex
 }
 
 export interface ProcessWikitextDataProvider<TPage extends WikitextPageContext> {
+  /**
+   * Read only authorized, registered ratings in context.page, including declarations
+   * from its includes. Keys match exactly. Omit no-rate and inaccessible references.
+   * Main category policy and custom policies are independent. Votes belong to page + ref.
+   * ListPages/ListUsers output cannot declare ratings, including its nested includes.
+   * The host owns registration, aggregation and invalidation when included declarations change.
+   */
+  fetchRatings?: (
+    refs: readonly RatingRef[],
+    context: ProcessWikitextCallbackContext<TPage>,
+  ) => Promise<readonly RatingState[]>;
   fetchInclude?: (
     pageRef: PageRef,
     context: ProcessWikitextCallbackContext<TPage>,
@@ -50,12 +64,17 @@ export interface ProcessWikitextOptions<TPage extends WikitextPageContext> {
   settings?: WikitextSettings;
   dataProvider?: ProcessWikitextDataProvider<TPage>;
   includeMaxIterations?: number;
+  readableText?: ReadableTextOptions;
 }
 
 export interface ProcessedWikitextDocument<
   TPage extends WikitextPageContext = WikitextPageContext,
 > {
   ast: SyntaxTree;
+  readableText: string;
+  /** First nonempty body paragraph, excluding headings and appended footnotes. */
+  firstParagraph: string;
+  characterCount: number;
   page: TPage;
   settings: WikitextSettings;
   diagnostics: Diagnostic[];

package/src/parser/rules/block/math/content.ts

@@ -1,54 +0,0 @@
-import type { ParseContext } from "../../types";
-import { parseBlockName } from "../utils";
-
-export interface MathContentResult {
-  latexSource: string;
-  consumed: number;
-  foundClose: boolean;
-}
-
-export function collectMathContent(ctx: ParseContext, startPos: number): MathContentResult {
-  let latexSource = "";
-  let pos = startPos;
-  let consumed = 0;
-  let foundClose = false;
-
-  while (pos < ctx.tokens.length) {
-    const token = ctx.tokens[pos];
-    if (!token) break;
-
-    if (token.type === "BLOCK_END_OPEN") {
-      const closeNameResult = parseBlockName(ctx, pos + 1);
-      if (closeNameResult?.name === "math") {
-        foundClose = true;
-        break;
-      }
-    }
-
-    latexSource += token.type === "BACKSLASH_BREAK" ? "\\\n" : token.value;
-    pos++;
-    consumed++;
-  }
-
-  return { latexSource, consumed, foundClose };
-}
-
-export function consumeMathClose(ctx: ParseContext, startPos: number): number {
-  let pos = startPos + 1;
-  let consumed = 1;
-
-  const closeNameResult = parseBlockName(ctx, pos);
-  if (closeNameResult) {
-    pos += closeNameResult.consumed;
-    consumed += closeNameResult.consumed;
-  }
-  if (ctx.tokens[pos]?.type === "BLOCK_CLOSE") {
-    pos++;
-    consumed++;
-  }
-  if (ctx.tokens[pos]?.type === "NEWLINE") {
-    consumed++;
-  }
-
-  return consumed;
-}

package/src/parser/rules/block/math/name.ts

@@ -1,35 +0,0 @@
-import type { ParseContext } from "../../types";
-
-export interface MathNameResult {
-  name: string | null;
-  consumed: number;
-}
-
-export function parseMathName(ctx: ParseContext, startPos: number): MathNameResult {
-  let pos = startPos;
-  let consumed = 0;
-  let name = "";
-
-  const first = ctx.tokens[pos];
-  if (first?.type !== "IDENTIFIER" && first?.type !== "TEXT") {
-    return { name: null, consumed: 0 };
-  }
-
-  while (pos < ctx.tokens.length) {
-    const token = ctx.tokens[pos];
-    if (
-      !token ||
-      token.type === "BLOCK_CLOSE" ||
-      token.type === "WHITESPACE" ||
-      token.type === "NEWLINE"
-    ) {
-      break;
-    }
-
-    name += token.value;
-    pos++;
-    consumed++;
-  }
-
-  return { name: name === "" ? null : name, consumed };
-}

package/src/parser/rules/inline/footnote/child.ts

@@ -1,22 +0,0 @@
-import type { Element } from "@wdprlib/ast";
-import type { ParseContext } from "../../types";
-import { parseInlineUntil } from "../utils";
-
-export interface FootnoteChildResult {
-  elements: Element[];
-  consumed: number;
-}
-
-export function parseFootnoteChild(ctx: ParseContext, pos: number): FootnoteChildResult {
-  const token = ctx.tokens[pos];
-  if (!token) {
-    return { elements: [], consumed: 0 };
-  }
-
-  const inlineResult = parseInlineUntil({ ...ctx, pos }, "BLOCK_END_OPEN");
-  if (inlineResult.elements.length > 0) {
-    return { elements: inlineResult.elements, consumed: inlineResult.consumed };
-  }
-
-  return { elements: [{ element: "text", data: token.value }], consumed: 1 };
-}

package/src/parser/rules/inline/footnote/newline.ts

@@ -1,27 +0,0 @@
-import type { ParseContext } from "../../types";
-
-export interface FootnoteNewlineResult {
-  consumed: number;
-  paragraphBreak: boolean;
-}
-
-export function consumeFootnoteNewline(ctx: ParseContext, pos: number): FootnoteNewlineResult {
-  let nextPos = pos + 1;
-  let consumed = 1;
-
-  while (ctx.tokens[nextPos]?.type === "WHITESPACE") {
-    nextPos++;
-    consumed++;
-  }
-
-  if (ctx.tokens[nextPos]?.type !== "NEWLINE") {
-    return { consumed, paragraphBreak: false };
-  }
-
-  while (ctx.tokens[nextPos]?.type === "NEWLINE") {
-    nextPos++;
-    consumed++;
-  }
-
-  return { consumed, paragraphBreak: true };
-}