@wdprlib/parser 5.3.0 → 5.4.0

package/src/parser/rules/inline/button/syntax.ts

@@ -0,0 +1,56 @@
+import type { PageButtonData } from "@wdprlib/ast";
+import { findRawTagClose } from "../parsing/raw-tag";
+import type { ParseContext } from "../../types";
+import { parseButtonAttributes } from "./attributes";
+
+interface ButtonSyntax {
+  data: PageButtonData;
+  end: number;
+}
+
+/** Read a complete standalone button, including permitted separator newlines. */
+export function parseButtonSyntax(
+  ctx: ParseContext,
+  start: number,
+  end: number,
+): ButtonSyntax | null {
+  const tokens = ctx.tokens;
+  if (tokens[start]?.type !== "BLOCK_OPEN" || tokens[start + 1]?.value.toLowerCase() !== "button")
+    return null;
+  let pos = start + 2;
+  const skipSpace = () => {
+    const before = pos;
+    while (pos < end && /^\s+$/.test(tokens[pos]?.value ?? "")) pos++;
+    return pos > before;
+  };
+  if (!skipSpace()) return null;
+  let action = "";
+  while (
+    pos < end &&
+    tokens[pos]?.type !== "BLOCK_CLOSE" &&
+    !/^\s+$/.test(tokens[pos]?.value ?? "")
+  ) {
+    const part = tokens[pos]?.value ?? "";
+    if (!/^[a-z0-9_-]+$/i.test(part)) return null;
+    action += part;
+    pos++;
+  }
+  if (!action) return null;
+  action = action.replaceAll("_", "-");
+  skipSpace();
+  const close = findRawTagClose(tokens, pos, end);
+  if (close === null) return null;
+  const attrs = parseButtonAttributes(
+    tokens
+      .slice(pos, close)
+      .map((token) => token.value)
+      .join(""),
+  );
+  const value = (key: string) => (attrs[key] && attrs[key] !== "0" ? attrs[key]! : null);
+  const attributes: Record<string, string> = {};
+  for (const name of ["class", "style"]) {
+    const attr = value(name);
+    if (attr !== null) attributes[name] = attr;
+  }
+  return { data: { action, text: value("text"), attributes }, end: close + 1 };
+}

package/src/parser/rules/inline/date/index.ts

@@ -0,0 +1,17 @@
+import type { InlineRule } from "../../types";
+import { parseDateSyntax } from "./syntax";
+
+export const dateRule: InlineRule = {
+  name: "date",
+  startTokens: ["BLOCK_OPEN"],
+  parse(ctx) {
+    const parsed = parseDateSyntax(ctx, ctx.pos, ctx.scope.inlineEnd ?? ctx.tokens.length);
+    return parsed
+      ? {
+          success: true,
+          consumed: parsed.end - ctx.pos,
+          elements: [{ element: "date", data: parsed.data }],
+        }
+      : { success: false };
+  },
+};

package/src/parser/rules/inline/date/syntax.ts

@@ -0,0 +1,46 @@
+import type { DateData } from "@wdprlib/ast";
+import type { ParseContext } from "../../types";
+import { parseAttributesRaw } from "../../block/parsing/attributes";
+
+interface DateSyntax {
+  data: DateData;
+  end: number;
+}
+
+/** Read a complete date without registering any parser side effects. */
+export function parseDateSyntax(ctx: ParseContext, start: number, end: number): DateSyntax | null {
+  const tokens = ctx.tokens;
+  if (tokens[start]?.type !== "BLOCK_OPEN" || tokens[start + 1]?.value !== "date") return null;
+  let pos = start + 2;
+  const skipSpace = () => {
+    const before = pos;
+    while (pos < end && /^(?:\s+)$/.test(tokens[pos]?.value ?? "")) pos++;
+    return pos > before;
+  };
+  if (!skipSpace()) return null;
+  const value = tokens[pos]?.value ?? "";
+  if (pos >= end || !/^\d+$/.test(value)) return null;
+  const timestamp = Number(value);
+  if (!Number.isSafeInteger(timestamp) || timestamp > 8_640_000_000_000) return null;
+  pos++;
+  const separator = skipSpace();
+  if (tokens[pos]?.type !== "BLOCK_CLOSE" && !separator) return null;
+  let close = pos;
+  for (; close < end; close++) {
+    const token = tokens[close];
+    if (!token || token.type === "EOF" || token.type === "BLOCK_OPEN" || /[\r\n]/.test(token.value))
+      return null;
+    if (token.type === "BLOCK_CLOSE") break;
+  }
+  if (close >= end) return null;
+  const attributes = parseAttributesRaw(ctx, pos).attrs;
+  const format = attributes.format || null;
+  return {
+    data: {
+      value: { timestamp, timezone: "UTC" },
+      format,
+      hover: format?.split("|").slice(1).includes("agohover") ?? false,
+    },
+    end: close + 1,
+  };
+}

package/src/parser/rules/inline/email/candidates.ts

@@ -0,0 +1,134 @@
+import type { Token, TokenType } from "../../../../lexer";
+import { isLocalChar, scanEmail } from "./scan";
+
+export interface EmailCandidate {
+  start: number;
+  end: number;
+  endToken: number;
+  address: string;
+}
+
+interface EmailCache {
+  source: string;
+  offsets: number[];
+  candidates: Map<number, EmailCandidate | null>;
+  commentEnds: Map<number, number>;
+  unclosedComment: number;
+}
+
+const caches = new WeakMap<readonly Token[], EmailCache>();
+export const EMAIL_START_TOKENS: TokenType[] = [
+  "TEXT",
+  "IDENTIFIER",
+  "UNDERSCORE",
+  "UNDERLINE_MARKER",
+  "STRIKE_MARKER",
+];
+
+function getCache(tokens: readonly Token[]): EmailCache {
+  let cache = caches.get(tokens);
+  if (cache) return cache;
+  const offsets = [0];
+  const values: string[] = [];
+  for (const token of tokens) {
+    values.push(token.value);
+    offsets.push(offsets.at(-1)! + token.value.length);
+  }
+  cache = {
+    source: values.join(""),
+    offsets,
+    candidates: new Map(),
+    commentEnds: new Map(),
+    unclosedComment: Infinity,
+  };
+  caches.set(tokens, cache);
+  return cache;
+}
+
+function commentEnd(cache: EmailCache, pos: number): number {
+  const opener = cache.source[pos] === "\n" ? pos + 1 : pos;
+  if (!cache.source.startsWith("[!--", opener)) return pos;
+  const cached = cache.commentEnds.get(pos);
+  if (cached !== undefined) return cached;
+  const close = opener >= cache.unclosedComment ? -1 : cache.source.indexOf("--]", opener + 4);
+  if (close === -1) cache.unclosedComment = Math.min(cache.unclosedComment, opener);
+  const end = close === -1 ? pos : close + 3;
+  cache.commentEnds.set(pos, end);
+  return end;
+}
+
+export function getEmailCandidate(tokens: readonly Token[], index: number): EmailCandidate | null {
+  const token = tokens[index];
+  if (!token || !EMAIL_START_TOKENS.includes(token.type)) return null;
+  const cache = getCache(tokens);
+  if (cache.candidates.has(index)) return cache.candidates.get(index)!;
+  const value = tokens[index]?.value ?? "";
+  // Compact TEXT may include a prefix before the last possible local-part run.
+  let suffix = value.length;
+  while (suffix > 0 && (isLocalChar(value.charCodeAt(suffix - 1)) || value[suffix - 1] === "."))
+    suffix--;
+  const doubled = value.lastIndexOf("..");
+  if (doubled >= suffix) suffix = doubled + 2;
+  while (value[suffix] === ".") suffix++;
+  if (suffix >= value.length || !isLocalChar(value.charCodeAt(suffix))) {
+    cache.candidates.set(index, null);
+    return null;
+  }
+  const start = cache.offsets[index]! + suffix;
+  const scanned = scanEmail(cache.source, start, (pos) => commentEnd(cache, pos));
+  if (scanned.end === undefined) {
+    cache.candidates.set(index, null);
+    let comment = 0;
+    // A failed local run cannot produce an address from one of its suffix tokens.
+    // Do not poison positions inside skipped comments: another parse scope may see them as raw text.
+    for (let i = index + 1; i < tokens.length && cache.offsets[i + 1]! <= scanned.localEnd; i++) {
+      const offset = cache.offsets[i]!;
+      while (scanned.comments[comment] && scanned.comments[comment]!.end <= offset) comment++;
+      const region = scanned.comments[comment];
+      if (!region || offset < region.start) cache.candidates.set(i, null);
+    }
+    return null;
+  }
+  let endToken = index;
+  while (cache.offsets[endToken + 1]! < scanned.end) endToken++;
+  const candidate = { start, end: scanned.end, endToken, address: scanned.address! };
+  cache.candidates.set(index, candidate);
+  return candidate;
+}
+
+interface EmailGroup {
+  candidates: EmailCandidate[];
+  source: string;
+  start: number;
+  end: number;
+  endToken: number;
+}
+
+export function getEmailGroup(
+  tokens: readonly Token[],
+  index: number,
+  limit: number,
+): EmailGroup | null {
+  const first = getEmailCandidate(tokens, index);
+  if (!first || first.endToken >= limit) return null;
+  const candidates = [first];
+  let last = first;
+  while (last.endToken > index) {
+    const next = getEmailCandidate(tokens, last.endToken);
+    if (!next || next.start < last.end || next.endToken >= limit) break;
+    candidates.push(next);
+    last = next;
+  }
+  const cache = getCache(tokens);
+  return {
+    candidates,
+    source: cache.source,
+    start: cache.offsets[index]!,
+    end: cache.offsets[last.endToken + 1]!,
+    endToken: last.endToken + 1,
+  };
+}
+
+export function emailRegionEnd(tokens: readonly Token[], index: number, limit: number): number {
+  return getEmailGroup(tokens, index, limit)?.endToken ?? index;
+}

package/src/parser/rules/inline/email/index.ts

@@ -0,0 +1,36 @@
+import type { Element } from "@wdprlib/ast";
+import type { InlineRule } from "../../types";
+import { EMAIL_START_TOKENS, getEmailGroup } from "./candidates";
+
+export const emailRule: InlineRule = {
+  name: "email",
+  startTokens: EMAIL_START_TOKENS,
+  parse(ctx) {
+    const group = getEmailGroup(ctx.tokens, ctx.pos, ctx.scope.inlineEnd ?? ctx.tokens.length);
+    if (!group) return { success: false };
+    const elements: Element[] = [];
+    let copied = group.start;
+    for (const candidate of group.candidates) {
+      if (copied < candidate.start)
+        elements.push({ element: "text", data: group.source.slice(copied, candidate.start) });
+      elements.push(
+        ctx.scope.suppressEmailLinks
+          ? { element: "text", data: candidate.address }
+          : {
+              element: "link",
+              data: {
+                type: "direct",
+                link: `mailto:${candidate.address}`,
+                label: { text: candidate.address },
+                target: null,
+                extra: null,
+              },
+            },
+      );
+      copied = candidate.end;
+    }
+    if (copied < group.end)
+      elements.push({ element: "text", data: group.source.slice(copied, group.end) });
+    return { success: true, elements, consumed: group.endToken - ctx.pos };
+  },
+};

package/src/parser/rules/inline/email/scan.ts

@@ -0,0 +1,76 @@
+export interface EmailScan {
+  end?: number;
+  address?: string;
+  localEnd: number;
+  comments: Array<{ start: number; end: number }>;
+}
+
+export function isDomainChar(code: number): boolean {
+  return (
+    (code >= 48 && code <= 57) ||
+    (code >= 65 && code <= 90) ||
+    (code >= 97 && code <= 122) ||
+    code === 45
+  );
+}
+
+export function isLocalChar(code: number): boolean {
+  return isDomainChar(code) || code === 95;
+}
+
+/** Read Email.php's grammar from one possible start, ignoring complete comments. */
+export function scanEmail(
+  source: string,
+  start: number,
+  commentEnd: (pos: number) => number,
+): EmailScan {
+  const comments: EmailScan["comments"] = [];
+  let pos = start;
+  const skipComments = () => {
+    let end = commentEnd(pos);
+    while (end > pos) {
+      comments.push({ start: pos, end });
+      pos = end;
+      end = commentEnd(pos);
+    }
+  };
+  let localPart = false;
+  while (pos < source.length) {
+    skipComments();
+    if (isLocalChar(source.charCodeAt(pos))) {
+      localPart = true;
+      pos++;
+    } else if (source[pos] === "." && localPart) {
+      localPart = false;
+      pos++;
+    } else break;
+  }
+  const localEnd = pos;
+  if (!localPart || source[pos] !== "@") return { localEnd, comments };
+  pos++;
+  let domainPart = false;
+  let dots = 0;
+  let end = -1;
+  while (pos < source.length) {
+    skipComments();
+    if (isDomainChar(source.charCodeAt(pos))) {
+      domainPart = true;
+      pos++;
+      if (dots > 0) end = pos;
+    } else if (source[pos] === "." && domainPart) {
+      domainPart = false;
+      dots++;
+      pos++;
+    } else break;
+  }
+  if (end === -1) return { localEnd, comments };
+  const parts: string[] = [];
+  let copied = start;
+  for (const comment of comments) {
+    if (comment.start >= end) break;
+    parts.push(source.slice(copied, comment.start));
+    copied = comment.end;
+  }
+  parts.push(source.slice(copied, end));
+  return { end, address: parts.join(""), localEnd, comments };
+}

package/src/parser/rules/inline/expr/branch.ts

@@ -1,3 +1,4 @@
+import { stripAutomaticLineBreak } from "../parsing/automatic-line-break";
 import type { Element } from "@wdprlib/ast";
 import type { ParseContext } from "../../types";
 import { getCandidateInlineRules } from "../utils";
@@ -52,7 +53,8 @@ export function parseInlineBranch(ctx: ParseContext, startPos: number): BranchPa
       }
       const result = rule.parse(inlineCtx);
       if (result.success) {
-        elements.push(...result.elements);
+        stripAutomaticLineBreak(elements, result.stripLeadingLineBreak);
+        for (const element of result.elements) elements.push(element);
         consumed += result.consumed;
         pos += result.consumed;
         matched = true;

package/src/parser/rules/inline/footnote/boundary.ts

@@ -0,0 +1,56 @@
+import { createOpaqueProbe, opaqueRuleEnd } from "../../opaque-probe";
+import type { ParseContext } from "../../types";
+import { codeBlockRule } from "../../block/code";
+import { htmlBlockRule } from "../../block/html";
+import { mathBlockRule } from "../../block/math";
+import { mathInlineRule } from "../math-inline";
+import { linkTripleRule } from "../link-triple";
+import { linkSingleRule } from "../link-single";
+import { linkStarRule } from "../link-star";
+import { linkAnchorRule } from "../link-anchor";
+import { protectedInlineRegionEnd } from "../raw/end";
+import { tryConsumeFootnoteClose } from "./close";
+import { parseFootnoteOpen } from "./open";
+
+const opaqueRules = [
+  codeBlockRule,
+  htmlBlockRule,
+  mathBlockRule,
+  mathInlineRule,
+  linkTripleRule,
+  linkSingleRule,
+  linkStarRule,
+  linkAnchorRule,
+];
+
+/** Locate the enclosing close without parsing code, HTML or math as footnote syntax. */
+export function findFootnoteEnd(ctx: ParseContext, start: number): number {
+  const probe = createOpaqueProbe(ctx);
+  let depth = 0;
+  for (let pos = start; pos < ctx.tokens.length; pos++) {
+    if (ctx.tokens[pos]?.type === "EOF") return pos;
+    const protectedEnd = protectedInlineRegionEnd(ctx.tokens, pos, ctx.tokens.length);
+    if (protectedEnd > pos) {
+      pos = protectedEnd - 1;
+      continue;
+    }
+
+    probe.pos = pos;
+    const close = tryConsumeFootnoteClose(probe, pos);
+    if (close) {
+      if (depth === 0) return pos;
+      depth--;
+      pos += close.consumed - 1;
+      continue;
+    }
+    const open = parseFootnoteOpen(probe);
+    if (open) {
+      depth++;
+      pos += open.consumed - 1;
+      continue;
+    }
+    const opaqueEnd = opaqueRuleEnd(probe, pos, opaqueRules);
+    if (opaqueEnd > pos) pos = opaqueEnd - 1;
+  }
+  return ctx.tokens.length;
+}

package/src/parser/rules/inline/footnote/content.ts

@@ -1,54 +1,42 @@
 import type { Element } from "@wdprlib/ast";
 import type { ParseContext } from "../../types";
-import { parseFootnoteChild } from "./child";
+import { parseBlocksUntil } from "../../block/parsing/content";
 import { tryConsumeFootnoteClose } from "./close";
-import { consumeFootnoteNewline } from "./newline";
+import { findFootnoteEnd } from "./boundary";
 
 export interface FootnoteContentResult {
-  paragraphs: Element[][];
+  elements: Element[];
   consumed: number;
   foundClose: boolean;
+  leadingParagraphBreak: boolean;
 }
 
 export function parseFootnoteContent(ctx: ParseContext, startPos: number): FootnoteContentResult {
-  const paragraphs: Element[][] = [[]];
-  let currentParagraph = 0;
-  let pos = startPos;
-  let consumed = 0;
-
-  while (pos < ctx.tokens.length) {
-    const token = ctx.tokens[pos];
-    if (!token || token.type === "EOF") {
-      break;
-    }
-
-    const closeResult = tryConsumeFootnoteClose(ctx, pos);
-    if (closeResult) {
-      return {
-        paragraphs,
-        consumed: consumed + closeResult.consumed,
-        foundClose: true,
-      };
-    }
-
-    if (token.type === "NEWLINE") {
-      const newlineResult = consumeFootnoteNewline(ctx, pos);
-      pos += newlineResult.consumed;
-      consumed += newlineResult.consumed;
-      if (newlineResult.paragraphBreak) {
-        currentParagraph++;
-        paragraphs[currentParagraph] = [];
-      } else {
-        paragraphs[currentParagraph]!.push({ element: "line-break" });
-      }
-      continue;
-    }
-
-    const child = parseFootnoteChild(ctx, pos);
-    paragraphs[currentParagraph]!.push(...child.elements);
-    pos += child.consumed;
-    consumed += child.consumed;
+  const end = findFootnoteEnd(ctx, startPos);
+  let leadingNewlines = 0;
+  for (let pos = startPos; pos < end; pos++) {
+    const type = ctx.tokens[pos]?.type;
+    if (type === "NEWLINE") leadingNewlines++;
+    else if (type !== "WHITESPACE") break;
   }
 
-  return { paragraphs, consumed, foundClose: false };
+  const bodyCtx: ParseContext = {
+    ...ctx,
+    tokens: ctx.tokens.slice(startPos, end),
+    pos: 0,
+    scope: {
+      ...ctx.scope,
+      inlineEnd: undefined,
+      tableFormatting: undefined,
+      blockCloseCondition: undefined,
+    },
+  };
+  const result = parseBlocksUntil(bodyCtx, () => false);
+  const close = tryConsumeFootnoteClose(ctx, end);
+  return {
+    elements: result.elements,
+    consumed: end - startPos + (close?.consumed ?? 0),
+    foundClose: close !== null,
+    leadingParagraphBreak: leadingNewlines >= 2,
+  };
 }

package/src/parser/rules/inline/footnote/elements.ts

@@ -1,38 +1,13 @@
 import type { Element } from "@wdprlib/ast";
 
-export function buildFootnoteChildren(paragraphs: Element[][]): Element[] {
-  const children: Element[] = [];
-
-  for (let i = 0; i < paragraphs.length; i++) {
-    const para = trimLineBreaks(paragraphs[i] ?? []);
-    if (para.length === 0) {
-      continue;
-    }
-
-    if (i === 0) {
-      children.push(...para);
-    } else {
-      children.push({
-        element: "container",
-        data: {
-          type: "paragraph",
-          attributes: {},
-          elements: para,
-        },
-      });
-    }
-  }
-
-  return children;
-}
-
-function trimLineBreaks(elements: Element[]): Element[] {
-  const result = [...elements];
-  while (result.length > 0 && result[0]?.element === "line-break") {
-    result.shift();
-  }
-  while (result.length > 0 && result[result.length - 1]?.element === "line-break") {
-    result.pop();
+/** Wikidot leaves the initial footnote paragraph unwrapped unless a blank line precedes it. */
+export function buildFootnoteChildren(
+  elements: Element[],
+  leadingParagraphBreak: boolean,
+): Element[] {
+  const first = elements[0];
+  if (!leadingParagraphBreak && first?.element === "container" && first.data.type === "paragraph") {
+    return [...first.data.elements, ...elements.slice(1)];
   }
-  return result;
+  return elements;
 }

package/src/parser/rules/inline/footnote/index.ts

@@ -28,7 +28,10 @@ export const footnoteRule: InlineRule = {
 
     const contentResult = parseFootnoteContent(ctx, openResult.bodyStart);
     const consumed = openResult.consumed + contentResult.consumed;
-    const children = buildFootnoteChildren(contentResult.paragraphs);
+    const children = buildFootnoteChildren(
+      contentResult.elements,
+      contentResult.leadingParagraphBreak,
+    );
 
     if (!contentResult.foundClose) {
       ctx.diagnostics.push({

package/src/parser/rules/inline/formatting/close.ts

@@ -1,3 +1,6 @@
+import { emailRegionEnd } from "../email/candidates";
+import { parseButtonSyntax } from "../button/syntax";
+import { parseSocialSyntax } from "../social/syntax";
 import { protectedInlineRegionEnd } from "../raw/end";
 import type { TokenType } from "../../../../lexer";
 import type { ParseContext } from "../../types";
@@ -20,6 +23,15 @@ export function findFormattingClose(
       getParagraphNewlineBoundary(ctx, pos, true).shouldBreak
     )
       return null;
+    const inlineRegionEnd = Math.max(
+      emailRegionEnd(ctx.tokens, pos, end),
+      parseButtonSyntax(ctx, pos, end)?.end ?? pos,
+      parseSocialSyntax(ctx, pos, end)?.end ?? pos,
+    );
+    if (inlineRegionEnd > pos) {
+      pos = inlineRegionEnd - 1;
+      continue;
+    }
     if (token.type === marker && !table?.suppressedClosers.has(pos)) return pos;
     const protectedEnd = protectedInlineRegionEnd(ctx.tokens, pos, end);
     if (protectedEnd > pos) pos = protectedEnd - 1;

package/src/parser/rules/inline/index.ts

@@ -31,6 +31,8 @@ import { linkTripleRule } from "./link-triple";
 import { linkSingleRule } from "./link-single";
 import { linkAnchorRule } from "./link-anchor";
 import { linkStarRule } from "./link-star";
+import { autolinkRule } from "./autolink";
+import { emailRule } from "./email";
 import { colorRule } from "./color";
 import {
   backslashLineBreakRule,
@@ -41,6 +43,9 @@ import { commentRule } from "./comment";
 import { htmlInlineRule } from "./html";
 import { rawRule } from "./raw";
 import { spanRule, closeSpanRule } from "./span";
+import { buttonRule } from "./button";
+import { socialRule } from "./social";
+import { dateRule } from "./date";
 import { sizeRule } from "./size";
 import { footnoteRule } from "./footnote";
 import { imageRule } from "./image";
@@ -65,6 +70,7 @@ export { linkTripleRule } from "./link-triple";
 export { linkSingleRule } from "./link-single";
 export { linkAnchorRule } from "./link-anchor";
 export { linkStarRule } from "./link-star";
+export { autolinkRule } from "./autolink";
 export { colorRule } from "./color";
 export {
   backslashLineBreakRule,
@@ -75,6 +81,9 @@ export { commentRule } from "./comment";
 export { htmlInlineRule } from "./html";
 export { rawRule } from "./raw";
 export { spanRule, closeSpanRule } from "./span";
+export { buttonRule } from "./button";
+export { socialRule } from "./social";
+export { dateRule } from "./date";
 export { sizeRule } from "./size";
 export { footnoteRule } from "./footnote";
 export { imageRule } from "./image";
@@ -112,6 +121,7 @@ export { textRule, fallbackRule } from "./text";
  * separate last-resort handler.
  */
 export const inlineRules: InlineRule[] = [
+  emailRule,
   boldRule,
   italicRule,
   underlineRule,
@@ -123,6 +133,7 @@ export const inlineRules: InlineRule[] = [
   linkSingleRule,
   linkAnchorRule,
   linkStarRule,
+  autolinkRule,
   colorRule,
   backslashLineBreakRule,
   underscoreLineBreakRule,
@@ -132,6 +143,9 @@ export const inlineRules: InlineRule[] = [
   rawRule,
   imageRule,
   embedBlockRule,
+  buttonRule,
+  socialRule,
+  dateRule,
   sizeRule,
   footnoteRule,
   spanRule,