@wdprlib/render 2.1.0 → 3.0.1

package/src/libs/highlighter/engine/tokenizer.ts

@@ -0,0 +1,266 @@
+import type { LanguageDefinition } from "../types";
+import { buildEndPattern } from "./end-pattern";
+import { resolveKeywordClass } from "./keywords";
+import { buildPartTokens } from "./parts";
+import { preprocessHighlightInput } from "./preprocess";
+import type { HighlightToken } from "./token";
+import { findGroupPosition } from "./utils";
+
+interface HighlighterState {
+  state: number;
+  lastdelim: string;
+  lastinner: string;
+  endpattern: RegExp | null;
+}
+
+/**
+ * Tokenize source code using a language definition's state machine.
+ */
+export function tokenize(def: LanguageDefinition, input: string): HighlightToken[] {
+  const str = preprocessHighlightInput(input);
+  const len = str.length;
+  if (len === 0) return [];
+
+  let state = -1;
+  let pos = 0;
+  let lastinner = def.defClass;
+  let lastdelim = def.defClass;
+  let endpattern: RegExp | null = null;
+  const stateStack: HighlighterState[] = [];
+  const tokenStack: HighlightToken[] = [];
+  const result: HighlightToken[] = [];
+
+  function getToken(): HighlightToken | null {
+    if (tokenStack.length > 0) {
+      return tokenStack.pop()!;
+    }
+    if (pos >= len) {
+      return null;
+    }
+
+    const endStateMatch = findEndStateMatch(str, pos, state, endpattern);
+    const token = matchStateToken({
+      def,
+      str,
+      pos,
+      state,
+      lastinner,
+      lastdelim,
+      endpattern,
+      stateStack,
+      tokenStack,
+      endStateMatch,
+      setPosition: (nextPos) => {
+        pos = nextPos;
+      },
+      setState: (next) => {
+        state = next.state;
+        lastinner = next.lastinner;
+        lastdelim = next.lastdelim;
+        endpattern = next.endpattern;
+      },
+    });
+    if (token) {
+      return token;
+    }
+
+    if (endStateMatch.endpos > -1) {
+      tokenStack.push({ class: lastdelim, content: endStateMatch.endmatch });
+      if (endStateMatch.endpos > pos) {
+        tokenStack.push({ class: lastinner, content: str.substring(pos, endStateMatch.endpos) });
+      }
+      const prev = stateStack.pop()!;
+      state = prev.state;
+      lastdelim = prev.lastdelim;
+      lastinner = prev.lastinner;
+      endpattern = prev.endpattern;
+      pos = endStateMatch.endpos + endStateMatch.endmatch.length;
+      if (tokenStack.length > 0) {
+        return tokenStack.pop()!;
+      }
+      return getToken();
+    }
+
+    const p = pos;
+    pos = len;
+    return { class: lastinner, content: str.substring(p) };
+  }
+
+  let token: HighlightToken | null;
+  while ((token = getToken()) !== null) {
+    result.push(token);
+  }
+
+  return result;
+}
+
+interface EndStateMatch {
+  endpos: number;
+  endmatch: string;
+}
+
+function findEndStateMatch(
+  str: string,
+  pos: number,
+  state: number,
+  endpattern: RegExp | null,
+): EndStateMatch {
+  if (state === -1 || !endpattern) {
+    return { endpos: -1, endmatch: "" };
+  }
+
+  endpattern.lastIndex = pos;
+  const match = endpattern.exec(str);
+  return match ? { endpos: match.index, endmatch: match[0] } : { endpos: -1, endmatch: "" };
+}
+
+interface MatchStateTokenArgs {
+  def: LanguageDefinition;
+  str: string;
+  pos: number;
+  state: number;
+  lastinner: string;
+  lastdelim: string;
+  endpattern: RegExp | null;
+  stateStack: HighlighterState[];
+  tokenStack: HighlightToken[];
+  endStateMatch: EndStateMatch;
+  setPosition(pos: number): void;
+  setState(state: HighlighterState): void;
+}
+
+function matchStateToken(args: MatchStateTokenArgs): HighlightToken | null {
+  const reg = args.def.regs[args.state];
+  if (!reg) return null;
+
+  reg.lastIndex = args.pos;
+  const match = reg.exec(args.str);
+  if (!match) return null;
+
+  const countsArr = args.def.counts[args.state]!;
+  let captureIndex = 1;
+
+  for (let patternIndex = 0; patternIndex < countsArr.length; patternIndex++) {
+    const count = countsArr[patternIndex]!;
+    if (captureIndex >= match.length) break;
+
+    if (
+      match[captureIndex] != null &&
+      (args.endStateMatch.endpos === -1 || match.index < args.endStateMatch.endpos)
+    ) {
+      return emitMatchedPattern(args, match, patternIndex, captureIndex, count);
+    }
+
+    captureIndex += count + 1;
+  }
+
+  return null;
+}
+
+function emitMatchedPattern(
+  args: MatchStateTokenArgs,
+  match: RegExpExecArray,
+  patternIndex: number,
+  captureIndex: number,
+  count: number,
+): HighlightToken {
+  const statesArr = args.def.states[args.state]!;
+  const delimArr = args.def.delim[args.state]!;
+  const matchStart = match.index;
+  const matchStr = match[captureIndex]!;
+  const groupStart = findGroupPosition(args.str, match, captureIndex, matchStart);
+
+  if (statesArr[patternIndex] !== -1) {
+    args.tokenStack.push({ class: delimArr[patternIndex]!, content: matchStr });
+  } else {
+    pushNonTransitionTokens(args, match, patternIndex, captureIndex, count, groupStart, matchStr);
+  }
+
+  if (groupStart > args.pos) {
+    args.tokenStack.push({
+      class: args.lastinner,
+      content: args.str.substring(args.pos, groupStart),
+    });
+  }
+
+  args.setPosition(groupStart + matchStr.length);
+
+  if (statesArr[patternIndex] !== -1) {
+    enterState(args, match, patternIndex, captureIndex, count);
+  }
+
+  return args.tokenStack.pop()!;
+}
+
+function pushNonTransitionTokens(
+  args: MatchStateTokenArgs,
+  match: RegExpExecArray,
+  patternIndex: number,
+  captureIndex: number,
+  count: number,
+  groupStart: number,
+  matchStr: string,
+): void {
+  let inner = args.def.inner[args.state]![patternIndex]!;
+  const partDef = args.def.parts[args.state]?.[patternIndex];
+  if (partDef) {
+    pushPartTokens(args, match, partDef, captureIndex, count, groupStart, matchStr, inner);
+    return;
+  }
+
+  inner = resolveKeywordClass(args.def, args.state, patternIndex, matchStr, inner);
+  args.tokenStack.push({ class: inner, content: matchStr });
+}
+
+function pushPartTokens(
+  args: MatchStateTokenArgs,
+  match: RegExpExecArray,
+  partDef: Record<number, string>,
+  captureIndex: number,
+  count: number,
+  groupStart: number,
+  matchStr: string,
+  inner: string,
+): void {
+  const parts: HighlightToken[] = [];
+  parts.push(
+    ...buildPartTokens(args.str, match, partDef, captureIndex, count, groupStart, matchStr, inner),
+  );
+  args.tokenStack.push(...parts);
+}
+
+function enterState(
+  args: MatchStateTokenArgs,
+  match: RegExpExecArray,
+  patternIndex: number,
+  captureIndex: number,
+  count: number,
+): void {
+  const statesArr = args.def.states[args.state]!;
+  const delimArr = args.def.delim[args.state]!;
+  const innerArr = args.def.inner[args.state]!;
+  args.stateStack.push({
+    state: args.state,
+    lastdelim: args.lastdelim,
+    lastinner: args.lastinner,
+    endpattern: args.endpattern,
+  });
+
+  const prevState = args.state;
+  const nextState = statesArr[patternIndex]!;
+  const endRe = args.def.end[nextState];
+  args.setState({
+    state: nextState,
+    lastinner: innerArr[patternIndex]!,
+    lastdelim: delimArr[patternIndex]!,
+    endpattern: buildEndPattern(
+      args.def,
+      prevState,
+      patternIndex,
+      count,
+      captureIndex,
+      match,
+      endRe ?? undefined,
+    ),
+  });
+}

package/src/libs/highlighter/engine/utils.ts

@@ -0,0 +1,38 @@
+/**
+ * Find the actual position of capture group `n` within the source string.
+ */
+export function findGroupPosition(
+  str: string,
+  match: RegExpExecArray,
+  groupIndex: number,
+  matchStart: number,
+): number {
+  const groupStr = match[groupIndex]!;
+  const idx = str.indexOf(groupStr, matchStart);
+  return idx >= 0 ? idx : matchStart;
+}
+
+/**
+ * Escape regex special characters in a string for safe use in `new RegExp()`.
+ */
+export function escapeRegex(str: string): string {
+  return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+/**
+ * Swap bracket characters to their matching counterparts.
+ */
+export function matchingBrackets(str: string): string {
+  return str.replace(/[()<>[\]{}]/g, (char) => MATCHING_BRACKETS[char] ?? char);
+}
+
+const MATCHING_BRACKETS: Record<string, string> = {
+  "(": ")",
+  ")": "(",
+  "<": ">",
+  ">": "<",
+  "[": "]",
+  "]": "[",
+  "{": "}",
+  "}": "{",
+};

package/src/render/collected-styles.ts

@@ -0,0 +1,22 @@
+import { STYLE_SLOT_PREFIX } from "@wdprlib/ast";
+import type { RenderContext } from "../context";
+import { pushStyleTag } from "./style-tag";
+
+export function renderCollectedStyles(ctx: RenderContext, styles: string[] | undefined): void {
+  if (!ctx.settings.allowStyleElements || !styles?.length) return;
+
+  for (const style of styles) {
+    if (style.startsWith(STYLE_SLOT_PREFIX)) {
+      renderStyleSlot(ctx, style);
+    } else {
+      pushStyleTag(ctx, style);
+    }
+  }
+}
+
+function renderStyleSlot(ctx: RenderContext, marker: string): void {
+  const slotId = parseInt(marker.slice(STYLE_SLOT_PREFIX.length), 10);
+  for (const css of ctx.getStyleSlotContents(slotId)) {
+    pushStyleTag(ctx, css);
+  }
+}

package/src/render/dispatch.ts

@@ -0,0 +1,181 @@
+import type { Element } from "@wdprlib/ast";
+import { RenderContext } from "../context";
+import { renderBibliographyBlock, renderBibliographyCite } from "../elements/bibliography";
+import { renderClearFloat } from "../elements/clear-float";
+import { renderCode } from "../elements/code";
+import { renderCollapsible } from "../elements/collapsible";
+import { renderContainer } from "../elements/container";
+import { renderColor } from "../elements/color";
+import { renderDate } from "../elements/date";
+import { renderEmbed } from "../elements/embed";
+import { renderEmbedBlock } from "../elements/embed-block";
+import { renderExpr, renderIf, renderIfExpr } from "../elements/expr";
+import { renderFootnoteBlock, renderFootnoteRef } from "../elements/footnote";
+import { renderHtmlBlock } from "../elements/html";
+import { renderIframe } from "../elements/iframe";
+import { renderIfTags } from "../elements/iftags";
+import { renderImage } from "../elements/image";
+import { renderInclude } from "../elements/include";
+import { renderLineBreaks } from "../elements/line-break";
+import { renderAnchor, renderAnchorName, renderLink } from "../elements/link";
+import { renderDefinitionList, renderList } from "../elements/list";
+import { renderMath, renderMathInline, renderEquationRef } from "../elements/math";
+import { renderModule } from "../elements/module/index";
+import { renderTable } from "../elements/table";
+import { renderTabView } from "../elements/tab-view";
+import { renderRaw, renderEmail, renderText } from "../elements/text";
+import { renderTableOfContents } from "../elements/toc";
+import { renderUser } from "../elements/user";
+import {
+  renderContentSeparator,
+  renderHorizontalRule,
+  renderLineBreak,
+  renderTextNode,
+} from "./primitives";
+import { renderStyleElement } from "./style";
+
+/**
+ * Render a list of sibling AST elements in document order.
+ */
+export function renderElements(ctx: RenderContext, elements: Element[]): void {
+  for (const element of elements) {
+    renderElement(ctx, element);
+  }
+}
+
+/**
+ * Dispatch a single AST element to its type-specific renderer.
+ */
+export function renderElement(ctx: RenderContext, element: Element): void {
+  switch (element.element) {
+    case "text":
+      renderTextNode(ctx, element.data);
+      break;
+    case "raw":
+      renderRaw(ctx, element.data);
+      break;
+    case "variable":
+      renderText(ctx, element.data);
+      break;
+    case "email":
+      renderEmail(ctx, element.data);
+      break;
+    case "container":
+      renderContainer(ctx, element.data);
+      break;
+    case "link":
+      renderLink(ctx, element.data);
+      break;
+    case "anchor":
+      renderAnchor(ctx, element.data);
+      break;
+    case "anchor-name":
+      renderAnchorName(ctx, element.data);
+      break;
+    case "image":
+      renderImage(ctx, element.data);
+      break;
+    case "list":
+      renderList(ctx, element.data);
+      break;
+    case "definition-list":
+      renderDefinitionList(ctx, element.data);
+      break;
+    case "table":
+      renderTable(ctx, element.data);
+      break;
+    case "collapsible":
+      renderCollapsible(ctx, element.data);
+      break;
+    case "code":
+      renderCode(ctx, element.data);
+      break;
+    case "tab-view":
+      renderTabView(ctx, element.data);
+      break;
+    case "footnote":
+      renderFootnoteRef(ctx, ctx.nextFootnoteIndex() + 1);
+      break;
+    case "footnote-ref":
+      renderFootnoteRef(ctx, element.data);
+      break;
+    case "footnote-block":
+      renderFootnoteBlock(ctx, element.data);
+      break;
+    case "bibliography-cite":
+      renderBibliographyCite(ctx, element.data);
+      break;
+    case "bibliography-block":
+      renderBibliographyBlock(ctx, element.data, renderElements);
+      break;
+    case "table-of-contents":
+      renderTableOfContents(ctx, element.data);
+      break;
+    case "math":
+      renderMath(ctx, element.data);
+      break;
+    case "math-inline":
+      renderMathInline(ctx, element.data);
+      break;
+    case "module":
+      renderModule(ctx, element.data);
+      break;
+    case "embed":
+      renderEmbed(ctx, element.data);
+      break;
+    case "embed-block":
+      renderEmbedBlock(ctx, element.data);
+      break;
+    case "user":
+      renderUser(ctx, element.data);
+      break;
+    case "date":
+      renderDate(ctx, element.data);
+      break;
+    case "color":
+      renderColor(ctx, element.data);
+      break;
+    case "html":
+      renderHtmlBlock(ctx, element.data);
+      break;
+    case "iframe":
+      renderIframe(ctx, element.data);
+      break;
+    case "include":
+      renderInclude(ctx, element.data);
+      break;
+    case "if-tags":
+      renderIfTags(ctx, element.data);
+      break;
+    case "style":
+      renderStyleElement(ctx, element.data);
+      break;
+    case "line-break":
+      renderLineBreak(ctx);
+      break;
+    case "line-breaks":
+      renderLineBreaks(ctx, element.data);
+      break;
+    case "clear-float":
+      renderClearFloat(ctx, element.data);
+      break;
+    case "horizontal-rule":
+      renderHorizontalRule(ctx);
+      break;
+    case "content-separator":
+      renderContentSeparator(ctx);
+      break;
+    case "expr":
+      renderExpr(ctx, element.data);
+      break;
+    case "if":
+      renderIf(ctx, element.data);
+      break;
+    case "ifexpr":
+      renderIfExpr(ctx, element.data);
+      break;
+    case "equation-reference":
+      renderEquationRef(ctx, element.data);
+      break;
+  }
+}

package/src/render/index.ts

@@ -0,0 +1,28 @@
+import type { SyntaxTree } from "@wdprlib/ast";
+import { RenderContext } from "../context";
+import type { RenderOptions } from "../types";
+import { renderCollectedStyles } from "./collected-styles";
+import { renderElements } from "./dispatch";
+
+export { renderElement, renderElements } from "./dispatch";
+
+/**
+ * Render a {@link SyntaxTree} to an HTML string.
+ *
+ * This is the main entry point of `@wdprlib/render`. It walks the AST
+ * produced by `@wdprlib/parser`, serialises each element to HTML, and
+ * appends any collected `[[module CSS]]` styles at the end (when
+ * `WikitextSettings.allowStyleElements` is `true`).
+ *
+ * @param tree - Parsed AST (from `parse()` or `resolveModules()`)
+ * @param options - Rendering configuration
+ * @returns Complete HTML string
+ *
+ * @group Render
+ */
+export function renderToHtml(tree: SyntaxTree, options: RenderOptions = {}): string {
+  const ctx = new RenderContext(tree, options);
+  renderElements(ctx, tree.elements);
+  renderCollectedStyles(ctx, tree.styles);
+  return ctx.getOutput();
+}

package/src/render/primitives.ts

@@ -0,0 +1,17 @@
+import { RenderContext } from "../context";
+
+export function renderTextNode(ctx: RenderContext, text: string): void {
+  ctx.pushEscaped(text);
+}
+
+export function renderLineBreak(ctx: RenderContext): void {
+  ctx.push("<br />");
+}
+
+export function renderHorizontalRule(ctx: RenderContext): void {
+  ctx.push("<hr />");
+}
+
+export function renderContentSeparator(ctx: RenderContext): void {
+  ctx.push(`<div class="content-separator" style="display: none:"></div>`);
+}

package/src/render/style-tag.ts

@@ -0,0 +1,6 @@
+import type { RenderContext } from "../context";
+import { escapeStyleContent } from "../escape";
+
+export function pushStyleTag(ctx: RenderContext, css: string): void {
+  ctx.push(`<style>${escapeStyleContent(css)}</style>`);
+}

package/src/render/style.ts

@@ -0,0 +1,15 @@
+import { RenderContext } from "../context";
+import { pushStyleTag } from "./style-tag";
+
+export function renderStyleElement(ctx: RenderContext, css: string): void {
+  if (!ctx.renderInlineStyles || !ctx.settings.allowStyleElements) {
+    return;
+  }
+
+  if (ctx.hasActiveStyleSlot()) {
+    ctx.pushToStyleSlot(css);
+    return;
+  }
+
+  pushStyleTag(ctx, css);
+}

package/src/types.ts

@@ -13,10 +13,14 @@ import type { EmbedAllowlistEntry } from "./elements/embed-block";
 export interface PageContext {
   /** Full page name including category prefix (e.g. `"secret:test2"`) */
   pageName: string;
-  /** Site slug used to build inter-site URLs (e.g. `"scp-wiki"`) */
+  /** Site slug for the current page (e.g. `"scp-wiki"`) */
   site?: string;
-  /** Site domain used for absolute URL generation (e.g. `"scp-wiki.wikidot.com"`) */
+  /** Current site domain used for absolute URL generation (e.g. `"scp-wiki.example.org"`) */
   domain?: string;
+  /** Known domains for cross-site page references keyed by site slug. */
+  siteDomains?: Record<string, string>;
+  /** Resolve a cross-site page reference site slug to a domain. */
+  resolveSiteDomain?: (site: string) => string | null | undefined;
   /**
    * Returns whether a page exists on the site.
    * When a target page does not exist, the renderer adds `class="newpage"`