@abreen/tada 1.17.1 → 1.18.0

package/build/utils/code.ts

@@ -1,10 +1,10 @@
 import MarkdownIt from 'markdown-it';
 import path from 'path';
 import { parse as parseJava } from 'java-parser';
-import { JSDOM } from 'jsdom';
+import { hastToHtml } from 'shiki';
 import { makeLogger } from '../log';
 import { getExtensionToShikiLanguage } from '../site-variables';
-import { highlightCode } from './shiki-highlighter';
+import { highlightCodeToHast } from './shiki-highlighter';
 import externalLinksPlugin from '../external-links-plugin';
 import { createApplyBasePath } from './paths';
 import katexPlugin from './katex';
@@ -360,104 +360,42 @@ function escapeHtml(text: string): string {
     .replace(/>/g, '>');
 }
 
-function createCodeLine(document: Document): HTMLSpanElement {
-  const line = document.createElement('span');
-  line.className = 'code-line';
-  return line;
+interface HastNode {
+  type: string;
+  value?: string;
+  children?: HastNode[];
 }
 
-function cloneOpenElements(
-  openElements: Node[],
-  line: HTMLSpanElement,
-): Node[] {
-  const containers: Node[] = [line];
-
-  for (const openElement of openElements) {
-    const clone = openElement.cloneNode(false);
-    containers[containers.length - 1].appendChild(clone);
-    containers.push(clone);
+function hasText(node: HastNode): boolean {
+  if (node.type === 'text') {
+    return Boolean(node.value);
   }
-
-  return containers;
+  return node.children?.some(hasText) ?? false;
 }
 
-function splitHighlightedHtmlIntoLines(
-  highlightedHtml: string,
-  lineCount: number,
-): string[] {
-  const fragment = JSDOM.fragment(`<code>${highlightedHtml}</code>`);
-  const codeEl = fragment.firstChild as HTMLElement;
-  const document = codeEl.ownerDocument;
-  const lines: HTMLSpanElement[] = [];
-  const openElements: Node[] = [];
-  let currentLine = createCodeLine(document);
-  let currentContainers: Node[] = [currentLine];
-  let currentLineHasContent = false;
-
-  function finishCurrentLine(): void {
-    if (!currentLineHasContent) {
-      currentContainers[currentContainers.length - 1].appendChild(
-        document.createTextNode('\u00A0'),
-      );
-    }
-    lines.push(currentLine);
-    currentLine = createCodeLine(document);
-    currentContainers = cloneOpenElements(openElements, currentLine);
-    currentLineHasContent = false;
-  }
-
-  function visit(node: Node): void {
-    if (node.nodeType === 3) {
-      const parts = (node.textContent || '').split('\n');
-      for (let i = 0; i < parts.length; i++) {
-        if (parts[i].length > 0) {
-          currentContainers[currentContainers.length - 1].appendChild(
-            document.createTextNode(parts[i]),
-          );
-          currentLineHasContent = true;
-        }
-        if (i < parts.length - 1) {
-          finishCurrentLine();
-        }
-      }
-      return;
-    }
-
-    if (node.nodeType !== 1) {
-      return;
-    }
-
-    const clone = node.cloneNode(false);
-    currentContainers[currentContainers.length - 1].appendChild(clone);
-    openElements.push(node);
-    currentContainers.push(clone);
-
-    for (const child of Array.from(node.childNodes)) {
-      visit(child);
-    }
-
-    currentContainers.pop();
-    openElements.pop();
-  }
-
-  for (const child of Array.from(codeEl.childNodes)) {
-    visit(child);
+/**
+ * Returns the HTML of each highlighted line. Shiki tokenizes line by line, so
+ * each `span.line` is balanced on its own, even inside multi-line tokens such
+ * as block comments. Empty lines get a non-breaking space so that their row
+ * keeps its height.
+ */
+function highlightLines(source: string, lang: string): string[] {
+  const pre = highlightCodeToHast(source, lang).children[0];
+  const code = pre?.type === 'element' ? pre.children[0] : undefined;
+  if (code?.type !== 'element') {
+    throw new Error('unexpected highlighter output');
   }
 
-  if (currentLineHasContent || lines.length < lineCount) {
-    if (!currentLineHasContent) {
-      currentContainers[currentContainers.length - 1].appendChild(
-        document.createTextNode('\u00A0'),
-      );
+  const lines: string[] = [];
+  for (const line of code.children) {
+    if (line.type === 'element') {
+      const html = hastToHtml(line, {
+        characterReferences: { useNamedReferences: true },
+      });
+      lines.push(hasText(line) ? html : `${html}&nbsp;`);
     }
-    lines.push(currentLine);
-  }
-
-  while (lines.length < lineCount) {
-    lines.push(createCodeLine(document));
   }
-
-  return lines.map(line => line.innerHTML);
+  return lines;
 }
 
 export function renderCodeSegment(
@@ -466,20 +404,19 @@ export function renderCodeSegment(
   lang: string,
   { linkLineNumbers = true }: { linkLineNumbers?: boolean } = {},
 ): string {
-  const source = lines.join('\n');
-  let lineHtml: string[] | undefined;
+  // Lines from CRLF sources end with CR. Shiki treats CRLF as a line break
+  // but would keep the last line's CR as text.
+  const sourceLines = lines.map(line =>
+    line.endsWith('\r') ? line.slice(0, -1) : line,
+  );
+  let lineHtml: string[];
 
   try {
-    const html = highlightCode(source, lang);
-    const fragment = JSDOM.fragment(html);
-    const inner = (fragment.querySelector('code') as HTMLElement).innerHTML;
-    lineHtml = splitHighlightedHtmlIntoLines(inner, lines.length);
+    const highlighted = highlightLines(sourceLines.join('\n'), lang);
+    lineHtml = sourceLines.map((_, i) => highlighted[i] ?? '');
   } catch (err: unknown) {
     log.error`Failed to highlight code block: ${(err as Error).message}`;
-  }
-
-  if (!lineHtml) {
-    lineHtml = lines.map(line => escapeHtml(line));
+    lineHtml = sourceLines.map(line => escapeHtml(line));
   }
 
   const rows = lineHtml.map((line, i) => {

package/build/utils/final-html.ts

@@ -1,5 +1,5 @@
 import path from 'path';
-import { JSDOM } from 'jsdom';
+import { decodeHTMLAttribute } from 'entities';
 import { getExtensionToShikiLanguage } from '../site-variables';
 import { createApplyBasePath, normalizeOutputPath } from './paths';
 import { isInternalLink } from './link';
@@ -14,8 +14,10 @@ interface FinalizeHtmlPageOptions {
   html: string;
   siteVariables: SiteVariables;
   sourceUrlPath: string;
-  validInternalTargets: Set<string>;
-  literateJavaOutputPaths?: Set<string>;
+  validInternalTargets: ReadonlySet<string>;
+  generatedPageTargets?: ReadonlySet<string>;
+  codePageSourceTargets?: ReadonlySet<string>;
+  literateJavaOutputPaths?: ReadonlySet<string>;
   dependencyCollector?: RenderDependencyCollector;
 }
 
@@ -24,6 +26,10 @@ interface FinalizedHtmlPage {
   analysis: HtmlOutputAnalysis;
 }
 
+type RewriterElement = HTMLRewriterTypes.Element;
+
+const HTML_NAMESPACE = 'http://www.w3.org/1999/xhtml';
+
 function splitHref(href: string): { pathname: string; suffix: string } {
   const match = href.match(/^([^?#]*)(.*)$/);
   return { pathname: match ? match[1] : href, suffix: match ? match[2] : '' };
@@ -85,16 +91,16 @@ function rewriteAbsoluteSrcWithBasePath(
 function resolveAnchorTarget({
   href,
   sourceUrlPath,
-  validInternalTargets,
   codeExtensions,
+  codePageSourceTargets,
   literateJavaOutputPaths,
   skipCodeLinkRewrite,
 }: {
   href: string;
   sourceUrlPath: string;
-  validInternalTargets: Set<string>;
   codeExtensions: string[];
-  literateJavaOutputPaths?: Set<string>;
+  codePageSourceTargets?: ReadonlySet<string>;
+  literateJavaOutputPaths?: ReadonlySet<string>;
   skipCodeLinkRewrite: boolean;
 }): { finalHref: string; resolvedTarget: string | null } {
   if (!isInternalLink(href)) {
@@ -116,19 +122,38 @@ function resolveAnchorTarget({
     const htmlTarget = `${resolvedPath}.html`;
     if (literateJavaOutputPaths?.has(resolvedPath)) {
       resolvedTarget = resolvedPath;
-    } else if (validInternalTargets.has(htmlTarget)) {
+    } else if (codePageSourceTargets?.has(resolvedPath)) {
       finalPathname = `${pathname}.html`;
       resolvedTarget = htmlTarget;
-    } else if (validInternalTargets.has(resolvedPath)) {
-      resolvedTarget = resolvedPath;
-    } else {
-      resolvedTarget = htmlTarget;
     }
   }
 
   return { finalHref: `${finalPathname}${suffix}`, resolvedTarget };
 }
 
+// HTMLRewriter exposes attribute values exactly as authored, without decoding
+// character references, so decode them the way an HTML parser would. It also
+// reports empty values as null.
+function getAttributeValue(
+  element: RewriterElement,
+  name: string,
+): string | null {
+  const value = element.getAttribute(name);
+  if (value === null) {
+    return element.hasAttribute(name) ? '' : null;
+  }
+  return value.includes('&') ? decodeHTMLAttribute(value) : value;
+}
+
+// HTMLRewriter writes new values verbatim apart from escaping double quotes.
+function setAttributeValue(
+  element: RewriterElement,
+  name: string,
+  value: string,
+): void {
+  element.setAttribute(name, value.replaceAll('&', '&amp;'));
+}
+
 export function finalizeHtmlPage({
   filePath,
   html,
@@ -136,31 +161,43 @@ export function finalizeHtmlPage({
   sourceUrlPath,
   validInternalTargets,
   literateJavaOutputPaths,
+  generatedPageTargets,
+  codePageSourceTargets,
   dependencyCollector,
 }: FinalizeHtmlPageOptions): FinalizedHtmlPage {
-  const dom = new JSDOM(html);
-  const document = dom.window.document;
   const applyBasePath = createApplyBasePath(siteVariables);
   const codeExtensions = Object.keys(
     getExtensionToShikiLanguage(siteVariables),
   );
   const analysis: HtmlOutputAnalysis = { outgoingTargets: new Set() };
+  // Content link targets are recorded before classification targets, matching
+  // the order of the original separate passes.
+  const classificationTargets: string[] = [];
 
-  for (const element of document.querySelectorAll('[href]')) {
-    const href = element.getAttribute('href');
-    if (!href) {
-      continue;
-    }
+  const origin = new URL(siteVariables.base).origin;
+  // Source paths contain no query or fragment. Escape filesystem delimiters
+  // while preserving the percent-encoded segments supplied by code pages.
+  const sourcePathname = sourceUrlPath.replace(/[?#]/g, encodeURIComponent);
+  const sourceUrl = new URL(applyBasePath(sourcePathname), origin);
+  const basePath = (siteVariables.basePath || '/').replace(/\/$/, '');
 
-    const isAnchor = element.tagName === 'A';
-    const isContentAnchor = isAnchor && element.closest('main.body') !== null;
+  // Rewrites an href and returns its final value. Anchors inside `main.body`
+  // are page content: they are resolved, rewritten to code pages, and
+  // validated. Other hrefs only receive the base path.
+  function rewriteHref(
+    element: RewriterElement,
+    href: string,
+    isContent: boolean,
+  ): string {
+    const isAnchor =
+      element.tagName === 'a' && element.namespaceURI === HTML_NAMESPACE;
 
-    if (isContentAnchor) {
+    if (isAnchor && isContent) {
       const { finalHref, resolvedTarget } = resolveAnchorTarget({
         href,
         sourceUrlPath,
-        validInternalTargets,
         codeExtensions,
+        codePageSourceTargets,
         literateJavaOutputPaths,
         skipCodeLinkRewrite: element.hasAttribute('download'),
       });
@@ -169,13 +206,11 @@ export function finalizeHtmlPage({
         : finalHref;
 
       if (rewrittenHref !== href) {
-        element.setAttribute('href', rewrittenHref);
+        setAttributeValue(element, 'href', rewrittenHref);
       }
 
       if (resolvedTarget) {
-        if (isAnchor) {
-          analysis.outgoingTargets.add(resolvedTarget);
-        }
+        analysis.outgoingTargets.add(resolvedTarget);
 
         const directoryIndexPath = getDirectoryIndexPath(resolvedTarget);
         if (
@@ -196,7 +231,7 @@ export function finalizeHtmlPage({
         dependencyCollector?.internalTargets?.add(resolvedTarget);
       }
 
-      continue;
+      return rewrittenHref;
     }
 
     if (isAnchor && isInternalLink(href)) {
@@ -209,31 +244,120 @@ export function finalizeHtmlPage({
 
     const rewrittenHref = rewriteAbsoluteHrefWithBasePath(href, applyBasePath);
     if (rewrittenHref !== href) {
-      element.setAttribute('href', rewrittenHref);
+      setAttributeValue(element, 'href', rewrittenHref);
     }
+    return rewrittenHref;
   }
 
-  for (const element of document.querySelectorAll('[src]')) {
-    const src = element.getAttribute('src');
-    if (!src) {
-      continue;
+  // Recomputes the generated-page marker from an anchor's final href.
+  function classifyAnchor(anchor: RewriterElement, href: string | null): void {
+    if (anchor.hasAttribute('data-tada-page')) {
+      anchor.removeAttribute('data-tada-page');
     }
-
-    const rewrittenSrc = rewriteAbsoluteSrcWithBasePath(src, applyBasePath);
-    if (rewrittenSrc !== src) {
-      element.setAttribute('src', rewrittenSrc);
+    if (href === null) {
+      return;
+    }
+    let url: URL;
+    try {
+      url = new URL(href, sourceUrl);
+    } catch {
+      return;
+    }
+    if (url.origin !== origin) {
+      return;
+    }
+    if (
+      basePath &&
+      url.pathname !== basePath &&
+      !url.pathname.startsWith(`${basePath}/`)
+    ) {
+      return;
+    }
+    let target = url.pathname.slice(basePath.length) || '/';
+    try {
+      target = decodeURIComponent(target);
+    } catch {
+      // Preserve malformed encodings for classification as authored.
+    }
+    target = normalizeOutputPath(target);
+    if (generatedPageTargets) {
+      classificationTargets.push(target);
+    }
+    if (
+      generatedPageTargets?.has(target) &&
+      !anchor.hasAttribute('target') &&
+      !anchor.hasAttribute('download')
+    ) {
+      anchor.setAttribute('data-tada-page', '');
     }
   }
 
-  const doctype = document.doctype;
-  const doctypePrefix = doctype
-    ? `<!DOCTYPE ${doctype.name}${
-        doctype.publicId ? ` PUBLIC "${doctype.publicId}"` : ''
-      }${doctype.systemId ? ` "${doctype.systemId}"` : ''}>`
-    : '';
-
-  return {
-    html: `${doctypePrefix}${document.documentElement.outerHTML}`,
-    analysis,
-  };
+  // HTMLRewriter edits the original markup in place. It reads `<noscript>`
+  // contents as raw text, so a nested rewriter parses them as markup (as a
+  // parser with scripting disabled would), inheriting whether the element is
+  // inside `main.body`.
+  function rewrite(input: string, initialContentDepth: number): string {
+    let contentDepth = initialContentDepth;
+    let noscriptSource = '';
+
+    return new HTMLRewriter()
+      .on('main.body', {
+        element(element) {
+          contentDepth++;
+          element.onEndTag(() => {
+            contentDepth--;
+          });
+        },
+      })
+      .on('[href]', {
+        element(element) {
+          const href = getAttributeValue(element, 'href');
+          const finalHref = href
+            ? rewriteHref(element, href, contentDepth > 0)
+            : href;
+          if (element.tagName === 'a') {
+            classifyAnchor(element, finalHref);
+          }
+        },
+      })
+      .on('a:not([href])', {
+        element(element) {
+          classifyAnchor(element, null);
+        },
+      })
+      .on('[src]', {
+        element(element) {
+          const src = getAttributeValue(element, 'src');
+          if (!src) {
+            return;
+          }
+          const rewrittenSrc = rewriteAbsoluteSrcWithBasePath(
+            src,
+            applyBasePath,
+          );
+          if (rewrittenSrc !== src) {
+            setAttributeValue(element, 'src', rewrittenSrc);
+          }
+        },
+      })
+      .on('noscript', {
+        text(chunk) {
+          noscriptSource += chunk.text;
+          if (!chunk.lastInTextNode) {
+            chunk.remove();
+            return;
+          }
+          chunk.replace(rewrite(noscriptSource, contentDepth), { html: true });
+          noscriptSource = '';
+        },
+      })
+      .transform(input);
+  }
+
+  const finalizedHtml = rewrite(html, 0);
+  for (const target of classificationTargets) {
+    dependencyCollector?.internalTargets?.add(target);
+  }
+
+  return { html: finalizedHtml, analysis };
 }

package/build/utils/literate-java.ts

@@ -1,4 +1,5 @@
 import fs from 'fs';
+import type Token from 'markdown-it/lib/token.mjs';
 import os from 'os';
 import path from 'path';
 import { execFileSync } from 'child_process';
@@ -63,6 +64,15 @@ export function parseLiterateJava(
   });
   const tokens = md.parse(content, {});
 
+  return { pageVariables, content, ...extractLiterateJavaCode(tokens) };
+}
+
+export function extractLiterateJavaCode(
+  tokens: Token[],
+): Pick<
+  LiterateJavaParseResult,
+  'javaSource' | 'codeBlocks' | 'visibleBlockIndices'
+> {
   const codeBlocks: LiterateCodeBlock[] = [];
   let javaLine = 1;
 
@@ -98,13 +108,7 @@ export function parseLiterateJava(
   const hiddenCount = codeBlocks.length - visibleBlockIndices.length;
   log.debug`Parsed ${codeBlocks.length} code block(s) (${hiddenCount} hidden), ${javaSource.split('\n').length} Java line(s)`;
 
-  return {
-    pageVariables,
-    content,
-    javaSource,
-    codeBlocks,
-    visibleBlockIndices,
-  };
+  return { javaSource, codeBlocks, visibleBlockIndices };
 }
 
 export function hasMainMethod(javaSource: string): boolean {

package/build/utils/markdown-partials.ts

@@ -1,6 +1,6 @@
 import fs from 'fs';
 import path from 'path';
-import _ from 'lodash';
+import { compileTemplate } from '../lodash-template';
 import type MarkdownIt from 'markdown-it';
 import type StateCore from 'markdown-it/lib/rules_core/state_core.mjs';
 import type StateBlock from 'markdown-it/lib/rules_block/state_block.mjs';
@@ -26,6 +26,7 @@ interface MarkdownPartialsEnv {
 interface MarkdownPartialsOptions {
   filePath: string;
   templateParams: Record<string, unknown>;
+  preserveHtmlComments?: boolean;
   dependencyCollector?: RenderDependencyCollector;
 }
 
@@ -77,12 +78,12 @@ function renderPartialContent(
   }
 
   const raw = fs.readFileSync(resolvedPath, 'utf-8');
-  const content = stripHtmlComments(raw);
+  const content = options.preserveHtmlComments ? raw : stripHtmlComments(raw);
 
   try {
     return {
       resolvedPath,
-      content: _.template(content)(options.templateParams),
+      content: compileTemplate(content)(options.templateParams),
     };
   } catch (err) {
     throw new Error(

package/build/utils/markdown.ts

@@ -7,7 +7,6 @@ import markdownItAnchor from 'markdown-it-anchor';
 import markdownItFootnote from 'markdown-it-footnote';
 import markdownItDeflist from 'markdown-it-deflist';
 import markdownItContainer from 'markdown-it-container';
-import textToId, { deduplicateId } from '../text-to-id';
 import { highlightCode } from './shiki-highlighter';
 import { isBundledLanguage, isPlainTextLanguage } from '../site-variables';
 import headingSubtitlePlugin from '../heading-subtitle-plugin';
@@ -28,10 +27,11 @@ interface CreateMarkdownOptions {
   filePath?: string;
   slides?: boolean;
   validatorOptions?: Record<string, unknown>;
-  literateJavaOutputPaths?: Set<string>;
+  literateJavaOutputPaths?: ReadonlySet<string>;
   sourceUrlPath?: string;
-  validTargets?: Set<string>;
+  validTargets?: ReadonlySet<string>;
   templateParams?: Record<string, unknown>;
+  preserveHtmlComments?: boolean;
   dependencyCollector?: RenderDependencyCollector;
 }
 
@@ -102,6 +102,7 @@ export function createMarkdown(
     markdown.use(markdownPartialsPlugin, {
       filePath,
       templateParams: options.templateParams,
+      preserveHtmlComments: options.preserveHtmlComments,
       dependencyCollector: options.dependencyCollector,
     });
   }
@@ -458,18 +459,12 @@ export function createMarkdown(
     state.tokens = transformedTokens;
   });
 
-  const usedIds = new Map<string, number>();
   markdown.use(markdownItContainer, 'alert', {
     marker: '!',
     validate: function (params: string) {
       return ALERT_PATTERN.test(params.trim());
     },
-    render: function (
-      tokens: Token[],
-      idx: number,
-      _options: unknown,
-      env: Record<string, unknown>,
-    ) {
+    render: function (tokens: Token[], idx: number) {
       const matches = tokens[idx].info.trim().match(ALERT_PATTERN);
 
       if (tokens[idx].nesting === 1) {
@@ -483,13 +478,7 @@ export function createMarkdown(
         const displayTitle = title
           ? markdown.utils.escapeHtml(curlyQuote(title))
           : capitalize(type || '');
-        const baseId = textToId(title || type || '');
-        const titleId = deduplicateId(usedIds, baseId);
-
-        if (!env.alertIds) {
-          env.alertIds = [];
-        }
-        (env.alertIds as string[]).push(titleId);
+        const titleId = tokens[idx].attrGet('id')!;
 
         let html = `<div class="${classNames.join(' ')}">`;
         html += `<p class="title" id="${titleId}">${displayTitle}</p>\n`;

package/build/utils/paths.ts

@@ -2,6 +2,9 @@ import path from 'path';
 import { globals } from '../globals';
 import type { SiteVariables } from '../types';
 
+/** The search index directory inside the output directory */
+export const SEARCH_INDEX_DIR = 'pagefind';
+
 export function getPackageDir(): string {
   return path.resolve(import.meta.dir, '..', '..');
 }

package/build/utils/plain-text.ts

@@ -0,0 +1,18 @@
+import { decodeHTML } from 'entities';
+
+/**
+ * Converts rendered inline HTML (such as a front matter title rendered as
+ * Markdown) into the plain text a reader sees: tags and comments are removed
+ * by an HTML parser, and character references are decoded exactly once.
+ */
+export function htmlToPlainText(html: string): string {
+  let text = '';
+  new HTMLRewriter()
+    .onDocument({
+      text(chunk) {
+        text += chunk.text;
+      },
+    })
+    .transform(html);
+  return decodeHTML(text);
+}