@abreen/tada 1.17.0 → 1.18.0

package/build/utils/code.ts

@@ -1,10 +1,10 @@
 import MarkdownIt from 'markdown-it';
 import path from 'path';
 import { parse as parseJava } from 'java-parser';
-import { JSDOM } from 'jsdom';
+import { hastToHtml } from 'shiki';
 import { makeLogger } from '../log';
 import { getExtensionToShikiLanguage } from '../site-variables';
-import { highlightCode } from './shiki-highlighter';
+import { highlightCodeToHast } from './shiki-highlighter';
 import externalLinksPlugin from '../external-links-plugin';
 import { createApplyBasePath } from './paths';
 import katexPlugin from './katex';
@@ -122,16 +122,7 @@ const JAVA_TYPE_DECLARATION_NODES = new Set([
   'recordDeclaration',
 ]);
 
-function extractJavaMethodMeta(
-  methodNode: CstNode,
-  requireBody = true,
-): MethodMeta | null {
-  const methodBody = methodNode.children?.methodBody?.[0];
-  const hasBody = Boolean(methodBody?.children?.block?.length);
-  if (requireBody && !hasBody) {
-    return null;
-  }
-
+function extractJavaMethodMeta(methodNode: CstNode): MethodMeta | null {
   const methodHeader = methodNode.children?.methodHeader?.[0];
   const methodDeclarator = methodHeader?.children?.methodDeclarator?.[0];
   const identifier = methodDeclarator?.children?.Identifier?.[0];
@@ -293,16 +284,15 @@ export function extractJavaMethodToc(sourceCode: string): JavaTocEntry[] {
       return;
     }
 
-    if (node.name === 'methodDeclaration' && typeDepth <= 1) {
+    if (
+      (node.name === 'methodDeclaration' ||
+        node.name === 'interfaceMethodDeclaration') &&
+      typeDepth <= 1
+    ) {
       const method = extractJavaMethodMeta(node);
       if (method) {
         callables.push({ ...method, kind: 'method' });
       }
-    } else if (node.name === 'interfaceMethodDeclaration' && typeDepth <= 1) {
-      const method = extractJavaMethodMeta(node, false);
-      if (method) {
-        callables.push({ ...method, kind: 'method' });
-      }
     } else if (node.name === 'constructorDeclaration' && typeDepth === 1) {
       const constructor = extractJavaConstructorMeta(node);
       if (constructor) {
@@ -325,6 +315,13 @@ export function extractJavaMethodToc(sourceCode: string): JavaTocEntry[] {
     for (const value of Object.values(children)) {
       for (const child of value) {
         if (child && child.name) {
+          // Anonymous classes have a classBody without a type declaration.
+          if (
+            node.name === 'unqualifiedClassInstanceCreationExpression' &&
+            child.name === 'classBody'
+          ) {
+            continue;
+          }
           visit(child, nextTypeDepth);
         }
       }
@@ -363,104 +360,42 @@ function escapeHtml(text: string): string {
     .replace(/>/g, '&gt;');
 }
 
-function createCodeLine(document: Document): HTMLSpanElement {
-  const line = document.createElement('span');
-  line.className = 'code-line';
-  return line;
+interface HastNode {
+  type: string;
+  value?: string;
+  children?: HastNode[];
 }
 
-function cloneOpenElements(
-  openElements: Node[],
-  line: HTMLSpanElement,
-): Node[] {
-  const containers: Node[] = [line];
-
-  for (const openElement of openElements) {
-    const clone = openElement.cloneNode(false);
-    containers[containers.length - 1].appendChild(clone);
-    containers.push(clone);
+function hasText(node: HastNode): boolean {
+  if (node.type === 'text') {
+    return Boolean(node.value);
   }
-
-  return containers;
+  return node.children?.some(hasText) ?? false;
 }
 
-function splitHighlightedHtmlIntoLines(
-  highlightedHtml: string,
-  lineCount: number,
-): string[] {
-  const fragment = JSDOM.fragment(`<code>${highlightedHtml}</code>`);
-  const codeEl = fragment.firstChild as HTMLElement;
-  const document = codeEl.ownerDocument;
-  const lines: HTMLSpanElement[] = [];
-  const openElements: Node[] = [];
-  let currentLine = createCodeLine(document);
-  let currentContainers: Node[] = [currentLine];
-  let currentLineHasContent = false;
-
-  function finishCurrentLine(): void {
-    if (!currentLineHasContent) {
-      currentContainers[currentContainers.length - 1].appendChild(
-        document.createTextNode('\u00A0'),
-      );
-    }
-    lines.push(currentLine);
-    currentLine = createCodeLine(document);
-    currentContainers = cloneOpenElements(openElements, currentLine);
-    currentLineHasContent = false;
-  }
-
-  function visit(node: Node): void {
-    if (node.nodeType === 3) {
-      const parts = (node.textContent || '').split('\n');
-      for (let i = 0; i < parts.length; i++) {
-        if (parts[i].length > 0) {
-          currentContainers[currentContainers.length - 1].appendChild(
-            document.createTextNode(parts[i]),
-          );
-          currentLineHasContent = true;
-        }
-        if (i < parts.length - 1) {
-          finishCurrentLine();
-        }
-      }
-      return;
-    }
-
-    if (node.nodeType !== 1) {
-      return;
-    }
-
-    const clone = node.cloneNode(false);
-    currentContainers[currentContainers.length - 1].appendChild(clone);
-    openElements.push(node);
-    currentContainers.push(clone);
-
-    for (const child of Array.from(node.childNodes)) {
-      visit(child);
-    }
-
-    currentContainers.pop();
-    openElements.pop();
-  }
-
-  for (const child of Array.from(codeEl.childNodes)) {
-    visit(child);
+/**
+ * Returns the HTML of each highlighted line. Shiki tokenizes line by line, so
+ * each `span.line` is balanced on its own, even inside multi-line tokens such
+ * as block comments. Empty lines get a non-breaking space so that their row
+ * keeps its height.
+ */
+function highlightLines(source: string, lang: string): string[] {
+  const pre = highlightCodeToHast(source, lang).children[0];
+  const code = pre?.type === 'element' ? pre.children[0] : undefined;
+  if (code?.type !== 'element') {
+    throw new Error('unexpected highlighter output');
   }
 
-  if (currentLineHasContent || lines.length < lineCount) {
-    if (!currentLineHasContent) {
-      currentContainers[currentContainers.length - 1].appendChild(
-        document.createTextNode('\u00A0'),
-      );
+  const lines: string[] = [];
+  for (const line of code.children) {
+    if (line.type === 'element') {
+      const html = hastToHtml(line, {
+        characterReferences: { useNamedReferences: true },
+      });
+      lines.push(hasText(line) ? html : `${html}&nbsp;`);
     }
-    lines.push(currentLine);
-  }
-
-  while (lines.length < lineCount) {
-    lines.push(createCodeLine(document));
   }
-
-  return lines.map(line => line.innerHTML);
+  return lines;
 }
 
 export function renderCodeSegment(
@@ -469,20 +404,19 @@ export function renderCodeSegment(
   lang: string,
   { linkLineNumbers = true }: { linkLineNumbers?: boolean } = {},
 ): string {
-  const source = lines.join('\n');
-  let lineHtml: string[] | undefined;
+  // Lines from CRLF sources end with CR. Shiki treats CRLF as a line break
+  // but would keep the last line's CR as text.
+  const sourceLines = lines.map(line =>
+    line.endsWith('\r') ? line.slice(0, -1) : line,
+  );
+  let lineHtml: string[];
 
   try {
-    const html = highlightCode(source, lang);
-    const fragment = JSDOM.fragment(html);
-    const inner = (fragment.querySelector('code') as HTMLElement).innerHTML;
-    lineHtml = splitHighlightedHtmlIntoLines(inner, lines.length);
+    const highlighted = highlightLines(sourceLines.join('\n'), lang);
+    lineHtml = sourceLines.map((_, i) => highlighted[i] ?? '');
   } catch (err: unknown) {
     log.error`Failed to highlight code block: ${(err as Error).message}`;
-  }
-
-  if (!lineHtml) {
-    lineHtml = lines.map(line => escapeHtml(line));
+    lineHtml = sourceLines.map(line => escapeHtml(line));
   }
 
   const rows = lineHtml.map((line, i) => {

package/build/utils/final-html.ts

@@ -1,5 +1,5 @@
 import path from 'path';
-import { JSDOM } from 'jsdom';
+import { decodeHTMLAttribute } from 'entities';
 import { getExtensionToShikiLanguage } from '../site-variables';
 import { createApplyBasePath, normalizeOutputPath } from './paths';
 import { isInternalLink } from './link';
@@ -14,8 +14,10 @@ interface FinalizeHtmlPageOptions {
   html: string;
   siteVariables: SiteVariables;
   sourceUrlPath: string;
-  validInternalTargets: Set<string>;
-  literateJavaOutputPaths?: Set<string>;
+  validInternalTargets: ReadonlySet<string>;
+  generatedPageTargets?: ReadonlySet<string>;
+  codePageSourceTargets?: ReadonlySet<string>;
+  literateJavaOutputPaths?: ReadonlySet<string>;
   dependencyCollector?: RenderDependencyCollector;
 }
 
@@ -24,6 +26,10 @@ interface FinalizedHtmlPage {
   analysis: HtmlOutputAnalysis;
 }
 
+type RewriterElement = HTMLRewriterTypes.Element;
+
+const HTML_NAMESPACE = 'http://www.w3.org/1999/xhtml';
+
 function splitHref(href: string): { pathname: string; suffix: string } {
   const match = href.match(/^([^?#]*)(.*)$/);
   return { pathname: match ? match[1] : href, suffix: match ? match[2] : '' };
@@ -85,16 +91,16 @@ function rewriteAbsoluteSrcWithBasePath(
 function resolveAnchorTarget({
   href,
   sourceUrlPath,
-  validInternalTargets,
   codeExtensions,
+  codePageSourceTargets,
   literateJavaOutputPaths,
   skipCodeLinkRewrite,
 }: {
   href: string;
   sourceUrlPath: string;
-  validInternalTargets: Set<string>;
   codeExtensions: string[];
-  literateJavaOutputPaths?: Set<string>;
+  codePageSourceTargets?: ReadonlySet<string>;
+  literateJavaOutputPaths?: ReadonlySet<string>;
   skipCodeLinkRewrite: boolean;
 }): { finalHref: string; resolvedTarget: string | null } {
   if (!isInternalLink(href)) {
@@ -116,19 +122,38 @@ function resolveAnchorTarget({
     const htmlTarget = `${resolvedPath}.html`;
     if (literateJavaOutputPaths?.has(resolvedPath)) {
       resolvedTarget = resolvedPath;
-    } else if (validInternalTargets.has(htmlTarget)) {
+    } else if (codePageSourceTargets?.has(resolvedPath)) {
       finalPathname = `${pathname}.html`;
       resolvedTarget = htmlTarget;
-    } else if (validInternalTargets.has(resolvedPath)) {
-      resolvedTarget = resolvedPath;
-    } else {
-      resolvedTarget = htmlTarget;
     }
   }
 
   return { finalHref: `${finalPathname}${suffix}`, resolvedTarget };
 }
 
+// HTMLRewriter exposes attribute values exactly as authored, without decoding
+// character references, so decode them the way an HTML parser would. It also
+// reports empty values as null.
+function getAttributeValue(
+  element: RewriterElement,
+  name: string,
+): string | null {
+  const value = element.getAttribute(name);
+  if (value === null) {
+    return element.hasAttribute(name) ? '' : null;
+  }
+  return value.includes('&') ? decodeHTMLAttribute(value) : value;
+}
+
+// HTMLRewriter writes new values verbatim apart from escaping double quotes.
+function setAttributeValue(
+  element: RewriterElement,
+  name: string,
+  value: string,
+): void {
+  element.setAttribute(name, value.replaceAll('&', '&amp;'));
+}
+
 export function finalizeHtmlPage({
   filePath,
   html,
@@ -136,31 +161,43 @@ export function finalizeHtmlPage({
   sourceUrlPath,
   validInternalTargets,
   literateJavaOutputPaths,
+  generatedPageTargets,
+  codePageSourceTargets,
   dependencyCollector,
 }: FinalizeHtmlPageOptions): FinalizedHtmlPage {
-  const dom = new JSDOM(html);
-  const document = dom.window.document;
   const applyBasePath = createApplyBasePath(siteVariables);
   const codeExtensions = Object.keys(
     getExtensionToShikiLanguage(siteVariables),
   );
   const analysis: HtmlOutputAnalysis = { outgoingTargets: new Set() };
+  // Content link targets are recorded before classification targets, matching
+  // the order of the original separate passes.
+  const classificationTargets: string[] = [];
 
-  for (const element of document.querySelectorAll('[href]')) {
-    const href = element.getAttribute('href');
-    if (!href) {
-      continue;
-    }
+  const origin = new URL(siteVariables.base).origin;
+  // Source paths contain no query or fragment. Escape filesystem delimiters
+  // while preserving the percent-encoded segments supplied by code pages.
+  const sourcePathname = sourceUrlPath.replace(/[?#]/g, encodeURIComponent);
+  const sourceUrl = new URL(applyBasePath(sourcePathname), origin);
+  const basePath = (siteVariables.basePath || '/').replace(/\/$/, '');
 
-    const isAnchor = element.tagName === 'A';
-    const isContentAnchor = isAnchor && element.closest('main.body') !== null;
+  // Rewrites an href and returns its final value. Anchors inside `main.body`
+  // are page content: they are resolved, rewritten to code pages, and
+  // validated. Other hrefs only receive the base path.
+  function rewriteHref(
+    element: RewriterElement,
+    href: string,
+    isContent: boolean,
+  ): string {
+    const isAnchor =
+      element.tagName === 'a' && element.namespaceURI === HTML_NAMESPACE;
 
-    if (isContentAnchor) {
+    if (isAnchor && isContent) {
       const { finalHref, resolvedTarget } = resolveAnchorTarget({
         href,
         sourceUrlPath,
-        validInternalTargets,
         codeExtensions,
+        codePageSourceTargets,
         literateJavaOutputPaths,
         skipCodeLinkRewrite: element.hasAttribute('download'),
       });
@@ -169,13 +206,11 @@ export function finalizeHtmlPage({
         : finalHref;
 
       if (rewrittenHref !== href) {
-        element.setAttribute('href', rewrittenHref);
+        setAttributeValue(element, 'href', rewrittenHref);
       }
 
       if (resolvedTarget) {
-        if (isAnchor) {
-          analysis.outgoingTargets.add(resolvedTarget);
-        }
+        analysis.outgoingTargets.add(resolvedTarget);
 
         const directoryIndexPath = getDirectoryIndexPath(resolvedTarget);
         if (
@@ -196,7 +231,7 @@ export function finalizeHtmlPage({
         dependencyCollector?.internalTargets?.add(resolvedTarget);
       }
 
-      continue;
+      return rewrittenHref;
     }
 
     if (isAnchor && isInternalLink(href)) {
@@ -209,31 +244,120 @@ export function finalizeHtmlPage({
 
     const rewrittenHref = rewriteAbsoluteHrefWithBasePath(href, applyBasePath);
     if (rewrittenHref !== href) {
-      element.setAttribute('href', rewrittenHref);
+      setAttributeValue(element, 'href', rewrittenHref);
     }
+    return rewrittenHref;
   }
 
-  for (const element of document.querySelectorAll('[src]')) {
-    const src = element.getAttribute('src');
-    if (!src) {
-      continue;
+  // Recomputes the generated-page marker from an anchor's final href.
+  function classifyAnchor(anchor: RewriterElement, href: string | null): void {
+    if (anchor.hasAttribute('data-tada-page')) {
+      anchor.removeAttribute('data-tada-page');
     }
-
-    const rewrittenSrc = rewriteAbsoluteSrcWithBasePath(src, applyBasePath);
-    if (rewrittenSrc !== src) {
-      element.setAttribute('src', rewrittenSrc);
+    if (href === null) {
+      return;
+    }
+    let url: URL;
+    try {
+      url = new URL(href, sourceUrl);
+    } catch {
+      return;
+    }
+    if (url.origin !== origin) {
+      return;
+    }
+    if (
+      basePath &&
+      url.pathname !== basePath &&
+      !url.pathname.startsWith(`${basePath}/`)
+    ) {
+      return;
+    }
+    let target = url.pathname.slice(basePath.length) || '/';
+    try {
+      target = decodeURIComponent(target);
+    } catch {
+      // Preserve malformed encodings for classification as authored.
+    }
+    target = normalizeOutputPath(target);
+    if (generatedPageTargets) {
+      classificationTargets.push(target);
+    }
+    if (
+      generatedPageTargets?.has(target) &&
+      !anchor.hasAttribute('target') &&
+      !anchor.hasAttribute('download')
+    ) {
+      anchor.setAttribute('data-tada-page', '');
     }
   }
 
-  const doctype = document.doctype;
-  const doctypePrefix = doctype
-    ? `<!DOCTYPE ${doctype.name}${
-        doctype.publicId ? ` PUBLIC "${doctype.publicId}"` : ''
-      }${doctype.systemId ? ` "${doctype.systemId}"` : ''}>`
-    : '';
-
-  return {
-    html: `${doctypePrefix}${document.documentElement.outerHTML}`,
-    analysis,
-  };
+  // HTMLRewriter edits the original markup in place. It reads `<noscript>`
+  // contents as raw text, so a nested rewriter parses them as markup (as a
+  // parser with scripting disabled would), inheriting whether the element is
+  // inside `main.body`.
+  function rewrite(input: string, initialContentDepth: number): string {
+    let contentDepth = initialContentDepth;
+    let noscriptSource = '';
+
+    return new HTMLRewriter()
+      .on('main.body', {
+        element(element) {
+          contentDepth++;
+          element.onEndTag(() => {
+            contentDepth--;
+          });
+        },
+      })
+      .on('[href]', {
+        element(element) {
+          const href = getAttributeValue(element, 'href');
+          const finalHref = href
+            ? rewriteHref(element, href, contentDepth > 0)
+            : href;
+          if (element.tagName === 'a') {
+            classifyAnchor(element, finalHref);
+          }
+        },
+      })
+      .on('a:not([href])', {
+        element(element) {
+          classifyAnchor(element, null);
+        },
+      })
+      .on('[src]', {
+        element(element) {
+          const src = getAttributeValue(element, 'src');
+          if (!src) {
+            return;
+          }
+          const rewrittenSrc = rewriteAbsoluteSrcWithBasePath(
+            src,
+            applyBasePath,
+          );
+          if (rewrittenSrc !== src) {
+            setAttributeValue(element, 'src', rewrittenSrc);
+          }
+        },
+      })
+      .on('noscript', {
+        text(chunk) {
+          noscriptSource += chunk.text;
+          if (!chunk.lastInTextNode) {
+            chunk.remove();
+            return;
+          }
+          chunk.replace(rewrite(noscriptSource, contentDepth), { html: true });
+          noscriptSource = '';
+        },
+      })
+      .transform(input);
+  }
+
+  const finalizedHtml = rewrite(html, 0);
+  for (const target of classificationTargets) {
+    dependencyCollector?.internalTargets?.add(target);
+  }
+
+  return { html: finalizedHtml, analysis };
 }

package/build/utils/jdi-runner/LiterateRunner.java

@@ -244,7 +244,15 @@ public class LiterateRunner {
                 case '\n' -> sb.append("\\n");
                 case '\r' -> sb.append("\\r");
                 case '\t' -> sb.append("\\t");
-                default -> sb.append(c);
+                default -> {
+                    if (c < 0x20) {
+                        sb.append("\\u00");
+                        sb.append(Character.forDigit(c >> 4, 16));
+                        sb.append(Character.forDigit(c & 0xf, 16));
+                    } else {
+                        sb.append(c);
+                    }
+                }
             }
         }
         sb.append('"');

package/build/utils/literate-java.ts

@@ -1,4 +1,5 @@
 import fs from 'fs';
+import type Token from 'markdown-it/lib/token.mjs';
 import os from 'os';
 import path from 'path';
 import { execFileSync } from 'child_process';
@@ -63,6 +64,15 @@ export function parseLiterateJava(
   });
   const tokens = md.parse(content, {});
 
+  return { pageVariables, content, ...extractLiterateJavaCode(tokens) };
+}
+
+export function extractLiterateJavaCode(
+  tokens: Token[],
+): Pick<
+  LiterateJavaParseResult,
+  'javaSource' | 'codeBlocks' | 'visibleBlockIndices'
+> {
   const codeBlocks: LiterateCodeBlock[] = [];
   let javaLine = 1;
 
@@ -98,13 +108,7 @@ export function parseLiterateJava(
   const hiddenCount = codeBlocks.length - visibleBlockIndices.length;
   log.debug`Parsed ${codeBlocks.length} code block(s) (${hiddenCount} hidden), ${javaSource.split('\n').length} Java line(s)`;
 
-  return {
-    pageVariables,
-    content,
-    javaSource,
-    codeBlocks,
-    visibleBlockIndices,
-  };
+  return { javaSource, codeBlocks, visibleBlockIndices };
 }
 
 export function hasMainMethod(javaSource: string): boolean {

package/build/utils/markdown-partials.ts

@@ -1,6 +1,6 @@
 import fs from 'fs';
 import path from 'path';
-import _ from 'lodash';
+import { compileTemplate } from '../lodash-template';
 import type MarkdownIt from 'markdown-it';
 import type StateCore from 'markdown-it/lib/rules_core/state_core.mjs';
 import type StateBlock from 'markdown-it/lib/rules_block/state_block.mjs';
@@ -26,6 +26,7 @@ interface MarkdownPartialsEnv {
 interface MarkdownPartialsOptions {
   filePath: string;
   templateParams: Record<string, unknown>;
+  preserveHtmlComments?: boolean;
   dependencyCollector?: RenderDependencyCollector;
 }
 
@@ -77,12 +78,12 @@ function renderPartialContent(
   }
 
   const raw = fs.readFileSync(resolvedPath, 'utf-8');
-  const content = stripHtmlComments(raw);
+  const content = options.preserveHtmlComments ? raw : stripHtmlComments(raw);
 
   try {
     return {
       resolvedPath,
-      content: _.template(content)(options.templateParams),
+      content: compileTemplate(content)(options.templateParams),
     };
   } catch (err) {
     throw new Error(