@abreen/tada 1.17.0 → 1.18.0

package/build/output-publication.ts

@@ -0,0 +1,238 @@
+import fs from 'fs';
+import path from 'path';
+import { globals } from './globals';
+import { toPosix } from './utils/paths';
+import type { OutputContent } from './types';
+
+type FileMutation =
+  | { kind: 'write'; path: string; content: OutputContent }
+  | { kind: 'delete'; path: string };
+
+/** An output file and the source that produced it */
+export interface OutputFile {
+  sourcePath: string;
+  content: OutputContent;
+}
+
+// EBUSY means a file is briefly held open. On Windows, EPERM and EACCES also
+// mean that (an indexer, antivirus, or the dev server holding the file); on
+// other platforms they are permanent permission errors and are not retried.
+const BUSY_ERROR_CODES = new Set(
+  process.platform === 'win32' ? ['EBUSY', 'EPERM', 'EACCES'] : ['EBUSY'],
+);
+const BUSY_RETRY_COUNT = 4;
+const BUSY_RETRY_DELAY_MS = 50;
+
+function retryWhileBusy(action: () => void): void {
+  for (let attempt = 0; ; attempt++) {
+    try {
+      action();
+      return;
+    } catch (error) {
+      const code = (error as NodeJS.ErrnoException).code;
+      if (attempt >= BUSY_RETRY_COUNT || !code || !BUSY_ERROR_CODES.has(code)) {
+        throw error;
+      }
+      globals.sleepSync(BUSY_RETRY_DELAY_MS);
+    }
+  }
+}
+
+/** Writes rendered text or bytes, or copies a referenced source file. */
+export function writeOutputFile(
+  filePath: string,
+  content: OutputContent,
+): void {
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+  // Older builds may have copied a source's read-only permissions.
+  if (fs.lstatSync(filePath, { throwIfNoEntry: false })?.isFile()) {
+    fs.chmodSync(filePath, 0o644);
+  }
+  if (typeof content === 'string' || content instanceof Uint8Array) {
+    fs.writeFileSync(filePath, content);
+  } else {
+    fs.copyFileSync(content.copyFrom, filePath);
+    // Distribution output is writable regardless of the source permissions.
+    fs.chmodSync(filePath, 0o644);
+  }
+}
+
+export function sameOutputContent(
+  left: OutputContent,
+  right: OutputContent,
+): boolean {
+  if (typeof left === 'string' || typeof right === 'string') {
+    return left === right;
+  }
+  if (left instanceof Uint8Array || right instanceof Uint8Array) {
+    return (
+      left instanceof Uint8Array &&
+      right instanceof Uint8Array &&
+      Buffer.compare(left, right) === 0
+    );
+  }
+  // Copied files compare by source path; a source whose contents changed is
+  // always rebuilt, which forces a write.
+  return left.copyFrom === right.copyFrom;
+}
+
+/**
+ * Mutations from one build's outputs to the next. Outputs from a source in
+ * `forceSourcePaths` are rewritten even when they look unchanged.
+ */
+export function computeMutations(
+  previous: ReadonlyMap<string, OutputFile>,
+  next: ReadonlyMap<string, OutputFile>,
+  forceSourcePaths: ReadonlySet<string> = new Set(),
+): FileMutation[] {
+  const mutations: FileMutation[] = [];
+  for (const outputPath of new Set([...previous.keys(), ...next.keys()])) {
+    const before = previous.get(outputPath);
+    const after = next.get(outputPath);
+    if (!after) {
+      mutations.push({ kind: 'delete', path: outputPath });
+    } else if (
+      !before ||
+      !sameOutputContent(before.content, after.content) ||
+      forceSourcePaths.has(after.sourcePath)
+    ) {
+      mutations.push({
+        kind: 'write',
+        path: outputPath,
+        content: after.content,
+      });
+    }
+  }
+  return mutations;
+}
+
+/**
+ * Mutations that make `rootDir` hold exactly `outputs`: every output is
+ * written, and every other file and empty directory is deleted except those
+ * under `keep`.
+ */
+export function planFullWrite(
+  rootDir: string,
+  outputs: ReadonlyMap<string, OutputFile>,
+  keep: readonly string[] = [],
+): FileMutation[] {
+  const files: string[] = [];
+  const dirs: string[] = [];
+  // Walk one level at a time: a recursive listing would follow symbolic links
+  // to directories. A link is listed as a file, so deleting it removes only
+  // the link, never what it points to.
+  const walk = (dir: string): void => {
+    for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+      const fullPath = path.join(dir, entry.name);
+      const relPath = toPosix(path.relative(rootDir, fullPath));
+      if (entry.isDirectory()) {
+        dirs.push(relPath);
+        walk(fullPath);
+      } else {
+        files.push(relPath);
+      }
+    }
+  };
+  if (fs.existsSync(rootDir)) {
+    walk(rootDir);
+  }
+  const isKept = (relPath: string) =>
+    keep.some(prefix => `${relPath}/`.startsWith(prefix));
+  const dirsWithFiles = new Set<string>();
+  for (const file of files) {
+    const segments = file.split('/');
+    for (let i = 1; i < segments.length; i++) {
+      dirsWithFiles.add(segments.slice(0, i).join('/'));
+    }
+  }
+
+  const deletes: FileMutation[] = [
+    ...files.filter(file => !isKept(file) && !outputs.has(file)),
+    ...dirs.filter(dir => !isKept(dir) && !dirsWithFiles.has(dir)),
+  ].map(relPath => ({ kind: 'delete', path: relPath }));
+  const writes: FileMutation[] = [...outputs].map(([outputPath, output]) => ({
+    kind: 'write',
+    path: outputPath,
+    content: output.content,
+  }));
+  return [...deletes, ...writes];
+}
+
+function pruneEmptyDirs(rootDir: string, filePath: string): void {
+  const root = path.resolve(rootDir);
+  for (
+    let dir = path.dirname(filePath);
+    dir !== root;
+    dir = path.dirname(dir)
+  ) {
+    try {
+      fs.rmdirSync(dir);
+    } catch {
+      return;
+    }
+  }
+}
+
+/** Throws if a path would be reached through a symbolic link below `rootDir`. */
+function createSymlinkGuard(rootDir: string): (target: string) => void {
+  const root = path.resolve(rootDir);
+  const safeDirs = new Set<string>([root]);
+  return target => {
+    const visited: string[] = [];
+    for (
+      let dir = path.dirname(target);
+      !safeDirs.has(dir) && dir.startsWith(root + path.sep);
+      dir = path.dirname(dir)
+    ) {
+      if (fs.lstatSync(dir, { throwIfNoEntry: false })?.isSymbolicLink()) {
+        throw new Error(`Refusing to write through the symbolic link ${dir}`);
+      }
+      visited.push(dir);
+    }
+    for (const dir of visited) {
+      safeDirs.add(dir);
+    }
+  };
+}
+
+/**
+ * Applies mutations to the files in `rootDir` in place. Deletes run first,
+ * deepest paths first, and remove directories they leave empty, so an output
+ * can change between a file and a directory. Nothing is renamed, staged, or
+ * rolled back: if a write fails, the error is thrown and earlier writes stay.
+ * Nothing is written or deleted through a symbolic link.
+ */
+export function applyMutations(
+  rootDir: string,
+  mutations: readonly FileMutation[],
+): void {
+  const assertNotThroughSymlink = createSymlinkGuard(rootDir);
+  const deletes = mutations
+    .filter(mutation => mutation.kind === 'delete')
+    .sort((a, b) => b.path.length - a.path.length);
+  for (const mutation of deletes) {
+    const target = path.resolve(rootDir, mutation.path);
+    assertNotThroughSymlink(target);
+    retryWhileBusy(() => {
+      if (fs.lstatSync(target, { throwIfNoEntry: false })?.isDirectory()) {
+        fs.rmdirSync(target);
+      } else {
+        fs.rmSync(target, { force: true });
+      }
+    });
+    pruneEmptyDirs(rootDir, target);
+  }
+  for (const mutation of mutations) {
+    if (mutation.kind === 'write') {
+      const target = path.resolve(rootDir, mutation.path);
+      assertNotThroughSymlink(target);
+      retryWhileBusy(() => {
+        // Replace a link at the target itself; writing would follow it
+        if (fs.lstatSync(target, { throwIfNoEntry: false })?.isSymbolicLink()) {
+          fs.rmSync(target, { force: true });
+        }
+        writeOutputFile(target, mutation.content);
+      });
+    }
+  }
+}

package/build/pagefind.ts

@@ -1,17 +1,26 @@
+import fs from 'fs';
 import path from 'path';
 import { makeLogger } from './log';
 import { collectReachableSiteAssets } from './reachability';
-import {
-  getContentDir,
-  getFilesByExtensions,
-  normalizeOutputPath,
-} from './util';
+import { normalizeOutputPath, SEARCH_INDEX_DIR } from './utils/paths';
+import { isFeatureEnabled } from './features';
+import type { SiteVariables } from './types';
+import type { TadaProjectScan } from './source-model';
 import { assertMutoolAvailable, extractPdfPages } from './pdf-text';
 import type { HtmlOutputAnalysis } from './types';
 
 const log = makeLogger(import.meta.url);
 const PAGEFIND_VERBOSE = process.env.TADA_LOG_LEVEL === 'debug';
-const PAGEFIND_OUTPUT_SUBDIR = 'pagefind';
+
+/**
+ * Output path prefixes a full write leaves alone: the search index, which is
+ * written after the build.
+ */
+export function getKeptOutputPrefixes(siteVariables: SiteVariables): string[] {
+  return isFeatureEnabled(siteVariables, 'search')
+    ? [`${SEARCH_INDEX_DIR}/`]
+    : [];
+}
 
 type PagefindModule = typeof import('pagefind');
 type PagefindIndex = Awaited<
@@ -71,16 +80,21 @@ async function addPdfRecord(
   }
 }
 
-function getPdfSourceByOutputPath(): Map<string, string> {
-  const contentDir = getContentDir();
-  const pdfFiles: string[] = getFilesByExtensions(contentDir, ['pdf']);
-
-  return new Map(
-    pdfFiles.map((filePath: string) => {
-      const relPath = path.relative(contentDir, filePath);
-      return [normalizeOutputPath(`/${relPath}`), filePath] as const;
-    }),
-  );
+/** PDFs copied from `content/`, keyed by their root-relative output path */
+export function getPdfSources(scan: TadaProjectScan): Map<string, string> {
+  const pdfSources = new Map<string, string>();
+  for (const [filePath, entry] of scan.sources) {
+    if (
+      entry.renderKind !== 'content-copy' ||
+      path.extname(filePath).toLowerCase() !== '.pdf'
+    ) {
+      continue;
+    }
+    for (const outputPath of entry.outputs) {
+      pdfSources.set(normalizeOutputPath(`/${outputPath}`), filePath);
+    }
+  }
+  return pdfSources;
 }
 
 interface IndexTargets {
@@ -120,6 +134,13 @@ interface BuildIndexOptions {
   loadPagefind?: () => Promise<PagefindModule>;
   checkMutool?: () => Promise<void>;
   extractPages?: typeof extractPdfPages;
+  prepareOutputDir?: (dir: string) => void;
+}
+
+function prepareSearchIndexDir(dir: string): void {
+  if (fs.lstatSync(dir, { throwIfNoEntry: false })?.isFile()) {
+    fs.unlinkSync(dir);
+  }
 }
 
 async function buildIndex({
@@ -131,6 +152,7 @@ async function buildIndex({
   loadPagefind = getPagefind,
   checkMutool = assertMutoolAvailable,
   extractPages = extractPdfPages,
+  prepareOutputDir = prepareSearchIndexDir,
 }: BuildIndexOptions): Promise<void> {
   const pagefind = await loadPagefind();
   const { index, errors: createErrors } = await pagefind.createIndex({
@@ -200,9 +222,10 @@ async function buildIndex({
       }
     }
 
-    const { errors: writeErrors } = await index.writeFiles({
-      outputPath: path.join(distPath, PAGEFIND_OUTPUT_SUBDIR),
-    });
+    // Keep older hashed files available to browsers using the previous index.
+    const outputPath = path.join(distPath, SEARCH_INDEX_DIR);
+    prepareOutputDir(outputPath);
+    const { errors: writeErrors } = await index.writeFiles({ outputPath });
     const writeError = formatPagefindErrors('index.writeFiles()', writeErrors);
     if (writeError) {
       throw new Error(writeError);
@@ -216,14 +239,15 @@ interface RunPagefindOptions {
   distPath: string;
   htmlAssetsByPath: Map<string, string>;
   htmlAnalysisByPath: Map<string, HtmlOutputAnalysis>;
+  pdfSourceByOutputPath: Map<string, string>;
 }
 
 export async function runPagefind({
   distPath,
   htmlAssetsByPath,
   htmlAnalysisByPath,
+  pdfSourceByOutputPath,
 }: RunPagefindOptions): Promise<void> {
-  const pdfSourceByOutputPath = getPdfSourceByOutputPath();
   const start = Date.now();
 
   log.debug`Finding reachable pages for search index`;
@@ -258,6 +282,7 @@ export class WatchPagefindRunner {
   private distPath: string | null;
   private htmlCacheByAssetPath: Map<string, string>;
   private htmlAnalysisByPath: Map<string, HtmlOutputAnalysis>;
+  private pdfSourceByOutputPath: Map<string, string>;
 
   constructor() {
     this.watchRunInProgress = false;
@@ -265,16 +290,19 @@ export class WatchPagefindRunner {
     this.distPath = null;
     this.htmlCacheByAssetPath = new Map();
     this.htmlAnalysisByPath = new Map();
+    this.pdfSourceByOutputPath = new Map();
   }
 
   update(
     distPath: string,
     htmlAssetsByPath: Map<string, string>,
     htmlAnalysisByPath: Map<string, HtmlOutputAnalysis>,
+    pdfSourceByOutputPath: Map<string, string>,
   ): void {
     this.distPath = distPath;
     this.htmlCacheByAssetPath = htmlAssetsByPath;
     this.htmlAnalysisByPath = htmlAnalysisByPath;
+    this.pdfSourceByOutputPath = pdfSourceByOutputPath;
   }
 
   run(): void {
@@ -289,7 +317,7 @@ export class WatchPagefindRunner {
     const distPath = this.distPath!;
     const htmlAssetsByPath = new Map(this.htmlCacheByAssetPath);
     const htmlAnalysisByPath = new Map(this.htmlAnalysisByPath);
-    const pdfSourceByOutputPath = getPdfSourceByOutputPath();
+    const pdfSourceByOutputPath = this.pdfSourceByOutputPath;
     const start = Date.now();
 
     log.debug`Preparing search index background snapshot`;

package/build/pdf-text.ts

@@ -137,7 +137,7 @@ export function buildPdfPageRecords(pageTexts: string[]): PdfExtractResult {
   return { pages, hasExtractedText: pages.length > 0 };
 }
 
-export async function extractPdfPages(
+async function extractPdfPagesUncached(
   pdfPath: string,
 ): Promise<PdfExtractResult> {
   await assertMutoolAvailable();
@@ -181,3 +181,43 @@ export async function extractPdfPages(
     }
   }
 }
+
+interface FileSignature {
+  size: number;
+  mtimeMs: number;
+}
+
+/**
+ * Wraps a PDF text extractor so each PDF is extracted again only after its size
+ * or modification time changes. Concurrent requests for one file share a
+ * single extraction, and failed extractions are not cached.
+ */
+export function createCachedPdfExtractor(
+  extract: (pdfPath: string) => Promise<PdfExtractResult>,
+  stat: (pdfPath: string) => FileSignature,
+): (pdfPath: string) => Promise<PdfExtractResult> {
+  const cache = new Map<
+    string,
+    FileSignature & { result: Promise<PdfExtractResult> }
+  >();
+  return pdfPath => {
+    const { size, mtimeMs } = stat(pdfPath);
+    const cached = cache.get(pdfPath);
+    if (cached && cached.size === size && cached.mtimeMs === mtimeMs) {
+      return cached.result;
+    }
+    const entry = { size, mtimeMs, result: extract(pdfPath) };
+    cache.set(pdfPath, entry);
+    entry.result.catch(() => {
+      if (cache.get(pdfPath) === entry) {
+        cache.delete(pdfPath);
+      }
+    });
+    return entry.result;
+  };
+}
+
+export const extractPdfPages = createCachedPdfExtractor(
+  extractPdfPagesUncached,
+  pdfPath => fs.statSync(pdfPath),
+);

package/build/pipeline.ts

@@ -1,123 +1,117 @@
 import fs from 'fs';
+import path from 'path';
 import { getDevSiteVariables, getProdSiteVariables } from './site-variables';
-import { compileTemplates } from './templates';
 import { getDistDir, getProdDistDir } from './utils/paths';
 import { isFeatureEnabled } from './features';
-import { bundle } from './bundle';
-import { copyFonts } from './generate-fonts';
-import { copyKatexAssets } from './generate-katex-assets';
-import { generateFavicons } from './generate-favicon';
-import { generateWebAppManifest } from './generate-web-app-manifest';
-import { copyPublicFiles, copyContentAssets } from './copy';
-import { ContentRenderer } from './generate-content-assets';
-import { runPagefind } from './pagefind';
+import { getKeptOutputPrefixes, runPagefind } from './pagefind';
 import { makeLogger, printFlair } from './log';
-import { generateBuildManifest, getNextVersion } from './build-manifest';
-import { scanProject } from './source-model';
-import { validateConfig } from './watch/validation';
-import { applyCommitPlan } from './watch/fs-commit';
-import path from 'path';
+import {
+  generateBuildManifest,
+  getNextVersion,
+  MANIFEST_FILE_NAME,
+} from './build-manifest';
+import { buildSite, createBuildMeta, type TadaSnapshot } from './site-build';
+import {
+  BuildFailedError,
+  diagnosticFromError,
+  printDiagnostics,
+} from './build-validation';
+import { applyMutations, planFullWrite } from './output-publication';
+import type { BuildDiagnostic } from './build-types';
+import type { SiteVariables } from './types';
 
 const log = makeLogger(import.meta.url);
 
-function makeTempOutputDir(targetDir: string): string {
-  const parentDir = path.dirname(targetDir);
-  fs.mkdirSync(parentDir, { recursive: true });
-  return fs.mkdtempSync(
-    path.join(parentDir, `${path.basename(targetDir)}-build-`),
+function fail(diagnostics: BuildDiagnostic[]): never {
+  printDiagnostics(diagnostics);
+  const noun = diagnostics.length === 1 ? 'error' : 'errors';
+  throw new BuildFailedError(`${diagnostics.length} ${noun}`);
+}
+
+async function indexSearch(
+  outputDir: string,
+  snapshot: TadaSnapshot,
+): Promise<void> {
+  if (isFeatureEnabled(snapshot.siteVariables, 'search')) {
+    const meta = createBuildMeta(snapshot);
+    await runPagefind({ distPath: outputDir, ...meta });
+  }
+}
+
+/** Updates `dist/` in place to hold exactly this build's outputs. */
+async function publishDev(snapshot: TadaSnapshot): Promise<void> {
+  const distDir = getDistDir();
+  applyMutations(
+    distDir,
+    planFullWrite(
+      distDir,
+      snapshot.outputs,
+      getKeptOutputPrefixes(snapshot.siteVariables),
+    ),
   );
+  await indexSearch(distDir, snapshot);
 }
 
-function removeDirIfExists(dir: string): void {
-  fs.rmSync(dir, { recursive: true, force: true });
+/** Writes a new `dist-prod/v{N}/`; it counts as a version once its manifest is written. */
+async function publishProd(snapshot: TadaSnapshot): Promise<void> {
+  const prodBase = getProdDistDir();
+  const version = getNextVersion(prodBase);
+  const versionDir = path.join(prodBase, `v${version}`);
+  // A directory without a manifest is left over from a failed build.
+  fs.rmSync(versionDir, { recursive: true, force: true });
+  try {
+    applyMutations(versionDir, planFullWrite(versionDir, snapshot.outputs));
+    await indexSearch(versionDir, snapshot);
+    await generateBuildManifest(
+      versionDir,
+      path.join(versionDir, MANIFEST_FILE_NAME),
+      version,
+    );
+  } catch (error) {
+    try {
+      fs.rmSync(versionDir, { recursive: true, force: true });
+    } catch {
+      // Without a manifest, the leftover directory is ignored and replaced.
+    }
+    throw error;
+  }
+  log.info`Built dist-prod/v${version}/`;
 }
 
 export async function runPipeline(
   mode: 'development' | 'production',
 ): Promise<void> {
   const isDev = mode === 'development';
-  const siteVariables = isDev ? getDevSiteVariables() : getProdSiteVariables();
-  const scan = scanProject(siteVariables);
-  const publishDir = isDev
-    ? getDistDir()
-    : path.join(getProdDistDir(), 'v-next');
-  const distDir = makeTempOutputDir(publishDir);
-  let published = false;
-
+  let siteVariables: SiteVariables;
   try {
-    const configDiagnostics = validateConfig(scan, siteVariables);
-    if (configDiagnostics.length > 0) {
-      throw new Error(configDiagnostics[0].message);
-    }
-
-    compileTemplates(siteVariables);
-    const contentRenderer = new ContentRenderer(siteVariables);
-    await contentRenderer.initHighlighter();
-
-    const parallelTasks: (Promise<unknown> | void)[] = [
-      bundle(siteVariables, { mode, distDir }),
-      copyFonts(distDir),
-      copyKatexAssets(distDir),
-    ];
-
-    if (isFeatureEnabled(siteVariables, 'favicon')) {
-      parallelTasks.push(generateFavicons(siteVariables, distDir));
-    }
-
-    const results = await Promise.all(parallelTasks);
-    const assetFiles = results[0] as string[]; // bundle output filenames
-
-    if (isFeatureEnabled(siteVariables, 'favicon')) {
-      generateWebAppManifest(siteVariables, distDir);
-    }
-
-    const publicRelPaths = copyPublicFiles(scan.publicDir, distDir);
-    copyContentAssets(
-      scan.contentDir,
-      distDir,
-      [...scan.processedExts],
-      publicRelPaths,
-    );
-
-    const { errors, htmlAssetsByPath, htmlAnalysisByPath } =
-      contentRenderer.processContent({ distDir, assetFiles, scan });
-
-    for (const err of errors) {
-      log.error`${err.message}`;
-    }
-    if (errors.length > 0) {
-      throw new Error(errors[0].message);
-    }
-
-    if (isFeatureEnabled(siteVariables, 'search')) {
-      await runPagefind({
-        distPath: distDir,
-        htmlAssetsByPath,
-        htmlAnalysisByPath,
-      });
-    }
-
-    let finalOutputDir = getDistDir();
-
-    if (!isDev) {
-      const prodBase = getProdDistDir();
-      const prodVersion = getNextVersion(prodBase);
-      finalOutputDir = path.join(prodBase, `v${prodVersion}`);
-      const manifestPath = path.join(distDir, 'tada.manifest.json');
-      await generateBuildManifest(distDir, manifestPath, prodVersion);
-      log.info`Built dist-prod/v${prodVersion}/`;
-    }
+    siteVariables = isDev ? getDevSiteVariables() : getProdSiteVariables();
+  } catch (error) {
+    fail([diagnosticFromError(error)]);
+  }
 
-    applyCommitPlan({
-      kind: 'replace-root',
-      stagedPath: distDir,
-      targetPath: finalOutputDir,
+  let result: Awaited<ReturnType<typeof buildSite>>;
+  try {
+    result = await buildSite({
+      siteVariables,
+      mode,
+      isWatchMode: false,
+      traceCache: new Map(),
     });
-    published = true;
-    printFlair();
-  } finally {
-    if (!published) {
-      removeDirIfExists(distDir);
-    }
+  } catch (error) {
+    fail([diagnosticFromError(error)]);
+  }
+  if (!result.ok) {
+    fail(result.diagnostics);
+  }
+
+  try {
+    await (isDev ? publishDev(result.snapshot) : publishProd(result.snapshot));
+  } catch (error) {
+    fail([
+      diagnosticFromError(
+        new Error(`Failed to write output: ${(error as Error).message}`),
+      ),
+    ]);
   }
+  printFlair();
 }