@abreen/tada 1.17.1 → 1.18.1

package/build/pdf-text.ts

@@ -137,7 +137,7 @@ export function buildPdfPageRecords(pageTexts: string[]): PdfExtractResult {
   return { pages, hasExtractedText: pages.length > 0 };
 }
 
-export async function extractPdfPages(
+async function extractPdfPagesUncached(
   pdfPath: string,
 ): Promise<PdfExtractResult> {
   await assertMutoolAvailable();
@@ -181,3 +181,43 @@ export async function extractPdfPages(
     }
   }
 }
+
+interface FileSignature {
+  size: number;
+  mtimeMs: number;
+}
+
+/**
+ * Wraps a PDF text extractor so each PDF is extracted again only after its size
+ * or modification time changes. Concurrent requests for one file share a
+ * single extraction, and failed extractions are not cached.
+ */
+export function createCachedPdfExtractor(
+  extract: (pdfPath: string) => Promise<PdfExtractResult>,
+  stat: (pdfPath: string) => FileSignature,
+): (pdfPath: string) => Promise<PdfExtractResult> {
+  const cache = new Map<
+    string,
+    FileSignature & { result: Promise<PdfExtractResult> }
+  >();
+  return pdfPath => {
+    const { size, mtimeMs } = stat(pdfPath);
+    const cached = cache.get(pdfPath);
+    if (cached && cached.size === size && cached.mtimeMs === mtimeMs) {
+      return cached.result;
+    }
+    const entry = { size, mtimeMs, result: extract(pdfPath) };
+    cache.set(pdfPath, entry);
+    entry.result.catch(() => {
+      if (cache.get(pdfPath) === entry) {
+        cache.delete(pdfPath);
+      }
+    });
+    return entry.result;
+  };
+}
+
+export const extractPdfPages = createCachedPdfExtractor(
+  extractPdfPagesUncached,
+  pdfPath => fs.statSync(pdfPath),
+);

package/build/pipeline.ts

@@ -1,123 +1,117 @@
 import fs from 'fs';
+import path from 'path';
 import { getDevSiteVariables, getProdSiteVariables } from './site-variables';
-import { compileTemplates } from './templates';
 import { getDistDir, getProdDistDir } from './utils/paths';
 import { isFeatureEnabled } from './features';
-import { bundle } from './bundle';
-import { copyFonts } from './generate-fonts';
-import { copyKatexAssets } from './generate-katex-assets';
-import { generateFavicons } from './generate-favicon';
-import { generateWebAppManifest } from './generate-web-app-manifest';
-import { copyPublicFiles, copyContentAssets } from './copy';
-import { ContentRenderer } from './generate-content-assets';
-import { runPagefind } from './pagefind';
+import { getKeptOutputPrefixes, runPagefind } from './pagefind';
 import { makeLogger, printFlair } from './log';
-import { generateBuildManifest, getNextVersion } from './build-manifest';
-import { scanProject } from './source-model';
-import { validateConfig } from './watch/validation';
-import { applyCommitPlan } from './watch/fs-commit';
-import path from 'path';
+import {
+  generateBuildManifest,
+  getNextVersion,
+  MANIFEST_FILE_NAME,
+} from './build-manifest';
+import { buildSite, createBuildMeta, type TadaSnapshot } from './site-build';
+import {
+  BuildFailedError,
+  diagnosticFromError,
+  printDiagnostics,
+} from './build-validation';
+import { applyMutations, planFullWrite } from './output-publication';
+import type { BuildDiagnostic } from './build-types';
+import type { SiteVariables } from './types';
 
 const log = makeLogger(import.meta.url);
 
-function makeTempOutputDir(targetDir: string): string {
-  const parentDir = path.dirname(targetDir);
-  fs.mkdirSync(parentDir, { recursive: true });
-  return fs.mkdtempSync(
-    path.join(parentDir, `${path.basename(targetDir)}-build-`),
+function fail(diagnostics: BuildDiagnostic[]): never {
+  printDiagnostics(diagnostics);
+  const noun = diagnostics.length === 1 ? 'error' : 'errors';
+  throw new BuildFailedError(`${diagnostics.length} ${noun}`);
+}
+
+async function indexSearch(
+  outputDir: string,
+  snapshot: TadaSnapshot,
+): Promise<void> {
+  if (isFeatureEnabled(snapshot.siteVariables, 'search')) {
+    const meta = createBuildMeta(snapshot);
+    await runPagefind({ distPath: outputDir, ...meta });
+  }
+}
+
+/** Updates `dist/` in place to hold exactly this build's outputs. */
+async function publishDev(snapshot: TadaSnapshot): Promise<void> {
+  const distDir = getDistDir();
+  applyMutations(
+    distDir,
+    planFullWrite(
+      distDir,
+      snapshot.outputs,
+      getKeptOutputPrefixes(snapshot.siteVariables),
+    ),
   );
+  await indexSearch(distDir, snapshot);
 }
 
-function removeDirIfExists(dir: string): void {
-  fs.rmSync(dir, { recursive: true, force: true });
+/** Writes a new `dist-prod/v{N}/`; it counts as a version once its manifest is written. */
+async function publishProd(snapshot: TadaSnapshot): Promise<void> {
+  const prodBase = getProdDistDir();
+  const version = getNextVersion(prodBase);
+  const versionDir = path.join(prodBase, `v${version}`);
+  // A directory without a manifest is left over from a failed build.
+  fs.rmSync(versionDir, { recursive: true, force: true });
+  try {
+    applyMutations(versionDir, planFullWrite(versionDir, snapshot.outputs));
+    await indexSearch(versionDir, snapshot);
+    await generateBuildManifest(
+      versionDir,
+      path.join(versionDir, MANIFEST_FILE_NAME),
+      version,
+    );
+  } catch (error) {
+    try {
+      fs.rmSync(versionDir, { recursive: true, force: true });
+    } catch {
+      // Without a manifest, the leftover directory is ignored and replaced.
+    }
+    throw error;
+  }
+  log.info`Built dist-prod/v${version}/`;
 }
 
 export async function runPipeline(
   mode: 'development' | 'production',
 ): Promise<void> {
   const isDev = mode === 'development';
-  const siteVariables = isDev ? getDevSiteVariables() : getProdSiteVariables();
-  const scan = scanProject(siteVariables);
-  const publishDir = isDev
-    ? getDistDir()
-    : path.join(getProdDistDir(), 'v-next');
-  const distDir = makeTempOutputDir(publishDir);
-  let published = false;
-
+  let siteVariables: SiteVariables;
   try {
-    const configDiagnostics = validateConfig(scan, siteVariables);
-    if (configDiagnostics.length > 0) {
-      throw new Error(configDiagnostics[0].message);
-    }
-
-    compileTemplates(siteVariables);
-    const contentRenderer = new ContentRenderer(siteVariables);
-    await contentRenderer.initHighlighter();
-
-    const parallelTasks: (Promise<unknown> | void)[] = [
-      bundle(siteVariables, { mode, distDir }),
-      copyFonts(distDir),
-      copyKatexAssets(distDir),
-    ];
-
-    if (isFeatureEnabled(siteVariables, 'favicon')) {
-      parallelTasks.push(generateFavicons(siteVariables, distDir));
-    }
-
-    const results = await Promise.all(parallelTasks);
-    const assetFiles = results[0] as string[]; // bundle output filenames
-
-    if (isFeatureEnabled(siteVariables, 'favicon')) {
-      generateWebAppManifest(siteVariables, distDir);
-    }
-
-    const publicRelPaths = copyPublicFiles(scan.publicDir, distDir);
-    copyContentAssets(
-      scan.contentDir,
-      distDir,
-      [...scan.processedExts],
-      publicRelPaths,
-    );
-
-    const { errors, htmlAssetsByPath, htmlAnalysisByPath } =
-      contentRenderer.processContent({ distDir, assetFiles, scan });
-
-    for (const err of errors) {
-      log.error`${err.message}`;
-    }
-    if (errors.length > 0) {
-      throw new Error(errors[0].message);
-    }
-
-    if (isFeatureEnabled(siteVariables, 'search')) {
-      await runPagefind({
-        distPath: distDir,
-        htmlAssetsByPath,
-        htmlAnalysisByPath,
-      });
-    }
-
-    let finalOutputDir = getDistDir();
-
-    if (!isDev) {
-      const prodBase = getProdDistDir();
-      const prodVersion = getNextVersion(prodBase);
-      finalOutputDir = path.join(prodBase, `v${prodVersion}`);
-      const manifestPath = path.join(distDir, 'tada.manifest.json');
-      await generateBuildManifest(distDir, manifestPath, prodVersion);
-      log.info`Built dist-prod/v${prodVersion}/`;
-    }
+    siteVariables = isDev ? getDevSiteVariables() : getProdSiteVariables();
+  } catch (error) {
+    fail([diagnosticFromError(error)]);
+  }
 
-    applyCommitPlan({
-      kind: 'replace-root',
-      stagedPath: distDir,
-      targetPath: finalOutputDir,
+  let result: Awaited<ReturnType<typeof buildSite>>;
+  try {
+    result = await buildSite({
+      siteVariables,
+      mode,
+      isWatchMode: false,
+      traceCache: new Map(),
     });
-    published = true;
-    printFlair();
-  } finally {
-    if (!published) {
-      removeDirIfExists(distDir);
-    }
+  } catch (error) {
+    fail([diagnosticFromError(error)]);
+  }
+  if (!result.ok) {
+    fail(result.diagnostics);
+  }
+
+  try {
+    await (isDev ? publishDev(result.snapshot) : publishProd(result.snapshot));
+  } catch (error) {
+    fail([
+      diagnosticFromError(
+        new Error(`Failed to write output: ${(error as Error).message}`),
+      ),
+    ]);
   }
+  printFlair();
 }

package/build/site-assets.ts

@@ -0,0 +1,46 @@
+import { bundle, bundleReloadClient } from './bundle';
+import { isFeatureEnabled } from './features';
+import { generateFavicons } from './generate-favicon';
+import { getFontOutputs } from './generate-fonts';
+import { getKatexOutputs } from './generate-katex-assets';
+import { createWebAppManifest } from './generate-web-app-manifest';
+import type { OutputContent, SiteVariables } from './types';
+
+/** Files Tada generates itself, as opposed to files rendered from sources */
+interface SiteAssets {
+  /** CSS and JavaScript bundles that every page links to */
+  assetFiles: string[];
+  /** Every generated file, keyed by output path */
+  outputs: Map<string, OutputContent>;
+}
+
+export async function generateSiteAssets(
+  siteVariables: SiteVariables,
+  {
+    mode,
+    isWatchMode,
+  }: { mode: 'development' | 'production'; isWatchMode: boolean },
+): Promise<SiteAssets> {
+  const generateFaviconFiles =
+    isFeatureEnabled(siteVariables, 'favicon') && !siteVariables.favicon;
+  const [bundles, reloadClient, favicons] = await Promise.all([
+    bundle(siteVariables, { mode }),
+    isWatchMode ? bundleReloadClient() : new Map<string, string>(),
+    generateFaviconFiles
+      ? generateFavicons(siteVariables)
+      : new Map<string, string | Uint8Array>(),
+  ]);
+  const bundleFiles = new Map([...bundles, ...reloadClient]);
+
+  const outputs = new Map<string, OutputContent>([
+    ...bundleFiles,
+    ...getFontOutputs(),
+    ...getKatexOutputs(),
+    ...favicons,
+  ]);
+  if (generateFaviconFiles) {
+    outputs.set('manifest.json', createWebAppManifest(siteVariables));
+  }
+
+  return { assetFiles: [...bundleFiles.keys()], outputs };
+}

package/build/site-build.ts

@@ -0,0 +1,267 @@
+import path from 'path';
+import { compileTemplates, config } from './templates';
+import { getProjectDir, toPosix } from './utils/paths';
+import { makeLogger } from './log';
+import { getRuntimeBundledShikiLanguages } from './site-variables';
+import { initHighlighter } from './utils/shiki-highlighter';
+import { generateSiteAssets } from './site-assets';
+import { getPdfSources } from './pagefind';
+import { checkTraceToolAvailability } from './utils/trace';
+import { scanProject, sourcePaths, type TadaProjectScan } from './source-model';
+import {
+  createContentRecord,
+  createPublicRecord,
+  type TadaSourceRecord,
+} from './source-records';
+import {
+  diagnosticFromError,
+  findOutputConflicts,
+  validateConfig,
+  validateProjectConfigLinks,
+} from './build-validation';
+import type { OutputFile } from './output-publication';
+import type {
+  OutputContent,
+  SiteVariables,
+  TraceToolAvailability,
+} from './types';
+import type { BuildDiagnostic, TadaBuildMeta, TraceCache } from './build-types';
+
+const log = makeLogger(import.meta.url);
+
+/** The `sourcePath` recorded for files Tada generates itself */
+export const GENERATED_SOURCE = 'tada';
+
+/** Everything one successful build produced, kept by watch mode */
+export interface TadaSnapshot {
+  siteVariables: SiteVariables;
+  assetFiles: string[];
+  generatedOutputs: ReadonlyMap<string, OutputContent>;
+  authorsData: unknown;
+  records: ReadonlyMap<string, TadaSourceRecord>;
+  outputs: ReadonlyMap<string, OutputFile>;
+  fileDependents: ReadonlyMap<string, ReadonlySet<string>>;
+  targetDependents: ReadonlyMap<string, ReadonlySet<string>>;
+  authorDependents: ReadonlyMap<string, ReadonlySet<string>>;
+  scan: TadaProjectScan;
+}
+
+type SnapshotInputs = Pick<
+  TadaSnapshot,
+  'siteVariables' | 'assetFiles' | 'authorsData' | 'records' | 'scan'
+> & { generatedOutputs?: ReadonlyMap<string, OutputContent> };
+
+export type SiteBuildResult =
+  | { ok: true; snapshot: TadaSnapshot }
+  | { ok: false; diagnostics: BuildDiagnostic[] };
+
+export interface RenderContext {
+  siteVariables: SiteVariables;
+  scan: TadaProjectScan;
+  assetFiles: string[];
+  isWatchMode: boolean;
+  traceCache: TraceCache;
+  traceToolAvailability: TraceToolAvailability;
+}
+
+export function createSnapshot(inputs: SnapshotInputs): TadaSnapshot {
+  const generatedOutputs = inputs.generatedOutputs ?? new Map();
+  const outputs = new Map<string, OutputFile>();
+  for (const [outputPath, content] of generatedOutputs) {
+    outputs.set(outputPath, { sourcePath: GENERATED_SOURCE, content });
+  }
+  const fileDependents = new Map<string, Set<string>>();
+  const targetDependents = new Map<string, Set<string>>();
+  const authorDependents = new Map<string, Set<string>>();
+  for (const record of inputs.records.values()) {
+    for (const [outputPath, content] of record.outputs) {
+      outputs.set(outputPath, { sourcePath: record.sourcePath, content });
+    }
+    for (const [index, keys] of [
+      [fileDependents, [...record.partialDeps, ...record.traceDeps]],
+      [targetDependents, record.internalTargets],
+      [authorDependents, record.authorKey ? [record.authorKey] : []],
+    ] as const) {
+      for (const key of keys) {
+        if (!index.has(key)) {
+          index.set(key, new Set());
+        }
+        index.get(key)!.add(record.sourcePath);
+      }
+    }
+  }
+  return {
+    ...inputs,
+    generatedOutputs,
+    outputs,
+    fileDependents,
+    targetDependents,
+    authorDependents,
+  };
+}
+
+export function createBuildMeta(snapshot: TadaSnapshot): TadaBuildMeta {
+  const htmlAssetsByPath = new Map<string, string>();
+  const htmlAnalysisByPath: TadaBuildMeta['htmlAnalysisByPath'] = new Map();
+  for (const record of snapshot.records.values()) {
+    if (record.kind !== 'content') {
+      continue;
+    }
+    for (const [outputPath, content] of record.outputs) {
+      if (outputPath.endsWith('.html') && typeof content === 'string') {
+        htmlAssetsByPath.set(outputPath, content);
+      }
+    }
+    for (const [outputPath, analysis] of record.htmlAnalysisByOutputPath ??
+      []) {
+      htmlAnalysisByPath.set(outputPath, analysis);
+    }
+  }
+  return {
+    htmlAssetsByPath,
+    htmlAnalysisByPath,
+    pdfSourceByOutputPath: getPdfSources(snapshot.scan),
+    siteVariables: snapshot.siteVariables,
+  };
+}
+
+export async function ensureHighlighter(
+  siteVariables: SiteVariables,
+): Promise<void> {
+  await initHighlighter(getRuntimeBundledShikiLanguages(siteVariables));
+}
+
+/**
+ * Renders each source into a record. A source that fails to render becomes a
+ * diagnostic, and rendering continues so every page error is reported.
+ */
+export function renderSources(
+  filePaths: Iterable<string>,
+  context: RenderContext,
+): { records: Map<string, TadaSourceRecord>; diagnostics: BuildDiagnostic[] } {
+  const records = new Map<string, TadaSourceRecord>();
+  const diagnostics: BuildDiagnostic[] = [];
+  for (const filePath of filePaths) {
+    try {
+      const record =
+        context.scan.sources.get(filePath)?.kind === 'public'
+          ? createPublicRecord(filePath, context.scan.publicDir)
+          : createContentRecord({
+              filePath,
+              siteVariables: context.siteVariables,
+              scan: context.scan,
+              assetFiles: context.assetFiles,
+              isWatchMode: context.isWatchMode,
+              traceCache: context.traceCache,
+              traceToolAvailability: context.traceToolAvailability,
+              skipLiterateJavaExecution: !context.traceToolAvailability.java,
+            });
+      if (record.outputs.size > 0) {
+        records.set(filePath, record);
+      }
+    } catch (error) {
+      const { message } = diagnosticFromError(error);
+      // Every page error names its file.
+      const relPath = toPosix(path.relative(getProjectDir(), filePath));
+      diagnostics.push({
+        message:
+          message.includes(filePath) || message.includes(relPath)
+            ? message
+            : `${filePath}: ${message}`,
+      });
+    }
+  }
+  return { records, diagnostics };
+}
+
+/** Validates rendered records and assembles the snapshot of a build. */
+export function finishBuild({
+  siteVariables,
+  scan,
+  assetFiles,
+  generatedOutputs,
+  records,
+  diagnostics,
+}: {
+  siteVariables: SiteVariables;
+  scan: TadaProjectScan;
+  assetFiles: string[];
+  generatedOutputs: ReadonlyMap<string, OutputContent>;
+  records: ReadonlyMap<string, TadaSourceRecord>;
+  diagnostics: BuildDiagnostic[];
+}): SiteBuildResult {
+  const allDiagnostics = [
+    ...diagnostics,
+    ...validateProjectConfigLinks(scan.validTargets),
+    ...findOutputConflicts(scan, siteVariables, generatedOutputs, records),
+  ];
+  if (allDiagnostics.length > 0) {
+    return { ok: false, diagnostics: allDiagnostics };
+  }
+  return {
+    ok: true,
+    snapshot: createSnapshot({
+      siteVariables,
+      assetFiles,
+      generatedOutputs,
+      authorsData: config('authors'),
+      scan,
+      records,
+    }),
+  };
+}
+
+/** Builds every source and generated file in memory. Nothing is written. */
+export async function buildSite({
+  siteVariables,
+  mode,
+  isWatchMode,
+  traceCache,
+  traceToolAvailability,
+  scan = scanProject(siteVariables),
+}: {
+  siteVariables: SiteVariables;
+  mode: 'development' | 'production';
+  isWatchMode: boolean;
+  traceCache: TraceCache;
+  /** Probed only if omitted and the site has content pages */
+  traceToolAvailability?: TraceToolAvailability;
+  /** A scan the caller already made with the same site variables */
+  scan?: TadaProjectScan;
+}): Promise<SiteBuildResult> {
+  const configDiagnostics = validateConfig(scan, siteVariables);
+  if (configDiagnostics.length > 0) {
+    return { ok: false, diagnostics: configDiagnostics };
+  }
+
+  compileTemplates(siteVariables);
+  await ensureHighlighter(siteVariables);
+  const assets = await generateSiteAssets(siteVariables, { mode, isWatchMode });
+
+  const pageCount = [...sourcePaths(scan, 'content', true)].length;
+  if (pageCount > 0) {
+    log.info`Processing ${pageCount} content ${pageCount === 1 ? 'file' : 'files'}`;
+  }
+  if (!traceToolAvailability) {
+    traceToolAvailability = pageCount > 0 ? checkTraceToolAvailability() : {};
+    if (pageCount > 0 && !traceToolAvailability.java) {
+      log.warn`javac was not found; literate Java pages will not include execution output`;
+    }
+  }
+  const { records, diagnostics } = renderSources(scan.sources.keys(), {
+    siteVariables,
+    scan,
+    assetFiles: assets.assetFiles,
+    isWatchMode,
+    traceCache,
+    traceToolAvailability,
+  });
+  return finishBuild({
+    siteVariables,
+    scan,
+    assetFiles: assets.assetFiles,
+    generatedOutputs: assets.outputs,
+    records,
+    diagnostics,
+  });
+}