@abreen/tada 1.17.0 → 1.18.0

package/build/watch/compiler.ts

@@ -1,27 +1,33 @@
 import path from 'path';
 import { getProjectConfigDir } from '../templates';
-import { getContentDir, getPublicDir } from '../util';
-import type {
-  ChangeBatch,
-  CompilerBuildResult,
-  WatchCompiler,
-  WatchTarget,
-} from './types';
-import { buildFull } from './build-full';
+import { getContentDir, getDistDir, getPublicDir } from '../utils/paths';
+import type { CompileOutcome, WatchTarget, WatchBuildResult } from './types';
+import type { TadaBuildMeta, TraceCache } from '../build-types';
+import type { TraceToolAvailability } from '../types';
+import {
+  applyMutations,
+  computeMutations,
+  planFullWrite,
+} from '../output-publication';
+import { loadProjectConfig } from '../config-loader';
+import { getKeptOutputPrefixes } from '../pagefind';
+import { getDevSiteVariables } from '../site-variables';
+import { scanProject } from '../source-model';
+import { buildSite, createBuildMeta, type TadaSnapshot } from '../site-build';
 import { buildIncremental } from './build-incremental';
-import type { TraceCache, WatchTraceOptions } from './compiler-types';
+import type { TadaSourceRecord } from '../source-records';
 import { checkTraceToolAvailability, isTraceSourceFile } from '../utils/trace';
-import { createTadaWatchPlan } from './planner';
-import { updateProjectScan } from '../source-model';
-import type { TadaSnapshot } from './snapshot';
-import { getWatchConfigFilePaths } from './config-paths';
+import { createTadaWatchPlan, diffAuthorKeys } from './planner';
+import {
+  getWatchConfigFilePaths,
+  classifyWatchConfigPath,
+} from './config-paths';
 
 export function invalidateTraceCacheForBatch(
   traceCache: TraceCache,
-  batch: ChangeBatch,
+  paths: ReadonlySet<string>,
 ): void {
-  for (const change of batch.changes) {
-    const sourcePath = path.resolve(change.path);
+  for (const sourcePath of paths) {
     if (isTraceSourceFile(sourcePath)) {
       for (const [cacheKey, entry] of traceCache) {
         if (sourcePath in entry.sourceMtims) {
@@ -32,13 +38,39 @@ export function invalidateTraceCacheForBatch(
   }
 }
 
-export class TadaWatchCompiler implements WatchCompiler {
+/** Drops cached traces whose source files no page in the build traces. */
+export function pruneTraceCache(
+  traceCache: TraceCache,
+  records: ReadonlyMap<string, TadaSourceRecord>,
+): void {
+  const traced = new Set<string>();
+  for (const record of records.values()) {
+    for (const sourcePath of record.traceDeps) {
+      traced.add(sourcePath);
+    }
+  }
+  for (const [cacheKey, entry] of traceCache) {
+    if (!Object.keys(entry.sourceMtims).every(file => traced.has(file))) {
+      traceCache.delete(cacheKey);
+    }
+  }
+}
+
+export class TadaWatchCompiler {
   private traceCache: TraceCache;
-  private traceOptions: WatchTraceOptions;
+  private traceToolAvailability: TraceToolAvailability;
+
+  readonly build = createBuildSession(async (snapshot, paths) => {
+    const outcome = await this.compile(snapshot, paths);
+    if (outcome.ok) {
+      pruneTraceCache(this.traceCache, outcome.snapshot.records);
+    }
+    return outcome;
+  });
 
   constructor() {
     this.traceCache = new Map();
-    this.traceOptions = { toolAvailability: checkTraceToolAvailability() };
+    this.traceToolAvailability = checkTraceToolAvailability();
   }
 
   getWatchTargets(): WatchTarget[] {
@@ -58,35 +90,124 @@ export class TadaWatchCompiler implements WatchCompiler {
     ];
   }
 
-  async build(
+  private async buildFull(
+    reuse?: Pick<TadaSnapshot, 'siteVariables' | 'scan'>,
+  ): Promise<CompileOutcome> {
+    const result = await buildSite({
+      siteVariables: reuse?.siteVariables ?? getDevSiteVariables(),
+      scan: reuse?.scan,
+      mode: 'development',
+      isWatchMode: true,
+      traceCache: this.traceCache,
+      traceToolAvailability: this.traceToolAvailability,
+    });
+    return result.ok ? { ...result, full: true } : result;
+  }
+
+  private async compile(
     snapshot: TadaSnapshot | undefined,
-    batch?: ChangeBatch,
-  ): Promise<CompilerBuildResult> {
-    if (batch) {
-      invalidateTraceCacheForBatch(this.traceCache, batch);
+    paths?: ReadonlySet<string>,
+  ): Promise<CompileOutcome> {
+    if (paths) {
+      invalidateTraceCacheForBatch(this.traceCache, paths);
     }
-
-    if (!snapshot || !batch) {
-      return buildFull({
-        traceCache: this.traceCache,
-        traceOptions: this.traceOptions,
-      });
+    const configKinds = new Set(
+      [...(paths ?? [])].map(classifyWatchConfigPath),
+    );
+    if (
+      !snapshot ||
+      !paths ||
+      configKinds.has('site') ||
+      configKinds.has('nav')
+    ) {
+      return this.buildFull();
     }
 
-    const scan = updateProjectScan(snapshot.scan, batch);
-    const plan = createTadaWatchPlan({ snapshot, batch, scan });
+    const authorsChanged = configKinds.has('authors');
+    const nextAuthors = authorsChanged
+      ? loadProjectConfig(
+          getProjectConfigDir(),
+          'authors',
+          snapshot.siteVariables,
+        )?.value
+      : snapshot.authorsData;
+    // Every build rescans the source directories instead of reconciling
+    // individual file and directory events.
+    const scan = scanProject(snapshot.siteVariables);
+    const plan = createTadaWatchPlan({
+      snapshot,
+      paths,
+      scan,
+      full:
+        authorsChanged &&
+        (snapshot.authorsData === undefined || nextAuthors === undefined),
+      changedAuthors: authorsChanged
+        ? diffAuthorKeys(snapshot.authorsData, nextAuthors)
+        : new Set(),
+    });
     if (plan.kind === 'full') {
-      return buildFull({
-        traceCache: this.traceCache,
-        traceOptions: this.traceOptions,
-      });
+      // Only the authors file changed, so the site config and scan still hold.
+      return this.buildFull({ siteVariables: snapshot.siteVariables, scan });
     }
 
     return buildIncremental({
       plan,
       snapshot,
       traceCache: this.traceCache,
-      traceOptions: this.traceOptions,
+      traceToolAvailability: this.traceToolAvailability,
     });
   }
 }
+
+/**
+ * Writes a successful build into `dist/` in place. Full builds compare
+ * against the files on disk; incremental builds against the previous build.
+ */
+function writeBuild(
+  outcome: Extract<CompileOutcome, { ok: true }>,
+  previous: TadaSnapshot | undefined,
+  distDir: string = getDistDir(),
+): void {
+  const keep = getKeptOutputPrefixes(outcome.snapshot.siteVariables);
+  applyMutations(
+    distDir,
+    outcome.full || !previous
+      ? planFullWrite(distDir, outcome.snapshot.outputs, keep)
+      : computeMutations(
+          previous.outputs,
+          outcome.snapshot.outputs,
+          outcome.forceSourcePaths,
+        ),
+  );
+}
+
+export function createBuildSession(
+  compile: (
+    snapshot: TadaSnapshot | undefined,
+    paths?: ReadonlySet<string>,
+  ) => Promise<CompileOutcome>,
+  write: typeof writeBuild = writeBuild,
+): (paths?: ReadonlySet<string>) => Promise<WatchBuildResult<TadaBuildMeta>> {
+  let snapshot: TadaSnapshot | undefined;
+  return async (
+    paths?: ReadonlySet<string>,
+  ): Promise<WatchBuildResult<TadaBuildMeta>> => {
+    const outcome = await compile(snapshot, paths);
+    if (!outcome.ok) {
+      return outcome;
+    }
+    try {
+      write(outcome, snapshot);
+    } catch (error) {
+      // The next change rebuilds everything against what is on disk.
+      snapshot = undefined;
+      const message = error instanceof Error ? error.message : String(error);
+      return {
+        ok: false,
+        diagnostics: [{ message: `Failed to write output: ${message}` }],
+      };
+    }
+    snapshot = outcome.snapshot;
+    return { ok: true, meta: createBuildMeta(outcome.snapshot) };
+  };
+}

package/build/watch/engine.ts

@@ -1,199 +1,244 @@
+import path from 'path';
+import fs from 'fs';
 import chokidar from 'chokidar';
-import { applyCommitPlan } from './fs-commit';
 import type {
-  ChangeBatch,
-  CompilerBuildResult,
+  WatchBuildResult,
+  WatchClock,
   WatchEngineOptions,
-  WatchEventKind,
+  WatchHandle,
   WatchLifecycleEvent,
 } from './types';
-import type { TadaSnapshot } from './snapshot';
 
-function normalizeBatch(changes: Map<string, WatchEventKind>): ChangeBatch {
-  return {
-    changes: [...changes.entries()]
-      .sort(([a], [b]) => a.localeCompare(b))
-      .map(([path, kind]) => ({ path, kind })),
-  };
-}
-
-function mapBatch(batch: ChangeBatch): Map<string, WatchEventKind> {
-  return new Map(batch.changes.map(change => [change.path, change.kind]));
-}
-
-function mergeChangeKind(
-  previous: WatchEventKind | undefined,
-  next: WatchEventKind,
-): WatchEventKind {
-  if (!previous) {
-    return next;
-  }
-  if (previous === 'add' && next === 'unlink') {
-    return 'change';
-  }
-  if (previous === 'unlink' && next === 'add') {
-    return 'change';
-  }
-  if (previous === 'add' || next === 'add') {
-    return 'add';
-  }
-  if (previous === 'unlink' || next === 'unlink') {
-    return 'unlink';
+export function runWatchEngine<Meta>(
+  options: WatchEngineOptions<Meta>,
+  dependencies = {
+    watch: chokidar.watch,
+    stat: (filePath: string): Pick<fs.Stats, 'isFile'> | undefined =>
+      fs.statSync(filePath, { throwIfNoEntry: false }),
+    clock: { setTimeout, clearTimeout } as WatchClock,
+  },
+): WatchHandle {
+  const { clock } = dependencies;
+  const debounceMs = options.debounceMs ?? 300;
+  const watchers: ReturnType<typeof chokidar.watch>[] = [];
+  const pending = new Set<string>();
+  let uncommitted = new Set<string>();
+  let closed = false;
+  let fatal: { error: unknown } | undefined;
+  let closing: Promise<PromiseSettledResult<void>[]> | undefined;
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  let wake: (() => void) | undefined;
+  const stopped = Promise.withResolvers<void>();
+
+  function stop(): void {
+    closed = true;
+    pending.clear();
+    if (timer !== undefined) {
+      clock.clearTimeout(timer);
+    }
+    timer = undefined;
+    wake?.();
+    stopped.resolve();
+    closing ??= Promise.allSettled(
+      watchers.map(async watcher => watcher.close()),
+    );
   }
-  return 'change';
-}
 
-function toDiagnostics(error: unknown): { message: string }[] {
-  if (error instanceof Error) {
-    return [{ message: error.message }];
+  function fail(error: unknown): void {
+    fatal ??= { error };
+    stop();
   }
-  return [{ message: String(error) }];
-}
 
-export async function runWatchEngine(
-  options: WatchEngineOptions,
-): Promise<void> {
-  const debounceMs = options.debounceMs ?? 300;
-  let snapshot: TadaSnapshot | undefined;
-  let uncommitted = new Map<string, WatchEventKind>();
-  let pending = new Map<string, WatchEventKind>();
-  let timer: ReturnType<typeof setTimeout> | null = null;
-  let running = false;
+  function wait(quiet = false): Promise<void> {
+    if (closed) {
+      return Promise.resolve();
+    }
+    return new Promise(resolve => {
+      wake = () => {
+        wake = undefined;
+        timer = undefined;
+        resolve();
+      };
+      if (quiet) {
+        timer = clock.setTimeout(() => wake?.(), debounceMs);
+      }
+    });
+  }
 
-  async function emit(event: WatchLifecycleEvent): Promise<void> {
-    await options.onEvent?.(event);
+  function changed(filePath: string): void {
+    if (closed) {
+      return;
+    }
+    pending.add(path.resolve(filePath));
+    if (timer !== undefined) {
+      clock.clearTimeout(timer);
+      timer = clock.setTimeout(() => wake?.(), debounceMs);
+    } else {
+      wake?.();
+    }
   }
 
-  async function waitForQuietPeriod(): Promise<void> {
-    await new Promise(resolve => setTimeout(resolve, debounceMs));
+  async function emit(event: WatchLifecycleEvent<Meta>): Promise<void> {
+    if (!closed) {
+      await options.onEvent?.(event);
+    }
   }
 
-  async function runBuild(batch?: ChangeBatch): Promise<CompilerBuildResult> {
-    await emit({ kind: 'build-started', batch });
+  async function build(
+    paths?: ReadonlySet<string>,
+  ): Promise<WatchBuildResult<Meta> | undefined> {
+    await emit({ kind: 'build-started', paths });
+    if (closed) {
+      return;
+    }
+    let outcome: WatchBuildResult<Meta>;
     try {
-      return await options.compiler.build(snapshot, batch);
+      outcome = await options.build(paths);
     } catch (error) {
-      return { ok: false, diagnostics: toDiagnostics(error) };
+      outcome = {
+        ok: false,
+        diagnostics: [
+          { message: error instanceof Error ? error.message : String(error) },
+        ],
+      };
+    }
+    if (!outcome.ok) {
+      await emit({
+        kind: 'build-failed',
+        paths,
+        diagnostics: outcome.diagnostics,
+      });
     }
+    return outcome;
   }
 
-  async function flush(): Promise<void> {
-    if (running) {
+  async function run(): Promise<void> {
+    if (closed) {
       return;
     }
-    running = true;
-    try {
-      let finalSuccess:
-        | {
-            batch?: ChangeBatch;
-            result: Extract<CompilerBuildResult, { ok: true }>;
+    const ready = options.targets.map(target => {
+      if (closed) {
+        return Promise.resolve();
+      }
+      const watcher = dependencies.watch(target.path, {
+        ignoreInitial: true,
+        atomic: true,
+        awaitWriteFinish: { stabilityThreshold: 100 },
+        ...target.chokidar,
+      });
+      watchers.push(watcher);
+      const included = (filePath: string) => {
+        if (closed) {
+          return;
+        }
+        try {
+          if (!target.filter || target.filter(filePath)) {
+            changed(filePath);
           }
-        | undefined;
-
-      while (pending.size > 0) {
-        const changes = new Map(uncommitted);
-        for (const [filePath, kind] of pending) {
-          changes.set(filePath, mergeChangeKind(changes.get(filePath), kind));
+        } catch (error) {
+          fail(error);
         }
-
-        const batch = normalizeBatch(changes);
-        pending = new Map();
-        const outcome = await runBuild(batch);
-
-        if (!outcome.ok) {
-          uncommitted = mapBatch(batch);
-          await emit({
-            kind: 'build-failed',
-            batch,
-            diagnostics: outcome.diagnostics,
-          });
+      };
+      watcher.on('add', included).on('unlink', included);
+      watcher.on('change', (filePath, stats) => {
+        if (closed) {
           return;
         }
-
-        applyCommitPlan(outcome.commit);
-        snapshot = outcome.snapshot;
-        uncommitted = new Map();
-        finalSuccess = { batch, result: outcome };
-
-        if (pending.size === 0) {
-          await waitForQuietPeriod();
+        // A polling file subscription does not become recursive on its own.
+        try {
+          if (stats?.isDirectory()) {
+            watcher.add(filePath);
+          }
+          included(filePath);
+        } catch (error) {
+          fail(error);
         }
-      }
-
-      if (finalSuccess) {
-        await emit({
-          kind: 'build-succeeded',
-          batch: finalSuccess.batch,
-          meta: finalSuccess.result.meta,
+      });
+      watcher.on('addDir', included);
+      watcher.on('unlinkDir', filePath => {
+        included(filePath);
+        // Wait until Chokidar finishes closing the old directory subscription.
+        queueMicrotask(() => {
+          if (closed) {
+            return;
+          }
+          try {
+            if (dependencies.stat(filePath)?.isFile()) {
+              watcher.add(filePath);
+            }
+          } catch (error) {
+            if ((error as NodeJS.ErrnoException).code !== 'ENOTDIR') {
+              fail(error);
+            }
+          }
         });
+      });
+      watcher.on('error', fail);
+      return new Promise<void>(resolve => watcher.once('ready', resolve));
+    });
+    await Promise.race([Promise.all(ready), stopped.promise]);
+    if (closed) {
+      return;
+    }
+    const startup = await build();
+    if (startup?.ok) {
+      await emit({ kind: 'build-succeeded', meta: startup.meta });
+    }
+    await emit({ kind: 'watching' });
+    while (!closed) {
+      if (!pending.size) {
+        await wait();
       }
-    } finally {
-      running = false;
-      if (pending.size > 0) {
-        schedule();
+      if (closed) {
+        break;
+      }
+      await wait(true);
+      let success: { paths: ReadonlySet<string>; meta: Meta } | undefined;
+      while (!closed && pending.size) {
+        const paths = new Set([...uncommitted, ...pending]);
+        pending.clear();
+        const outcome = await build(paths);
+        if (!outcome?.ok) {
+          uncommitted = paths;
+          success = undefined;
+          break;
+        }
+        uncommitted.clear();
+        success = { paths, meta: outcome.meta };
+        if (!pending.size) {
+          await wait(true);
+        }
+      }
+      if (success) {
+        await emit({ kind: 'build-succeeded', ...success });
       }
     }
   }
 
-  function schedule(): void {
-    if (timer) {
-      clearTimeout(timer);
+  const done = (async () => {
+    // Defer setup so callers can install their completion handler first.
+    await Promise.resolve();
+    try {
+      await run();
+    } catch (error) {
+      fatal ??= { error };
+    } finally {
+      stop();
+      const results = await closing!;
+      const failure = results.find(result => result.status === 'rejected');
+      if (failure?.status === 'rejected') {
+        fatal ??= { error: failure.reason };
+      }
     }
-    timer = setTimeout(() => {
-      timer = null;
-      void flush();
-    }, debounceMs);
-  }
-
-  function onFileChange(filePath: string, kind: WatchEventKind): void {
-    pending.set(filePath, mergeChangeKind(pending.get(filePath), kind));
-    schedule();
-  }
-
-  const startupOutcome = await runBuild();
-  if (!startupOutcome.ok) {
-    await emit({
-      kind: 'build-failed',
-      diagnostics: startupOutcome.diagnostics,
-    });
-  } else {
-    applyCommitPlan(startupOutcome.commit);
-    snapshot = startupOutcome.snapshot;
-    await emit({ kind: 'build-succeeded', meta: startupOutcome.meta });
-  }
-
-  const watcherEntries = options.compiler
-    .getWatchTargets()
-    .map(target => ({
-      target,
-      watcher: chokidar.watch(target.path, {
-        ignoreInitial: true,
-        atomic: true,
-        awaitWriteFinish: { stabilityThreshold: 100 },
-        ...target.chokidar,
-      }),
-    }));
-
-  let readyCount = 0;
-  const readyGoal = watcherEntries.length;
-  const onReady = async () => {
-    readyCount += 1;
-    if (readyCount === readyGoal) {
-      await emit({ kind: 'watching' });
+    if (fatal) {
+      throw fatal.error;
     }
+  })();
+  return {
+    done,
+    close: () => {
+      stop();
+      return done;
+    },
   };
-
-  for (const { watcher, target } of watcherEntries) {
-    const emitIfIncluded = (filePath: string, kind: WatchEventKind) => {
-      if (!target.filter || target.filter(filePath)) {
-        onFileChange(filePath, kind);
-      }
-    };
-    watcher.on('add', filePath => emitIfIncluded(filePath, 'add'));
-    watcher.on('change', filePath => emitIfIncluded(filePath, 'change'));
-    watcher.on('unlink', filePath => emitIfIncluded(filePath, 'unlink'));
-    watcher.on('ready', () => void onReady());
-  }
-
-  return new Promise(() => {});
 }

package/build/watch/index.ts

@@ -9,9 +9,16 @@ export async function runWatch(options: { httpPort?: number }): Promise<void> {
     distDir: getDistDir(),
   });
 
-  await runWatchEngine({
-    compiler: new TadaWatchCompiler(),
+  const compiler = new TadaWatchCompiler();
+  const handle = runWatchEngine({
+    targets: compiler.getWatchTargets(),
+    build: paths => compiler.build(paths),
     onEvent: event => runtime.onEvent(event),
     debounceMs: 300,
   });
+  try {
+    await handle.done;
+  } finally {
+    runtime.close();
+  }
 }