@akanjs/cli 3.0.0-beta.9 → 3.0.0

package/incrementalBuilder.proc.js

@@ -1,41 +1,41 @@
 // @bun
+import {
+  ApplicationBuildReporter,
+  HmrWatcher,
+  WatchRootResolver
+} from "./index-48k96p55.js";
 import {
   CodegenLock,
   WorkspaceExecutor,
   AppExecutor
-} from "./index-2tv3fdx4.js";
-import"./index-q0y2jq9j.js";
-import"./index-6wwzs1bc.js";
+} from "./index-3w94e7n3.js";
+import"./index-52w0fvh8.js";
 import {
   GraphClientEntryDiscovery,
   RouteClientBuilder,
   AutoImportSync,
+  ServerGraphFile,
+  CsrDevPatcher,
+  CsrDevBundler,
   DevChangePlanner,
-  DevGeneratedIndexSync
-} from "./index-w8kmx1b6.js";
-import {
-  HmrWatcher,
-  WatchRootResolver
-} from "./index-hwkygn9s.js";
+  DevGeneratedIndexSync,
+  SsrDevBundler
+} from "./index-6he6gvfg.js";
 import {
   prepareDevWatchBatch
 } from "./index-a5vmm6x4.js";
 
 // pkgs/@akanjs/devkit/incrementalBuilder/incrementalBuilder.proc.ts
+import { rm } from "fs/promises";
 import path2 from "path";
 import { Logger as Logger2 } from "akanjs/common";
+import { CSR_DEV_PATCHING_MARKER } from "akanjs/server/hmr/csrDevManifest";
 
 // pkgs/@akanjs/devkit/incrementalBuilder/buildBatchRunner.ts
 import path from "path";
 import { Logger } from "akanjs/common";
 
 class BuildBatchRunner {
-  static #entryCandidates = (workspaceRoot) => [
-    path.join(workspaceRoot, "pkgs/@akanjs/devkit/incrementalBuilder/buildBatch.proc.ts"),
-    path.join(workspaceRoot, "node_modules/@akanjs/devkit/incrementalBuilder/buildBatch.proc.ts"),
-    path.join(import.meta.dir, "buildBatch.proc.js"),
-    path.join(import.meta.dir, "buildBatch.proc.ts")
-  ];
   #logger = new Logger("BuildBatchRunner");
   #entry = null;
   #workspaceRoot;
@@ -47,7 +47,12 @@ class BuildBatchRunner {
   async#resolveEntry() {
     if (this.#entry)
       return this.#entry;
-    const candidates = BuildBatchRunner.#entryCandidates(this.#workspaceRoot);
+    const candidates = [
+      path.join(this.#workspaceRoot, "pkgs/@akanjs/devkit/incrementalBuilder/buildBatch.proc.ts"),
+      path.join(this.#workspaceRoot, "node_modules/@akanjs/devkit/incrementalBuilder/buildBatch.proc.ts"),
+      path.join(import.meta.dir, "buildBatch.proc.js"),
+      path.join(import.meta.dir, "buildBatch.proc.ts")
+    ];
     for (const candidate of candidates) {
       if (!await Bun.file(candidate).exists())
         continue;
@@ -62,7 +67,9 @@ class BuildBatchRunner {
     const started = Date.now();
     const entry = await this.#resolveEntry();
     let result = null;
-    const proc = Bun.spawn(["bun", entry, JSON.stringify(request)], {
+    const reported = new Map;
+    const payload = request.trace ? { ...request, trace: { ...request.trace, spawnAt: Date.now() } } : request;
+    const proc = Bun.spawn(["bun", entry, JSON.stringify(payload)], {
       cwd: this.#cwd,
       env: process.env,
       stdio: ["ignore", "inherit", "inherit"],
@@ -72,8 +79,11 @@ class BuildBatchRunner {
           return;
         if (message.type === "build-batch-result")
           result = message.data;
-        else
+        else {
+          if (message.type === "build-status")
+            reported.set(message.data.phase, message.data.message);
           onMessage(message);
+        }
       }
     });
     const exitCode = await proc.exited;
@@ -83,11 +93,14 @@ class BuildBatchRunner {
     }
     const message = proc.signalCode ? `build worker was killed by ${proc.signalCode} before reporting a result${proc.signalCode === "SIGKILL" ? " \u2014 most often the kernel OOM killer; check the sandbox's memory limit" : ""}` : `build worker exited with code ${exitCode} before reporting a result`;
     this.#logger.error(`[build-batch] generation=${request.generation} ${message}`);
-    return {
-      generation: request.generation,
-      errors: Object.fromEntries(request.needs.map((need) => [need, message])),
-      crashed: true
-    };
+    const crashedNeeds = request.needs.filter((need) => !reported.has(need));
+    const failed = request.needs.flatMap((need) => {
+      if (!reported.has(need))
+        return [[need, message]];
+      const error = reported.get(need);
+      return error ? [[need, error]] : [];
+    });
+    return { generation: request.generation, errors: Object.fromEntries(failed), crashed: true, crashedNeeds };
   }
 }
 
@@ -124,6 +137,90 @@ class BuilderChannel {
   }
 }
 
+// pkgs/@akanjs/devkit/incrementalBuilder/builderWorkQueue.ts
+class BuilderWorkQueue {
+  #runBatch;
+  #onSettled;
+  #pending = [];
+  #running = null;
+  #idle = Promise.resolve();
+  #wakeIdle = null;
+  constructor({ runBatch, onSettled }) {
+    this.#runBatch = runBatch;
+    this.#onSettled = onSettled;
+  }
+  get size() {
+    return this.#pending.length + (this.#running ? 1 : 0);
+  }
+  enqueueBatch(batch) {
+    const tail = this.#pending.at(-1);
+    if (tail?.batch) {
+      tail.batch = BuilderWorkQueue.merge(tail.batch, batch);
+      return tail.settle.promise;
+    }
+    const job = this.#job("batch", batch, async () => {
+      return;
+    });
+    job.run = async () => await this.#runBatch(job.batch ?? batch);
+    return this.#push(job);
+  }
+  enqueue(label, run) {
+    let result;
+    const job = this.#job(label, null, async () => {
+      result = await run();
+    });
+    return this.#push(job).then(() => result);
+  }
+  async drain() {
+    while (this.size > 0)
+      await this.#idle;
+  }
+  static merge(into, next) {
+    const discovery = into.discovery || next.discovery ? {
+      files: [...new Set([...into.discovery?.files ?? [], ...next.discovery?.files ?? []])],
+      refresh: !!into.discovery?.refresh || !!next.discovery?.refresh,
+      generation: Math.max(into.discovery?.generation ?? 0, next.discovery?.generation ?? 0)
+    } : undefined;
+    return {
+      generation: Math.max(into.generation, next.generation),
+      needs: [...new Set([...into.needs, ...next.needs])],
+      changedFiles: [...new Set([...into.changedFiles, ...next.changedFiles])],
+      trace: next.trace ?? into.trace,
+      ...discovery ? { discovery } : {}
+    };
+  }
+  #job(label, batch, run) {
+    return { label, batch, run, settle: Promise.withResolvers() };
+  }
+  #push(job) {
+    if (this.size === 0)
+      this.#idle = new Promise((resolve) => {
+        this.#wakeIdle = resolve;
+      });
+    this.#pending.push(job);
+    if (!this.#running)
+      this.#pump();
+    return job.settle.promise;
+  }
+  async#pump() {
+    for (let job = this.#pending.shift();job; job = this.#pending.shift()) {
+      this.#running = job;
+      const started = Date.now();
+      try {
+        await job.run();
+        job.settle.resolve();
+      } catch (error) {
+        job.settle.reject(error);
+      } finally {
+        this.#running = null;
+        this.#onSettled?.(job.label, Date.now() - started);
+      }
+    }
+    this.#wakeIdle?.();
+    this.#wakeIdle = null;
+  }
+}
+
 // pkgs/@akanjs/devkit/incrementalBuilder/incrementalBuilder.proc.ts
 class IncrementalBuilder {
   #logger = new Logger2("IncrementalBuilder");
@@ -138,8 +235,29 @@ class IncrementalBuilder {
   #autoImportSync;
   #watcher = null;
   #generation = 0;
+  #discoveredThrough = 0;
   #csrActive = IncrementalBuilder.#csrArmedByEnv();
-  #workQueue = Promise.resolve();
+  #csrBundler;
+  #patcher;
+  #csrGate = Promise.resolve();
+  #csrArming = false;
+  #csrArms = 0;
+  #crashRetryTimer = null;
+  #crashRetryEntries = new Set;
+  #csrWasActive = false;
+  #csrArmSucceeded = false;
+  #ssrBundler;
+  #ssrPatcher;
+  #ssrLock = Promise.resolve();
+  #delegations = new Map;
+  #ssrArming = null;
+  #ssrBroken = false;
+  #ssrCrashedAt = null;
+  static #crashRetryMs = 1e4;
+  #beside = new Set;
+  #barrelFailed = false;
+  #fastQueue = Promise.resolve();
+  #workQueue;
   #inFlight = 0;
   #workCount = 0;
   #shuttingDown = false;
@@ -153,6 +271,7 @@ class IncrementalBuilder {
     this.#optimizedFonts = options.optimizedFonts;
     this.#discovery = options.discovery;
     this.#generation = options.initialGeneration ?? 0;
+    this.#discoveredThrough = this.#generation;
     this.#batchRunner = new BuildBatchRunner({
       workspaceRoot: options.app.workspace.workspaceRoot,
       cwd: options.app.cwdPath
@@ -160,25 +279,41 @@ class IncrementalBuilder {
     this.#changePlanner = new DevChangePlanner({ workspaceRoot: options.app.workspace.workspaceRoot });
     this.#generatedIndexSync = new DevGeneratedIndexSync({ workspaceRoot: options.app.workspace.workspaceRoot });
     this.#autoImportSync = new AutoImportSync({ workspaceRoot: options.app.workspace.workspaceRoot });
+    this.#csrBundler = new CsrDevBundler(options.app);
+    this.#patcher = process.env.AKAN_DEV_CSR_PATCHER === "off" ? null : new CsrDevPatcher(this.#csrBundler, { resident: true });
+    this.#ssrBundler = new SsrDevBundler(options.app, { discovery: () => this.#discovery });
+    this.#ssrPatcher = process.env.AKAN_DEV_CSR_PATCHER === "off" ? null : new CsrDevPatcher(this.#ssrBundler, { resident: true });
+    this.#workQueue = new BuilderWorkQueue({
+      runBatch: async (batch) => await this.#runQueuedBatch(batch),
+      onSettled: (label, ms) => {
+        this.#workCount += 1;
+        this.#logger.verbose(`[work-queue] ${label} finished in ${ms}ms`);
+        this.#reportMetrics();
+      }
+    });
   }
   get #artifactDir() {
     return `${this.#app.cwdPath}/.akan/artifact`;
   }
   async handleBuildRoute(msg) {
-    await this.#enqueueWork(`build-route:${msg.routeId}`, async () => BuilderChannel.send(await this.#handleBuildRoute(msg)));
+    await this.#workQueue.enqueue(`build-route:${msg.routeId}`, async () => BuilderChannel.send(await this.#handleBuildRoute(msg)));
   }
   async#handleBuildRoute(msg) {
+    const seenGeneration = this.#discoveredThrough;
     try {
       const delta = await new RouteClientBuilder({
         app: this.#app,
         routeId: msg.routeId,
         seeds: msg.seeds,
+        graphSeeds: msg.graphSeeds,
         artifact: this.#artifact,
         knownEntries: new Set(msg.knownEntries),
-        discovery: this.#discovery
+        discovery: this.#discovery,
+        browser: "registry"
       }).build();
+      await this.#ensureSsrEntries(delta.registryEntries ?? []);
       this.#logger.verbose(`build-route ok routeId=${msg.routeId} newEntries=${delta.newEntries.length}`);
-      this.#sendBuildStatus("route", { generation: msg.generation, ok: true, files: msg.seeds });
+      this.#sendBuildStatus("route", { generation: this.#generation, ok: true, files: msg.seeds, scope: msg.routeId });
       return {
         type: "build-route-res",
         id: msg.id,
@@ -191,35 +326,42 @@ class IncrementalBuilder {
           clientDeps: delta.clientDeps,
           clientDepsByEntry: delta.clientDepsByEntry,
           routeId: msg.routeId,
-          generation: msg.generation
+          generation: msg.generation,
+          seenGeneration
         }
       };
     } catch (err) {
-      const errMsg = err instanceof Error ? err.message : String(err);
+      const errMsg = ApplicationBuildReporter.formatError(err, this.#app.workspace.workspaceRoot);
       this.#logger.error(`build-route failed routeId=${msg.routeId}: ${errMsg}`);
-      this.#sendBuildStatus("route", { generation: msg.generation, ok: false, files: msg.seeds, message: errMsg });
+      this.#sendBuildStatus("route", {
+        generation: this.#generation,
+        ok: false,
+        files: msg.seeds,
+        message: errMsg,
+        scope: msg.routeId
+      });
       return { type: "build-route-res", id: msg.id, ok: false, error: errMsg };
     }
   }
-  #sendBuildStatus(phase, { generation, ok, files, message }) {
+  #sendBuildStatus(phase, {
+    generation,
+    ok,
+    files,
+    message,
+    scope
+  }) {
     if (typeof generation !== "number")
       return;
     BuilderChannel.emit({
       type: "build-status",
-      data: {
-        generation,
-        phase,
-        ok,
-        files: files ?? [],
-        message
-      }
+      data: { generation, phase, ok, files: files ?? [], message, ...scope !== undefined ? { scope } : {} }
     });
   }
-  async#enqueueWork(label, fn) {
+  async#enqueueFast(label, fn) {
     const started = Date.now();
     this.#inFlight += 1;
-    const run = this.#workQueue.then(fn, fn);
-    this.#workQueue = run.then(() => {
+    const run = this.#fastQueue.then(fn, fn);
+    this.#fastQueue = run.then(() => {
       return;
     }).catch(() => {
       return;
@@ -229,12 +371,22 @@ class IncrementalBuilder {
     } finally {
       this.#inFlight -= 1;
       this.#workCount += 1;
-      this.#logger.verbose(`[work-queue] ${label} finished in ${Date.now() - started}ms`);
+      this.#logger.verbose(`[fast-queue] ${label} finished in ${Date.now() - started}ms`);
       this.#reportMetrics();
     }
   }
   get #idle() {
-    return this.#inFlight === 0 && this.#cssRebuildTimer === null;
+    return this.#inFlight === 0 && this.#workQueue.size === 0 && this.#cssRebuildTimer === null;
+  }
+  #runBeside(work) {
+    this.#inFlight += 1;
+    const tracked = work.finally(() => {
+      this.#inFlight -= 1;
+      this.#beside.delete(tracked);
+      this.#reportMetrics();
+    });
+    this.#beside.add(tracked);
+    return tracked;
   }
   #reportMetrics() {
     if (!this.#idle || this.#shuttingDown)
@@ -256,12 +408,17 @@ class IncrementalBuilder {
       this.#cssRebuildTimer = null;
       this.#pendingCssRebuild = null;
     }
-    await this.#workQueue.catch(() => {
+    await this.#fastQueue.catch(() => {
       return;
     });
+    await this.#workQueue.drain();
     await this.#cssRebuildQueue.catch(() => {
       return;
     });
+    while (this.#beside.size > 0)
+      await Promise.all([...this.#beside]);
+    if (this.#delegations.size > 0)
+      this.#logger.verbose(`registry saves handed to build workers this session: ${[...this.#delegations].map(([reason, count]) => `${reason}=${count}`).join(", ")}`);
     const flushed = await BuilderChannel.drain();
     this.#logger.debug(`drained in ${Date.now() - started}ms${flushed ? ` after flushing ${flushed} ipc write(s)` : ""}; exiting for recycle`);
     process.exit(0);
@@ -335,24 +492,24 @@ class IncrementalBuilder {
     }, 150);
   }
   async installWatcher() {
-    const artifactDir = this.#artifactDir;
     const roots = await new WatchRootResolver(this.#app).resolve();
     const watcher = new HmrWatcher({
       roots,
       logger: this.#logger,
       onBatch: async (batch) => {
-        await this.#enqueueWork("hmr-batch", async () => this.#handleWatchBatch(artifactDir, batch));
+        await this.#enqueueFast("hmr-batch", async () => this.#handleWatchBatch(batch));
       }
     });
     await watcher.start();
     this.#watcher = watcher;
     this.#logger.verbose(`watching ${roots.length} roots`);
   }
-  async#handleWatchBatch(artifactDir, batch) {
+  async#handleWatchBatch(batch) {
     const rawKinds = new Set(batch.kinds);
     if (rawKinds.size === 0)
       return;
     const generation = ++this.#generation;
+    const trace = { ...batch.trace, batchAt: Date.now() };
     const [autoImport, indexSync] = await CodegenLock.run(this.#app.workspace.workspaceRoot, `hmr-batch:${this.#app.name}`, async () => {
       const auto = await this.#autoImportSync.syncForBatch(batch.files);
       return [auto, await this.#generatedIndexSync.syncForBatch(batch.files)];
@@ -371,22 +528,19 @@ class IncrementalBuilder {
     this.#logger.verbose(`[hmr] batch generation=${generation} kinds=${kinds.join(",")} files=${files.length} generated=${indexSync.changedFiles.length} roles=${devPlan.roles.join(",") || "(none)"} actions=${devPlan.actions.join(",") || "(none)"}`);
     for (const error of indexSync.errors)
       this.#logger.error(error);
-    if (kinds.includes("code")) {
-      const started = Date.now();
-      if (kinds.includes("config"))
-        this.#discovery = await GraphClientEntryDiscovery.create(this.#app);
-      else
-        this.#discovery.invalidate?.(files);
-      this.#logger.verbose(`client-entry-discovery ${kinds.includes("config") ? "refreshed" : "invalidated"} (${Date.now() - started}ms)`);
-    }
+    const discovery = kinds.includes("code") ? { files, refresh: kinds.includes("config"), generation } : undefined;
     if (hasSyncErrors) {
+      this.#barrelFailed = true;
       this.#sendBuildStatus("barrel", { generation, ok: false, files, message: indexSync.errors.join(`
 `) });
       BuilderChannel.emit(event);
+      if (discovery)
+        this.#workQueue.enqueue("discovery", async () => await this.#refreshDiscovery(discovery)).catch(this.#slowLaneFailed("discovery"));
       return;
     }
-    if (indexSync.changedFiles.length > 0)
+    if (indexSync.changedFiles.length > 0 || this.#barrelFailed)
       this.#sendBuildStatus("barrel", { generation, ok: true, files });
+    this.#barrelFailed = false;
     const rebuildClient = devPlan.actions.includes("rebuild-client");
     if (kinds.includes("code") && !rebuildClient) {
       this.#logger.verbose(`client rebuild skipped; devPlan actions=${devPlan.actions.join(",") || "(none)"}`);
@@ -399,48 +553,318 @@ class IncrementalBuilder {
     } else if (kinds.includes("code") && rebuildClient && this.batchTouchesPagesTree(pageRoots, expandedBatch)) {
       this.#logger.verbose("pageKeys refresh skipped; changed page source cannot add/remove a route key");
     }
-    const needs = [];
+    BuilderChannel.emit(event);
     if (kinds.includes("code") && rebuildClient) {
-      if (this.#shouldRebuildCsr())
-        needs.push("csr");
-      else
+      const needs = ["pages", "css"];
+      if (!this.#csrActive)
         this.#logger.verbose(`csr-rebundle skipped; request /__csr or ?csr=true (or set AKAN_DEV_CSR_REBUILD=1) to enable per-save CSR rebuilds`);
-      needs.push("pages");
-      needs.push("css");
+      const patchCsr = async () => this.#csrActive ? await this.#patchCsr(generation, files, trace) : false;
+      const patchSsr = async () => await this.#patchSsr(generation, files, trace);
+      let csrToWorker;
+      let ssrToWorker;
+      if (this.#csrArming) {
+        ssrToWorker = await patchSsr();
+        csrToWorker = await patchCsr();
+      } else {
+        csrToWorker = await patchCsr();
+        ssrToWorker = await patchSsr();
+      }
+      if (csrToWorker)
+        needs.unshift("csr");
+      if (ssrToWorker)
+        needs.unshift("ssr");
+      const batch = { generation, needs, changedFiles: files, trace, ...discovery ? { discovery } : {} };
+      if (needs.includes("csr") || needs.includes("ssr"))
+        await this.#workQueue.enqueueBatch(batch);
+      else
+        this.#workQueue.enqueueBatch(batch).catch(this.#slowLaneFailed("batch"));
+      return;
     }
-    BuilderChannel.emit(event);
-    if (needs.length > 0)
-      await this.#runBatch({ generation, needs, changedFiles: files });
-    else if (kinds.includes("css")) {
+    if (discovery)
+      this.#workQueue.enqueue("discovery", async () => await this.#refreshDiscovery(discovery)).catch(this.#slowLaneFailed("discovery"));
+    if (kinds.includes("css")) {
       this.scheduleCssRebuild({ generation, changedFiles: files });
       this.#logger.verbose(`css-rebuild scheduled generation=${generation}`);
     }
   }
-  async#runBatch({
-    generation,
-    needs,
-    changedFiles
-  }) {
+  async#patchCsr(generation, files, trace) {
+    const patcher = this.#patcher;
+    if (!patcher)
+      return true;
+    await this.#csrGate;
+    if (!this.#csrActive)
+      return false;
     const started = Date.now();
-    const result = await this.#batchRunner.run(await this.#batchRequest({ generation, needs, changedFiles }), (msg) => BuilderChannel.emit(msg));
+    const marker = path2.join(this.#csrBundler.outDir, CSR_DEV_PATCHING_MARKER);
+    await Bun.write(marker, String(process.pid));
+    try {
+      const result = await patcher.update(files, {
+        announce: (update) => {
+          const now = Date.now();
+          BuilderChannel.emit({
+            type: "csr-updated",
+            data: {
+              generation: update.generation,
+              mode: "registry",
+              reload: update.reload,
+              reason: update.reason,
+              patchUrl: update.patchUrl,
+              changedIds: update.changedIds,
+              trace: { ...trace, patchAt: now, sentAt: now }
+            }
+          });
+        }
+      });
+      if (result.kind === "delegate") {
+        this.#logger.verbose(`csr-patch handed to a build worker (${this.#countDelegation("csr", result.reason)}): ${result.reason}`);
+        return true;
+      }
+      this.#sendBuildStatus("csr", { generation, ok: true, files });
+      this.#logger.verbose(result.kind === "unchanged" ? `csr-patch unchanged (${Date.now() - started}ms)` : `csr-patch generation=${result.update.generation} ${result.update.reload ? `reload (${result.update.reason})` : `patch modules=${result.update.changedIds.length}`} (${Date.now() - started}ms)`);
+      return false;
+    } catch (err) {
+      const message = ApplicationBuildReporter.formatError(err, this.#app.workspace.workspaceRoot);
+      if (IncrementalBuilder.#isCompileError(err)) {
+        this.#sendBuildStatus("csr", { generation, ok: false, files, message });
+        return false;
+      }
+      this.#logger.verbose(`csr-patch threw; a build worker takes this save: ${message}`);
+      patcher.forget();
+      return true;
+    } finally {
+      await rm(marker, { force: true });
+    }
+  }
+  async#patchSsr(generation, files, trace) {
+    const patcher = this.#ssrPatcher;
+    if (!patcher)
+      return true;
+    const hold = await ServerGraphFile.touches(await ServerGraphFile.read(this.#artifactDir), files, (file) => ServerGraphFile.clientExportsOf(file));
+    this.#discovery.invalidate?.(files);
+    return await this.#withSsrLock(async () => await this.#runSsrPatcher(patcher, files, { trace, hold, batchGeneration: generation }));
+  }
+  async#ensureSsrEntries(entries, { crashRetry = false } = {}) {
+    if (entries.length === 0)
+      return;
+    const arming = this.#ssrArming;
+    if (arming) {
+      this.#runBeside(arming.then(async () => await this.#ensureSsrEntries(entries, { crashRetry })).catch(this.#slowLaneFailed("ssr-ensure")));
+      return;
+    }
+    await this.#withSsrLock(async () => {
+      const patcher = this.#ssrPatcher;
+      const generation = this.#generation;
+      if (patcher && !await this.#runSsrPatcher(patcher, [], { roots: entries, onlyRoots: true, batchGeneration: generation }))
+        return;
+      if (!patcher && await this.#ssrBundler.holds(entries))
+        return;
+      if (this.#ssrBroken)
+        return;
+      if (this.#ssrCrashedAt !== null && Date.now() - this.#ssrCrashedAt < IncrementalBuilder.#crashRetryMs) {
+        patcher?.releaseHandedOver();
+        this.#retryAfterCrash(this.#ssrCrashedAt, entries);
+        return;
+      }
+      await this.#runSsrWorker(generation).catch((err) => {
+        this.#logger.error(`ssr-registry could not take a route's entries; the save that fixes it rebuilds it: ${ApplicationBuildReporter.formatError(err, this.#app.workspace.workspaceRoot)}`);
+      });
+      if (this.#ssrCrashedAt !== null && !crashRetry)
+        this.#retryAfterCrash(this.#ssrCrashedAt, entries);
+    });
+  }
+  #retryAfterCrash(crashedAt, entries) {
+    for (const entry of entries)
+      this.#crashRetryEntries.add(entry);
+    if (this.#crashRetryTimer)
+      return;
+    const timer = setTimeout(() => {
+      this.#crashRetryTimer = null;
+      const retried = [...this.#crashRetryEntries];
+      this.#crashRetryEntries.clear();
+      if (this.shuttingDown || retried.length === 0)
+        return;
+      this.#runBeside(this.#ensureSsrEntries(retried, { crashRetry: true })).catch(this.#slowLaneFailed("ssr-crash-retry"));
+    }, Math.max(0, crashedAt + IncrementalBuilder.#crashRetryMs - Date.now()));
+    timer.unref?.();
+    this.#crashRetryTimer = timer;
+  }
+  async armSsrRegistry() {
+    const delayMs = Number(process.env.AKAN_DEV_SSR_ARM_DELAY_MS);
+    if (Number.isInteger(delayMs) && delayMs > 0)
+      await Bun.sleep(delayMs);
+    const arm = async () => await this.#withSsrLock(async () => {
+      const generation = this.#generation;
+      if (this.#ssrPatcher && !await this.#runSsrPatcher(this.#ssrPatcher, [], { batchGeneration: generation }))
+        return;
+      await this.#runSsrWorker(generation);
+    }).catch((err) => {
+      this.#logger.error(`ssr-registry boot build failed; the next save retries it: ${ApplicationBuildReporter.formatError(err, this.#app.workspace.workspaceRoot)}`);
+    });
+    const arming = this.#runBeside((async () => {
+      await arm();
+      const crashedAt = this.#ssrCrashedAt;
+      if (crashedAt === null)
+        return;
+      while (!this.shuttingDown && Date.now() < crashedAt + IncrementalBuilder.#crashRetryMs)
+        await Bun.sleep(250);
+      if (!this.shuttingDown)
+        await arm();
+    })());
+    this.#ssrArming = arming;
+    await arming;
+    this.#ssrArming = null;
+  }
+  static #isCompileError(error) {
+    return error instanceof AggregateError;
+  }
+  #slowLaneFailed(label) {
+    return (error) => this.#logger.error(`${label} failed in the slow lane: ${ApplicationBuildReporter.formatError(error, this.#app.workspace.workspaceRoot)}`);
+  }
+  #countDelegation(registry, reason) {
+    const key = `${registry}:${reason.split(":")[0]}`;
+    const count = (this.#delegations.get(key) ?? 0) + 1;
+    this.#delegations.set(key, count);
+    return `#${count} for this reason`;
+  }
+  async#withSsrLock(fn) {
+    const run = this.#ssrLock.then(fn, fn);
+    this.#ssrLock = run.then(() => {
+      return;
+    }, () => {
+      return;
+    });
+    return await run;
+  }
+  async#runSsrWorker(generation) {
+    let result;
+    try {
+      result = await this.#runBatch({ generation, needs: ["ssr"], changedFiles: [] });
+    } finally {
+      this.#ssrPatcher?.forget();
+    }
+    this.#noteSsrWorker(result);
+    if (result.errors.ssr)
+      throw new Error(result.errors.ssr);
+  }
+  #noteSsrWorker(result) {
+    const crashed = !!result.crashed && (result.crashedNeeds ?? ["ssr"]).includes("ssr");
+    this.#ssrBroken = !!result.errors.ssr && !crashed;
+    this.#ssrCrashedAt = crashed ? Date.now() : null;
+  }
+  async#runSsrPatcher(patcher, files, {
+    roots,
+    onlyRoots,
+    trace,
+    hold,
+    batchGeneration
+  } = {}) {
+    const started = Date.now();
+    const marker = path2.join(this.#ssrBundler.outDir, CSR_DEV_PATCHING_MARKER);
+    await Bun.write(marker, String(process.pid));
+    try {
+      const result = await patcher.update(files, {
+        roots,
+        onlyRoots,
+        announce: (update) => {
+          const now = Date.now();
+          BuilderChannel.emit({
+            type: "ssr-updated",
+            data: {
+              generation: update.generation,
+              reload: update.reload,
+              reason: update.reason,
+              patchUrl: update.patchUrl,
+              changedIds: update.changedIds,
+              ...trace ? { trace: { ...trace, patchAt: now, sentAt: now } } : {},
+              ...hold ? { hold } : {},
+              ...batchGeneration !== undefined ? { batchGeneration } : {},
+              ...update.epoch !== undefined ? { epoch: update.epoch } : {},
+              ...update.first ? { first: true } : {}
+            }
+          });
+        }
+      });
+      if (result.kind === "delegate") {
+        this.#logger.verbose(`ssr-patch handed to a build worker (${this.#countDelegation("ssr", result.reason)}): ${result.reason}`);
+        return true;
+      }
+      if (!onlyRoots) {
+        this.#ssrBroken = false;
+        this.#ssrCrashedAt = null;
+        this.#sendBuildStatus("ssr", { generation: batchGeneration, ok: true, files });
+      }
+      this.#logger.verbose(result.kind === "unchanged" ? `ssr-patch unchanged (${Date.now() - started}ms)` : `ssr-patch generation=${result.update.generation} ${result.update.reload ? `reload (${result.update.reason})` : `patch modules=${result.update.changedIds.length}`} (${Date.now() - started}ms)`);
+      return false;
+    } catch (err) {
+      const message = ApplicationBuildReporter.formatError(err, this.#app.workspace.workspaceRoot);
+      if (IncrementalBuilder.#isCompileError(err)) {
+        this.#sendBuildStatus("ssr", { generation: batchGeneration, ok: false, files, message });
+        return false;
+      }
+      this.#logger.verbose(`ssr-patch threw; a build worker takes it: ${message}`);
+      patcher.forget();
+      return true;
+    } finally {
+      await rm(marker, { force: true });
+    }
+  }
+  async#runQueuedBatch({ discovery, ...work }) {
+    if (discovery)
+      await this.#refreshDiscovery(discovery);
+    this.#discoveredThrough = Math.max(this.#discoveredThrough, work.generation);
+    try {
+      if (work.needs.includes("ssr"))
+        await this.#withSsrLock(async () => {
+          try {
+            this.#noteSsrWorker(await this.#runBatch(work));
+          } finally {
+            this.#ssrPatcher?.forget();
+          }
+        });
+      else
+        await this.#runBatch(work);
+    } finally {
+      if (work.needs.includes("csr"))
+        this.#patcher?.forget();
+    }
+  }
+  async#refreshDiscovery({ files, refresh, generation }) {
+    const started = Date.now();
+    if (refresh)
+      this.#discovery = await GraphClientEntryDiscovery.create(this.#app);
+    else
+      this.#discovery.invalidate?.(files);
+    this.#discoveredThrough = Math.max(this.#discoveredThrough, generation);
+    this.#logger.verbose(`client-entry-discovery ${refresh ? "refreshed" : "invalidated"} (${Date.now() - started}ms)`);
+  }
+  async#runBatch(work) {
+    const { generation, needs, changedFiles } = work;
+    const started = Date.now();
+    const result = await this.#batchRunner.run(await this.#batchRequest(work), (msg) => BuilderChannel.emit(msg));
     if (result.optimizedFonts)
       this.#optimizedFonts = result.optimizedFonts;
     if (result.cssAssets)
       this.#artifact = { ...this.#artifact, cssAssets: result.cssAssets };
     if (result.crashed) {
-      for (const need of needs)
+      const crashedNeeds = result.crashedNeeds ?? needs;
+      if (crashedNeeds.includes("csr"))
+        this.#patcher?.releaseHandedOver();
+      if (crashedNeeds.includes("ssr"))
+        this.#ssrPatcher?.releaseHandedOver();
+      for (const need of crashedNeeds)
         if (need !== "base")
-          this.#sendBuildStatus(need, { generation, ok: false, files: changedFiles, message: result.errors[need] });
+          this.#sendBuildStatus(need, {
+            generation,
+            ok: false,
+            files: changedFiles,
+            message: `${result.errors[need] ?? "the build worker crashed"}; the next save builds it again`
+          });
     }
     if (needs.includes("css"))
       this.#logger.verbose(`css-rebuild checked (${Date.now() - started}ms)`);
     return result;
   }
-  async#batchRequest({
-    generation,
-    needs,
-    changedFiles
-  }) {
+  async#batchRequest({ generation, needs, changedFiles, trace }) {
     return {
       appName: this.#app.name,
       workspaceRoot: this.#app.workspace.workspaceRoot,
@@ -451,7 +875,8 @@ class IncrementalBuilder {
       pageKeys: await this.#app.getPageKeys(),
       optimizedFonts: this.#optimizedFonts,
       cssAssets: this.#artifact.cssAssets ?? null,
-      artifactDir: path2.resolve(this.#artifactDir)
+      artifactDir: path2.resolve(this.#artifactDir),
+      ...trace ? { trace } : {}
     };
   }
   async boot() {
@@ -462,7 +887,7 @@ class IncrementalBuilder {
   }
   async announceRecoveredState(changedFiles) {
     const generation = ++this.#generation;
-    await this.#enqueueWork("boot-recovered", async () => {
+    await this.#workQueue.enqueue("boot-recovered", async () => {
       await this.#runBatch({ generation, needs: ["pages", "css"], changedFiles });
     });
   }
@@ -491,22 +916,47 @@ class IncrementalBuilder {
     this.#logger.verbose(`announced boot state after recycle generation=${generation}`);
   }
   async handleBuildCsr(msg) {
-    await this.#enqueueWork("build-csr", async () => {
+    this.#beginCsrArm();
+    this.#csrActive = true;
+    const armed = this.#workQueue.enqueue("build-csr", async () => {
       const started = Date.now();
-      const result = await this.#batchRunner.run(await this.#batchRequest({ generation: this.#generation, needs: ["csr"], changedFiles: [] }));
-      const error = result.errors.csr;
+      let error;
+      try {
+        const result = await this.#batchRunner.run(await this.#batchRequest({ generation: this.#generation, needs: ["csr"], changedFiles: [] }));
+        error = result.errors.csr;
+      } catch (err) {
+        error = ApplicationBuildReporter.formatError(err, this.#app.workspace.workspaceRoot);
+      }
       if (error) {
         this.#logger.error(`csr-build failed: ${error}`);
         await BuilderChannel.send({ type: "build-csr-res", id: msg.id, ok: false, error });
         return;
       }
-      this.#csrActive = true;
+      this.#csrArmSucceeded = true;
+      this.#patcher?.forget();
       this.#logger.info(`csr-build ok on demand (${Date.now() - started}ms); rebuilding CSR on every save now`);
       await BuilderChannel.send({ type: "build-csr-res", id: msg.id, ok: true });
     });
+    this.#csrGate = this.#armingCsr(armed);
+    await armed;
   }
-  #shouldRebuildCsr() {
-    return this.#csrActive;
+  #beginCsrArm() {
+    if (this.#csrArms === 0) {
+      this.#csrWasActive = this.#csrActive;
+      this.#csrArmSucceeded = false;
+    }
+    this.#csrArms += 1;
+    this.#csrArming = true;
+  }
+  #armingCsr(armed) {
+    const settle = () => {
+      this.#csrArms -= 1;
+      if (this.#csrArms > 0)
+        return;
+      this.#csrArming = false;
+      this.#csrActive = this.#csrWasActive || this.#csrArmSucceeded;
+    };
+    return armed.then(settle, settle);
   }
   static #csrArmedByEnv() {
     return process.env.AKAN_DEV_CSR_REBUILD === "1";
@@ -534,19 +984,24 @@ class IncrementalBuilder {
   async rearmCsrFromEnv() {
     if (!IncrementalBuilder.#csrArmedByEnv())
       return;
+    this.#beginCsrArm();
     this.#csrActive = true;
-    await this.#enqueueWork("build-csr-rearm", async () => {
-      const result = await this.#batchRunner.run(await this.#batchRequest({ generation: this.#generation, needs: ["csr"], changedFiles: [] }));
+    this.#csrArmSucceeded = true;
+    const rearmed = this.#workQueue.enqueue("build-csr-rearm", async () => {
+      const result = await this.#batchRunner.run(await this.#batchRequest({ generation: this.#generation, needs: ["csr"], changedFiles: [] }), (msg) => BuilderChannel.emit(msg));
+      this.#patcher?.forget();
       if (result.errors.csr)
         this.#logger.error(`csr-rearm failed: ${result.errors.csr}`);
       else
         this.#logger.verbose("csr-rearm ok; this session had CSR armed before the builder restarted");
     });
+    this.#csrGate = this.#armingCsr(rearmed);
+    await rearmed.catch((err) => this.#logger.error(`csr-rearm failed: ${ApplicationBuildReporter.formatError(err, this.#app.workspace.workspaceRoot)}`));
   }
-  static #recoverBoot(app, bootError, logger, runner) {
+  static #recoverBoot(app, bootError, logger, runner, initialGeneration) {
     const firstMessage = bootError instanceof Error ? bootError.message : String(bootError);
     logger.error(`boot build failed; entering degraded watch mode until the error is fixed: ${firstMessage}`);
-    let generation = 0;
+    let generation = initialGeneration;
     const sendFailure = (files, message) => {
       BuilderChannel.emit({
         type: "build-status",
@@ -555,6 +1010,7 @@ class IncrementalBuilder {
     };
     sendFailure([], firstMessage);
     BuilderChannel.emit({ type: "builder-ready" });
+    BuilderChannel.emit({ type: "boot-armed" });
     return new Promise((resolve, reject) => {
       (async () => {
         const roots = await new WatchRootResolver(app).resolve();
@@ -606,23 +1062,15 @@ class IncrementalBuilder {
         builder.shutdown(msg.reason);
         return;
       }
-      if (msg.type === "build-route") {
-        const error = builder?.shuttingDown ? recyclingError : bootingError;
-        if (!builder || builder.shuttingDown) {
-          BuilderChannel.emit({ type: "build-route-res", id: msg.id, ok: false, error });
-          return;
-        }
-        builder.handleBuildRoute(msg);
+      if (msg.type !== "build-route" && msg.type !== "build-csr")
         return;
-      }
-      if (msg.type === "build-csr") {
+      if (!builder || builder.shuttingDown) {
         const error = builder?.shuttingDown ? recyclingError : bootingError;
-        if (!builder || builder.shuttingDown) {
-          BuilderChannel.emit({ type: "build-csr-res", id: msg.id, ok: false, error });
-          return;
-        }
-        builder.handleBuildCsr(msg);
+        BuilderChannel.emit({ type: `${msg.type}-res`, id: msg.id, ok: false, error });
+        return;
       }
+      const handled = msg.type === "build-route" ? builder.handleBuildRoute(msg) : builder.handleBuildCsr(msg);
+      handled.catch((err) => logger.error(`${msg.type} failed: ${err instanceof Error ? err.message : String(err)}`));
     });
     process.on("disconnect", () => {
       logger.warn("host IPC channel closed; exiting builder");
@@ -630,12 +1078,14 @@ class IncrementalBuilder {
     });
     let recoveredFiles = null;
     const bootRunner = new BuildBatchRunner({ workspaceRoot, cwd: app.cwdPath });
+    const initialGeneration = IncrementalBuilder.#initialGeneration();
     try {
-      builder = new IncrementalBuilder({ app, watch, ...await IncrementalBuilder.#buildBootDeps(app, bootRunner) });
+      const deps = await IncrementalBuilder.#buildBootDeps(app, bootRunner);
+      builder = new IncrementalBuilder({ app, watch, initialGeneration, ...deps });
     } catch (err) {
       if (!watch)
         throw err;
-      const recovered = await IncrementalBuilder.#recoverBoot(app, err, logger, bootRunner);
+      const recovered = await IncrementalBuilder.#recoverBoot(app, err, logger, bootRunner, initialGeneration);
       builder = recovered.builder;
       recoveredFiles = recovered.changedFiles;
     }
@@ -644,10 +1094,15 @@ class IncrementalBuilder {
       await builder.announceRecoveredState(recoveredFiles);
     else if (process.env.AKAN_BUILDER_ANNOUNCE_BOOT === "1")
       await builder.announceBootState();
-    await builder.rearmCsrFromEnv();
+    await Promise.all([builder.rearmCsrFromEnv(), builder.armSsrRegistry()]);
+    BuilderChannel.emit({ type: "boot-armed" });
+  }
+  static #initialGeneration() {
+    const generation = Number(process.env.AKAN_BUILDER_INITIAL_GENERATION);
+    return Number.isInteger(generation) && generation > 0 ? generation + 1 : 0;
   }
 }
-IncrementalBuilder.main().catch((err) => {
+await IncrementalBuilder.main().catch((err) => {
   console.error(err);
   process.exit(1);
 });