@akanjs/devkit 2.4.1-rc.2 → 2.4.1-rc.4

package/incrementalBuilder/builderReply.test.ts

@@ -0,0 +1,73 @@
+import { afterEach, describe, expect, test } from "bun:test";
+import { mkdtemp, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { BuilderReply } from "./builderReply";
+
+const tempDirs: string[] = [];
+
+afterEach(async () => {
+  for (const dir of tempDirs.splice(0)) await rm(dir, { recursive: true, force: true });
+});
+
+/**
+ * Run a child that answers a build request and exits immediately — the shape of the recycle drain
+ * finishing its last work item — and report whatever the parent actually received.
+ */
+const replyThenExit = async (
+  bytes: number,
+  { awaitFlush }: { awaitFlush: boolean },
+): Promise<{ id?: number } | null> => {
+  const dir = await mkdtemp(path.join(os.tmpdir(), "builder-reply-"));
+  tempDirs.push(dir);
+  const entry = path.join(dir, "child.ts");
+  const modulePath = JSON.stringify(path.join(import.meta.dir, "builderReply"));
+  const reply = awaitFlush ? "await BuilderReply.send(res as never);" : "process.send?.(res);";
+  await Bun.write(
+    entry,
+    [
+      `import { BuilderReply } from ${modulePath};`,
+      'const chunk = "x".repeat(200);',
+      "const moduleMap: Record<string, string> = {};",
+      `for (let i = 0; i * 210 < ${bytes}; i++) moduleMap["chunk-" + i] = chunk;`,
+      'const res = { type: "build-route-res", id: 7, ok: true, data: { ssrManifestDelta: moduleMap } };',
+      reply,
+      "process.exit(0);",
+    ].join("\n"),
+  );
+  let received: { id?: number } | null = null;
+  const proc = Bun.spawn(["bun", entry], {
+    stdio: ["ignore", "inherit", "inherit"],
+    serialization: "advanced",
+    ipc: (message) => {
+      received = message as { id?: number };
+    },
+  });
+  await proc.exited;
+  // Replies land before the exit callback, never after, but leave room for a straggler to prove it.
+  await Bun.sleep(50);
+  return received;
+};
+
+describe("BuilderReply.send", () => {
+  test("delivers a reply too large for the pipe buffer before the process exits", async () => {
+    expect(await replyThenExit(1_000_000, { awaitFlush: true })).toMatchObject({ id: 7 });
+    // A manifest delta is usually well past the buffer, but the small case must keep working too.
+    expect(await replyThenExit(0, { awaitFlush: true })).toMatchObject({ id: 7 });
+  });
+
+  test("without the flush wait the same reply is lost, which is why this class exists", async () => {
+    // A control, not a requirement: if a future bun flushes ipc writes on exit, this fails and says so.
+    expect(await replyThenExit(1_000_000, { awaitFlush: false })).toBeNull();
+  });
+
+  test("resolves instead of hanging when there is no ipc channel", async () => {
+    const send = process.send;
+    try {
+      (process as { send?: typeof process.send }).send = undefined;
+      await BuilderReply.send({ type: "build-csr-res", id: 1, ok: true });
+    } finally {
+      (process as { send?: typeof process.send }).send = send;
+    }
+  });
+});

package/incrementalBuilder/builderReply.ts

@@ -0,0 +1,30 @@
+import type { BuilderCsrRes, BuilderRes } from "akanjs/server";
+
+/**
+ * Sends a builder's answer to a backend request and reports when it has actually left the process.
+ *
+ * `process.exit` truncates an ipc write that has not flushed yet, and anything past the pipe buffer
+ * (~64KiB) needs the sender to stay alive to drain it — a `build-route-res` carrying a manifest delta is
+ * routinely larger than that. Measured on bun 1.3: a 100KB reply sent immediately before
+ * `process.exit(0)` was lost 20/20 times, and arrived 20/20 times when the exit waited for the flush
+ * callback. That made the recycle drain, which exists to release bundler memory, eat the answer of
+ * whatever route build it happened to be draining.
+ */
+export class BuilderReply {
+  /** Bound so a runtime that ever stops invoking the callback costs one late reply instead of wedging
+   * the shutdown drain until the host's kill watchdog fires. */
+  static readonly #flushTimeoutMs = 5_000;
+
+  static async send(res: BuilderRes | BuilderCsrRes): Promise<void> {
+    const send = process.send;
+    if (!send) return;
+    await new Promise<void>((resolve) => {
+      const timer = setTimeout(resolve, BuilderReply.#flushTimeoutMs);
+      // A failed send has nothing to retry — the host answers the id itself when this process exits.
+      send.call(process, res, undefined, undefined, () => {
+        clearTimeout(timer);
+        resolve();
+      });
+    });
+  }
+}

package/incrementalBuilder/incrementalBuilder.host.test.ts

@@ -10,17 +10,24 @@ afterEach(() => {
   mock.restore();
 });
 
+interface SpawnRecord {
+  proc: { pid: number; send: ReturnType<typeof mock>; kill: ReturnType<typeof mock>; killed: boolean };
+  options: { ipc?: (message: unknown) => void; onExit?: () => void; env?: Record<string, string> };
+}
+
+const mockSpawns = (): SpawnRecord[] => {
+  const spawns: SpawnRecord[] = [];
+  (Bun as unknown as { spawn: typeof Bun.spawn }).spawn = mock((_, options) => {
+    const proc = { pid: 10_000 + spawns.length, send: mock(), kill: mock(), killed: false };
+    spawns.push({ proc, options: options as SpawnRecord["options"] });
+    return proc as never;
+  }) as never;
+  return spawns;
+};
+
 describe("IncrementalBuilderHost", () => {
   test("restarts after a ready builder exits", async () => {
-    const spawns: Array<{
-      proc: { pid: number; send: ReturnType<typeof mock>; kill: ReturnType<typeof mock>; killed: boolean };
-      options: { ipc?: (message: unknown) => void; onExit?: () => void };
-    }> = [];
-    (Bun as unknown as { spawn: typeof Bun.spawn }).spawn = mock((_, options) => {
-      const proc = { pid: 10_000 + spawns.length, send: mock(), kill: mock(), killed: false };
-      spawns.push({ proc, options: options as { ipc?: (message: unknown) => void; onExit?: () => void } });
-      return proc as never;
-    }) as never;
+    const spawns = mockSpawns();
 
     const onReady = mock();
     const onRestartReady = mock();
@@ -48,4 +55,187 @@ describe("IncrementalBuilderHost", () => {
 
     host.stop();
   });
+
+  test("recycles a ready builder gracefully and replaces it immediately", async () => {
+    const spawns = mockSpawns();
+    const onRestartReady = mock();
+    const host = new IncrementalBuilderHost({
+      app: { cwdPath: "/tmp/app" } as never,
+      entry: "/tmp/builder.ts",
+      env: {},
+      onMessage: () => undefined,
+    });
+
+    host.start({ onRestartReady });
+    spawns[0]?.options.ipc?.({ type: "builder-ready" });
+    expect(spawns[0]?.options.env?.AKAN_BUILDER_ANNOUNCE_BOOT).toBeUndefined();
+
+    const reason = "rss=1300MiB>=1200MiB after 3 build(s)";
+    expect(host.recycle(reason)).toBe(true);
+    // Graceful: the builder is asked to drain, not killed, so a rebuild in flight still completes.
+    expect(spawns[0]?.proc.send).toHaveBeenCalledWith({ type: "builder-shutdown", reason });
+    expect(spawns[0]?.proc.kill).not.toHaveBeenCalled();
+    expect(host.recycle("second request")).toBe(false);
+
+    // A planned exit skips the crash backoff — the dev server has no file watcher until it is back.
+    spawns[0]?.options.onExit?.();
+    expect(spawns).toHaveLength(2);
+    expect(spawns[1]?.options.env?.AKAN_BUILDER_ANNOUNCE_BOOT).toBe("1");
+    spawns[1]?.options.ipc?.({ type: "builder-ready" });
+    expect(onRestartReady).toHaveBeenCalledTimes(1);
+    expect(host.status).toBe("ready");
+
+    // The flag is per-recycle: a later crash restart must not re-announce a boot artifact.
+    spawns[1]?.options.onExit?.();
+    expect(spawns).toHaveLength(2);
+    await wait(1_050);
+    expect(spawns).toHaveLength(3);
+    expect(spawns[2]?.options.env?.AKAN_BUILDER_ANNOUNCE_BOOT).toBeUndefined();
+
+    host.stop();
+  });
+
+  test("fails the requests a departing builder never answered", async () => {
+    const spawns = mockSpawns();
+    const messages: unknown[] = [];
+    const host = new IncrementalBuilderHost({
+      app: { cwdPath: "/tmp/app" } as never,
+      entry: "/tmp/builder.ts",
+      env: {},
+      onMessage: (message) => messages.push(message),
+    });
+
+    host.start();
+    spawns[0]?.options.ipc?.({ type: "builder-ready" });
+    expect(host.send({ type: "build-route", id: 1, routeId: "a", seeds: [], knownEntries: [] })).toBe(true);
+    expect(host.send({ type: "build-csr", id: 2, reason: "device webview" })).toBe(true);
+    // Answered before the exit, so this one must not be failed again afterwards.
+    expect(host.send({ type: "build-route", id: 3, routeId: "c", seeds: [], knownEntries: [] })).toBe(true);
+    spawns[0]?.options.ipc?.({ type: "build-route-res", id: 3, ok: true, data: { routeId: "c" } });
+    messages.length = 0;
+
+    host.recycle("rss=1300MiB>=1200MiB after 3 build(s)");
+    spawns[0]?.options.onExit?.();
+
+    // Nothing else answers these: the builder only refuses requests that arrive after it starts shutting
+    // down, and a kill or a truncated write sends nothing at all.
+    expect(messages).toEqual([
+      {
+        type: "build-route-res",
+        id: 1,
+        ok: false,
+        error: "builder exited to release bundler memory before answering; reload to retry",
+      },
+      {
+        type: "build-csr-res",
+        id: 2,
+        ok: false,
+        error: "builder exited to release bundler memory before answering; reload to retry",
+      },
+    ]);
+
+    // The replacement owes nothing, so its own exit stays quiet.
+    spawns[1]?.options.ipc?.({ type: "builder-ready" });
+    messages.length = 0;
+    spawns[1]?.options.onExit?.();
+    expect(messages).toEqual([]);
+    await wait(1_050);
+
+    host.stop();
+  });
+
+  test("names a crash rather than a recycle, and answers on stop too", async () => {
+    const spawns = mockSpawns();
+    const messages: unknown[] = [];
+    const host = new IncrementalBuilderHost({
+      app: { cwdPath: "/tmp/app" } as never,
+      entry: "/tmp/builder.ts",
+      env: {},
+      onMessage: (message) => messages.push(message),
+    });
+
+    host.start();
+    spawns[0]?.options.ipc?.({ type: "builder-ready" });
+    host.send({ type: "build-route", id: 1, routeId: "a", seeds: [], knownEntries: [] });
+    messages.length = 0;
+    spawns[0]?.options.onExit?.();
+    expect(messages).toEqual([
+      {
+        type: "build-route-res",
+        id: 1,
+        ok: false,
+        error: "builder exited unexpectedly before answering; reload once it is back",
+      },
+    ]);
+
+    // `stop()` clears the process before its exit callback runs, so the callback bails on its identity
+    // check and cannot be the only place this happens.
+    await wait(1_050);
+    spawns[1]?.options.ipc?.({ type: "builder-ready" });
+    messages.length = 0;
+    expect(host.send({ type: "build-route", id: 2, routeId: "b", seeds: [], knownEntries: [] })).toBe(true);
+    host.stop();
+    expect(messages).toEqual([
+      { type: "build-route-res", id: 2, ok: false, error: "builder was stopped before answering" },
+    ]);
+  });
+
+  test("only recycles a builder that is ready", () => {
+    const spawns = mockSpawns();
+    const host = new IncrementalBuilderHost({
+      app: { cwdPath: "/tmp/app" } as never,
+      entry: "/tmp/builder.ts",
+      env: {},
+      onMessage: () => undefined,
+    });
+
+    host.start();
+    expect(host.recycle("while still booting")).toBe(false);
+    expect(spawns[0]?.proc.send).not.toHaveBeenCalled();
+
+    host.stop();
+    expect(host.recycle("after stop")).toBe(false);
+  });
+});
+
+describe("IncrementalBuilderHost.maxRssBytes", () => {
+  const withEnv = (values: Record<string, string | undefined>, fn: () => void) => {
+    const previous = Object.fromEntries(Object.keys(values).map((key) => [key, process.env[key]]));
+    try {
+      for (const [key, value] of Object.entries(values)) {
+        if (value === undefined) delete process.env[key];
+        else process.env[key] = value;
+      }
+      fn();
+    } finally {
+      for (const [key, value] of Object.entries(previous)) {
+        if (value === undefined) delete process.env[key];
+        else process.env[key] = value;
+      }
+    }
+  };
+
+  test("defaults to a dev ceiling well above a fresh boot", () => {
+    withEnv(
+      { AKAN_BUILDER_MAX_RSS_MB: undefined, AKAN_BUILDER_MAX_RSS: undefined, AKAN_MEMORY_LIMIT: undefined },
+      () => {
+        expect(IncrementalBuilderHost.maxRssBytes()).toBe(1_200 * 1024 * 1024);
+      },
+    );
+  });
+
+  test("honors an explicit override and treats 0 as unbounded", () => {
+    withEnv({ AKAN_BUILDER_MAX_RSS_MB: "700" }, () => {
+      expect(IncrementalBuilderHost.maxRssBytes()).toBe(700 * 1024 * 1024);
+    });
+    withEnv({ AKAN_BUILDER_MAX_RSS_MB: "0" }, () => {
+      expect(IncrementalBuilderHost.maxRssBytes()).toBeNull();
+    });
+  });
+
+  test("takes a share of a declared memory limit, leaving room for the other dev processes", () => {
+    withEnv({ AKAN_BUILDER_MAX_RSS_MB: undefined, AKAN_BUILDER_MAX_RSS: undefined, AKAN_MEMORY_LIMIT: "4gb" }, () => {
+      expect(IncrementalBuilderHost.maxRssBytes()).toBe(Math.floor(4 * 1024 ** 3 * 0.35));
+    });
+  });
 });

package/incrementalBuilder/incrementalBuilder.host.ts

@@ -1,6 +1,7 @@
 import path from "node:path";
 import { Logger } from "akanjs/common";
 import type { BuilderMessage } from "akanjs/server";
+import { MemoryLimit } from "akanjs/server/memoryLimit";
 import type { App } from "../commandDecorators";
 
 const builderMsgTypeSet = new Set<BuilderMessage["type"]>([
@@ -11,6 +12,7 @@ const builderMsgTypeSet = new Set<BuilderMessage["type"]>([
   "css-updated",
   "pages-updated",
   "build-status",
+  "builder-metrics",
 ]);
 interface IncrementalBuilderHostOptions {
   app: App;
@@ -25,11 +27,26 @@ interface IncrementalBuilderStartOptions {
   onExit?: () => void;
   onReady?: () => void;
   onRestartReady?: () => void;
+  /**
+   * Ask the builder to re-announce the artifact it boots with. Needed whenever a *previous* builder's
+   * artifact may still be live in a running backend — after an rss recycle, and after an idle wake.
+   */
+  announceBootState?: boolean;
 }
 
 export class IncrementalBuilderHost {
   static readonly #restartBaseDelayMs = 1_000;
   static readonly #restartMaxDelayMs = 30_000;
+  /**
+   * A builder that has stopped answering has to be replaced anyway; killing it after this long turns
+   * a wedged drain into an ordinary restart instead of leaving the recycle stuck forever.
+   */
+  static readonly #recycleDrainTimeoutMs = 30_000;
+  /**
+   * Dev has no memory limit to derive a fraction from, so this is what bounds the builder there. Well
+   * above a fresh boot (~300-600MB depending on app size) with room for one full rebuild on top.
+   */
+  static readonly #devMaxRssBytes = 1_200 * 1024 * 1024;
   logger = new Logger("IncrementalBuilderHost");
   entry: string;
   env: Record<string, string>;
@@ -40,7 +57,21 @@ export class IncrementalBuilderHost {
   #status: IncrementalBuilderStatus = "stopped";
   #restartAttempts = 0;
   #restartTimer: ReturnType<typeof setTimeout> | null = null;
+  #recycleTimer: ReturnType<typeof setTimeout> | null = null;
+  #recycleRequested: boolean = false;
+  #spawnAfterRecycle: boolean = false;
   #manualStop = false;
+  /**
+   * Requests handed to the running builder that it has not answered yet, keyed by the backend's
+   * correlation id.
+   *
+   * Nothing else answers a request whose builder exits while holding it: the builder only refuses
+   * requests that arrive *after* it starts shutting down, a kill or crash sends nothing at all, and even
+   * a clean drain races its own `process.exit`. The builder exits routinely — it is recycled whenever its
+   * RSS passes the ceiling — so a page request that happened to be mid route-build left the backend's
+   * promise pending and the browser tab spinning with no error and nothing to retry.
+   */
+  readonly #inFlight = new Map<number, "build-route" | "build-csr">();
   #startOptions: IncrementalBuilderStartOptions = {};
   constructor({ app, entry, env, onMessage }: IncrementalBuilderHostOptions) {
     this.app = app;
@@ -51,24 +82,38 @@ export class IncrementalBuilderHost {
   get status() {
     return this.#status;
   }
+  /**
+   * The running builder's pid, so the host can read its RSS from the OS between builds. The builder
+   * only reports its own metrics at work-completion points, which is the *peak*; on a platform that
+   * returns bundler arenas to the OS while idle, that sample goes stale within seconds.
+   */
+  get pid(): number | null {
+    return this.#proc?.pid ?? null;
+  }
   start(options: IncrementalBuilderStartOptions = {}) {
     if (this.#proc) this.stop();
     this.#manualStop = false;
     this.#startOptions = options;
+    this.#spawnAfterRecycle = options.announceBootState ?? false;
     this.#spawn(false);
     return this;
   }
   #spawn(isRestart: boolean) {
     this.#status = isRestart ? "restarting" : "starting";
     this.ready = false;
+    // A fresh builder rebuilds every artifact while the running backend still holds the previous one;
+    // the flag is what tells it to re-announce what it booted with.
+    const afterRecycle = this.#spawnAfterRecycle;
+    this.#spawnAfterRecycle = false;
     let proc!: Bun.Subprocess<"ignore", "inherit", "inherit">;
     proc = Bun.spawn(["bun", this.entry], {
       cwd: this.app.cwdPath,
-      env: { ...this.env, AKAN_WATCH: "1" },
+      env: { ...this.env, AKAN_WATCH: "1", ...(afterRecycle ? { AKAN_BUILDER_ANNOUNCE_BOOT: "1" } : {}) },
       stdio: ["ignore", "inherit", "inherit"],
       ipc: (msg: BuilderMessage) => {
         if (this.#proc !== proc) return;
         if (!msg || typeof msg !== "object") return;
+        if (msg.type === "build-route-res" || msg.type === "build-csr-res") this.#inFlight.delete(msg.id);
         if (builderMsgTypeSet.has(msg.type)) this.#onMessage(msg);
         if (msg.type === "builder-ready" && !this.ready) {
           this.ready = true;
@@ -83,13 +128,28 @@ export class IncrementalBuilderHost {
         if (this.#proc !== proc) return;
         this.#proc = null;
         const wasReady = this.ready;
+        const wasRecycle = this.#recycleRequested;
+        this.#clearRecycle();
         this.ready = false;
+        this.#failInFlight(
+          wasRecycle
+            ? "builder exited to release bundler memory before answering; reload to retry"
+            : "builder exited unexpectedly before answering; reload once it is back",
+        );
         if (this.#manualStop || this.#status === "stopped") return;
         if (!wasReady) {
           this.#status = "stopped";
           this.#startOptions.onExit?.();
           return;
         }
+        // A recycle is a planned exit, so it neither counts as a failed attempt nor waits out the
+        // crash backoff — the dev server is without a watcher until the replacement is up.
+        if (wasRecycle) {
+          this.logger.verbose("builder exited for a recycle; spawning its replacement now");
+          this.#spawnAfterRecycle = true;
+          this.#spawn(true);
+          return;
+        }
         this.#scheduleRestart();
       },
     });
@@ -112,6 +172,47 @@ export class IncrementalBuilderHost {
       this.#spawn(true);
     }, delay);
   }
+  /**
+   * Ask the builder to drain its queues and exit so the OS reclaims the bundler arenas `Bun.build`
+   * never gives back; `onExit` then spawns the replacement. Graceful rather than `kill()` so a rebuild
+   * in flight is never truncated, with a watchdog for a builder that stops answering.
+   */
+  recycle(reason: string): boolean {
+    if (!this.#proc || this.#status !== "ready" || this.#recycleRequested) return false;
+    const proc = this.#proc;
+    if (!this.send({ type: "builder-shutdown", reason })) return false;
+    this.#recycleRequested = true;
+    this.logger.info(`recycling builder pid=${proc.pid} (${reason})`);
+    this.#recycleTimer = setTimeout(() => {
+      this.#recycleTimer = null;
+      if (this.#proc !== proc) return;
+      this.logger.warn(
+        `builder pid=${proc.pid} did not exit within ${IncrementalBuilderHost.#recycleDrainTimeoutMs}ms of the recycle request; killing it`,
+      );
+      proc.kill();
+    }, IncrementalBuilderHost.#recycleDrainTimeoutMs);
+    return true;
+  }
+  #clearRecycle() {
+    this.#recycleRequested = false;
+    if (!this.#recycleTimer) return;
+    clearTimeout(this.#recycleTimer);
+    this.#recycleTimer = null;
+  }
+  /**
+   * RSS at which the builder is recycled: an explicit override, else a share of the container's limit
+   * (the builder is one of several dev processes), else the dev default. Set
+   * `AKAN_BUILDER_MAX_RSS_MB=0` to leave it unbounded.
+   */
+  static maxRssBytes(): number | null {
+    if (process.env.AKAN_BUILDER_MAX_RSS_MB === "0") return null;
+    return MemoryLimit.resolveMaxRssBytes({
+      megabytesEnv: "AKAN_BUILDER_MAX_RSS_MB",
+      bytesEnv: "AKAN_BUILDER_MAX_RSS",
+      limitFraction: 0.35,
+      fallbackBytes: IncrementalBuilderHost.#devMaxRssBytes,
+    });
+  }
   send(message: BuilderMessage): boolean {
     if (!this.#proc || this.#status !== "ready") {
       this.logger.warn(`incrementalBuilderHost is ${this.#status}; cannot send ${message.type}`);
@@ -119,6 +220,7 @@ export class IncrementalBuilderHost {
     }
     try {
       this.#proc.send(message);
+      if (message.type === "build-route" || message.type === "build-csr") this.#inFlight.set(message.id, message.type);
       return true;
     } catch (error) {
       this.logger.warn(
@@ -127,8 +229,24 @@ export class IncrementalBuilderHost {
       return false;
     }
   }
+  /**
+   * Answer every request the departing builder still owed, as if it had failed them itself. `onExit`
+   * cannot do this alone: `stop()` clears `#proc` first, so the exit callback bails on its identity check.
+   */
+  #failInFlight(reason: string): void {
+    if (!this.#inFlight.size) return;
+    const lost = [...this.#inFlight];
+    this.#inFlight.clear();
+    this.logger.warn(`failing ${lost.length} unanswered builder request(s): ${reason}`);
+    for (const [id, type] of lost) {
+      if (type === "build-route") this.#onMessage({ type: "build-route-res", id, ok: false, error: reason });
+      else this.#onMessage({ type: "build-csr-res", id, ok: false, error: reason });
+    }
+  }
   stop() {
     this.#manualStop = true;
+    this.#clearRecycle();
+    this.#failInFlight("builder was stopped before answering");
     if (this.#restartTimer) {
       clearTimeout(this.#restartTimer);
       this.#restartTimer = null;