akanjs 3.0.0-alpha.7 → 3.0.0-alpha.71

package/server/akanApp.ts

@@ -2,15 +2,23 @@
 
 import { mkdir, readdir, rm } from "node:fs/promises";
 import path from "node:path";
-import { Logger } from "akanjs/common";
+import { Logger, logSeverity } from "akanjs/common";
 import type { AkanChildRole, AkanChildStatus, AkanIpcMessage, AkanMetricsReport, AkanUpstream } from "akanjs/service";
-import { isTraceEnabled } from "akanjs/signal";
+import { isTraceEnabled } from "../signal/trace";
 import { makeAkanChildProxyHeaders } from "./akanAppHeaders";
 import type { BuilderCsrReq, BuilderCsrRes, BuilderMessage, BuilderReq, BuilderRes } from "./artifact";
 import { resolveEncodedSidecar } from "./assetEncoding";
+import { compressResponse } from "./contentEncoding";
 import { isPortInUseError } from "./lifecycle/portInUse";
+import { resolveRuntimeDir } from "./lifecycle/runtimeDir";
+import { ChildOutputReader } from "./logging/childOutputReader";
+import { HubFileSink } from "./logging/hubFileSink";
+import { LogControlSocket } from "./logging/logControlSocket";
+import { LogHub } from "./logging/logHub";
+import { LogStreamRoute } from "./logging/logStreamRoute";
 import { RotatingLogWriter } from "./logging/rotatingLogWriter";
 import { ProcessMetricsCollector } from "./processMetricsCollector";
+import { getWebConfigFromEnv } from "./types";
 
 interface ChildState {
   idx: number;
@@ -35,9 +43,13 @@ interface ChildState {
   lastErrorMessage?: string;
 }
 
+/** The shape `#startSolo` needs off `server.ts`; the module itself is loaded by path, so it has no type here. */
+interface SoloServer {
+  start: (options: { listen: boolean }) => Promise<unknown>;
+}
+
 interface GatewayWsData {
   childIdx: number;
-  socketId: string;
   upstream: WebSocket;
 }
 
@@ -46,6 +58,19 @@ type GatewayUpstream = {
   ws?: Extract<AkanUpstream, { type: "tcp" }>;
 };
 
+/**
+ * Received-only close codes cannot be sent in a close frame. The gateway deliberately normalizes
+ * every unsendable code, including semantically distinct 1005 and 1006 events, to 1001 in both
+ * relay directions. In particular, Bun's global client `WebSocket.close()` throws an
+ * InvalidAccessError for these codes at the client-to-upstream relay; normalization at the
+ * upstream-to-client `Bun.ServerWebSocket.close()` relay is defensive and keeps behavior symmetric.
+ */
+const relayableCloseCode = (code: number): number => {
+  if (code >= 3000 && code <= 4999) return code;
+  if (code >= 1000 && code <= 1014 && code !== 1004 && code !== 1005 && code !== 1006) return code;
+  return 1001;
+};
+
 /** Options for the Akan gateway that launches child server replicas and listens for traffic. */
 export interface AkanAppOptions {
   replica?: number | string;
@@ -54,6 +79,18 @@ export interface AkanAppOptions {
   port?: number;
   wsBasePort?: number;
   openapi?: boolean;
+  /**
+   * Boot only these modules and the ones they reach, in every child. Omitted or empty mounts every enabled
+   * module. Handed down as `AKAN_MODULES`, since each replica builds its own container.
+   */
+  modules?: string[];
+  /**
+   * Run the one replica in this process instead of spawning it — see `#startSolo`. Defaults on for a single
+   * traffic replica the environment configured, and off whenever `replica` is passed here, because code that
+   * states a topology is asking for the gateway that serves it. `AKAN_SOLO=false` turns it off; like
+   * `AKAN_SSR`, the env can only narrow, so it cannot force a gateway's replicas into one process.
+   */
+  solo?: boolean;
 }
 
 interface AkanReplicaConfig {
@@ -76,6 +113,9 @@ export class AkanApp {
   /** Hosted by `akan start`: crash loops should yield to the dev host, which restarts on file edits. */
   readonly #devHosted = process.env.AKAN_COMMAND_TYPE === "start";
   readonly #healthTimeoutMs = AkanApp.#parseHealthTimeoutMs();
+  readonly #upstreamWaitMs = AkanApp.#parseUpstreamWaitMs();
+  /** Child stderr is bundler/runtime noise the gateway cannot act on; it still reaches the rotating log file. */
+  readonly #printChildStderr = process.env.AKAN_CHILD_STDERR === "1";
   readonly #serverPath: string;
   readonly #artifactDir: string;
   readonly #replica: AkanReplicaConfig;
@@ -84,6 +124,10 @@ export class AkanApp {
   readonly #port: number;
   readonly #wsBasePort: number;
   readonly #openapi?: boolean;
+  /** The gateway hands `/_akan/client|styles|fonts` straight off disk, so it needs the same answer its children do. */
+  readonly #web = getWebConfigFromEnv();
+  readonly #modules: string[];
+  readonly #solo: boolean;
   readonly #children = new Map<number, ChildState>();
   readonly #roomChildren = new Map<string, Set<number>>();
   readonly #childRooms = new Map<number, Set<string>>();
@@ -98,9 +142,10 @@ export class AkanApp {
   #metricsTimer: Timer | null = null;
   #logWriter: RotatingLogWriter | null = null;
   #removeLogSink: (() => void) | null = null;
-  readonly #childOutputBuffers = new Map<string, string>();
-  readonly #childStderrBlockBuffers = new Map<string, string[]>();
-  readonly #childStderrBlockTimers = new Map<string, ReturnType<typeof setTimeout>>();
+  #logHub: LogHub | null = null;
+  #logControl: LogControlSocket | null = null;
+  #hubFileSink: HubFileSink | null = null;
+  #logStream: LogStreamRoute | null = null;
   static readonly #ansiPattern = new RegExp(`${String.fromCharCode(27)}\\[[0-?]*[ -/]*[@-~]`, "g");
   #gatewayMetrics: AkanMetricsReport = {};
   #proxyHopCount = 0;
@@ -115,16 +160,30 @@ export class AkanApp {
     this.#serverPath = AkanApp.#resolveServerPath(resolvedOptions.serverPath ?? serverPath);
     this.#artifactDir = path.resolve(path.dirname(this.#serverPath), ".akan", "artifact");
     this.#replica = AkanApp.#parseReplicaConfig(resolvedOptions.replica);
-    this.#runtimeDir = path.resolve(
-      resolvedOptions.runtimeDir ??
-        process.env.AKAN_RUNTIME_DIR ??
-        (process.env.NODE_ENV === "production"
-          ? path.resolve(process.cwd(), "runtime")
-          : path.resolve(process.cwd(), "local", "apps", process.env.AKAN_PUBLIC_APP_NAME ?? "unknown", "runtime")),
-    );
+    this.#runtimeDir = resolveRuntimeDir(resolvedOptions.runtimeDir);
     this.#port = Number(resolvedOptions.port ?? process.env.PORT ?? 8282);
     this.#wsBasePort = Number(resolvedOptions.wsBasePort ?? process.env.AKAN_WS_BASE_PORT ?? this.#port + 10_000);
     this.#openapi = resolvedOptions.openapi;
+    this.#modules = resolvedOptions.modules ?? [];
+    this.#solo = AkanApp.#resolveSolo(resolvedOptions, this.#replica);
+    this.#logHub = LogHub.attach();
+  }
+
+  /**
+   * One replica that serves traffic means there is nothing to balance and nothing to fan pubsub out to, so the
+   * gateway is a second Bun runtime and a proxy hop in front of the only server there is. Measured on
+   * `apps/akan`: 28MB of RSS and half the requests per second.
+   *
+   * A batch-only replica is excluded because it never listens — keeping the gateway is what leaves something
+   * bound to answer `/_akan/app/health`. `akan start` is excluded because the gateway is also the dev host's
+   * builder relay, its crash page, and what holds the port across a child restart.
+   */
+  static #resolveSolo(options: AkanAppOptions, replica: AkanReplicaConfig) {
+    if (options.solo !== undefined) return options.solo;
+    if (process.env.AKAN_SOLO === "false" || process.env.AKAN_SOLO === "0") return false;
+    if (process.env.AKAN_COMMAND_TYPE === "start") return false;
+    if (options.replica !== undefined) return false;
+    return replica.total === 1 && replica.batch === 0;
   }
 
   static #resolveServerPath(serverPath: string) {
@@ -157,10 +216,16 @@ export class AkanApp {
     return Math.max(min, parsed);
   }
 
+  /**
+   * The command that started this gateway outranks an inherited `NODE_ENV`: the command is a statement about
+   * this run, while the env var can arrive from a shell export, a CI image, or the workspace `.env` Bun loads
+   * on its own. Handing a dev child `NODE_ENV=production` makes it serve from the production route cache and
+   * throw on every route, since only `akan build` writes the routes manifest that cache expects.
+   */
   static #defaultChildNodeEnv() {
-    if (process.env.NODE_ENV) return process.env.NODE_ENV;
     if (process.env.AKAN_COMMAND_TYPE === "start") return "development";
     if (process.env.AKAN_COMMAND_TYPE === "build") return "production";
+    if (process.env.NODE_ENV) return process.env.NODE_ENV;
     return Bun.main.endsWith(".js") ? "production" : "development";
   }
 
@@ -174,6 +239,17 @@ export class AkanApp {
     return process.env.AKAN_COMMAND_TYPE === "start" ? 15_000 : 5_000;
   }
 
+  /**
+   * How long a request waits for a replica that is booting or restarting before the gateway answers
+   * 503. Dev pays first-touch transpiles on the boot path, so it gets the wider budget; `0` restores
+   * the old fail-immediately behavior.
+   */
+  static #parseUpstreamWaitMs() {
+    const configured = Number(process.env.AKAN_UPSTREAM_WAIT_MS);
+    if (Number.isFinite(configured) && configured >= 0) return configured;
+    return process.env.AKAN_COMMAND_TYPE === "start" ? 15_000 : 5_000;
+  }
+
   /**
    * Must exceed the child's own shutdown timeout (see `AkanServer.#defaultShutdownTimeoutMs`) so
    * children always get to exit on their own before this gateway stops waiting.
@@ -185,7 +261,10 @@ export class AkanApp {
   }
 
   async start() {
+    if (this.#solo) return await this.#startSolo();
+    Logger.role = "gateway";
     await this.#prepareRuntimeDir();
+    await this.#startLogHub();
     this.#startFileLogging();
     for (let idx = 0; idx < this.#replica.total; idx++) this.#spawn(idx);
     try {
@@ -212,6 +291,25 @@ export class AkanApp {
     });
   }
 
+  async #startSolo() {
+    const role = this.#getRole(0);
+
+    Object.assign(process.env, {
+      NODE_ENV: AkanApp.#defaultChildNodeEnv(),
+      AKAN_REPLICA: this.#replica.value,
+      AKAN_REPLICA_IDX: "0",
+      AKAN_APP_DIR: path.dirname(this.#serverPath),
+      SERVER_MODE: role,
+      ...(this.#openapi === undefined ? {} : { AKAN_OPENAPI: this.#openapi ? "true" : "false" }),
+      ...(this.#modules.length ? { AKAN_MODULES: this.#modules.join(",") } : {}),
+    });
+    this.logger.info(`Starting ${role} replica in this process (solo); set AKAN_SOLO=false for the gateway`);
+    const mod = (await import(this.#serverPath)) as { server?: SoloServer; app?: SoloServer };
+    const server = mod.server ?? mod.app;
+    if (!server?.start) throw new Error("server.ts must export server or app with start()");
+    await server.start({ listen: true });
+  }
+
   async stop(signal = "SIGTERM") {
     if (this.#stopping) return;
     this.#stopping = true;
@@ -245,6 +343,7 @@ export class AkanApp {
     for (const child of stragglers) child.proc.kill("SIGKILL");
     await Promise.all(stragglers.map((child) => child.proc.exited.catch(() => undefined)));
     this.#children.clear();
+    await this.#stopLogHub();
     await this.#stopFileLogging();
     this.#resolveStopped?.();
     this.#resolveStopped = null;
@@ -289,6 +388,7 @@ export class AkanApp {
         AKAN_CHILD_SOCKET: upstream.http.socketPath,
         AKAN_CHILD_WS_PORT: upstream.ws ? String(upstream.ws.port) : "",
         ...(this.#openapi === undefined ? {} : { AKAN_OPENAPI: this.#openapi ? "true" : "false" }),
+        ...(this.#modules.length ? { AKAN_MODULES: this.#modules.join(",") } : {}),
       },
       ipc: (message) => this.#handleMessage(idx, message as AkanIpcMessage, proc),
       stdout: "pipe",
@@ -312,8 +412,12 @@ export class AkanApp {
       lastRestartReason: previous?.lastRestartReason,
     });
     this.#invalidateFederationChildCache();
-    this.#pipeOutput(idx, role, proc.stdout, "stdout");
-    this.#pipeOutput(idx, role, proc.stderr, "stderr");
+    const output = new ChildOutputReader({
+      onLine: (line) => this.#writeChildLine(idx, role, proc.pid, "stdout", line),
+      onBlock: (lines) => this.#writeChildStderrBlock(idx, role, proc.pid, lines),
+    });
+    void output.pipe(proc.stdout, "stdout");
+    void output.pipe(proc.stderr, "stderr");
     proc.exited.then((code) => this.#handleChildExit(idx, proc, code));
   }
 
@@ -463,7 +567,7 @@ export class AkanApp {
     const entries = await readdir(this.#runtimeDir).catch(() => []);
     await Promise.all(
       entries
-        .filter((name) => /^akan-child-.*\.sock$/.test(name))
+        .filter((name) => /^akan-(child-.*|control)\.sock$/.test(name))
         .map((name) => rm(path.join(this.#runtimeDir, name), { force: true }).catch(() => undefined)),
     );
   }
@@ -479,6 +583,13 @@ export class AkanApp {
   #startFileLogging() {
     this.#logWriter = RotatingLogWriter.fromRuntimeDir(this.#runtimeDir);
     if (!this.#logWriter) return;
+    if (this.#logHub && Logger.isNdjson) {
+      this.#hubFileSink = new HubFileSink(this.#logHub, this.#logWriter, {
+        minSev: logSeverity[Logger.fileLevel],
+        json: Logger.format === "ndjson-only",
+      });
+      return;
+    }
     this.#removeLogSink = Logger.addSink((entry) => {
       this.#logWriter?.write("gateway", entry.plainMessage);
     });
@@ -487,11 +598,38 @@ export class AkanApp {
   async #stopFileLogging() {
     this.#removeLogSink?.();
     this.#removeLogSink = null;
+    this.#hubFileSink?.close();
+    this.#hubFileSink = null;
     const writer = this.#logWriter;
     this.#logWriter = null;
     await writer?.close();
   }
 
+  /** The journal every child forwards into, and the socket `akan logs` reads it from. Additive to the text file. */
+  async #startLogHub() {
+    const hub = LogHub.attach();
+    this.#logHub = hub;
+    hub.onFloorChange((minSev) => this.#fanoutToAll({ type: "log.level", minSev }));
+    this.#logStream = LogStreamRoute.fromEnv(() => this.#logHub);
+    const control = new LogControlSocket(hub, this.#runtimeDir);
+    try {
+      await control.start();
+      this.#logControl = control;
+    } catch (error) {
+      this.logger.warn(
+        `Log control socket unavailable at ${control.path}: ${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
+  }
+
+  async #stopLogHub() {
+    await this.#logControl?.stop();
+    this.#logControl = null;
+    this.#logStream = null;
+    this.#logHub?.close();
+    this.#logHub = null;
+  }
+
   #listen() {
     this.#server = Bun.serve({
       idleTimeout: 0,
@@ -515,11 +653,12 @@ export class AkanApp {
     const url = new URL(req.url);
     if (url.pathname === "/_akan/app/health") return Response.json(this.#getHealthStatus());
     if (url.pathname === "/_akan/app/metrics") return Response.json(this.#getMetricsStatus());
+    if (url.pathname === LogStreamRoute.path && this.#logStream) return this.#logStream.handle(req);
     if (url.pathname === "/_akan/bench/ping") return new Response("ok");
     if (this.#isWebSocketPath(url.pathname)) return this.#upgradeWebSocket(req, server);
     const assetResponse = await this.#serveImmutableArtifact(req, url);
     if (assetResponse) return assetResponse;
-    return await this.#proxyHttp(req);
+    return await this.#proxyHttp(req, server);
   }
 
   #isWebSocketPath(pathname: string) {
@@ -527,6 +666,7 @@ export class AkanApp {
   }
 
   async #serveImmutableArtifact(req: Request, url: URL): Promise<Response | null> {
+    if (!this.#web.ssr) return null;
     const clientPrefix = "/_akan/client/";
     if (url.pathname.startsWith(clientPrefix)) {
       const filePath = this.#safeResolve(
@@ -564,10 +704,10 @@ export class AkanApp {
     if (!child || !upstream) return new Response("No websocket upstream is ready", { status: 503 });
     const url = new URL(req.url);
     const upstreamWs = new WebSocket(`ws://${upstream.host}:${upstream.port}${url.pathname}${url.search}`, {
-      headers: this.#makeProxyHeaders(req, child.idx),
+      headers: this.#makeProxyHeaders(req, child.idx, server),
     } as unknown as string[]);
-    const socketId = crypto.randomUUID();
-    const upgraded = server.upgrade(req, { data: { childIdx: child.idx, socketId, upstream: upstreamWs } });
+
+    const upgraded = server.upgrade(req, { data: { childIdx: child.idx, upstream: upstreamWs } });
     if (!upgraded) {
       upstreamWs.close();
       return new Response("WebSocket upgrade failed", { status: 500 });
@@ -586,7 +726,9 @@ export class AkanApp {
       const result = ws.send(event.data as string | ArrayBuffer);
       if (result === 0) upstream.close();
     });
-    upstream.addEventListener("close", (event) => ws.close(event.code, event.reason));
+    upstream.addEventListener("close", (event) => {
+      ws.close(relayableCloseCode(event.code), event.reason);
+    });
     upstream.addEventListener("error", () => ws.close(1011, "upstream websocket error"));
     Object.assign(ws.data, { pending });
   }
@@ -607,7 +749,7 @@ export class AkanApp {
   }
 
   #handleWsClose(ws: Bun.ServerWebSocket<GatewayWsData>, code: number, reason: string) {
-    ws.data.upstream.close(code, reason);
+    ws.data.upstream.close(relayableCloseCode(code), reason);
     const child = this.#children.get(ws.data.childIdx);
     if (child) child.metrics.activeWebSockets = Math.max(0, (child.metrics.activeWebSockets ?? 1) - 1);
   }
@@ -661,14 +803,12 @@ export class AkanApp {
     };
   }
 
-  async #proxyHttp(req: Request): Promise<Response> {
-    const child = this.#pickFederationChild();
-    if (!child?.upstream || child.upstream.type !== "unix") {
-      return this.#respondWithCrashPage(req) ?? new Response("No healthy federation child is ready", { status: 503 });
-    }
+  async #proxyHttp(req: Request, server: Bun.Server<GatewayWsData>): Promise<Response> {
+    const child = await this.#pickReadyFederationChild(req);
+    if (!child?.upstream || child.upstream.type !== "unix") return this.#respondWithUnavailable(req);
     const url = new URL(req.url);
     const upstreamUrl = `http://akan-child${url.pathname}${url.search}`;
-    const headers = this.#makeProxyHeaders(req, child.idx);
+    const headers = this.#makeProxyHeaders(req, child.idx, server);
     child.metrics.activeRequests = (child.metrics.activeRequests ?? 0) + 1;
     child.metrics.totalRequests = (child.metrics.totalRequests ?? 0) + 1;
     const traced = isTraceEnabled();
@@ -682,14 +822,14 @@ export class AkanApp {
         signal: req.signal,
         redirect: "manual",
       });
-      return this.#proxyResponse(upstreamRes);
+      return await this.#proxyResponse(req, upstreamRes);
     } catch (error) {
       if (AkanApp.#isUpstreamOpenFailure(error)) {
         this.logger.error(
           `Child ${child.idx}/${child.role} upstream is unreachable (${child.upstream.socketPath}); restarting`,
         );
         this.#scheduleChildRestart(child, child.proc, "upstream-open-failed");
-        return new Response("Federation child upstream is unreachable; restarting", { status: 503 });
+        return AkanApp.#unavailableResponse(req, "Federation child upstream is unreachable; restarting");
       }
       throw error;
     } finally {
@@ -705,26 +845,82 @@ export class AkanApp {
   }
 
   /**
-   * Dev-only: every traffic replica is in the crashed terminal state, so a bare 503 would hide the
-   * boot error from the browser. Surface it, and reload once a fixed gateway takes over the port.
+   * A replica that is booting or restarting is normally back within a second or two, and a browser
+   * handed a 503 stays on that dead page until somebody reloads by hand — so a request waits for the
+   * upstream instead of failing at it. A dev crash loop is terminal, so it is answered immediately.
    */
-  #respondWithCrashPage(req: Request): Response | null {
+  async #pickReadyFederationChild(req: Request): Promise<ChildState | null> {
+    const ready = this.#pickFederationChild();
+    if (ready?.upstream) return ready;
+    const deadline = performance.now() + this.#upstreamWaitMs;
+    while (performance.now() < deadline) {
+      if (this.#stopping || req.signal.aborted || this.#getCrashLoopDetail()) return null;
+      await Bun.sleep(50);
+      const child = this.#pickFederationChild();
+      if (child?.upstream) return child;
+    }
+    return null;
+  }
+
+  /** Dev-only terminal state: every traffic replica gave up booting. The detail is the boot error. */
+  #getCrashLoopDetail(): string | null {
     if (!this.#devHosted) return null;
     const trafficChildren = [...this.#children.values()].filter((child) => child.role !== "batch");
     if (trafficChildren.length === 0) return null;
     if (!trafficChildren.every((child) => child.status === "crashed")) return null;
-    const detail =
+    return (
       trafficChildren.map((child) => child.lastErrorMessage ?? child.lastRestartReason).find(Boolean) ??
-      "unknown boot error";
-    const message = `Backend failed to start after ${AkanApp.#devMaxChildBootFailures} boot attempts: ${detail}`;
-    if (!req.headers.get("accept")?.includes("text/html")) {
-      return new Response(message, { status: 503, headers: { "cache-control": "no-store" } });
+      "unknown boot error"
+    );
+  }
+
+  /**
+   * Both states answer 503, so an ingress or CDN reads them exactly as before; the HTML body is what
+   * a browser navigation needs, since nothing on a bare-text 503 can bring the page back on its own.
+   */
+  #respondWithUnavailable(req: Request): Response {
+    const crashDetail = this.#getCrashLoopDetail();
+    const page = crashDetail
+      ? {
+          heading: "Backend failed to start",
+          detail: crashDetail,
+          text: `Backend failed to start after ${AkanApp.#devMaxChildBootFailures} boot attempts: ${crashDetail}`,
+          note: `The dev server stopped retrying after ${AkanApp.#devMaxChildBootFailures} failed boots. Fix the error and save — this page reloads automatically.`,
+        }
+      : {
+          heading: "Backend is starting",
+          detail: "No healthy federation child is ready",
+          text: "No healthy federation child is ready",
+          note: "A replica is booting or restarting — this page reloads itself as soon as it answers.",
+        };
+    if (req.headers.get("accept")?.includes("text/html")) {
+      return new Response(AkanApp.#statusPageHtml(page), {
+        status: 503,
+        headers: { "content-type": "text/html; charset=utf-8", "cache-control": "no-store" },
+      });
     }
-    const html = `<!doctype html>
+    return AkanApp.#unavailableResponse(req, page.text);
+  }
+
+  /**
+   * An API caller asks for JSON, so it is answered with the payload `HttpClient` restores into an `Err` —
+   * a bare-text 503 reaches the browser as a JSON parse error naming the page body instead of the outage.
+   */
+  static #unavailableResponse(req: Request, detail: string): Response {
+    const headers = { "cache-control": "no-store" };
+    if (!req.headers.get("accept")?.includes("application/json")) return new Response(detail, { status: 503, headers });
+    return Response.json(
+      { error: "base.error.serverUnavailable", statusCode: 503, data: { status: 503 }, details: detail },
+      { status: 503, headers },
+    );
+  }
+
+  static #statusPageHtml({ heading, detail, note }: { heading: string; detail: string; note: string }) {
+    return `<!doctype html>
 <html>
 <head>
 <meta charset="utf-8" />
-<title>Backend failed to start</title>
+<title>${AkanApp.#escapeHtml(heading)}</title>
 <style>
   body { margin: 0; padding: 48px 24px; background: #111827; color: #e5e7eb; font-family: ui-sans-serif, system-ui, sans-serif; }
   main { max-width: 720px; margin: 0 auto; }
@@ -735,9 +931,9 @@ export class AkanApp {
 </head>
 <body>
 <main>
-<h1>Backend failed to start</h1>
+<h1>${AkanApp.#escapeHtml(heading)}</h1>
 <pre>${AkanApp.#escapeHtml(detail)}</pre>
-<p>The dev server stopped retrying after ${AkanApp.#devMaxChildBootFailures} failed boots. Fix the error and save &mdash; this page reloads automatically.</p>
+<p>${AkanApp.#escapeHtml(note)}</p>
 </main>
 <script>
   const poll = async () => {
@@ -751,10 +947,6 @@ export class AkanApp {
 </script>
 </body>
 </html>`;
-    return new Response(html, {
-      status: 503,
-      headers: { "content-type": "text/html; charset=utf-8", "cache-control": "no-store" },
-    });
   }
 
   static #escapeHtml(text: string) {
@@ -778,17 +970,23 @@ export class AkanApp {
     this.#proxyHopMaxMs = Math.max(this.#proxyHopMaxMs, durationMs);
   }
 
-  #proxyResponse(upstreamRes: Response): Response {
+  async #proxyResponse(req: Request, upstreamRes: Response): Promise<Response> {
     const headers = new Headers(upstreamRes.headers);
 
     headers.delete("content-encoding");
     headers.delete("content-length");
     this.#rewriteInternalLocation(headers);
-    return new Response(upstreamRes.body, {
+    const proxied = new Response(upstreamRes.body, {
       status: upstreamRes.status,
       statusText: upstreamRes.statusText,
       headers,
     });
+
+    return AkanApp.#isProxiedJson(headers) ? await compressResponse(req, proxied) : proxied;
+  }
+
+  static #isProxiedJson(headers: Headers): boolean {
+    return (headers.get("content-type") ?? "").split(";")[0]?.trim().toLowerCase() === "application/json";
   }
 
   #rewriteInternalLocation(headers: Headers) {
@@ -837,8 +1035,8 @@ export class AkanApp {
     return resolved;
   }
 
-  #makeProxyHeaders(req: Request, childIdx: number) {
-    return makeAkanChildProxyHeaders(req, childIdx);
+  #makeProxyHeaders(req: Request, childIdx: number, server?: Bun.Server<GatewayWsData>) {
+    return makeAkanChildProxyHeaders(req, childIdx, server?.requestIP(req));
   }
 
   #invalidateFederationChildCache() {
@@ -902,6 +1100,10 @@ export class AkanApp {
       case "metrics.report":
         this.#updateMetrics(idx, message.metrics);
         return;
+      case "log.records":
+        this.#logHub?.ingestMany(message.records);
+        if (message.dropped) this.logger.warn(`Child ${idx} dropped ${message.dropped} log records (ipc backpressure)`);
+        return;
       case "health.pong":
         this.#markHealthy(idx);
         return;
@@ -948,6 +1150,9 @@ export class AkanApp {
     child.healthPath = message.healthPath;
     child.lastPongAtMono = performance.now();
     child.restartAttempts = 0;
+    
+    if (this.#logHub && this.#logHub.floor !== null)
+      this.#sendToChild(child, { type: "log.level", minSev: this.#logHub.floor });
     child.restartPending = false;
     child.lastErrorMessage = undefined;
     this.#invalidateFederationChildCache();
@@ -1140,6 +1345,10 @@ export class AkanApp {
     }
   }
 
+  #fanoutToAll(message: AkanIpcMessage) {
+    for (const child of this.#children.values()) this.#sendToChild(child, message);
+  }
+
   #fanoutToBatch(message: AkanIpcMessage) {
     for (const child of this.#children.values()) {
       if (child.role === "batch" || child.role === "all") {
@@ -1165,99 +1374,30 @@ export class AkanApp {
     }
   }
 
-  async #pipeOutput(
-    idx: number,
-    role: AkanChildRole,
-    stream: ReadableStream<Uint8Array> | null,
-    type: "stdout" | "stderr",
-  ) {
-    if (!stream) return;
-    const bufferKey = `${idx}:${type}`;
-    const reader = stream.getReader();
-    const decoder = new TextDecoder();
-    try {
-      for (;;) {
-        const { done, value } = await reader.read();
-        if (done) break;
-        const text = decoder.decode(value, { stream: true });
-        this.#writeChildOutput(idx, role, type, bufferKey, text);
-      }
-      const remaining = decoder.decode();
-      if (remaining) this.#writeChildOutput(idx, role, type, bufferKey, remaining);
-    } finally {
-      this.#flushChildOutput(idx, role, type, bufferKey);
-      if (type === "stderr") this.#flushChildStderrBlock(idx, role, AkanApp.#childStderrBlockKey(idx, role));
-    }
-  }
-
-  #writeChildOutput(idx: number, role: AkanChildRole, type: "stdout" | "stderr", bufferKey: string, text: string) {
-    let buffered = `${this.#childOutputBuffers.get(bufferKey) ?? ""}${text}`;
-    for (;;) {
-      const newlineIdx = buffered.indexOf("\n");
-      if (newlineIdx === -1) break;
-      const line = buffered.slice(0, newlineIdx + 1);
-      buffered = buffered.slice(newlineIdx + 1);
-      this.#writeChildOutputLine(idx, role, type, line);
+  #writeChildLine(idx: number, role: AkanChildRole, pid: number | null, type: "stdout" | "stderr", line: string) {
+    if (Logger.isNdjson) {
+      this.#logHub?.ingest(ChildOutputReader.toRecord({ type, text: line, name: "child", role, replicaIdx: idx, pid }));
+      return;
     }
-    if (buffered) this.#childOutputBuffers.set(bufferKey, buffered);
-    else this.#childOutputBuffers.delete(bufferKey);
-  }
-
-  #flushChildOutput(idx: number, role: AkanChildRole, type: "stdout" | "stderr", bufferKey: string) {
-    const buffered = this.#childOutputBuffers.get(bufferKey);
-    if (!buffered) return;
-    this.#childOutputBuffers.delete(bufferKey);
-    this.#writeChildOutputLine(idx, role, type, `${buffered}\n`);
-  }
-
-  #writeChildOutputLine(idx: number, role: AkanChildRole, type: "stdout" | "stderr", line: string) {
-    if (type === "stderr" && this.#bufferChildStderrLine(idx, role, line)) return;
-    this.#writeChildOutputLineRaw(idx, role, type, line);
-  }
-
-  #writeChildOutputLineRaw(idx: number, role: AkanChildRole, type: "stdout" | "stderr", line: string) {
     const prefixedLine = `[child:${idx} ${role}] [${type}] ${line}`;
-    process[type].write(prefixedLine);
+    if (type === "stdout" || this.#printChildStderr) process[type].write(prefixedLine);
     this.#logWriter?.write(`${idx}-${role}`, AkanApp.#stripAnsi(prefixedLine));
   }
 
-  #bufferChildStderrLine(idx: number, role: AkanChildRole, line: string): boolean {
-    const key = AkanApp.#childStderrBlockKey(idx, role);
-    const block = this.#childStderrBlockBuffers.get(key) ?? [];
-    block.push(line);
-    this.#childStderrBlockBuffers.set(key, block);
-
-    const existingTimer = this.#childStderrBlockTimers.get(key);
-    if (existingTimer) clearTimeout(existingTimer);
-
-    if (line.trim() === "" || block.length >= 64) {
-      this.#flushChildStderrBlock(idx, role, key);
-      return true;
-    }
-
-    this.#childStderrBlockTimers.set(
-      key,
-      setTimeout(() => this.#flushChildStderrBlock(idx, role, key), 50),
-    );
-    return true;
-  }
-
-  #flushChildStderrBlock(idx: number, role: AkanChildRole, key: string) {
-    const timer = this.#childStderrBlockTimers.get(key);
-    if (timer) clearTimeout(timer);
-    this.#childStderrBlockTimers.delete(key);
-
-    const block = this.#childStderrBlockBuffers.get(key);
-    if (!block?.length) return;
-    this.#childStderrBlockBuffers.delete(key);
-
-    const text = block.join("");
+  /**
+   * In ndjson mode a stack trace is one record, and `AKAN_CHILD_STDERR` does not apply: the line that killed a
+   * child is the one the collector must not miss.
+   */
+  #writeChildStderrBlock(idx: number, role: AkanChildRole, pid: number | null, lines: string[]) {
+    const text = lines.join("");
     if (AkanApp.#isBenignRsdwConnectionClosedBlock(text)) return;
-    for (const blockLine of block) this.#writeChildOutputLineRaw(idx, role, "stderr", blockLine);
-  }
-
-  static #childStderrBlockKey(idx: number, role: AkanChildRole): string {
-    return `${idx}:${role}:stderr`;
+    if (Logger.isNdjson) {
+      this.#logHub?.ingest(
+        ChildOutputReader.toRecord({ type: "stderr", text, name: "child", role, replicaIdx: idx, pid }),
+      );
+      return;
+    }
+    for (const line of lines) this.#writeChildLine(idx, role, pid, "stderr", line);
   }
 
   static #isBenignRsdwConnectionClosedBlock(text: string): boolean {