akanjs 2.4.0-rc.0 → 2.4.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/base/baseEnv.ts +5 -1
- package/package.json +1 -1
- package/server/akanApp.ts +204 -34
- package/server/akanServer.ts +46 -8
- package/server/lifecycle/portInUse.ts +11 -0
- package/server/rscWorker.tsx +5 -0
- package/service/ipcTypes.ts +3 -1
- package/types/server/lifecycle/portInUse.d.ts +5 -0
- package/types/service/ipcTypes.d.ts +5 -1
package/base/baseEnv.ts
CHANGED
|
@@ -104,7 +104,11 @@ export const getEnv = (): ClientEnv => {
|
|
|
104
104
|
const clientHost =
|
|
105
105
|
process.env.AKAN_PUBLIC_CLIENT_HOST ??
|
|
106
106
|
(operationMode === "local" || side === "server" ? "localhost" : window.location.hostname);
|
|
107
|
-
const clientPort =
|
|
107
|
+
const clientPort =
|
|
108
|
+
side === "server"
|
|
109
|
+
? parseInt(process.env.AKAN_PUBLIC_CLIENT_PORT ?? (operationMode === "local" ? "8282" : "443"))
|
|
110
|
+
: parseInt(window.location.port || (window.location.protocol === "https:" ? "443" : "80"));
|
|
111
|
+
|
|
108
112
|
const clientHttpProtocol =
|
|
109
113
|
side === "client"
|
|
110
114
|
? (window.location.protocol as "http:" | "https:")
|
package/package.json
CHANGED
package/server/akanApp.ts
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import { mkdir, rm } from "node:fs/promises";
|
|
1
|
+
import { mkdir, readdir, rm } from "node:fs/promises";
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { Logger } from "akanjs/common";
|
|
4
4
|
import type { AkanChildRole, AkanChildStatus, AkanIpcMessage, AkanMetricsReport, AkanUpstream } from "akanjs/service";
|
|
5
5
|
import { isTraceEnabled } from "akanjs/signal";
|
|
6
6
|
import { makeAkanChildProxyHeaders } from "./akanAppHeaders";
|
|
7
7
|
import type { BuilderMessage, BuilderReq, BuilderRes } from "./artifact";
|
|
8
|
+
import { isPortInUseError } from "./lifecycle/portInUse";
|
|
8
9
|
import { RotatingLogWriter } from "./logging/rotatingLogWriter";
|
|
9
10
|
import { ProcessMetricsCollector } from "./processMetricsCollector";
|
|
10
11
|
|
|
@@ -16,9 +17,11 @@ interface ChildState {
|
|
|
16
17
|
status: AkanChildStatus;
|
|
17
18
|
pid?: number;
|
|
18
19
|
upstream?: AkanUpstream;
|
|
20
|
+
wsUpstream?: Extract<AkanUpstream, { type: "tcp" }>;
|
|
19
21
|
healthPath?: string;
|
|
20
22
|
metrics: AkanMetricsReport;
|
|
21
|
-
|
|
23
|
+
/** Monotonic (`performance.now()`) so sleep/wake wall-clock jumps cannot fake a health timeout. */
|
|
24
|
+
lastPongAtMono?: number;
|
|
22
25
|
restartAttempts: number;
|
|
23
26
|
restartCount: number;
|
|
24
27
|
restartTimer: Timer | null;
|
|
@@ -26,6 +29,7 @@ interface ChildState {
|
|
|
26
29
|
lastExitCode?: number | null;
|
|
27
30
|
lastRestartAt?: number;
|
|
28
31
|
lastRestartReason?: string;
|
|
32
|
+
lastErrorMessage?: string;
|
|
29
33
|
}
|
|
30
34
|
|
|
31
35
|
interface GatewayWsData {
|
|
@@ -62,8 +66,13 @@ export class AkanApp {
|
|
|
62
66
|
static readonly #childRestartBaseDelayMs = 1_000;
|
|
63
67
|
static readonly #childRestartMaxDelayMs = 30_000;
|
|
64
68
|
static readonly #childRestartGraceMs = 5_000;
|
|
69
|
+
/** In dev, stop restarting a replica that never boots after this many consecutive failures. */
|
|
70
|
+
static readonly #devMaxChildBootFailures = 3;
|
|
65
71
|
|
|
66
72
|
readonly logger = new Logger("AkanApp");
|
|
73
|
+
/** Hosted by `akan start`: crash loops should yield to the dev host, which restarts on file edits. */
|
|
74
|
+
readonly #devHosted = process.env.AKAN_COMMAND_TYPE === "start";
|
|
75
|
+
readonly #healthTimeoutMs = AkanApp.#parseHealthTimeoutMs();
|
|
67
76
|
readonly #serverPath: string;
|
|
68
77
|
readonly #artifactDir: string;
|
|
69
78
|
readonly #replica: AkanReplicaConfig;
|
|
@@ -103,10 +112,13 @@ export class AkanApp {
|
|
|
103
112
|
this.#serverPath = AkanApp.#resolveServerPath(resolvedOptions.serverPath ?? serverPath);
|
|
104
113
|
this.#artifactDir = path.resolve(path.dirname(this.#serverPath), ".akan", "artifact");
|
|
105
114
|
this.#replica = AkanApp.#parseReplicaConfig(resolvedOptions.replica);
|
|
106
|
-
this.#runtimeDir =
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
115
|
+
this.#runtimeDir = path.resolve(
|
|
116
|
+
resolvedOptions.runtimeDir ??
|
|
117
|
+
process.env.AKAN_RUNTIME_DIR ??
|
|
118
|
+
(process.env.NODE_ENV === "production"
|
|
119
|
+
? path.resolve(process.cwd(), "runtime")
|
|
120
|
+
: path.resolve(process.cwd(), "local", "apps", process.env.AKAN_PUBLIC_APP_NAME ?? "unknown", "runtime")),
|
|
121
|
+
);
|
|
110
122
|
this.#port = Number(resolvedOptions.port ?? process.env.PORT ?? 8282);
|
|
111
123
|
this.#wsBasePort = Number(resolvedOptions.wsBasePort ?? process.env.AKAN_WS_BASE_PORT ?? this.#port + 10_000);
|
|
112
124
|
this.#openapi = resolvedOptions.openapi;
|
|
@@ -149,15 +161,46 @@ export class AkanApp {
|
|
|
149
161
|
return Bun.main.endsWith(".js") ? "production" : "development";
|
|
150
162
|
}
|
|
151
163
|
|
|
164
|
+
/**
|
|
165
|
+
* Dev builds and first-touch transpiles can stall a child's event loop well past the production
|
|
166
|
+
* pong budget, so `akan start` runs with a wider timeout to avoid restarting healthy replicas.
|
|
167
|
+
*/
|
|
168
|
+
static #parseHealthTimeoutMs() {
|
|
169
|
+
const configured = Number(process.env.AKAN_HEALTH_TIMEOUT_MS);
|
|
170
|
+
if (Number.isFinite(configured) && configured > 0) return configured;
|
|
171
|
+
return process.env.AKAN_COMMAND_TYPE === "start" ? 15_000 : 5_000;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Must exceed the child's own shutdown timeout (see `AkanServer.#defaultShutdownTimeoutMs`) so
|
|
176
|
+
* children always get to exit on their own before this gateway stops waiting.
|
|
177
|
+
*/
|
|
178
|
+
static #childShutdownWaitMs() {
|
|
179
|
+
const configured = Number(process.env.AKAN_CHILD_SHUTDOWN_WAIT_MS);
|
|
180
|
+
if (Number.isFinite(configured) && configured > 0) return configured;
|
|
181
|
+
return process.env.AKAN_COMMAND_TYPE === "start" ? 5_000 : 30_000;
|
|
182
|
+
}
|
|
183
|
+
|
|
152
184
|
async start() {
|
|
153
185
|
await this.#prepareRuntimeDir();
|
|
154
186
|
this.#startFileLogging();
|
|
155
187
|
for (let idx = 0; idx < this.#replica.total; idx++) this.#spawn(idx);
|
|
156
|
-
|
|
188
|
+
try {
|
|
189
|
+
this.#listen();
|
|
190
|
+
} catch (error) {
|
|
191
|
+
if (isPortInUseError(error)) {
|
|
192
|
+
const message = `Port ${this.#port} is already in use — another \`akan start\` or an orphaned gateway may still be running (try: lsof -ti :${this.#port}).`;
|
|
193
|
+
this.logger.error(message);
|
|
194
|
+
this.#reportBackendBuildStatus({ ok: false, message });
|
|
195
|
+
}
|
|
196
|
+
await this.stop("listen-failed");
|
|
197
|
+
throw error;
|
|
198
|
+
}
|
|
157
199
|
this.#snapshotTimer = setInterval(() => this.#requestRoomSnapshots(), 30_000);
|
|
158
200
|
this.#healthTimer = setInterval(() => this.#checkHealth(), 2_000);
|
|
159
201
|
this.#startMetricsReporting();
|
|
160
202
|
process.on("message", (message) => this.#handleHostMessage(message as BuilderMessage));
|
|
203
|
+
process.on("disconnect", () => this.#handleHostDisconnect());
|
|
161
204
|
process.on("SIGINT", () => this.#handleShutdownSignal("SIGINT"));
|
|
162
205
|
process.on("SIGTERM", () => this.#handleShutdownSignal("SIGTERM"));
|
|
163
206
|
await new Promise<void>((resolve) => {
|
|
@@ -192,11 +235,12 @@ export class AkanApp {
|
|
|
192
235
|
}
|
|
193
236
|
await Promise.race([
|
|
194
237
|
Promise.all([...this.#children.values()].map((child) => child.proc.exited.catch(() => undefined))),
|
|
195
|
-
new Promise((resolve) => setTimeout(resolve,
|
|
238
|
+
new Promise((resolve) => setTimeout(resolve, AkanApp.#childShutdownWaitMs())),
|
|
196
239
|
]);
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
240
|
+
|
|
241
|
+
const stragglers = [...this.#children.values()].filter((child) => !child.proc.killed);
|
|
242
|
+
for (const child of stragglers) child.proc.kill("SIGKILL");
|
|
243
|
+
await Promise.all(stragglers.map((child) => child.proc.exited.catch(() => undefined)));
|
|
200
244
|
this.#children.clear();
|
|
201
245
|
await this.#stopFileLogging();
|
|
202
246
|
this.#resolveStopped?.();
|
|
@@ -213,6 +257,18 @@ export class AkanApp {
|
|
|
213
257
|
});
|
|
214
258
|
}
|
|
215
259
|
|
|
260
|
+
/**
|
|
261
|
+
* The IPC channel closes when the dev host dies (including SIGKILL). Exiting takes the children
|
|
262
|
+
* down too, so a dead host never strands a gateway tree that would block the next `akan start`.
|
|
263
|
+
*/
|
|
264
|
+
#handleHostDisconnect() {
|
|
265
|
+
if (this.#stopping) return;
|
|
266
|
+
this.logger.warn("Host IPC channel closed; shutting down gateway and children");
|
|
267
|
+
this.#exitAfterStop = true;
|
|
268
|
+
setTimeout(() => process.exit(1), AkanApp.#childShutdownWaitMs() + 5_000);
|
|
269
|
+
void this.stop("ipc-disconnect").catch(() => process.exit(1));
|
|
270
|
+
}
|
|
271
|
+
|
|
216
272
|
#spawn(idx: number) {
|
|
217
273
|
const role = this.#getRole(idx);
|
|
218
274
|
const upstream = this.#getChildUpstream(idx, role);
|
|
@@ -261,6 +317,7 @@ export class AkanApp {
|
|
|
261
317
|
#handleChildExit(idx: number, proc: Bun.Subprocess<"ignore", "pipe", "pipe">, code: number | null) {
|
|
262
318
|
const child = this.#children.get(idx);
|
|
263
319
|
if (!child || child.proc !== proc) return;
|
|
320
|
+
if (child.status === "crashed") return;
|
|
264
321
|
child.status = "exited";
|
|
265
322
|
child.lastExitCode = code;
|
|
266
323
|
this.#invalidateFederationChildCache();
|
|
@@ -276,11 +333,16 @@ export class AkanApp {
|
|
|
276
333
|
child.lastRestartReason = reason;
|
|
277
334
|
return;
|
|
278
335
|
}
|
|
336
|
+
if (this.#devHosted && child.restartAttempts >= AkanApp.#devMaxChildBootFailures - 1 && !child.ready) {
|
|
337
|
+
this.#markChildCrashed(child, reason);
|
|
338
|
+
return;
|
|
339
|
+
}
|
|
279
340
|
|
|
280
341
|
child.restartPending = true;
|
|
281
342
|
child.ready = false;
|
|
282
343
|
child.status = reason === "health-timeout" || reason === "upstream-open-failed" ? "unhealthy" : "exited";
|
|
283
344
|
child.upstream = undefined;
|
|
345
|
+
child.wsUpstream = undefined;
|
|
284
346
|
child.healthPath = undefined;
|
|
285
347
|
this.#invalidateFederationChildCache();
|
|
286
348
|
child.lastRestartReason = reason;
|
|
@@ -341,6 +403,41 @@ export class AkanApp {
|
|
|
341
403
|
this.#spawn(idx);
|
|
342
404
|
}
|
|
343
405
|
|
|
406
|
+
/**
|
|
407
|
+
* Dev-only terminal state: the same broken code fails every boot, so retrying is pure churn.
|
|
408
|
+
* The dev host replaces this whole gateway on the next server-side edit, which clears the state.
|
|
409
|
+
*/
|
|
410
|
+
#markChildCrashed(child: ChildState, reason: string) {
|
|
411
|
+
|
|
412
|
+
if (child.status === "crashed") return;
|
|
413
|
+
child.ready = false;
|
|
414
|
+
child.status = "crashed";
|
|
415
|
+
child.restartPending = false;
|
|
416
|
+
child.upstream = undefined;
|
|
417
|
+
child.wsUpstream = undefined;
|
|
418
|
+
child.healthPath = undefined;
|
|
419
|
+
child.lastRestartReason = reason;
|
|
420
|
+
this.#invalidateFederationChildCache();
|
|
421
|
+
this.#removeChildRooms(child.idx);
|
|
422
|
+
const attempts = child.restartAttempts + 1;
|
|
423
|
+
const detail = child.lastErrorMessage ?? reason;
|
|
424
|
+
const message = `Backend replica ${child.idx}/${child.role} failed ${attempts} consecutive boots; waiting for a code change to retry: ${detail}`;
|
|
425
|
+
this.logger.error(`[child-crash-loop] ${message}`);
|
|
426
|
+
this.#reportBackendBuildStatus({ ok: false, message });
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
/** Forwards a backend-phase build status to the dev host so failures reach the HMR overlay. */
|
|
430
|
+
#reportBackendBuildStatus({ ok, message }: { ok: boolean; message: string }) {
|
|
431
|
+
process.send?.({
|
|
432
|
+
type: "build-status",
|
|
433
|
+
data: { generation: -1, phase: "backend", ok, files: [], message },
|
|
434
|
+
} satisfies BuilderMessage);
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
#isChildUnavailable(child: ChildState): boolean {
|
|
438
|
+
return child.proc.killed || child.status === "exited" || child.status === "crashed";
|
|
439
|
+
}
|
|
440
|
+
|
|
344
441
|
async #stopChildForRestart(child: ChildState, proc: Bun.Subprocess<"ignore", "pipe", "pipe">, reason: string) {
|
|
345
442
|
if (reason.startsWith("exit:") || proc.killed) return;
|
|
346
443
|
if (!proc.killed) {
|
|
@@ -359,9 +456,13 @@ export class AkanApp {
|
|
|
359
456
|
|
|
360
457
|
async #prepareRuntimeDir() {
|
|
361
458
|
await mkdir(this.#runtimeDir, { recursive: true });
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
459
|
+
|
|
460
|
+
const entries = await readdir(this.#runtimeDir).catch(() => []);
|
|
461
|
+
await Promise.all(
|
|
462
|
+
entries
|
|
463
|
+
.filter((name) => /^akan-child-.*\.sock$/.test(name))
|
|
464
|
+
.map((name) => rm(path.join(this.#runtimeDir, name), { force: true }).catch(() => undefined)),
|
|
465
|
+
);
|
|
365
466
|
}
|
|
366
467
|
|
|
367
468
|
async #removeChildSocket(idx: number, role: AkanChildRole) {
|
|
@@ -455,11 +556,9 @@ export class AkanApp {
|
|
|
455
556
|
|
|
456
557
|
#upgradeWebSocket(req: Request, server: Bun.Server<GatewayWsData>): Response | undefined {
|
|
457
558
|
const child = this.#pickFederationChild();
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
}
|
|
461
|
-
const upstream = this.#getChildUpstream(child.idx, child.role).ws;
|
|
462
|
-
if (!upstream) return new Response("No websocket upstream is ready", { status: 503 });
|
|
559
|
+
|
|
560
|
+
const upstream = child?.upstream ? (child.wsUpstream ?? this.#getChildUpstream(child.idx, child.role).ws) : null;
|
|
561
|
+
if (!child || !upstream) return new Response("No websocket upstream is ready", { status: 503 });
|
|
463
562
|
const url = new URL(req.url);
|
|
464
563
|
const upstreamWs = new WebSocket(`ws://${upstream.host}:${upstream.port}${url.pathname}${url.search}`, {
|
|
465
564
|
headers: this.#makeProxyHeaders(req, child.idx),
|
|
@@ -513,6 +612,7 @@ export class AkanApp {
|
|
|
513
612
|
#getHealthStatus() {
|
|
514
613
|
return {
|
|
515
614
|
status: this.#stopping ? "stopping" : "running",
|
|
615
|
+
pid: process.pid,
|
|
516
616
|
children: [...this.#children.values()].map((child) => ({
|
|
517
617
|
idx: child.idx,
|
|
518
618
|
role: child.role,
|
|
@@ -526,6 +626,7 @@ export class AkanApp {
|
|
|
526
626
|
lastExitCode: child.lastExitCode,
|
|
527
627
|
lastRestartAt: child.lastRestartAt,
|
|
528
628
|
lastRestartReason: child.lastRestartReason,
|
|
629
|
+
lastErrorMessage: child.lastErrorMessage,
|
|
529
630
|
})),
|
|
530
631
|
};
|
|
531
632
|
}
|
|
@@ -560,7 +661,7 @@ export class AkanApp {
|
|
|
560
661
|
async #proxyHttp(req: Request): Promise<Response> {
|
|
561
662
|
const child = this.#pickFederationChild();
|
|
562
663
|
if (!child?.upstream || child.upstream.type !== "unix") {
|
|
563
|
-
return new Response("No healthy federation child is ready", { status: 503 });
|
|
664
|
+
return this.#respondWithCrashPage(req) ?? new Response("No healthy federation child is ready", { status: 503 });
|
|
564
665
|
}
|
|
565
666
|
const url = new URL(req.url);
|
|
566
667
|
const upstreamUrl = `http://akan-child${url.pathname}${url.search}`;
|
|
@@ -600,6 +701,70 @@ export class AkanApp {
|
|
|
600
701
|
return candidate.code === "FailedToOpenSocket" || String(candidate.message ?? "").includes("FailedToOpenSocket");
|
|
601
702
|
}
|
|
602
703
|
|
|
704
|
+
/**
|
|
705
|
+
* Dev-only: every traffic replica is in the crashed terminal state, so a bare 503 would hide the
|
|
706
|
+
* boot error from the browser. Surface it, and reload once a fixed gateway takes over the port.
|
|
707
|
+
*/
|
|
708
|
+
#respondWithCrashPage(req: Request): Response | null {
|
|
709
|
+
if (!this.#devHosted) return null;
|
|
710
|
+
const trafficChildren = [...this.#children.values()].filter((child) => child.role !== "batch");
|
|
711
|
+
if (trafficChildren.length === 0) return null;
|
|
712
|
+
if (!trafficChildren.every((child) => child.status === "crashed")) return null;
|
|
713
|
+
const detail =
|
|
714
|
+
trafficChildren.map((child) => child.lastErrorMessage ?? child.lastRestartReason).find(Boolean) ??
|
|
715
|
+
"unknown boot error";
|
|
716
|
+
const message = `Backend failed to start after ${AkanApp.#devMaxChildBootFailures} boot attempts: ${detail}`;
|
|
717
|
+
if (!req.headers.get("accept")?.includes("text/html")) {
|
|
718
|
+
return new Response(message, { status: 503, headers: { "cache-control": "no-store" } });
|
|
719
|
+
}
|
|
720
|
+
const html = `<!doctype html>
|
|
721
|
+
<html>
|
|
722
|
+
<head>
|
|
723
|
+
<meta charset="utf-8" />
|
|
724
|
+
<title>Backend failed to start</title>
|
|
725
|
+
<style>
|
|
726
|
+
body { margin: 0; padding: 48px 24px; background: #111827; color: #e5e7eb; font-family: ui-sans-serif, system-ui, sans-serif; }
|
|
727
|
+
main { max-width: 720px; margin: 0 auto; }
|
|
728
|
+
h1 { font-size: 20px; color: #f87171; }
|
|
729
|
+
pre { padding: 16px; border-radius: 8px; background: #1f2937; color: #fca5a5; white-space: pre-wrap; word-break: break-word; }
|
|
730
|
+
p { color: #9ca3af; font-size: 14px; }
|
|
731
|
+
</style>
|
|
732
|
+
</head>
|
|
733
|
+
<body>
|
|
734
|
+
<main>
|
|
735
|
+
<h1>Backend failed to start</h1>
|
|
736
|
+
<pre>${AkanApp.#escapeHtml(detail)}</pre>
|
|
737
|
+
<p>The dev server stopped retrying after ${AkanApp.#devMaxChildBootFailures} failed boots. Fix the error and save — this page reloads automatically.</p>
|
|
738
|
+
</main>
|
|
739
|
+
<script>
|
|
740
|
+
const poll = async () => {
|
|
741
|
+
try {
|
|
742
|
+
const res = await fetch(location.href, { cache: "no-store" });
|
|
743
|
+
if (res.ok) { location.reload(); return; }
|
|
744
|
+
} catch {}
|
|
745
|
+
setTimeout(poll, 1000);
|
|
746
|
+
};
|
|
747
|
+
setTimeout(poll, 1000);
|
|
748
|
+
</script>
|
|
749
|
+
</body>
|
|
750
|
+
</html>`;
|
|
751
|
+
return new Response(html, {
|
|
752
|
+
status: 503,
|
|
753
|
+
headers: { "content-type": "text/html; charset=utf-8", "cache-control": "no-store" },
|
|
754
|
+
});
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
static #escapeHtml(text: string) {
|
|
758
|
+
const replacements: Record<string, string> = {
|
|
759
|
+
"&": "&",
|
|
760
|
+
"<": "<",
|
|
761
|
+
">": ">",
|
|
762
|
+
'"': """,
|
|
763
|
+
"'": "'",
|
|
764
|
+
};
|
|
765
|
+
return text.replace(/[&<>"']/g, (ch) => replacements[ch] ?? ch);
|
|
766
|
+
}
|
|
767
|
+
|
|
603
768
|
/**
|
|
604
769
|
* Gateway-observed upstream round-trip time. The pure proxy overhead is this value
|
|
605
770
|
* minus the child handler time captured in the per-request trace.
|
|
@@ -709,8 +874,7 @@ export class AkanApp {
|
|
|
709
874
|
(child.role === "federation" || child.role === "all") &&
|
|
710
875
|
child.ready &&
|
|
711
876
|
child.status !== "unhealthy" &&
|
|
712
|
-
child
|
|
713
|
-
!child.proc.killed,
|
|
877
|
+
!this.#isChildUnavailable(child),
|
|
714
878
|
);
|
|
715
879
|
this.#federationChildCache = candidates;
|
|
716
880
|
if (candidates.length === 0) return null;
|
|
@@ -767,6 +931,7 @@ export class AkanApp {
|
|
|
767
931
|
return;
|
|
768
932
|
case "error":
|
|
769
933
|
this.logger.error(message.message);
|
|
934
|
+
if (child) child.lastErrorMessage = message.message;
|
|
770
935
|
if (child && proc) void this.#scheduleChildRestart(child, proc, "child-error");
|
|
771
936
|
return;
|
|
772
937
|
case "build-route":
|
|
@@ -798,13 +963,19 @@ export class AkanApp {
|
|
|
798
963
|
child.status = "ready";
|
|
799
964
|
child.pid = message.pid;
|
|
800
965
|
child.upstream = message.upstream;
|
|
966
|
+
child.wsUpstream = message.wsUpstream;
|
|
801
967
|
child.healthPath = message.healthPath;
|
|
802
|
-
child.
|
|
968
|
+
child.lastPongAtMono = performance.now();
|
|
803
969
|
child.restartAttempts = 0;
|
|
804
970
|
child.restartPending = false;
|
|
971
|
+
child.lastErrorMessage = undefined;
|
|
805
972
|
this.#invalidateFederationChildCache();
|
|
806
|
-
|
|
973
|
+
|
|
974
|
+
const trafficChildren = [...this.#children.values()].filter((item) => item.role !== "batch");
|
|
975
|
+
if (child.role !== "batch" && trafficChildren.every((item) => item.ready)) {
|
|
807
976
|
process.send?.({ type: "backend-ready", pid: process.pid } satisfies AkanIpcMessage);
|
|
977
|
+
}
|
|
978
|
+
if ([...this.#children.values()].every((item) => item.ready)) {
|
|
808
979
|
this.logger.verbose(`All ${this.#children.size} child process(es) are ready`);
|
|
809
980
|
}
|
|
810
981
|
}
|
|
@@ -813,7 +984,7 @@ export class AkanApp {
|
|
|
813
984
|
const child = this.#children.get(idx);
|
|
814
985
|
if (!child) return;
|
|
815
986
|
child.status = "healthy";
|
|
816
|
-
child.
|
|
987
|
+
child.lastPongAtMono = performance.now();
|
|
817
988
|
this.#invalidateFederationChildCache();
|
|
818
989
|
}
|
|
819
990
|
|
|
@@ -828,7 +999,7 @@ export class AkanApp {
|
|
|
828
999
|
for (const childIdx of targets) {
|
|
829
1000
|
if (childIdx === originIdx) continue;
|
|
830
1001
|
const child = this.#children.get(childIdx);
|
|
831
|
-
if (!child || child
|
|
1002
|
+
if (!child || this.#isChildUnavailable(child)) continue;
|
|
832
1003
|
if (
|
|
833
1004
|
!this.#sendToChild(child, {
|
|
834
1005
|
type: "pubsub.deliver",
|
|
@@ -935,31 +1106,30 @@ export class AkanApp {
|
|
|
935
1106
|
|
|
936
1107
|
#requestRoomSnapshots() {
|
|
937
1108
|
for (const child of this.#children.values()) {
|
|
938
|
-
if (child
|
|
1109
|
+
if (this.#isChildUnavailable(child)) continue;
|
|
939
1110
|
this.#sendToChild(child, { type: "pubsub.snapshot.request" } satisfies AkanIpcMessage);
|
|
940
1111
|
}
|
|
941
1112
|
}
|
|
942
1113
|
|
|
943
1114
|
#checkHealth() {
|
|
944
|
-
const
|
|
1115
|
+
const nowMono = performance.now();
|
|
945
1116
|
for (const child of this.#children.values()) {
|
|
946
|
-
if (child
|
|
947
|
-
if (child.
|
|
1117
|
+
if (this.#isChildUnavailable(child)) continue;
|
|
1118
|
+
if (child.lastPongAtMono && nowMono - child.lastPongAtMono > this.#healthTimeoutMs) {
|
|
948
1119
|
child.status = "unhealthy";
|
|
949
1120
|
this.#invalidateFederationChildCache();
|
|
950
1121
|
void this.#scheduleChildRestart(child, child.proc, "health-timeout");
|
|
951
|
-
|
|
1122
|
+
continue;
|
|
952
1123
|
}
|
|
953
1124
|
const sent = this.#sendToChild(child, {
|
|
954
1125
|
type: "health.ping",
|
|
955
1126
|
nonce: crypto.randomUUID(),
|
|
956
|
-
sentAt: now,
|
|
1127
|
+
sentAt: Date.now(),
|
|
957
1128
|
} satisfies AkanIpcMessage);
|
|
958
1129
|
if (!sent) {
|
|
959
1130
|
child.status = "unhealthy";
|
|
960
1131
|
this.#invalidateFederationChildCache();
|
|
961
1132
|
void this.#scheduleChildRestart(child, child.proc, "health-send-failed");
|
|
962
|
-
return;
|
|
963
1133
|
}
|
|
964
1134
|
}
|
|
965
1135
|
}
|
|
@@ -1000,7 +1170,7 @@ export class AkanApp {
|
|
|
1000
1170
|
}
|
|
1001
1171
|
|
|
1002
1172
|
#sendToChild(child: ChildState, message: AkanIpcMessage | BuilderMessage): boolean {
|
|
1003
|
-
if (child
|
|
1173
|
+
if (this.#isChildUnavailable(child)) return false;
|
|
1004
1174
|
try {
|
|
1005
1175
|
child.proc.send(message);
|
|
1006
1176
|
return true;
|
package/server/akanServer.ts
CHANGED
|
@@ -16,6 +16,7 @@ import type { AkanLib, AkanLibProps } from "./akanLib";
|
|
|
16
16
|
import type { BuilderRpc } from "./artifact";
|
|
17
17
|
import { DiLifecycle } from "./di/diLifecycle";
|
|
18
18
|
import type { HmrWsData, HmrWsHub } from "./hmr/wsHub";
|
|
19
|
+
import { isPortInUseError } from "./lifecycle/portInUse";
|
|
19
20
|
import { ShutdownManager } from "./lifecycle/shutdownManager";
|
|
20
21
|
import { ProcessMetricsCollector } from "./processMetricsCollector";
|
|
21
22
|
import { WebProxyRunner } from "./proxy";
|
|
@@ -74,7 +75,7 @@ export class AkanServer {
|
|
|
74
75
|
websocketPrefix = "/ws";
|
|
75
76
|
openapi = AkanServer.#isOpenApiEnvEnabled();
|
|
76
77
|
serverMode: "federation" | "batch" | "all";
|
|
77
|
-
shutdownTimeoutMs =
|
|
78
|
+
shutdownTimeoutMs = AkanServer.#defaultShutdownTimeoutMs();
|
|
78
79
|
|
|
79
80
|
#di: DiLifecycle;
|
|
80
81
|
#localPublish: ((roomId: string, data: object | object[]) => void) | null = null;
|
|
@@ -257,10 +258,10 @@ export class AkanServer {
|
|
|
257
258
|
websocket: websocketHandlers,
|
|
258
259
|
} as Parameters<typeof Bun.serve>[0]);
|
|
259
260
|
if (unix && process.env.AKAN_CHILD_WS_PORT) {
|
|
260
|
-
const
|
|
261
|
-
|
|
261
|
+
const preferredWsPort = Number(process.env.AKAN_CHILD_WS_PORT);
|
|
262
|
+
const wsServeOptions = (port: number) => ({
|
|
262
263
|
idleTimeout: 0,
|
|
263
|
-
port
|
|
264
|
+
port,
|
|
264
265
|
routes: ApiRouter.buildRoutes({
|
|
265
266
|
prefix: this.prefix,
|
|
266
267
|
websocketPrefix: this.websocketPrefix,
|
|
@@ -268,12 +269,20 @@ export class AkanServer {
|
|
|
268
269
|
builtinRoutes,
|
|
269
270
|
routeOptions,
|
|
270
271
|
renderEnvRoutes,
|
|
271
|
-
upgradeAppWs: (req, data) => this.#wsServer?.upgrade(req, { data }) ?? false,
|
|
272
|
+
upgradeAppWs: (req: Request, data: { createdAt: number }) => this.#wsServer?.upgrade(req, { data }) ?? false,
|
|
272
273
|
webProxyRunner,
|
|
273
274
|
}),
|
|
274
275
|
websocket: websocketHandlers,
|
|
275
276
|
});
|
|
276
|
-
|
|
277
|
+
try {
|
|
278
|
+
this.#wsServer = Bun.serve(wsServeOptions(preferredWsPort));
|
|
279
|
+
} catch (error) {
|
|
280
|
+
if (!isPortInUseError(error)) throw error;
|
|
281
|
+
|
|
282
|
+
this.logger.warn(`ws port ${preferredWsPort} is in use; falling back to an ephemeral port`);
|
|
283
|
+
this.#wsServer = Bun.serve(wsServeOptions(0));
|
|
284
|
+
}
|
|
285
|
+
this.logger.verbose(`${this.name} websocket fallback is serving on port ${this.#wsServer.port}`);
|
|
277
286
|
}
|
|
278
287
|
|
|
279
288
|
const server = this.#server;
|
|
@@ -300,12 +309,14 @@ export class AkanServer {
|
|
|
300
309
|
this.#startMetricsReporting();
|
|
301
310
|
this.#di.registerSchedule(this.serverMode);
|
|
302
311
|
this.logger.verbose(`🚀 ${this.name} is running on ${unix ? `unix://${unix}` : `port ${port}`}`);
|
|
312
|
+
const wsPort = this.#wsServer?.port;
|
|
303
313
|
process.send?.({
|
|
304
314
|
type: "ready",
|
|
305
315
|
pid: process.pid,
|
|
306
316
|
replicaIdx: Number(process.env.AKAN_REPLICA_IDX ?? 0),
|
|
307
317
|
role: this.serverMode,
|
|
308
318
|
upstream: unix ? { type: "unix", socketPath: unix } : { type: "tcp", host: "127.0.0.1", port: Number(port) },
|
|
319
|
+
wsUpstream: typeof wsPort === "number" ? { type: "tcp", host: "127.0.0.1", port: wsPort } : undefined,
|
|
309
320
|
healthPath: "/_akan/app/child-health",
|
|
310
321
|
} satisfies AkanIpcMessage);
|
|
311
322
|
await this.#di.runSchedulerInit();
|
|
@@ -324,7 +335,7 @@ export class AkanServer {
|
|
|
324
335
|
if (!isNoListenCommand) {
|
|
325
336
|
this.#startMetricsReporting();
|
|
326
337
|
this.#di.registerSchedule(this.serverMode);
|
|
327
|
-
|
|
338
|
+
this.#registerParentIpc();
|
|
328
339
|
process.send?.({
|
|
329
340
|
type: "ready",
|
|
330
341
|
pid: process.pid,
|
|
@@ -336,7 +347,7 @@ export class AkanServer {
|
|
|
336
347
|
}
|
|
337
348
|
return this;
|
|
338
349
|
}
|
|
339
|
-
|
|
350
|
+
this.#registerParentIpc();
|
|
340
351
|
return this.listen();
|
|
341
352
|
}
|
|
342
353
|
async stop() {
|
|
@@ -368,6 +379,23 @@ export class AkanServer {
|
|
|
368
379
|
}
|
|
369
380
|
}
|
|
370
381
|
|
|
382
|
+
#registerParentIpc() {
|
|
383
|
+
process.on("message", (message) => this.#handleIpcMessage(message as AkanIpcMessage));
|
|
384
|
+
process.on("disconnect", () => this.#handleParentDisconnect());
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
/**
|
|
388
|
+
* The IPC channel closes when the parent gateway dies (including SIGKILL). Exiting here keeps a
|
|
389
|
+
* killed dev/gateway run from stranding replicas that would hold ports and break the next boot.
|
|
390
|
+
*/
|
|
391
|
+
#handleParentDisconnect() {
|
|
392
|
+
this.logger.warn("Parent IPC channel closed; shutting down to avoid an orphaned replica");
|
|
393
|
+
setTimeout(() => process.exit(1), this.shutdownTimeoutMs + 1_000);
|
|
394
|
+
void this.stop()
|
|
395
|
+
.then(() => process.exit(0))
|
|
396
|
+
.catch(() => process.exit(1));
|
|
397
|
+
}
|
|
398
|
+
|
|
371
399
|
#handleIpcMessage(message: AkanIpcMessage) {
|
|
372
400
|
if (!message || typeof message !== "object") return;
|
|
373
401
|
if (message.type === "pubsub.deliver") this.#localPublish?.(message.roomId, message.data as object | object[]);
|
|
@@ -461,6 +489,16 @@ export class AkanServer {
|
|
|
461
489
|
);
|
|
462
490
|
}
|
|
463
491
|
|
|
492
|
+
/**
|
|
493
|
+
* Shutdown must finish inside the gateway's child-wait budget or the layer above SIGKILLs this
|
|
494
|
+
* process and strands its resources; dev (`akan start`) keeps it short so edit-restarts stay snappy.
|
|
495
|
+
*/
|
|
496
|
+
static #defaultShutdownTimeoutMs() {
|
|
497
|
+
const configured = Number(process.env.AKAN_SHUTDOWN_TIMEOUT_MS);
|
|
498
|
+
if (Number.isFinite(configured) && configured > 0) return configured;
|
|
499
|
+
return process.env.AKAN_COMMAND_TYPE === "start" ? 3_000 : 30_000;
|
|
500
|
+
}
|
|
501
|
+
|
|
464
502
|
async #withShutdownTimeout<T>(promise: Promise<T>) {
|
|
465
503
|
let timeout: Timer | null = null;
|
|
466
504
|
try {
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Detects "address already in use" listen failures across the shapes Bun throws them in
|
|
3
|
+
* (`code: "EADDRINUSE"` on newer versions, message-only on older ones).
|
|
4
|
+
*/
|
|
5
|
+
export const isPortInUseError = (error: unknown): boolean => {
|
|
6
|
+
if (!error || typeof error !== "object") return false;
|
|
7
|
+
const candidate = error as { code?: unknown; message?: unknown };
|
|
8
|
+
if (candidate.code === "EADDRINUSE") return true;
|
|
9
|
+
const message = String(candidate.message ?? "");
|
|
10
|
+
return message.includes("EADDRINUSE") || message.toLowerCase().includes("in use");
|
|
11
|
+
};
|
package/server/rscWorker.tsx
CHANGED
|
@@ -216,6 +216,11 @@ export class RscRenderer {
|
|
|
216
216
|
}
|
|
217
217
|
this.#send = process.send.bind(process) as (message: unknown) => void;
|
|
218
218
|
process.on("message", (msg: InMsg) => this.#handleMessage(msg));
|
|
219
|
+
|
|
220
|
+
process.on("disconnect", () => {
|
|
221
|
+
this.#logger.warn("parent IPC channel closed; exiting rsc worker");
|
|
222
|
+
process.exit(0);
|
|
223
|
+
});
|
|
219
224
|
this.#logger.verbose(`constructed (pid=${process.pid})`);
|
|
220
225
|
}
|
|
221
226
|
|
package/service/ipcTypes.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export type AkanChildRole = "all" | "federation" | "batch";
|
|
2
2
|
|
|
3
|
-
export type AkanChildStatus = "starting" | "ready" | "healthy" | "draining" | "unhealthy" | "exited";
|
|
3
|
+
export type AkanChildStatus = "starting" | "ready" | "healthy" | "draining" | "unhealthy" | "exited" | "crashed";
|
|
4
4
|
|
|
5
5
|
export type AkanUpstream = { type: "unix"; socketPath: string } | { type: "tcp"; host: string; port: number };
|
|
6
6
|
|
|
@@ -110,6 +110,8 @@ export type AkanIpcMessage =
|
|
|
110
110
|
replicaIdx: number;
|
|
111
111
|
role: AkanChildRole;
|
|
112
112
|
upstream?: AkanUpstream;
|
|
113
|
+
/** Actual websocket upstream the child bound; may differ from the preferred port when it was in use. */
|
|
114
|
+
wsUpstream?: Extract<AkanUpstream, { type: "tcp" }>;
|
|
113
115
|
healthPath?: string;
|
|
114
116
|
}
|
|
115
117
|
| { type: "backend-ready"; pid: number }
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export type AkanChildRole = "all" | "federation" | "batch";
|
|
2
|
-
export type AkanChildStatus = "starting" | "ready" | "healthy" | "draining" | "unhealthy" | "exited";
|
|
2
|
+
export type AkanChildStatus = "starting" | "ready" | "healthy" | "draining" | "unhealthy" | "exited" | "crashed";
|
|
3
3
|
export type AkanUpstream = {
|
|
4
4
|
type: "unix";
|
|
5
5
|
socketPath: string;
|
|
@@ -122,6 +122,10 @@ export type AkanIpcMessage = {
|
|
|
122
122
|
replicaIdx: number;
|
|
123
123
|
role: AkanChildRole;
|
|
124
124
|
upstream?: AkanUpstream;
|
|
125
|
+
/** Actual websocket upstream the child bound; may differ from the preferred port when it was in use. */
|
|
126
|
+
wsUpstream?: Extract<AkanUpstream, {
|
|
127
|
+
type: "tcp";
|
|
128
|
+
}>;
|
|
125
129
|
healthPath?: string;
|
|
126
130
|
} | {
|
|
127
131
|
type: "backend-ready";
|