@teamscala/service-boot 0.23.300 → 0.23.302

package/package.json

@@ -6,7 +6,7 @@
   },
   "name": "@teamscala/service-boot",
   "type": "module",
-  "version": "0.23.300",
+  "version": "0.23.302",
   "sideEffects": false,
   "files": [
     "src"
@@ -882,6 +882,14 @@
       "types": "./src/resolve-service-context.d.ts",
       "default": "./src/resolve-service-context.ts"
     },
+    "./resolve-service-port": {
+      "types": "./src/resolve-service-port.d.ts",
+      "default": "./src/resolve-service-port.ts"
+    },
+    "./resolve-service-port.ts": {
+      "types": "./src/resolve-service-port.d.ts",
+      "default": "./src/resolve-service-port.ts"
+    },
     "./resolve-theme-mode": {
       "types": "./src/resolve-theme-mode.d.ts",
       "default": "./src/resolve-theme-mode.ts"

package/src/resolve-service-port.d.ts

@@ -0,0 +1,66 @@
+/**
+ * @system service-runtime
+ * @status handwritten
+ * @edit edit directly
+ *
+ * Decides which port a service binds, from the two sources that can claim one:
+ * the PORT environment variable and the port the registry declares for that
+ * service (`service_routing.local_port` for a backend, `service_install.port`
+ * for a UI service).
+ *
+ * This is the resolver for BOTH kinds. It lives in service-boot rather than
+ * service-runtime because the UI branch boots from here and may not import
+ * upward; keeping one copy is the point, since the second copy is exactly what
+ * went wrong (below).
+ *
+ * The rule, and why it is not "env always wins":
+ *
+ * Railway injects PORT and healthchecks that exact port, reporting the service
+ * unavailable if it listens elsewhere — so where there is NO routing row (a
+ * Railway-only backend), the env is the only truth and must win.
+ *
+ * But a `service_routing` row means this service has a DECLARED port on this
+ * host. There, a PORT that disagrees is a polluted launcher environment, never
+ * an intention, and obeying it binds someone else's port. That is not a
+ * hypothetical: mcp-ai-chat-system booted onto 3021 — session-picker's port —
+ * while its own MCP_SERVER_URL still pointed at 8080. Both processes bound 3021,
+ * SO_REUSEPORT load-balanced between them, and every symptom appeared somewhere
+ * else: a chat loopback that resolved to nothing, tool-description curation
+ * failing for every tool and disabling itself via its circuit breaker, RPC calls
+ * landing in the wrong service, and 59 restarts. Nothing failed loudly because
+ * nothing ever compared the two numbers.
+ *
+ * So the registry wins wherever it has an opinion, and the disagreement is
+ * REPORTED rather than silently resolved — a wrong port that boots cleanly is
+ * the expensive shape, because the damage surfaces far from the cause.
+ *
+ * That reasoning was written for the backend branch and wired only there, so
+ * the UI branch kept taking `process.env.PORT` unconditionally — and on
+ * 2026-07-31 reproduced the identical incident in the other direction:
+ * scala-quiz (kind:ui, declared 5203) inherited a leaked PORT=3021 from the
+ * tmux server environment and bound session-picker's port. Half of every RPC
+ * to the session daemon was answered by a quiz service with no /rpc routes.
+ *
+ * A rule enforced on one of two branches is not enforced. Both branches now
+ * call this function, and `port-resolution-is-registry-authoritative` fails
+ * the push if either stops.
+ */
+export interface PortResolution {
+    /** The port to bind. */
+    port: number;
+    /** Which source decided it — for the boot log, so the choice is auditable. */
+    source: "env" | "service_routing";
+    /** Set when PORT disagreed with the row; the caller logs it. */
+    conflict?: {
+        envPort: number;
+        declaredPort: number;
+    };
+}
+/**
+ * @param envPort   parsed PORT env, or undefined when unset/unparseable
+ * @param declaredPort `service_routing.local_port`, or undefined when the
+ *                     service has no routing row (the Railway-only case)
+ */
+export declare function resolveServicePort(envPort: number | undefined, declaredPort: number | undefined): PortResolution | {
+    error: "no-port-source";
+};

package/src/resolve-service-port.test.ts

@@ -0,0 +1,98 @@
+/**
+ * @system service-boot
+ * @status handwritten
+ * @edit edit directly
+ *
+ * The load-bearing case is the contradiction one: a PORT env that disagrees
+ * with the service's own declared port must NOT be obeyed. Obeying it put
+ * mcp-ai-chat-system on session-picker's port with a loopback pointing at a
+ * port nothing was listening on, and the failure surfaced as broken tool
+ * curation rather than as a port problem. It then did it again on 2026-07-31
+ * to scala-quiz — see the kind:ui block below.
+ */
+
+import { describe, expect, test } from "bun:test";
+import { resolveServicePort } from "./resolve-service-port.ts";
+
+/** Narrow off the error arm so a resolution's port can be asserted directly. */
+function resolvedPort(envPort: number | undefined, declaredPort: number | undefined): number {
+	const r = resolveServicePort(envPort, declaredPort);
+	if ("error" in r) throw new Error(`expected a resolution, got ${r.error}`);
+	return r.port;
+}
+
+describe("resolveServicePort", () => {
+	test("no routing row: Railway's injected PORT wins", () => {
+		// A Railway-only backend legitimately has no dev routing row, and Railway
+		// healthchecks the exact port it injected.
+		expect(resolveServicePort(8080, undefined)).toEqual({ port: 8080, source: "env" });
+	});
+
+	test("no routing row and no usable PORT is an error, not a guess", () => {
+		expect(resolveServicePort(undefined, undefined)).toEqual({ error: "no-port-source" });
+	});
+
+	test("routing row with no PORT env: the row wins", () => {
+		expect(resolveServicePort(undefined, 3021)).toEqual({ port: 3021, source: "service_routing" });
+	});
+
+	test("routing row and an agreeing PORT: no conflict reported", () => {
+		const r = resolveServicePort(3021, 3021);
+		expect(r).toEqual({ port: 3021, source: "service_routing" });
+		expect("conflict" in r).toBe(false);
+	});
+
+	test("a PORT contradicting the routing row is REFUSED and reported", () => {
+		// The exact incident: env said 3021 (session-picker's port), the row said
+		// 8080. Obeying the env bound another service's port.
+		expect(resolveServicePort(3021, 8080)).toEqual({
+			port: 8080,
+			source: "service_routing",
+			conflict: { envPort: 3021, declaredPort: 8080 },
+		});
+	});
+
+	test("an unparseable or out-of-range PORT never displaces a valid row", () => {
+		expect(resolvedPort(Number.NaN, 3021)).toBe(3021);
+		expect(resolvedPort(0, 3021)).toBe(3021);
+		expect(resolvedPort(70_000, 3021)).toBe(3021);
+	});
+
+	test("an out-of-range PORT with no row is an error rather than a bad bind", () => {
+		expect(resolveServicePort(70_000, undefined)).toEqual({ error: "no-port-source" });
+	});
+
+	// The UI branch reaches this same function via `service_install.port`. These
+	// cases are the 2026-07-31 incident, which the backend-only wiring missed.
+	describe("kind:ui services (declared port from service_install)", () => {
+		test("the live bug: a leaked PORT cannot move a UI service off its declared port", () => {
+			// scala-quiz declares 5203; PORT=3021 leaked in from the tmux server
+			// env and it bound session-picker's port, splitting that daemon's
+			// traffic via SO_REUSEPORT.
+			expect(resolveServicePort(3021, 5203)).toEqual({
+				port: 5203,
+				source: "service_routing",
+				conflict: { envPort: 3021, declaredPort: 5203 },
+			});
+		});
+
+		test("the conflict is reported, not swallowed — the boot log must be able to say so", () => {
+			const r = resolveServicePort(3021, 5203);
+			expect("error" in r).toBe(false);
+			if ("error" in r) return;
+			expect(r.conflict).toBeDefined();
+			expect(r.conflict?.envPort).toBe(3021);
+		});
+
+		test("Railway is unaffected: no declared port means the injected PORT still wins", () => {
+			expect(resolveServicePort(8080, undefined)).toEqual({ port: 8080, source: "env" });
+		});
+
+		test("an agreeing PORT reports no conflict", () => {
+			expect(resolveServicePort(5203, 5203)).toEqual({
+				port: 5203,
+				source: "service_routing",
+			});
+		});
+	});
+});

package/src/resolve-service-port.ts

@@ -0,0 +1,90 @@
+/**
+ * @system service-runtime
+ * @status handwritten
+ * @edit edit directly
+ *
+ * Decides which port a service binds, from the two sources that can claim one:
+ * the PORT environment variable and the port the registry declares for that
+ * service (`service_routing.local_port` for a backend, `service_install.port`
+ * for a UI service).
+ *
+ * This is the resolver for BOTH kinds. It lives in service-boot rather than
+ * service-runtime because the UI branch boots from here and may not import
+ * upward; keeping one copy is the point, since the second copy is exactly what
+ * went wrong (below).
+ *
+ * The rule, and why it is not "env always wins":
+ *
+ * Railway injects PORT and healthchecks that exact port, reporting the service
+ * unavailable if it listens elsewhere — so where there is NO routing row (a
+ * Railway-only backend), the env is the only truth and must win.
+ *
+ * But a `service_routing` row means this service has a DECLARED port on this
+ * host. There, a PORT that disagrees is a polluted launcher environment, never
+ * an intention, and obeying it binds someone else's port. That is not a
+ * hypothetical: mcp-ai-chat-system booted onto 3021 — session-picker's port —
+ * while its own MCP_SERVER_URL still pointed at 8080. Both processes bound 3021,
+ * SO_REUSEPORT load-balanced between them, and every symptom appeared somewhere
+ * else: a chat loopback that resolved to nothing, tool-description curation
+ * failing for every tool and disabling itself via its circuit breaker, RPC calls
+ * landing in the wrong service, and 59 restarts. Nothing failed loudly because
+ * nothing ever compared the two numbers.
+ *
+ * So the registry wins wherever it has an opinion, and the disagreement is
+ * REPORTED rather than silently resolved — a wrong port that boots cleanly is
+ * the expensive shape, because the damage surfaces far from the cause.
+ *
+ * That reasoning was written for the backend branch and wired only there, so
+ * the UI branch kept taking `process.env.PORT` unconditionally — and on
+ * 2026-07-31 reproduced the identical incident in the other direction:
+ * scala-quiz (kind:ui, declared 5203) inherited a leaked PORT=3021 from the
+ * tmux server environment and bound session-picker's port. Half of every RPC
+ * to the session daemon was answered by a quiz service with no /rpc routes.
+ *
+ * A rule enforced on one of two branches is not enforced. Both branches now
+ * call this function, and `port-resolution-is-registry-authoritative` fails
+ * the push if either stops.
+ */
+
+export interface PortResolution {
+	/** The port to bind. */
+	port: number;
+	/** Which source decided it — for the boot log, so the choice is auditable. */
+	source: "env" | "service_routing";
+	/** Set when PORT disagreed with the row; the caller logs it. */
+	conflict?: { envPort: number; declaredPort: number };
+}
+
+function isBindablePort(value: number | undefined): value is number {
+	return value !== undefined && Number.isFinite(value) && value > 0 && value <= 65535;
+}
+
+/**
+ * @param envPort   parsed PORT env, or undefined when unset/unparseable
+ * @param declaredPort `service_routing.local_port`, or undefined when the
+ *                     service has no routing row (the Railway-only case)
+ */
+export function resolveServicePort(
+	envPort: number | undefined,
+	declaredPort: number | undefined,
+): PortResolution | { error: "no-port-source" } {
+	const envUsable = isBindablePort(envPort);
+	const declaredUsable = isBindablePort(declaredPort);
+
+	// No routing row: Railway's injected PORT is the only truth available.
+	if (!declaredUsable) {
+		return envUsable ? { port: envPort, source: "env" } : { error: "no-port-source" };
+	}
+
+	// Routing row present and the env agrees (or is absent) — nothing to resolve.
+	if (!envUsable || envPort === declaredPort) {
+		return { port: declaredPort, source: "service_routing" };
+	}
+
+	// Routing row present and the env contradicts it: the row wins, loudly.
+	return {
+		port: declaredPort,
+		source: "service_routing",
+		conflict: { envPort, declaredPort },
+	};
+}

package/src/serve/serve.ts

@@ -66,7 +66,24 @@ function serveStartInternal(
 ): ServeHandle {
 	const server = Bun.serve({
 		port: options.port,
-		reusePort: true,
+		// reusePort is deliberately NOT set. SO_REUSEPORT lets a second process
+		// bind a port this service already owns; the kernel then load-balances
+		// between them and BOTH look healthy while each receives a fraction of
+		// the traffic. That is the worst available failure mode — it presents as
+		// intermittent, unattributable errors in whichever service is unlucky,
+		// far from the process that actually took the port.
+		//
+		// It has now cost two incidents. mcp-ai-chat-system bound session-picker's
+		// 3021 and broke tool curation; scala-quiz did the same on 2026-07-31 and
+		// answered ~half of every session-daemon RPC with a 404. Neither failed at
+		// bind, because reusePort made a collision legal.
+		//
+		// Nothing here needs it: PM2 fork mode stops before it starts, dev runs one
+		// process per service (`readiness-gated-boot` — the boot window is handled
+		// by the in-process readiness gate, explicitly not blue/green), and each
+		// Railway container is alone. So the only thing reusePort can express on
+		// this platform is a mistake, and without it that mistake is an immediate,
+		// legible EADDRINUSE naming the port.
 		// Bun's default idleTimeout (~10s, swept on a coarse timer wheel)
 		// FIN-closes every pending connection in one ~2ms sweep once their
 		// handlers exceed it — packet-captured 2026-06-06: concurrent SSR

package/src/ui-service.ts

@@ -14,6 +14,7 @@ import { BootFatalError } from "@teamscala/os/errors/boot-error";
 import { Hono } from "hono";
 import type { ServiceInstallConfig } from "./load-service-install-row.ts";
 import { setReady } from "./readiness-gate.ts";
+import { resolveServicePort } from "./resolve-service-port.ts";
 import { serveStart } from "./serve/serve";
 import { buildMinimalApp, buildFullApp, swapToFullApp } from "./two-phase-boot.ts";
 
@@ -66,12 +67,43 @@ export async function bootUiService(
 	// Phase 1: Build minimal tool-only app
 	const minimalAppResult = await buildMinimalApp(config, serviceSlug, databaseUrl);
 
-	// Get port from config
+	// Resolve the listen port through the SHARED resolver (`resolveServicePort`),
+	// the same one the backend branch uses. The registry wins wherever it has an
+	// opinion and a contradicting PORT env is reported, never silently obeyed —
+	// full reasoning in that file.
+	//
+	// This branch previously took `process.env.PORT` unconditionally, which is
+	// how scala-quiz (declared 5203) booted onto session-picker's 3021 from a
+	// leaked env and silently split that daemon's traffic (2026-07-31). Railway
+	// is unaffected by construction: it injects PORT and a Railway-only service
+	// has no declared port, so the env still wins there.
 	const envPort = process.env.PORT ? Number(process.env.PORT) : undefined;
-	const port = requireValidPort(
-		envPort && Number.isFinite(envPort) && envPort > 0 ? envPort : config.port,
-		serviceSlug,
-	);
+	// The APP binds its PRIVATE port (app_port) when present — for scale-to-zero
+	// services `port` is the ACTIVATOR's ingress (what callers resolve), not the
+	// app's bind. Falling back to `port` keeps RESIDENT + unmigrated services
+	// unchanged. Without this, a migrated row (port=activator) would make the app
+	// bind the activator's port → SO_REUSEPORT silent load-balancing (the
+	// service-ports-from-registry trap: clean boot, two owners in ss -ltnp).
+	const declaredAppPort = config.app_port ?? config.port;
+	const resolution = resolveServicePort(envPort, declaredAppPort);
+	if ("error" in resolution) {
+		throw new BootFatalError(
+			"ui-service.port-resolution",
+			serviceSlug,
+			`No port source: service_install declares no port and no valid PORT env is set. ` +
+				`Every UI service requires a port in its service_install row (or a PORT env when Railway-deployed).`,
+		);
+	}
+	if (resolution.conflict) {
+		log.warn(
+			`[ui-service-boot:${serviceSlug}] PORT env (${resolution.conflict.envPort}) contradicts its ` +
+				`service_install row (${resolution.conflict.declaredPort}) — using ` +
+				`${resolution.conflict.declaredPort}. A stale PORT in this service's launcher ` +
+				`environment is a polluted launcher env, never an intention; binding the env port ` +
+				`would take another service's port.`,
+		);
+	}
+	const port = requireValidPort(resolution.port, serviceSlug);
 
 	// Start serving with the minimal app (useMutableRef=true)
 	try {