cartethyia 2.5.10 → 2.6.0

package/src/boot/app.ts

@@ -45,6 +45,7 @@ import {
 import { createTokenCountHandler } from "../transport/dispatch/token-count";
 
 import { createTransportPipeline } from "../transport/guards/pipeline";
+import { defaultProcessConcurrencyController } from "../transport/guards/concurrency-guard";
 import { createDrainHandler, isLoopbackPeer, tokenMatches } from "../transport/drain-endpoint";
 import { createConsoleAccountResetHandler } from "../console/auth/account-reset";
 import { resetConsoleAuthState } from "../console/auth/account-reset-store";
@@ -244,6 +245,10 @@ export function createGatewayApp(deps: GatewayAppDeps) {
     deps.mode === "production" ? deps.shutdownCoordinator : undefined,
   );
   deps.runtimeMetricsSampler?.bindInFlight(() => requestStateStore.inFlightCount());
+  // The concurrency ceiling is derived (memory limit ÷ body budget), so it is
+  // published as a gauge: an operator reading "shedding at 16 in flight" needs
+  // to see that 16 IS the limit on this box, not that the process is broken.
+  deps.runtimeMetricsSampler?.bindConcurrencyLimit(defaultProcessConcurrencyController.limit());
   // Shutdown ordering: abort in-flight proxy controllers first so the
   // bounded drain observes cancellation and finalizers run before
   // telemetry flush and pool close. The reason is threaded through so the
@@ -379,6 +384,16 @@ export function createGatewayApp(deps: GatewayAppDeps) {
       telemetry: deps.telemetryBuffer,
       ...(deps.modelStrikes ? { modelStrikes: deps.modelStrikes } : {}),
       ...(deps.consoleApi?.redis ? { redis: deps.consoleApi.redis } : {}),
+      // Bound at construction so the guard holds the sampler's live snapshot
+      // rather than a value captured at boot.
+      ...(deps.runtimeMetricsSampler
+        ? { memorySnapshot: () => deps.runtimeMetricsSampler?.memorySnapshot() }
+        : {}),
+      // Reads the cached route snapshot, which the settings route invalidates
+      // when the switch is saved — so flipping it takes effect on the next
+      // request and adds no query to the proxy path.
+      isInvokeDisabled: async (tenantId) =>
+        (await deps.snapshotService.getSnapshot())?.invokeDisabledTenants?.has(tenantId),
     });
     transportPipeline.mountRoot(app);
     const proxyDeps: ProviderProxyHandlerDeps = {
@@ -657,7 +672,7 @@ export function createGatewayApp(deps: GatewayAppDeps) {
       createShareRouter({
         db: deps.db,
         shareStore: new DrizzleShareLinkStore(deps.db),
-        stats: createShareStatsPort({ redis: deps.consoleApi?.redis }),
+        stats: createShareStatsPort({ db: deps.db, redis: deps.consoleApi?.redis }),
         resolveClientIp: (request) => {
           const peer = peerAddresses.get(request);
           return peer

package/src/boot/dependencies.ts

@@ -3,6 +3,9 @@ import { assertPoolFitsServerCapacity, bootDatabase, ensureMigrated } from "../p
 import { poolMaxFromEnv } from "../persistence/db-pg";
 import type { CartethyiaDatabase } from "../persistence/db-handle";
 import { resolveRedisBackend, resolveRedisClient } from "../persistence/redis";
+import { resolveProcessTopology, type ProcessTopology } from "../config/runtime";
+import { createRedisMaintenanceLeaseStore } from "../workers/maintenance-lease";
+import { resolveDbMode } from "../persistence/db-mode";
 import type { RedisClient } from "../persistence/redis";
 import {
   bundledModelCatalog,
@@ -71,8 +74,6 @@ import { preferencesReaderFor } from "../persistence/tenant-preferences";
 import { sweepExpiredCooldowns } from "../providers/operations/account-health-service";
 import { DrizzleTelemetryStore } from "../persistence/stores/telemetry-store";
 import { TelemetryPayloadCapture } from "../observability/payload-capture";
-import { foldDayIndex, listEventDays } from "../observability/event-day-index";
-import { resolveTelemetryCompactIntervalMs } from "../config/telemetry";
 import { sweepExpiredPoolCooldowns } from "../network/pool-health-machine";
 import { TelemetryBatchBuffer } from "../observability/telemetry-buffer";
 import { RuntimeMetricsSampler } from "../observability/runtime-metrics";
@@ -86,6 +87,7 @@ import {
 import { resolveSsrfPolicy, resolveTrustedProxyBoundary } from "../config/network";
 import type { TrustedProxyBoundary } from "../config/network";
 import { resolveTelemetryRetentionDays } from "../config/telemetry";
+import { resolveConcurrencyTolerance } from "../config/resources";
 import { CONFIG_SPEC, readInt, readList } from "../config/spec";
 import type { ProviderAdapter } from "../providers/provider-contracts";
 import type { ProviderRegistry } from "../providers/provider-registry";
@@ -145,9 +147,89 @@ const OAUTH_REFRESH_INTER_ITEM_DELAY_MS = 1_500;
 const CLIENT_VERSION_MONITOR_INTERVAL_MS = 30 * 60_000;
 const CLIENT_VERSION_INITIAL_DELAY_MS = 10_000;
 
+/** Bound on the boot-time Redis probe: a start must fail, not hang. */
+const REDIS_PROBE_TIMEOUT_MS = 2_000;
+
+async function withTimeout<T>(promise: Promise<T>, ms: number): Promise<T> {
+  const { promise: timeout, reject } = Promise.withResolvers<never>();
+  const timer = setTimeout(() => reject(new Error("timeout")), ms);
+  try {
+    return await Promise.race([promise, timeout]);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+/** Bounded Redis reachability probe: fails rather than hangs. */
+async function redisIsReachable(): Promise<boolean> {
+  const client = resolveRedisClient();
+  if (!client) return false;
+  try {
+    const result = await withTimeout(client.ping(), REDIS_PROBE_TIMEOUT_MS);
+    return result === "PONG";
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Refuses to start a multi-process deployment whose shared state is not
+ * actually shared.
+ *
+ * Every process shares one port via `SO_REUSEPORT`, so the kernel balances
+ * across them — and each process keeps its own in-memory admission counters,
+ * quota cache, and cooldown map. Those are only correct when they live in
+ * Redis. Starting anyway does not fail loudly; it enforces every per-key limit
+ * N times over, which then reads as "the limit did not work".
+ *
+ * Redis is checked by PING with a hard bound, not merely for being configured:
+ * a configured-but-unreachable Redis is the failure mode that matters, and an
+ * unbounded connect at boot would hang the start instead of failing it.
+ */
+async function assertMultiProcessPreconditions(
+  topology: ProcessTopology,
+  redisReachable: () => Promise<boolean>,
+): Promise<void> {
+  if (!topology.multiProcess) return;
+  if (process.platform !== "linux") {
+    throw Object.assign(
+      new Error(
+        `CARTETHYIA_PROCESS_COUNT=${topology.count} requires Linux (SO_REUSEPORT); ` +
+          `this process is on ${process.platform}`,
+      ),
+      { code: "multi_process_unsupported_platform" },
+    );
+  }
+  if (resolveDbMode() !== "full") {
+    throw Object.assign(
+      new Error(
+        `CARTETHYIA_PROCESS_COUNT=${topology.count} requires CARTETHYIA_DB_MODE=full (PostgreSQL); ` +
+          "two processes on one SQLite file corrupt it — there is no cross-process lock in the driver",
+      ),
+      { code: "multi_process_requires_postgres" },
+    );
+  }
+  const reachable = await redisReachable();
+  if (!reachable) {
+    throw Object.assign(
+      new Error(
+        `CARTETHYIA_PROCESS_COUNT=${topology.count} requires a REACHABLE Redis: admission counters, ` +
+          "quotas, and cooldowns are per-process without one, so every per-key limit would be " +
+          `enforced ${topology.count} times over`,
+      ),
+      { code: "multi_process_requires_redis" },
+    );
+  }
+}
+
 export async function buildProductionDeps(): Promise<ProductionDeps> {
   const startupStartedAt = performance.now();
   const startupMemoryBefore = process.memoryUsage();
+  // Resolved first and validated before any connection is opened: a
+  // half-configured multi-process deploy that starts anyway looks healthy
+  // until a per-key limit is enforced N times over.
+  const topology = resolveProcessTopology();
+  await assertMultiProcessPreconditions(topology, redisIsReachable);
   const handle = await measureStartupPhase("database-connect", bootDatabase);
   const db = handle.db;
   // One rule, no mode flag: REDIS_URL set → shared client, unset → memory.
@@ -157,7 +239,7 @@ export async function buildProductionDeps(): Promise<ProductionDeps> {
     await ensureMigrated();
     // Pool sizing is an external-server concern; the embedded backend has no
     // connection budget to fit into.
-    if (handle.kind === "pg") await assertPoolFitsServerCapacity(poolMaxFromEnv());
+    if (handle.kind === "pg") await assertPoolFitsServerCapacity(poolMaxFromEnv(), topology.count);
   });
   await measureStartupPhase("provider-seed", () => seedBundledProviders(db));
   // Boot-only: retire global rows for providers the bundle has dropped, so a
@@ -193,7 +275,6 @@ export async function buildProductionDeps(): Promise<ProductionDeps> {
     tenantId: string | null,
   ): Promise<readonly { accountId: string; inflight: number }[]> => {
     if (providerId === "" || tenantId === null) return [];
-    const snapshot = await routingEngine.accountInflightSnapshot();
     const accounts = await new DrizzleProviderCatalogStore(db, {
       telemetryBuffer,
       bundledModelCatalog: bundledCatalog,
@@ -202,10 +283,20 @@ export async function buildProductionDeps(): Promise<ProductionDeps> {
       snapshotInvalidator: snapshotService,
     }).listAccounts(tenantId, providerId);
     const owned = new Set(accounts.map((account) => account.id));
+    // Scoped read: only the provider's own candidates for this tenant are
+    // queried, so the console read costs one exact MGET over the accounts it
+    // is about to report on — not a fleet-wide `KEYS` scan whose cost is the
+    // size of every bucket every process ever wrote.
+    const routeSnapshot = await snapshotService.getSnapshot();
+    const ownedCandidates = routeSnapshot.candidates.filter(
+      (candidate) =>
+        candidate.provider_id === providerId &&
+        candidate.provider_account_id !== undefined &&
+        owned.has(candidate.provider_account_id),
+    );
+    const snapshot = await routingEngine.accountInflightSnapshot(ownedCandidates);
     const totals = new Map<string, number>();
     for (const [bucket, count] of snapshot) {
-      const prefix = `${providerId}:`;
-      if (!bucket.startsWith(prefix)) continue;
       const accountId = bucket.slice(bucket.lastIndexOf(":") + 1);
       if (!owned.has(accountId)) continue;
       totals.set(accountId, (totals.get(accountId) ?? 0) + count);
@@ -260,6 +351,13 @@ export async function buildProductionDeps(): Promise<ProductionDeps> {
       const children = await store.sumChildrenConsumed(apiKeyId);
       return (row.lifetimeTokensConsumed ?? 0) + children;
     },
+    // Slack above every key's concurrency limit. A tenant preference an
+    // operator sets in the runtime settings modal, falling back to the
+    // deployment default so an unset row does not silently lose the allowance.
+    (tenantId) =>
+      preferencesReaderFor(db)
+        .readPreferences(tenantId)
+        .then((prefs) => prefs?.concurrencyTolerance ?? resolveConcurrencyTolerance()),
   );
   const ipStore = redis
     ? new RedisIpAbuseStore(redis)
@@ -282,8 +380,14 @@ export async function buildProductionDeps(): Promise<ProductionDeps> {
   // One list covers every periodic task: a small deployment can cut the
   // heaviest sweeps (quota refresh, OAuth refresh, client-version polling)
   // without the codebase growing a flag per task.
+  // Leases coordinate the SHARED sweeps across processes. Absent Redis there is
+  // nothing to coordinate with, and the registry's local one-at-a-time rule is
+  // the correct (and only) guard.
   const scheduledTasks = new ScheduledTaskRegistry(
     readList("CARTETHYIA_DISABLED_TASKS", CONFIG_SPEC.CARTETHYIA_DISABLED_TASKS) ?? [],
+    {
+      ...(redis ? { lease: createRedisMaintenanceLeaseStore(redis) } : {}),
+    },
   );
   if (redis) {
     scheduledTasks.register({
@@ -295,6 +399,8 @@ export async function buildProductionDeps(): Promise<ProductionDeps> {
   scheduledTasks.register({
     name: "account-health-sweep",
     intervalMs: 30_000,
+    // Recovers cooldowns deployment-wide and invalidates the shared snapshot.
+    shared: true,
     run: async () => {
       const recovered = await sweepExpiredCooldowns(db);
       const poolRecovered = await sweepExpiredPoolCooldowns(db);
@@ -306,27 +412,11 @@ export async function buildProductionDeps(): Promise<ProductionDeps> {
   scheduledTasks.register({
     name: "telemetry-payload-cleanup",
     intervalMs: 15 * 60_000,
+    // Deletes payload frames for every tenant: two passes over the same
+    // batches is duplicated work, not speedup.
+    shared: true,
     run: () => payloadCapture.cleanupExpired(),
   });
-  scheduledTasks.register({
-    name: "telemetry-index-fold",
-    intervalMs: resolveTelemetryCompactIntervalMs(),
-    // Brings each day's index up to date with the bytes appended since the last
-    // pass, so a usage read is a small JSON read rather than a parse of the
-    // whole period. Folding is incremental and idempotent: a crash only leaves
-    // the watermark behind, and the next pass re-reads the tail.
-    run: async () => {
-      const days = await listEventDays();
-      let folded = 0;
-      for (const day of days) {
-        const result = await foldDayIndex(day).catch(() => ({ folded: 0 }));
-        folded += result.folded;
-      }
-      if (folded > 0) {
-        log.info(`[telemetry] indexed ${folded} event(s) across ${days.length} day(s)`);
-      }
-    },
-  });
   scheduledTasks.register({
     name: "console-session-sweep",
     // Sessions live 48 h, so an hourly sweep deleted nothing 47 times out of
@@ -334,11 +424,15 @@ export async function buildProductionDeps(): Promise<ProductionDeps> {
     // be expired. Half the session lifetime is the point where a pass can
     // actually find work, and still leaves a full grace period after it.
     intervalMs: 12 * 60 * 60_000,
+    shared: true,
     run: () => deleteExpiredSessions(db),
   });
   scheduledTasks.register({
     name: "telemetry-retention",
     intervalMs: 6 * 60 * 60_000,
+    // Retention deletes rows and files for the whole deployment; a second
+    // concurrent pass would churn the same batches.
+    shared: true,
     run: () =>
       telemetryStore.pruneTelemetry(
         new Date(Date.now() - resolveTelemetryRetentionDays() * 24 * 60 * 60_000),
@@ -365,7 +459,16 @@ export async function buildProductionDeps(): Promise<ProductionDeps> {
     modelCatalogEntries: () => modelCatalogCacheSize(),
     // Late-bound via bindInFlight once the request state store exists.
     inFlight: () => 0,
+    // The buffer is built below; the closure keeps the sampler from holding a
+    // reference during construction.
+    telemetryBacklog: () => telemetryBuffer?.snapshot(),
   });
+  // One sample NOW, not on the first tick. The headroom guard reads
+  // `memorySnapshot()` on every gateway request and treats `undefined` as
+  // "unknown" — so without this the guard would spend its first interval
+  // (seconds, at the configured cadence) unable to shed at all, which is
+  // exactly the window a cold start is most likely to be short of memory.
+  runtimeMetricsSampler.sample();
   scheduledTasks.register({
     name: "runtime-metrics",
     intervalMs: readInt(

package/src/config/resources.ts

@@ -0,0 +1,137 @@
+/**
+ * Canonical resource policy: how much memory this process may use, and how
+ * much work it may hold at once.
+ *
+ * These two numbers were previously decided in three unrelated places — a
+ * hardcoded cap in `concurrency-guard.ts`, a host-wide `os.totalmem()` in the
+ * headroom guard, and a fallback constant in `runtime-metrics.ts` — which
+ * meant a container limited to 512 MiB admitted the same concurrency as a 4 GiB
+ * one and only started shedding once the *host* was nearly full. On a 1-core /
+ * 2 GiB VPS that is the difference between shedding early and being
+ * OOM-killed, and the OOM kills the dashboard too — the one surface that would
+ * have explained what happened.
+ *
+ * Everything here is derived from one limit and one body ceiling, so the
+ * numbers cannot disagree with each other:
+ *
+ *   concurrency = floor(limit × BUDGET_FRACTION / (TRANSIENT_COPIES × body))
+ *
+ * The transient-copies factor is why this is not simply "limit / body": a
+ * request body is held more than once between the socket read, the JSON parse,
+ * and the upstream re-serialization, so budgeting one copy per request
+ * understates the real hold by a factor of about two.
+ */
+import { CONFIG_SPEC, readOptionalInt } from "./spec";
+import { resolveMaxBodyBytes } from "./runtime";
+
+/** Used when no limit is discoverable at all (native dev, macOS, Windows). */
+export const FALLBACK_MEMORY_LIMIT_BYTES = 512 * 1024 * 1024;
+
+/**
+ * Ceiling on concurrent in-flight requests per process.
+ *
+ * A hard ceiling even when memory would allow more: this process is one core,
+ * and past this point queueing delay, not memory, is what degrades. It also
+ * bounds every per-request structure that is sized by concurrency.
+ */
+export const MAX_PROCESS_CONCURRENCY = 64;
+
+/**
+ * How many copies of a request body are resident at once.
+ *
+ * Measured, not assumed: the socket read buffer, the parsed object graph, and
+ * the upstream re-serialization each hold a body-sized allocation, and garbage
+ * collection lags rather than keeping them to one. Budgeting one copy halves
+ * the real peak and lets the process admit twice the concurrency it can hold.
+ */
+export const REQUEST_BODY_TRANSIENT_COPIES = 2;
+
+/** Share of the memory limit all in-flight request bodies together may hold. */
+export const REQUEST_BODY_BUDGET_FRACTION = 0.25;
+
+/** Share of the memory limit kept free for allocation spikes and GC slack. */
+export const MEMORY_HEADROOM_FRACTION = 0.2;
+
+/**
+ * Concurrency this process should admit, derived from the memory limit and the
+ * body ceiling rather than guessed.
+ *
+ * Worked examples, which are the point of having one formula:
+ * - 2 GiB limit, 4 MiB bodies → 64 (the ceiling; memory is not the binder)
+ * - 512 MiB limit, 4 MiB bodies → 16
+ * - 256 MiB limit, 16 MiB bodies → 2
+ *
+ * An explicit override is honoured, clamped to 1..MAX_PROCESS_CONCURRENCY. It
+ * is the operator's call and is never silently raised back to the derived
+ * value; exceeding the derived figure is a deliberate choice, so it is
+ * reported at the call site rather than corrected here.
+ */
+export function deriveProcessConcurrency(options: {
+  readonly memoryLimitBytes: number;
+  readonly maxBodyBytes: number;
+  readonly override?: number | undefined;
+}): number {
+  if (options.override !== undefined) {
+    if (!Number.isInteger(options.override)) return 1;
+    return Math.min(MAX_PROCESS_CONCURRENCY, Math.max(1, options.override));
+  }
+  const body = Math.max(1, options.maxBodyBytes);
+  const perRequest = REQUEST_BODY_TRANSIENT_COPIES * body;
+  const budget = Math.max(1, Math.floor(options.memoryLimitBytes * REQUEST_BODY_BUDGET_FRACTION));
+  const derived = Math.floor(budget / perRequest);
+  return Math.min(MAX_PROCESS_CONCURRENCY, Math.max(1, derived));
+}
+
+/**
+ * Bytes kept free below the memory limit.
+ *
+ * A guard that only reacts once RSS equals the limit has already lost: the
+ * allocation that breaches it happens between two samples, and the spike that
+ * matters — an upstream response landing all at once — arrives in far less
+ * than one sample interval.
+ */
+export function deriveMemoryHeadroomBytes(memoryLimitBytes: number): number {
+  return Math.ceil(memoryLimitBytes * MEMORY_HEADROOM_FRACTION);
+}
+
+/**
+ * The optional operator override, read through the config spec so "unset" and
+ * "out of range" are treated the way every other knob treats them.
+ */
+export function resolveProcessConcurrencyOverride(): number | undefined {
+  return readOptionalInt(
+    "CARTETHYIA_PROCESS_CONCURRENCY_LIMIT",
+    CONFIG_SPEC.CARTETHYIA_PROCESS_CONCURRENCY_LIMIT,
+  );
+}
+
+/**
+ * Slack added on top of every key's concurrency limit.
+ *
+ * Operator configuration, not something a key carries about itself: it is
+ * deliberate hidden allowance so a run that lands a few requests over the
+ * number an operator chose is not refused at that number. The effective
+ * ceiling is `limit + tolerance`, and only requests past it are refused.
+ */
+export function resolveConcurrencyTolerance(): number {
+  const value = readOptionalInt(
+    "CARTETHYIA_CONCURRENCY_TOLERANCE",
+    CONFIG_SPEC.CARTETHYIA_CONCURRENCY_TOLERANCE,
+  );
+  return value === undefined ? 0 : Math.max(0, value);
+}
+
+/**
+ * The effective concurrency for this process.
+ *
+ * Takes the limit as an argument rather than resolving it here: the limit
+ * owner is `resolveMemoryLimitBytes()` in `observability/runtime-metrics.ts`,
+ * and duplicating that resolution would let the two drift.
+ */
+export function resolveProcessConcurrency(memoryLimitBytes: number): number {
+  return deriveProcessConcurrency({
+    memoryLimitBytes,
+    maxBodyBytes: resolveMaxBodyBytes(),
+    override: resolveProcessConcurrencyOverride(),
+  });
+}

package/src/config/runtime.ts

@@ -1,76 +1,133 @@
-/**
- * HTTP listener, dashboard runtime, and graceful-shutdown drain resolvers.
- */
-
-import { CONFIG_SPEC, readInt, readOptionalText } from "./spec";
-import { resolvePackagePath } from "./package-paths";
-
-/** Resolves the HTTP listener port from the process environment. */
-export function resolvePort(): number {
-  return readInt("PORT", CONFIG_SPEC.PORT);
-}
-
-/** Resolves the dashboard static asset directory. */
-export function resolveDashboardDist(): string {
-  const raw = process.env.DASHBOARD_DIST?.trim();
-  if (!raw) return resolvePackagePath("dist", "dashboard");
-  return raw;
-}
-
-/**
- * Resolves the configured default gateway API key (`CARTETHYIA_API_KEY`), if
- * one is set. Consumers that need the raw secret — first-boot seeding and the
- * console's key recovery from existing installations — read it here instead of
- * probing `process.env` directly.
- */
-export function resolveDefaultGatewayApiKey(): string | undefined {
-  return readOptionalText("CARTETHYIA_API_KEY", CONFIG_SPEC.CARTETHYIA_API_KEY);
-}
-
-/** Resolves the request-body ceiling shared by the Bun listener and ingress middleware. */
-export function resolveMaxBodyBytes(): number {
-  return readInt("CARTETHYIA_SERVER_MAX_BODY_BYTES", CONFIG_SPEC.CARTETHYIA_SERVER_MAX_BODY_BYTES);
-}
-
-/** Resolves the listener idle-socket timeout (seconds). */
-export function resolveIdleTimeout(): number {
-  return readInt("CARTETHYIA_SERVER_IDLE_TIMEOUT", CONFIG_SPEC.CARTETHYIA_SERVER_IDLE_TIMEOUT);
-}
-
-/** Resolves the graceful-shutdown drain window: time for in-flight requests to finish before abort. */
-export function resolveShutdownDrainWindowMs(): number {
-  return readInt(
-    "CARTETHYIA_SHUTDOWN_DRAIN_WINDOW_MS",
-    CONFIG_SPEC.CARTETHYIA_SHUTDOWN_DRAIN_WINDOW_MS,
-  );
-}
-
-/** Resolves the operator drain token, or `undefined` when the drain route is disabled. */
-export function resolveDrainToken(): string | undefined {
-  return readOptionalText("CARTETHYIA_DRAIN_TOKEN", CONFIG_SPEC.CARTETHYIA_DRAIN_TOKEN);
-}
-
-/**
- * Whether Elysia precompiles route handlers at startup (AOT).
- *
- * Precompilation takes the eager validator branch, which calls TypeBox's
- * `SchemaCompile(schema)` and then reads `.buildResult.external` off the
- * result. In a source-run process that field is absent — the lazy
- * `require("typebox/schema")` resolves to a namespace that does not carry it —
- * and the process dies at the first schema-bearing route with
- * `undefined is not an object (evaluating 'this.tb.buildResult.external')`.
- * `scripts/build/aot.ts` avoids this by rewriting the call site and bundling
- * the namespaces statically, so only the built artifact can actually take the
- * eager path.
- *
- * So: precompilation is on for production *builds*, and off for a source run
- * even when `NODE_ENV=production`. A deployment that runs
- * `bun run src/main.ts` must not be pushed onto the eager branch.
- *
- * `CARTETHYIA_SOURCE_RUN` marks a source run explicitly. It is set by
- * `scripts/deploy/*.sh`, which are the only supported way to run from source.
- */
-export function resolveElysiaPrecompile(): boolean {
-  if (process.env.CARTETHYIA_SOURCE_RUN === "1") return false;
-  return process.env.NODE_ENV === "production";
-}
+/**
+ * HTTP listener, dashboard runtime, and graceful-shutdown drain resolvers.
+ */
+
+import { CONFIG_SPEC, readInt, readOptionalText } from "./spec";
+import { resolvePackagePath } from "./package-paths";
+
+/** Resolves the HTTP listener port from the process environment. */
+export function resolvePort(): number {
+  return readInt("PORT", CONFIG_SPEC.PORT);
+}
+
+/** Resolves the dashboard static asset directory. */
+export function resolveDashboardDist(): string {
+  const raw = process.env.DASHBOARD_DIST?.trim();
+  if (!raw) return resolvePackagePath("dist", "dashboard");
+  return raw;
+}
+
+/**
+ * Resolves the configured default gateway API key (`CARTETHYIA_API_KEY`), if
+ * one is set. Consumers that need the raw secret — first-boot seeding and the
+ * console's key recovery from existing installations — read it here instead of
+ * probing `process.env` directly.
+ */
+export function resolveDefaultGatewayApiKey(): string | undefined {
+  return readOptionalText("CARTETHYIA_API_KEY", CONFIG_SPEC.CARTETHYIA_API_KEY);
+}
+
+/** Resolves the request-body ceiling shared by the Bun listener and ingress middleware. */
+export function resolveMaxBodyBytes(): number {
+  return readInt("CARTETHYIA_SERVER_MAX_BODY_BYTES", CONFIG_SPEC.CARTETHYIA_SERVER_MAX_BODY_BYTES);
+}
+
+/** Resolves the listener idle-socket timeout (seconds). */
+export function resolveIdleTimeout(): number {
+  return readInt("CARTETHYIA_SERVER_IDLE_TIMEOUT", CONFIG_SPEC.CARTETHYIA_SERVER_IDLE_TIMEOUT);
+}
+
+/** Resolves the graceful-shutdown drain window: time for in-flight requests to finish before abort. */
+export function resolveShutdownDrainWindowMs(): number {
+  return readInt(
+    "CARTETHYIA_SHUTDOWN_DRAIN_WINDOW_MS",
+    CONFIG_SPEC.CARTETHYIA_SHUTDOWN_DRAIN_WINDOW_MS,
+  );
+}
+
+/** Resolves the operator drain token, or `undefined` when the drain route is disabled. */
+export function resolveDrainToken(): string | undefined {
+  return readOptionalText("CARTETHYIA_DRAIN_TOKEN", CONFIG_SPEC.CARTETHYIA_DRAIN_TOKEN);
+}
+
+/**
+ * Whether Elysia precompiles route handlers at startup (AOT).
+ *
+ * Precompilation takes the eager validator branch, which calls TypeBox's
+ * `SchemaCompile(schema)` and then reads `.buildResult.external` off the
+ * result. In a source-run process that field is absent — the lazy
+ * `require("typebox/schema")` resolves to a namespace that does not carry it —
+ * and the process dies at the first schema-bearing route with
+ * `undefined is not an object (evaluating 'this.tb.buildResult.external')`.
+ * `scripts/build/aot.ts` avoids this by rewriting the call site and bundling
+ * the namespaces statically, so only the built artifact can actually take the
+ * eager path.
+ *
+ * So: precompilation is on for production *builds*, and off for a source run
+ * even when `NODE_ENV=production`. A deployment that runs
+ * `bun run src/main.ts` must not be pushed onto the eager branch.
+ *
+ * `CARTETHYIA_SOURCE_RUN` marks a source run explicitly. It is set by
+ * `scripts/deploy/*.sh`, which are the only supported way to run from source.
+ */
+export function resolveElysiaPrecompile(): boolean {
+  if (process.env.CARTETHYIA_SOURCE_RUN === "1") return false;
+  return process.env.NODE_ENV === "production";
+}
+
+/**
+ * How many gateway processes this deployment runs, and which one this is.
+ *
+ * Multi-process is opt-in and deliberately narrow. Every process shares the
+ * port through `SO_REUSEPORT`, so the kernel load-balances across them — which
+ * means state that used to be process-local is now per-process unless it lives
+ * somewhere shared. A second process without that sharing is not "twice the
+ * capacity", it is two processes disagreeing:
+ *
+ * - **PostgreSQL, not SQLite.** Two processes on one SQLite file corrupt it;
+ *   there is no cross-process lock in the driver.
+ * - **Redis configured AND reachable.** Admission counters, quotas, and
+ *   cooldowns are per-process without it, so a per-key RPM limit silently
+ *   becomes N× the configured value.
+ * - **Linux.** `SO_REUSEPORT` is the mechanism; elsewhere it is a silent no-op
+ *   and the second process simply fails to bind.
+ *
+ * These are checked at resolution time rather than left to fail at bind time,
+ * because a half-configured multi-process deploy that starts anyway looks fine
+ * until a limit is enforced twice.
+ */
+export interface ProcessTopology {
+  readonly count: number;
+  readonly index: number;
+  /** True when more than one process serves this deployment. */
+  readonly multiProcess: boolean;
+}
+
+/**
+ * Resolves the topology from the environment.
+ *
+ * `count > 1` requires an explicit index: a process that does not know which
+ * one it is cannot own a shard of anything, and a guessed index would have two
+ * processes claiming the same one.
+ */
+export function resolveProcessTopology(): ProcessTopology {
+  const count = readInt("CARTETHYIA_PROCESS_COUNT", CONFIG_SPEC.CARTETHYIA_PROCESS_COUNT);
+  if (count === 1) return { count: 1, index: 0, multiProcess: false };
+  // Mandatory when the count is above one: a process that does not know which
+  // one it is cannot own a shard of anything, and a defaulted index would have
+  // two processes claiming the same number. The spec's default of 0 cannot
+  // express "absent", so the raw value is checked here before it is read.
+  const rawIndex = process.env.CARTETHYIA_PROCESS_INDEX?.trim();
+  if (rawIndex === undefined || rawIndex.length === 0) {
+    throw new Error(
+      `CARTETHYIA_PROCESS_INDEX is required when CARTETHYIA_PROCESS_COUNT is ${count} (0 – ${count - 1})`,
+    );
+  }
+  const index = readInt("CARTETHYIA_PROCESS_INDEX", CONFIG_SPEC.CARTETHYIA_PROCESS_INDEX);
+  if (index >= count) {
+    throw new Error(
+      `CARTETHYIA_PROCESS_INDEX must be an integer between 0 and ${count - 1} (got "${rawIndex}")`,
+    );
+  }
+  return { count, index, multiProcess: true };
+}