@cosmicdrift/kumiko-server-runtime 0.365.0 → 0.366.0

package/dist/boot/wait-for-database.d.ts

@@ -0,0 +1,13 @@
+import { type DbConnection } from "@cosmicdrift/kumiko-framework/db";
+export declare function isTransientConnectionError(error: unknown): boolean;
+export declare const WAIT_FOR_DATABASE_INITIAL_BACKOFF_MS = 1000;
+export declare const WAIT_FOR_DATABASE_MAX_BACKOFF_MS = 15000;
+export declare const WAIT_FOR_DATABASE_DEADLINE_MS = 90000;
+export type WaitForDatabaseOptions = {
+    readonly processName: string;
+    readonly log?: (line: string) => void;
+    readonly sleep?: (ms: number) => Promise<void>;
+    readonly now?: () => number;
+};
+export declare function waitForDatabase(db: DbConnection, options: WaitForDatabaseOptions): Promise<void>;
+export declare function waitUntilDatabaseReachable<T>(ping: () => Promise<T>, options: WaitForDatabaseOptions): Promise<T>;

package/dist/boot/wait-for-database.js

@@ -0,0 +1,64 @@
+import { extractPgError, tableExists } from "@cosmicdrift/kumiko-framework/db";
+const TRANSIENT_CODES = new Set([
+    "53300", // too_many_connections
+    "57P03", // cannot_connect_now (startup / recovery)
+    "ECONNREFUSED",
+    "ECONNRESET",
+    "ETIMEDOUT",
+    "ENOTFOUND",
+    "EAI_AGAIN",
+    "CONNECTION_CLOSED",
+    "CONNECT_TIMEOUT",
+    "CONNECTION_DESTROYED",
+    "ERR_POSTGRES_CONNECTION_REFUSED", // Bun.SQL
+    "ERR_POSTGRES_CONNECTION_CLOSED", // Bun.SQL
+    "ERR_POSTGRES_CONNECTION_TIMEOUT", // Bun.SQL
+    "ERR_POSTGRES_CONNECTION_FAILED", // Bun.SQL
+]);
+const CONNECTION_EXCEPTION_CLASS = "08";
+export function isTransientConnectionError(error) {
+    const code = extractPgError(error)?.code;
+    if (code === undefined)
+        return false;
+    return TRANSIENT_CODES.has(code) || code.startsWith(CONNECTION_EXCEPTION_CLASS);
+}
+export const WAIT_FOR_DATABASE_INITIAL_BACKOFF_MS = 1_000;
+export const WAIT_FOR_DATABASE_MAX_BACKOFF_MS = 15_000;
+// ponytail: the kubelet liveness probe restarts a pod without an open port after ~105s anyway,
+// so waiting longer buys nothing. Upgrade: make this configurable via env once a deployment
+// needs a longer outage window (e.g. a managed-DB failover that takes minutes).
+export const WAIT_FOR_DATABASE_DEADLINE_MS = 90_000;
+function logToConsole(line) {
+    // biome-ignore lint/suspicious/noConsole: boot-time progress hint, no logger configured this early
+    console.log(line);
+}
+function sleepReal(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+// Postgres can be briefly unreachable at boot (connection slots exhausted, failover, restart).
+// Crashing makes the orchestrator restart-loop and adds connection churn; wait and retry instead.
+export async function waitForDatabase(db, options) {
+    await waitUntilDatabaseReachable(() => tableExists(db, "pg_catalog.pg_class"), options);
+}
+export async function waitUntilDatabaseReachable(ping, options) {
+    const { processName, log = logToConsole, sleep = sleepReal, now = Date.now } = options;
+    const deadline = now() + WAIT_FOR_DATABASE_DEADLINE_MS;
+    let backoffMs = WAIT_FOR_DATABASE_INITIAL_BACKOFF_MS;
+    for (let attempt = 1;; attempt++) {
+        try {
+            return await ping();
+        }
+        catch (error) {
+            if (!isTransientConnectionError(error))
+                throw error;
+            const remainingMs = deadline - now();
+            if (remainingMs <= 0)
+                throw error;
+            const waitMs = Math.min(backoffMs, remainingMs);
+            const code = extractPgError(error)?.code ?? "unknown";
+            log(`[${processName}] database not reachable (code ${code}, attempt ${attempt}), retrying in ${waitMs}ms…`);
+            await sleep(waitMs);
+            backoffMs = Math.min(backoffMs * 2, WAIT_FOR_DATABASE_MAX_BACKOFF_MS);
+        }
+    }
+}

package/dist/boot/worker-boot-core.js

@@ -20,6 +20,7 @@ import { requireEnv } from "../run-prod-app.js";
 import { addConfigAccessorFactory, buildBootExtraContext, loggerFromExtraContext, } from "../run-prod-app-boot-context.js";
 import { resolveBootCrypto } from "./boot-crypto.js";
 import { jobRunLoggerCallbacks } from "./job-run-logger.js";
+import { waitForDatabase, waitUntilDatabaseReachable } from "./wait-for-database.js";
 import { assertWorkerMetricsOptions } from "./worker-metrics-server.js";
 function isSecretsContext(value) {
     if (typeof value !== "object" || value === null)
@@ -96,7 +97,10 @@ export async function bootWorkerProcess(options, envSource, profile) {
     }
     // Before the connections, so an abort leaks nothing.
     if (options.kms) {
-        const kmsHealth = await options.kms.health();
+        const kms = options.kms;
+        const kmsHealth = await waitUntilDatabaseReachable(() => kms.health(), {
+            processName: processName,
+        });
         if (!kmsHealth.ok) {
             throw new Error(`[${processName}] BOOT ABORTED — KMS health check failed (latency ${kmsHealth.latencyMs}ms)`);
         }
@@ -108,6 +112,16 @@ export async function bootWorkerProcess(options, envSource, profile) {
         redisUrl,
         clientOptions: redisClientOptionsFromEnv(envSource),
     });
+    try {
+        await waitForDatabase(db, { processName });
+    }
+    catch (err) {
+        await closeDb();
+        await closeOutsideTransactionDb();
+        redis.disconnect();
+        await cacheSync.close();
+        throw err;
+    }
     const resolvedEffectiveFeatures = options.effectiveFeatures ??
         (await buildTierEffectiveFeatures(features, db, registry, cacheSync));
     if (options.migrations !== false) {

package/dist/extra-routes-deps.d.ts

@@ -1,5 +1,6 @@
 import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
 import type { Registry, WriteResult } from "@cosmicdrift/kumiko-framework/engine";
+import type { MetricsHandle } from "@cosmicdrift/kumiko-framework/observability";
 import type { Redis } from "ioredis";
 export type { SystemDispatchArgs } from "@cosmicdrift/kumiko-framework/api";
 export { makeDispatchSystemQuery, makeDispatchSystemWrite, } from "@cosmicdrift/kumiko-framework/api";
@@ -15,6 +16,8 @@ export type SystemWireDeps = {
     /** Feature-registry — e.g. for plugin-lookups via
      *  `registry.getExtensionUsages("subscriptionProvider")`. */
     readonly registry: Registry;
+    /** Same contract as `ctx.metricsFor`: unregistered metric names no-op. */
+    readonly metricsFor: (featureName: string) => MetricsHandle;
     /** Writes through the /api/*-command-dispatcher (same idempotency/job
      *  hooks) as an auto-constructed SystemAdmin of the target tenant,
      *  WITHOUT the route's own access-check. Privilege-scope: SystemAdmin is

package/dist/run-prod-app.d.ts

@@ -8,7 +8,7 @@ import { type AllInOneEntrypoint, type ApiEntrypoint } from "@cosmicdrift/kumiko
 import { type ComposedEnvSchema, KumikoBootError } from "@cosmicdrift/kumiko-framework/env";
 import type { ErrorDocsConfig } from "@cosmicdrift/kumiko-framework/errors";
 import type { JobRunnerOptions } from "@cosmicdrift/kumiko-framework/jobs";
-import type { ObservabilityOptions, ObservabilityProvider } from "@cosmicdrift/kumiko-framework/observability";
+import { type ObservabilityOptions, type ObservabilityProvider } from "@cosmicdrift/kumiko-framework/observability";
 import type { MasterKeyProvider } from "@cosmicdrift/kumiko-framework/secrets";
 import type { PageHeadMeta, PageHeadResolver, PageHeadSystemQuery } from "@cosmicdrift/kumiko-headless/apex";
 import { Redis } from "ioredis";

package/dist/run-prod-app.js

@@ -58,6 +58,7 @@ import { renderDryRun } from "@cosmicdrift/kumiko-framework/env/dry-run";
 import { createEsOperationsTable, createSeedMigrationContext, runPendingSeedMigrations, } from "@cosmicdrift/kumiko-framework/es-ops";
 import { assertSeedModeNotInProduction } from "@cosmicdrift/kumiko-framework/event-store";
 import { assertKumikoSchemaCurrent, SchemaDriftError, } from "@cosmicdrift/kumiko-framework/migrations";
+import { createMetricsForFactory, } from "@cosmicdrift/kumiko-framework/observability";
 import { createDispatcher, createEntityCache, createEventDedup, createIdempotencyGuard, flushEscapeHatchAudits, } from "@cosmicdrift/kumiko-framework/pipeline";
 import { createRedisCacheSyncBus, redisClientOptionsFromEnv, } from "@cosmicdrift/kumiko-framework/redis";
 import { warnIfNonUtcServerTimeZone } from "@cosmicdrift/kumiko-framework/time";
@@ -67,6 +68,7 @@ import { resolveBootCrypto } from "./boot/boot-crypto.js";
 import { jobRunLoggerCallbacks } from "./boot/job-run-logger.js";
 import { startPiiEventBackfillOnBoot } from "./boot/pii-event-backfill-on-boot.js";
 import { recordAppStartedOnBoot } from "./boot/record-app-started-on-boot.js";
+import { waitForDatabase, waitUntilDatabaseReachable } from "./boot/wait-for-database.js";
 import { buildBunServeOptions, resolveDerivedMaxRequestBodySize } from "./bun-serve-options.js";
 import { buildComposeAuthOptions, composeFeatures } from "./compose-features.js";
 import { makeDispatchSystemWrite } from "./extra-routes-deps.js";
@@ -336,7 +338,10 @@ export async function runProdApp(options) {
     // against an unreachable key store — a silent start would answer every
     // PII read with 503. Runs BEFORE connections so an abort leaks nothing.
     if (options.kms) {
-        const kmsHealth = await options.kms.health();
+        const kms = options.kms;
+        const kmsHealth = await waitUntilDatabaseReachable(() => kms.health(), {
+            processName: "runProdApp",
+        });
         if (!kmsHealth.ok) {
             throw new Error(`[runProdApp] BOOT ABORTED — KMS health check failed (latency ${kmsHealth.latencyMs}ms)`);
         }
@@ -344,6 +349,7 @@ export async function runProdApp(options) {
     const { db, close: closeDb } = createDbConnection(databaseUrl, dbConnectionOptionsFromEnv(envSource));
     const { db: outsideTransactionDb, close: closeOutsideTransactionDb } = createDbConnection(databaseUrl, outsideTransactionPoolOptionsFromEnv(envSource));
     const redis = new Redis(redisUrl, { maxRetriesPerRequest: null });
+    await waitForDatabase(db, { processName: "runProdApp" });
     // Sprint-8a Tier-Composition auto-wire: scan features for a
     // tenantTierResolver-extension. If found AND user didn't supply own
     // effectiveFeatures, build the resolver here (db + registry are
@@ -680,6 +686,7 @@ export async function runProdApp(options) {
             db,
             redis,
             registry,
+            metricsFor: createMetricsForFactory(entrypoint.observability.meter),
             dispatchSystemWrite: makeDispatchSystemWrite(entrypoint.dispatcher),
         });
     }

package/dist/run-worker-app.d.ts

@@ -4,7 +4,7 @@ import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
 import type { EffectiveFeaturesResolver, FeatureDefinition, Registry, ValidateBootOptions } from "@cosmicdrift/kumiko-framework/engine";
 import type { WorkerEntrypoint } from "@cosmicdrift/kumiko-framework/entrypoint";
 import type { JobRunnerOptions } from "@cosmicdrift/kumiko-framework/jobs";
-import type { ObservabilityOptions, ObservabilityProvider } from "@cosmicdrift/kumiko-framework/observability";
+import { type ObservabilityOptions, type ObservabilityProvider } from "@cosmicdrift/kumiko-framework/observability";
 import type { MasterKeyProvider, SecretsContext } from "@cosmicdrift/kumiko-framework/secrets";
 import type { Redis } from "ioredis";
 import { type WorkerMetricsOptions, type WorkerMetricsServer } from "./boot/worker-metrics-server.js";

package/dist/run-worker-app.js

@@ -15,6 +15,7 @@
 // Container/Coolify sets the same env vars as runProdApp:
 //   DATABASE_URL, REDIS_URL, JWT_SECRET — PORT is not needed.
 import { assertSeedModeNotInProduction } from "@cosmicdrift/kumiko-framework/event-store";
+import { createMetricsForFactory, } from "@cosmicdrift/kumiko-framework/observability";
 import { startPiiEventBackfillOnBoot } from "./boot/pii-event-backfill-on-boot.js";
 import { recordAppStartedOnBoot } from "./boot/record-app-started-on-boot.js";
 import { bootWorkerProcess, resolveWorkerEnvSource } from "./boot/worker-boot-core.js";
@@ -70,6 +71,7 @@ export async function runWorkerApp(options) {
             db,
             redis,
             registry,
+            metricsFor: createMetricsForFactory(entrypoint.observability.meter),
             dispatchSystemWrite: makeDispatchSystemWrite(entrypoint.dispatcher),
             lifecycle: entrypoint.lifecycle,
             ...(boot.secrets && { secrets: boot.secrets }),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cosmicdrift/kumiko-server-runtime",
-  "version": "0.365.0",
+  "version": "0.366.0",
   "description": "Production server-boot runtime for Kumiko apps: connections, schema-drift-gate, seeds, lifecycle, graceful shutdown. Symmetric to kumiko-dev-server's runDevApp, without dev/scaffold/codegen tooling.",
   "license": "BUSL-1.1",
   "author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
@@ -84,12 +84,12 @@
     }
   },
   "dependencies": {
-    "@cosmicdrift/kumiko-bundled-features": "0.365.0",
-    "@cosmicdrift/kumiko-framework": "0.365.0",
-    "@cosmicdrift/kumiko-headless": "0.365.0",
-    "@cosmicdrift/kumiko-renderer-mail-html": "0.365.0",
-    "@cosmicdrift/kumiko-renderer-web": "0.365.0",
-    "@cosmicdrift/kumiko-types": "0.365.0",
+    "@cosmicdrift/kumiko-bundled-features": "0.366.0",
+    "@cosmicdrift/kumiko-framework": "0.366.0",
+    "@cosmicdrift/kumiko-headless": "0.366.0",
+    "@cosmicdrift/kumiko-renderer-mail-html": "0.366.0",
+    "@cosmicdrift/kumiko-renderer-web": "0.366.0",
+    "@cosmicdrift/kumiko-types": "0.366.0",
     "bun-types": "^1.3.13",
     "ioredis": "^5.10.1",
     "temporal-polyfill": "^0.3.2"