gencow 0.1.190 → 0.1.191

package/lib/app-delete-operation.mjs

@@ -16,6 +16,7 @@ const APP_DELETE_ACTIVE_STATES = new Set([
   "failed_tenant_db_cleanup",
   "failed_catalog_cleanup",
 ]);
+const APP_DELETE_TERMINAL_FAILURE_STATES = new Set(["blocked_catalog_outcome_unknown"]);
 const CORRELATION_ID_PATTERN = /^[A-Za-z0-9_-]{8,128}$/u;
 const DIAGNOSTIC_CODE_PATTERN = /^[A-Za-z][A-Za-z0-9_]{2,127}$/u;
 const APP_ID_PATTERN = /^[a-z][a-z0-9]*(?:-[a-z0-9]+){2}$/u;
@@ -74,6 +75,20 @@ function isMatchingTerminalResponse(value, accepted, expectedAppId) {
   );
 }
 
+function isMatchingTerminalFailureResponse(value, accepted, expectedAppId) {
+  return (
+    value &&
+    typeof value === "object" &&
+    !Array.isArray(value) &&
+    value.success === false &&
+    value.deleted === expectedAppId &&
+    value.operationId === accepted.operationId &&
+    value.correlationId === accepted.correlationId &&
+    APP_DELETE_TERMINAL_FAILURE_STATES.has(value.state) &&
+    DIAGNOSTIC_CODE_PATTERN.test(value.code ?? "")
+  );
+}
+
 export async function pollAppDeleteOperation({
   creds,
   initialBody,
@@ -99,6 +114,9 @@ export async function pollAppDeleteOperation({
       if (!statusRes?.ok || !statusData) return invalidPollResponse(accepted);
       successfulPollResponses += 1;
       if (isMatchingTerminalResponse(statusData, accepted, name)) return statusData;
+      if (isMatchingTerminalFailureResponse(statusData, accepted, name)) {
+        return { ...statusData, error: statusData.code };
+      }
       const operation = parseAppDeleteOperationEnvelope(statusData, name);
       if (
         !operation ||

package/lib/canonical-drizzle-generator-paths.mjs

@@ -0,0 +1,7 @@
+import { posix, win32 } from "node:path";
+
+export function toDrizzleProjectPath(projectRoot, canonicalFile) {
+  const pathApi = win32.isAbsolute(projectRoot) ? win32 : posix;
+  const projectPath = pathApi.relative(projectRoot, canonicalFile).replaceAll("\\", "/");
+  return projectPath.startsWith(".") ? projectPath : `./${projectPath}`;
+}

package/lib/canonical-drizzle-generator.config.mjs

@@ -1,6 +1,8 @@
 import { existsSync, lstatSync, realpathSync } from "node:fs";
 import { basename, dirname, isAbsolute, relative, resolve } from "node:path";
 
+import { toDrizzleProjectPath } from "./canonical-drizzle-generator-paths.mjs";
+
 export const GENCOW_DRIZZLE_CONFIG_CONTRACT_VERSION = 1;
 
 function requiredEnvironment(name) {
@@ -38,7 +40,7 @@ function resolveSchemaFiles(projectRoot) {
     }
     if (!seen.has(canonical)) {
       seen.add(canonical);
-      files.push(canonical);
+      files.push(toDrizzleProjectPath(projectRoot, canonical));
     }
   }
   if (files.length === 0) throw new Error("Gencow migration schema list is empty");

package/lib/cli-artifact-guard.mjs

@@ -6,6 +6,7 @@ export const REQUIRED_CLI_TARBALL_FILES = Object.freeze([
   "lib/cron-manifest.mjs",
   "lib/dev-cloud-bundle.mjs",
   "lib/canonical-drizzle-generator.config.mjs",
+  "lib/canonical-drizzle-generator-paths.mjs",
   "runtime/tooling.mjs",
   "runtime/server.mjs",
   "templateFeature/ai/manifest.json",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gencow",
-  "version": "0.1.190",
+  "version": "0.1.191",
   "description": "Gencow — AI Backend Engine",
   "type": "module",
   "bin": {
@@ -33,8 +33,8 @@
   "devDependencies": {
     "@types/node": "^25.9.5",
     "better-auth": "^1.6.23",
-    "@gencow/core": "0.1.42",
     "@gencow/client": "0.2.6",
+    "@gencow/core": "0.1.42",
     "@gencow/react": "0.2.6"
   },
   "scripts": {

package/runtime/server.mjs

@@ -88955,7 +88955,7 @@ var require_package = __commonJS({
 var require_libvips = __commonJS({
   "../../node_modules/.pnpm/sharp@0.34.5/node_modules/sharp/lib/libvips.js"(exports2, module2) {
     var { spawnSync } = __require("node:child_process");
-    var { createHash: createHash24 } = __require("node:crypto");
+    var { createHash: createHash25 } = __require("node:crypto");
     var semverCoerce = require_coerce();
     var semverGreaterThanOrEqualTo = require_gte();
     var semverSatisfies = require_satisfies();
@@ -89046,7 +89046,7 @@ var require_libvips = __commonJS({
       }
       return false;
     };
-    var sha512 = (s) => createHash24("sha512").update(s).digest("hex");
+    var sha512 = (s) => createHash25("sha512").update(s).digest("hex");
     var yarnLocator = () => {
       try {
         const identHash = sha512(`imgsharp-libvips-${buildPlatformArch()}`);
@@ -94229,7 +94229,29 @@ var init_platform_active_slot = __esm({
 });
 
 // ../server/dist/server-platform-slot-promotion.js
-import { timingSafeEqual as timingSafeEqual6 } from "crypto";
+import { createHash as createHash17, timingSafeEqual as timingSafeEqual6 } from "crypto";
+function parseRouteProjectionDegradedApprovalHeaders(input) {
+  if (input.count === void 0 && input.digest === void 0 && input.appIds === void 0)
+    return void 0;
+  if (!/^[1-9][0-9]{0,3}$/u.test(input.count ?? "") || !/^[a-f0-9]{64}$/u.test(input.digest ?? "") || !/^[1-9][0-9]{0,9}(,[1-9][0-9]{0,9}){0,999}$/u.test(input.appIds ?? "")) {
+    throw new Error("PLATFORM_ROUTE_DEGRADED_APPROVAL_INVALID");
+  }
+  const readyMismatchAppCount = Number(input.count);
+  const readyMismatchAppIds = input.appIds.split(",").map(Number);
+  const canonicalAppIds = [...new Set(readyMismatchAppIds)].sort((left, right) => left - right);
+  const canonicalDigest = createHash17("sha256").update(JSON.stringify(canonicalAppIds)).digest("hex");
+  if (!Number.isSafeInteger(readyMismatchAppCount) || readyMismatchAppCount > 1e3 || readyMismatchAppIds.some((appId) => !Number.isSafeInteger(appId) || appId <= 0) || readyMismatchAppIds.some((appId, index) => appId !== canonicalAppIds[index]) || readyMismatchAppCount !== canonicalAppIds.length || input.digest !== canonicalDigest) {
+    throw new Error("PLATFORM_ROUTE_DEGRADED_APPROVAL_INVALID");
+  }
+  return { readyMismatchAppCount, readyMismatchAppDigest: input.digest, readyMismatchAppIds };
+}
+function routeProjectionDegradedApprovalFromRequest(context2) {
+  return parseRouteProjectionDegradedApprovalHeaders({
+    count: context2.req.header(ROUTE_READY_MISMATCH_COUNT_HEADER),
+    digest: context2.req.header(ROUTE_READY_MISMATCH_DIGEST_HEADER),
+    appIds: context2.req.header(ROUTE_READY_MISMATCH_APP_IDS_HEADER)
+  });
+}
 function applyPlatformSlotPromotionRequestTimeout(request, server) {
   if (!["/internal/platform-slot/prepare-promotion", "/internal/platform-slot/promote"].includes(new URL(request.url).pathname))
     return false;
@@ -94274,8 +94296,9 @@ function registerPlatformSlotPromotionRoute(params) {
     }
     try {
       state2 = "preparing";
+      const approval = routeProjectionDegradedApprovalFromRequest(c);
       const result2 = await params.autoRecoveryRuntime.promote();
-      const evidence = await params.assertTenantRouteParity?.();
+      const evidence = await params.assertTenantRouteParity?.(approval);
       if (!evidence?.evidenceDigest || !Number.isSafeInteger(evidence.bindingCount)) {
         throw new Error("PLATFORM_TENANT_ROUTE_PARITY_EVIDENCE_INVALID");
       }
@@ -94306,7 +94329,8 @@ function registerPlatformSlotPromotionRoute(params) {
         return c.json({ status: "prepared", promoted: false, code: "PLATFORM_SLOT_NOT_ROUTED" }, 409);
       }
       state2 = "promoting";
-      const currentEvidence = await params.assertTenantRouteParity?.();
+      const approval = routeProjectionDegradedApprovalFromRequest(c);
+      const currentEvidence = await params.assertTenantRouteParity?.(approval);
       if (!currentEvidence || currentEvidence.bindingCount !== preparedEvidence.bindingCount || currentEvidence.evidenceDigest !== preparedEvidence.evidenceDigest)
         throw new Error("PLATFORM_TENANT_ROUTE_PARITY_EVIDENCE_CHANGED");
       params.setCandidateMode(false);
@@ -94322,11 +94346,14 @@ function registerPlatformSlotPromotionRoute(params) {
     }
   });
 }
-var PLATFORM_SLOT_PROMOTION_REQUEST_TIMEOUT_SECONDS;
+var ROUTE_READY_MISMATCH_COUNT_HEADER, ROUTE_READY_MISMATCH_DIGEST_HEADER, ROUTE_READY_MISMATCH_APP_IDS_HEADER, PLATFORM_SLOT_PROMOTION_REQUEST_TIMEOUT_SECONDS;
 var init_server_platform_slot_promotion = __esm({
   "../server/dist/server-platform-slot-promotion.js"() {
     "use strict";
     init_platform_active_slot();
+    ROUTE_READY_MISMATCH_COUNT_HEADER = "X-Gencow-Emergency-Route-Ready-Mismatch-Count";
+    ROUTE_READY_MISMATCH_DIGEST_HEADER = "X-Gencow-Emergency-Route-Ready-Mismatch-Digest";
+    ROUTE_READY_MISMATCH_APP_IDS_HEADER = "X-Gencow-Emergency-Route-Ready-Mismatch-App-Ids";
     PLATFORM_SLOT_PROMOTION_REQUEST_TIMEOUT_SECONDS = 240;
   }
 });
@@ -134090,7 +134117,7 @@ var init_official_custom_domains = __esm({
 });
 
 // ../server/dist/server-platform-app-proxy-domain.js
-import { createHash as createHash21 } from "crypto";
+import { createHash as createHash22 } from "crypto";
 function stripTrailingDots(host) {
   return host.replace(/\.+$/, "");
 }
@@ -134169,7 +134196,7 @@ function decodeActiveDomainApp(value, expectedHostname) {
   return { appName, verifiedBinding };
 }
 function hashCustomDomainHost(host) {
-  return createHash21("sha256").update(host).digest("hex").slice(0, 16);
+  return createHash22("sha256").update(host).digest("hex").slice(0, 16);
 }
 function isGenerationChangedError(error51) {
   return error51?.code === "GENCOW_DB_GENERATION_CHANGED";
@@ -134409,7 +134436,7 @@ var init_server_platform_media_worker_storage_runtime = __esm({
 });
 
 // ../server/dist/server-runtime-activity-store.js
-import { createHash as createHash22 } from "node:crypto";
+import { createHash as createHash23 } from "node:crypto";
 function rowsFromUnknown3(result2) {
   if (Array.isArray(result2))
     return result2;
@@ -134660,7 +134687,7 @@ async function checkpointRuntimeActivity(rawSql, checkpoint) {
   if (!Number.isSafeInteger(appId) || !Number.isSafeInteger(activityEpoch) || activityEpoch < 0 || !nodeId || !Number.isSafeInteger(nodeEpoch) || nodeEpoch < 0) {
     return null;
   }
-  const bindingHash = createHash22("sha256").update(JSON.stringify([
+  const bindingHash = createHash23("sha256").update(JSON.stringify([
     "runtime-activity-binding-v1",
     appId,
     typeof row?.activation_id === "string" ? row.activation_id : null,
@@ -149672,7 +149699,7 @@ var init_server_platform_notification_worker_runtime = __esm({
 });
 
 // ../server/src/server-node-proxy-signature.ts
-import { createHash as createHash23, createHmac as createHmac9, timingSafeEqual as timingSafeEqual11 } from "crypto";
+import { createHash as createHash24, createHmac as createHmac9, timingSafeEqual as timingSafeEqual11 } from "crypto";
 function sanitizeNodeProxyForwardHeaders(headers) {
   const sanitized = new Headers();
   for (const [key, value] of headers.entries()) {
@@ -149695,7 +149722,7 @@ function canonicalSignedHeaderHash2(headers) {
     const value = headers?.get(name) ?? "";
     return `${name}:${value.trim()}`;
   }).join("\n");
-  return createHash23("sha256").update(canonical).digest("base64url");
+  return createHash24("sha256").update(canonical).digest("base64url");
 }
 function buildSignaturePayload2(input) {
   return [
@@ -150030,17 +150057,22 @@ var init_server_platform_runtime_helpers = __esm({
 
 // ../server/dist/server-legacy-runtime-recovery-route.js
 import { resolve as resolve45 } from "node:path";
+function parseLegacyRuntimeRecoveryMode(value) {
+  return value === "preserve_replace" || value === "canonicalize_absent" || value === "retire_unadmitted" ? value : null;
+}
 function registerLegacyRuntimeRecoveryRoute(params) {
   const logger3 = params.logger ?? console;
   params.app.post("/internal/runtime-recovery/legacy-preserve-replace", async (c) => {
-    if (!params.internalToken || c.req.header("X-Internal-Token") !== params.internalToken) {
+    if (params.internalToken && c.req.header("X-Internal-Token") !== params.internalToken) {
       return c.json({ error: "Forbidden" }, 403);
     }
     const body = await c.req.json().catch(() => null);
     const appName = typeof body?.appName === "string" ? body.appName.trim() : "";
-    const mode = body?.mode === "canonicalize_absent" ? "canonicalize_absent" : "preserve_replace";
+    const mode = parseLegacyRuntimeRecoveryMode(body?.mode ?? "preserve_replace");
     if (!APP_NAME.test(appName))
       return c.json({ error: "appName required" }, 400);
+    if (!mode)
+      return c.json({ error: "recovery mode invalid" }, 400);
     if (!await params.isActiveController())
       return c.json({ error: "Controller unavailable" }, 409);
     try {
@@ -150060,6 +150092,63 @@ function registerLegacyRuntimeRecoveryRoute(params) {
         const result3 = await convergeMalformedLegacyAbsence({ db: params.db, appId: app2.id, appName, currentNodeId: params.currentNodeId, evidence });
         return c.json({ ok: true, result: result3 });
       }
+      if (mode === "retire_unadmitted") {
+        const { getRuntimeLifecycleActivityCoordinator } = await import(resolve45(params.functionsPath, "../src/runtime-lifecycle-activity-coordinator.ts"));
+        const { stopRuntimeAuthoritatively } = await import(resolve45(params.functionsPath, "../src/runtime-control-plane-lifecycle.ts"));
+        const { observeLegacyRuntimeEvidence } = await import(resolve45(params.functionsPath, "../src/legacy-runtime-evidence-observer.ts"));
+        const { convergeMalformedLegacyAbsence } = await import(resolve45(params.functionsPath, "../src/runtime-malformed-legacy-absence-recovery.ts"));
+        const currentRows = await params.db.select({
+          port: params.appsTable.port,
+          instanceId: params.appsTable.instanceId,
+          dataDir: params.appsTable.dataDir,
+          nodeId: params.appsTable.nodeId,
+          currentDeployId: params.appsTable.currentDeployId,
+          hasBackend: params.appsTable.hasBackend
+        }).from(params.appsTable).where(params.eqOp(params.appsTable.id, app2.id)).limit(1);
+        const current = currentRows[0];
+        const activity = getRuntimeLifecycleActivityCoordinator();
+        if (!current || !activity)
+          throw new Error("LEGACY_RUNTIME_RETIRE_COORDINATOR_UNAVAILABLE");
+        activity.closeExplicitDrainGate(app2.id, appName);
+        try {
+          await activity.synchronizeRuntimeBinding(app2.id, appName);
+          await activity.waitForQuiescence(app2.id, appName);
+          const before2 = await observeLegacyRuntimeEvidence({
+            appName,
+            appPort: current.port,
+            appNodeId: params.currentNodeId,
+            currentNodeId: params.currentNodeId
+          });
+          if (!before2.observationComplete || ![before2.directProcess, before2.warmPoolAssignment, before2.listener].includes("present") || before2.effectiveRoute !== "exact") {
+            throw new Error("LEGACY_RUNTIME_RETIRE_INCUMBENT_NOT_EXACT");
+          }
+          await stopRuntimeAuthoritatively({
+            db: params.db,
+            app: { id: app2.id, name: appName, ...current },
+            currentNodeId: params.currentNodeId,
+            reason: "operator_stop",
+            actorId: "legacy-runtime-recovery",
+            actorKind: "platform",
+            expectedAuthority: "observe"
+          });
+          const evidence = await observeLegacyRuntimeEvidence({
+            appName,
+            appPort: current.port,
+            appNodeId: params.currentNodeId,
+            currentNodeId: params.currentNodeId
+          });
+          const result3 = await convergeMalformedLegacyAbsence({
+            db: params.db,
+            appId: app2.id,
+            appName,
+            currentNodeId: params.currentNodeId,
+            evidence
+          });
+          return c.json({ ok: true, result: result3 });
+        } finally {
+          activity.reopenDrainGate(app2.id);
+        }
+      }
       const { convergeLegacyRuntimeAuthority } = await import(resolve45(params.functionsPath, "../src/legacy-runtime-bootstrap-orchestrator.ts"));
       const result2 = await convergeLegacyRuntimeAuthority({
         db: params.db,
@@ -150644,7 +150733,7 @@ async function bootstrapPlatformRuntime(params) {
   const convergeRuntimeRouteProjection = requirePlatformExplicitStopConverger(autoRecoveryRuntime.convergeRuntimeRouteProjection);
   routeProjectionRuntime.setAbsentRouteConverger(convergeRuntimeRouteProjection);
   params.app.get("/internal/platform-recovery/status", (c) => c.json(autoRecoveryRuntime.snapshot()));
-  registerProcessPlatformSlotPromotionRoute(params.app, params.rawSql, autoRecoveryRuntime, () => routeProjectionRuntime.convergeServingFleet(), logger3);
+  registerProcessPlatformSlotPromotionRoute(params.app, params.rawSql, autoRecoveryRuntime, (approval) => routeProjectionRuntime.convergeServingFleet(approval), logger3);
   const explicitStopReconciler = await preparePlatformExplicitStopReconcilerForRuntime({
     functionsPath: params.functionsPath,
     db: params.db,
@@ -152779,6 +152868,8 @@ function classifyPostgresClientFailure(error51) {
   }
   for (const item of chain2) {
     const code = errorCode(item);
+    if (code === "GENCOW_DB_GENERATION_CHANGED")
+      return "generation_changed";
     if (code === "ERR_POSTGRES_CONNECTION_CLOSED")
       return "connection_closed";
     if (code === "ECONNRESET" || code === "EPIPE")
@@ -152919,11 +153010,11 @@ function createRuntimePostgresGenerationManager(params) {
   let recoveryAttempts = 0;
   let recoveryTimer;
   const randomUnit = () => Math.min(0.999999999, Math.max(0, randomImpl()));
-  function createRecord(client, generation, jitterMs) {
+  function createRecord(client, generation, jitterMs, state2 = "healthy") {
     return {
       client,
       generation,
-      state: "healthy",
+      state: state2,
       inFlight: 0,
       closeStarted: false,
       admitAfter: Date.now() + jitterMs
@@ -152932,7 +153023,7 @@ function createRuntimePostgresGenerationManager(params) {
   function snapshot() {
     if (managerClosed)
       throw new RuntimePostgresGenerationError("GENCOW_DB_CLIENT_CLOSED");
-    if (active.state !== "healthy") {
+    if (active.state !== "healthy" && active.state !== "suspect") {
       throw new RuntimePostgresGenerationError("GENCOW_DB_RECOVERY_CIRCUIT_OPEN");
     }
     return { client: active.client, generation: active.generation };
@@ -152951,7 +153042,7 @@ function createRuntimePostgresGenerationManager(params) {
     void params.closeClient(record2.client).finally(() => {
       record2.state = "closed";
       quarantined.delete(record2.generation);
-      if (!managerClosed && recoveryCircuitOpen && active.state !== "healthy" && quarantined.size < maxQuarantinedGenerations) {
+      if (!managerClosed && recoveryCircuitOpen && active.state !== "healthy" && active.state !== "suspect" && quarantined.size < maxQuarantinedGenerations) {
         const reason = lastRecoveryReason ?? "generation_capacity_recovered";
         scheduleRecovery(reason);
       }
@@ -152965,14 +153056,20 @@ function createRuntimePostgresGenerationManager(params) {
     try {
       nextGeneration += 1;
       const jitterMs = recoveryJitterMaxMs === 0 ? 0 : Math.floor(randomUnit() * (recoveryJitterMaxMs + 1));
-      active = createRecord(params.createClient(), nextGeneration, jitterMs);
-      recoveryCircuitOpen = false;
-      recoveryAttempts = 0;
-      recoveryInFlight = false;
+      active = createRecord(params.createClient(), nextGeneration, jitterMs, params.verifyClient ? "suspect" : "healthy");
       if (recoveryTimer) {
         clearTimeoutImpl(recoveryTimer);
         recoveryTimer = void 0;
       }
+      if (params.verifyClient) {
+        recoveryCircuitOpen = true;
+        recoveryInFlight = true;
+        active.verification = verifyReplacement(active, reason);
+        return true;
+      }
+      recoveryCircuitOpen = false;
+      recoveryAttempts = 0;
+      recoveryInFlight = false;
       params.onRecovered?.({ generation: nextGeneration, reason });
       return true;
     } catch {
@@ -152980,8 +153077,39 @@ function createRuntimePostgresGenerationManager(params) {
       return false;
     }
   }
+  async function verifyReplacement(record2, reason) {
+    try {
+      const waitMs = Math.max(0, record2.admitAfter - Date.now());
+      if (waitMs > 0) {
+        await new Promise((resolve50) => {
+          const timer2 = setTimeoutImpl(resolve50, waitMs);
+          timer2.unref?.();
+        });
+      }
+      await params.verifyClient?.(record2.client);
+      if (managerClosed || record2 !== active || record2.state !== "suspect") {
+        return;
+      }
+      record2.state = "healthy";
+      recoveryCircuitOpen = false;
+      recoveryInFlight = false;
+      recoveryAttempts = 0;
+      params.onRecovered?.({ generation: record2.generation, reason });
+    } catch {
+      if (managerClosed || record2 !== active || record2.state !== "suspect") {
+        return;
+      }
+      record2.state = "quarantined";
+      quarantined.set(record2.generation, record2);
+      lastRecoveryReason = reason;
+      recoveryCircuitOpen = true;
+      recoveryInFlight = false;
+      closeWhenDrained(record2);
+      scheduleRecovery(reason);
+    }
+  }
   function scheduleRecovery(reason) {
-    if (managerClosed || active.state === "healthy" || recoveryTimer)
+    if (managerClosed || active.state === "healthy" || active.state === "suspect" || recoveryTimer)
       return;
     recoveryInFlight = true;
     recoveryAttempts += 1;
@@ -153007,12 +153135,18 @@ function createRuntimePostgresGenerationManager(params) {
     }
     record2.inFlight += 1;
     return Promise.resolve().then(async () => {
+      if (record2.state === "suspect") {
+        await record2.verification;
+      }
       const waitMs = Math.max(0, record2.admitAfter - Date.now());
       if (waitMs > 0)
         await new Promise((resolve50) => setTimeout(resolve50, waitMs));
-      if (managerClosed || record2.state !== "healthy" || record2 !== active) {
+      if (managerClosed || record2 !== active) {
         throw new RuntimePostgresGenerationError("GENCOW_DB_GENERATION_CHANGED");
       }
+      if (record2.state !== "healthy") {
+        throw new RuntimePostgresGenerationError("GENCOW_DB_RECOVERY_CIRCUIT_OPEN");
+      }
       return work();
     }).finally(() => {
       record2.inFlight = Math.max(0, record2.inFlight - 1);
@@ -153108,6 +153242,33 @@ async function closeWithBoundedWait(client, timeoutMs) {
   } catch {
   }
 }
+async function verifyRuntimePostgresConnectivity(client, timeoutMs) {
+  const pending = client.unsafe("SELECT 1");
+  let timeout;
+  try {
+    await Promise.race([
+      pending,
+      new Promise((_3, reject2) => {
+        timeout = setTimeout(() => {
+          try {
+            pending.cancel?.();
+          } catch {
+          }
+          reject2(new RuntimePostgresClientError({
+            reason: "operation_timeout",
+            operation: "unsafe",
+            generation: 0,
+            elapsedMs: timeoutMs
+          }));
+        }, timeoutMs);
+        timeout.unref?.();
+      })
+    ]);
+  } finally {
+    if (timeout !== void 0)
+      clearTimeout(timeout);
+  }
+}
 function rowsToValues(rows4) {
   if (!Array.isArray(rows4))
     return rows4;
@@ -153140,6 +153301,7 @@ function createRuntimePostgresClient(params) {
   let lastFailure;
   const generations = createRuntimePostgresGenerationManager({
     createClient: params.createClient,
+    verifyClient: (client) => verifyRuntimePostgresConnectivity(client, operationTimeoutMs > 0 ? operationTimeoutMs : 1e4),
     closeClient: (client) => closeWithBoundedWait(client, 250),
     recoveryJitterMaxMs: params.recoveryJitterMaxMs ?? (operationTimeoutMs > 0 ? Math.max(1, Math.min(25, Math.floor(operationTimeoutMs / 4))) : 25),
     onRecovered: ({ generation, reason }) => {
@@ -153177,7 +153339,7 @@ function createRuntimePostgresClient(params) {
     const causeCode = recordedFailure.causeCode ? ` causeCode=${recordedFailure.causeCode}` : "";
     const causeName = recordedFailure.causeName ? ` causeName=${recordedFailure.causeName}` : "";
     logger3.warn(`[db] postgres client quarantined label=${params.label}${dbInfo} reason=${recycleParams.reason} operation=${recycleParams.operation} elapsedMs=${recycleParams.elapsedMs} generation=${recycleParams.expectedGeneration} recoveries=${recoveries}${causeName}${causeCode}`);
-    if (!recovery.recovered) {
+    if (diagnostics.recoveryCircuitOpen) {
       logger3.warn(`[db] postgres client circuit_open label=${params.label}${dbInfo} reason=${recycleParams.reason} generation=${diagnostics.activeGeneration} recoveries=${recoveries}`);
     }
     return recovery.recovered;
@@ -184150,6 +184312,7 @@ var ProcedureRpcHandler = class {
 var RECYCLED_DB_ERROR_CODES = /* @__PURE__ */ new Set([
   "GENCOW_DB_CONNECTION_CLOSED",
   "GENCOW_DB_CONNECTION_TIMEOUT",
+  "GENCOW_DB_GENERATION_CHANGED",
   "GENCOW_DB_OPERATION_TIMEOUT",
   "GENCOW_DB_SOCKET_CLOSED"
 ]);
@@ -184250,6 +184413,7 @@ var APP_DELETE_ACTIVE_STATES = /* @__PURE__ */ new Set([
   "failed_tenant_db_cleanup",
   "failed_catalog_cleanup"
 ]);
+var APP_DELETE_TERMINAL_FAILURE_STATES = /* @__PURE__ */ new Set(["blocked_catalog_outcome_unknown"]);
 var CAPABILITY_PATTERN = /^[a-z0-9][a-z0-9-]{0,63}$/u;
 function createAppDeleteStatusLoader(getQueryDef2) {
   return async ({ ctx, args, result: result2 }) => {
@@ -184292,6 +184456,12 @@ function isMatchingNonterminalAppDeleteResult(value, accepted) {
   const result2 = value;
   return result2.operationId === accepted.operationId && result2.correlationId === accepted.correlationId && result2.deleted === accepted.deleted;
 }
+function isMatchingTerminalFailureAppDeleteResult(value, accepted) {
+  if (!value || typeof value !== "object" || Array.isArray(value))
+    return false;
+  const result2 = value;
+  return result2.success === false && APP_DELETE_TERMINAL_FAILURE_STATES.has(String(result2.state ?? "")) && result2.operationId === accepted.operationId && result2.correlationId === accepted.correlationId && result2.deleted === accepted.deleted;
+}
 async function resolveMutationSuccessResponse(input) {
   if (input.mutationName !== "apps.delete" || !isNonterminalAppDeleteResult(input.result)) {
     return { result: input.result, status: 201 };
@@ -184319,6 +184489,9 @@ async function resolveMutationSuccessResponse(input) {
     if (isMatchingTerminalAppDeleteResult(status, accepted)) {
       return { result: status, status: 201 };
     }
+    if (isMatchingTerminalFailureAppDeleteResult(status, accepted)) {
+      return { result: status, status: 409 };
+    }
     if (!isMatchingNonterminalAppDeleteResult(status, accepted)) {
       return { result: input.result, status: 409 };
     }
@@ -188192,6 +188365,16 @@ function registerServerHealthRoutes(input) {
     isCoreAdmissionReady: input.isCoreAdmissionReady
   }));
   input.app.get("/health/ready", createReadinessHandler({
+    rawSql: input.rawSql,
+    checkPlatformDatabaseCapabilities: input.platformDatabaseCapabilities ? () => input.platformDatabaseCapabilities.checkReadiness() : void 0,
+    requirePlatformDatabaseCapabilities: input.isPlatform,
+    getRegisteredQueries: input.getRegisteredQueries,
+    getRegisteredMutations: input.getRegisteredMutations,
+    getOperationInventory: input.getOperationInventory,
+    runtimeEnv: input.runtimeEnv,
+    readDeepHealth: platformDeepHealth?.read
+  }));
+  input.app.get("/health/route-ready", createReadinessHandler({
     rawSql: input.rawSql,
     checkPlatformDatabaseCapabilities: input.platformDatabaseCapabilities ? () => input.platformDatabaseCapabilities.checkReadiness() : void 0,
     requirePlatformDatabaseCapabilities: input.isPlatform,
@@ -189973,7 +190156,7 @@ async function recordAuthActivity(dbInstance2, c, params) {
 }
 
 // ../server/dist/auth-oauth-exchange-store.js
-import { createHash as createHash17 } from "crypto";
+import { createHash as createHash18 } from "crypto";
 import { sql as sql29 } from "drizzle-orm";
 
 // ../server/dist/crypto.js
@@ -189997,8 +190180,8 @@ function getSecretKey() {
     return cachedKey;
   const internalToken = process.env.GENCOW_INTERNAL_TOKEN;
   if (internalToken) {
-    const { createHash: createHash24 } = __require("crypto");
-    cachedKey = createHash24("sha256").update(`gencow-settings-key:${internalToken}`).digest().subarray(0, KEY_LENGTH);
+    const { createHash: createHash25 } = __require("crypto");
+    cachedKey = createHash25("sha256").update(`gencow-settings-key:${internalToken}`).digest().subarray(0, KEY_LENGTH);
     return cachedKey;
   }
   const keyPath = resolveSecretKeyPath();
@@ -190069,7 +190252,7 @@ function rowsFromResult17(result2) {
   return [];
 }
 function hashOAuthExchangeCode(code) {
-  return createHash17("sha256").update(code).digest("hex");
+  return createHash18("sha256").update(code).digest("hex");
 }
 async function createOAuthExchangeCode(db, record2) {
   const code = crypto.randomUUID();
@@ -191713,7 +191896,7 @@ async function resolveAuthMePayload(input) {
 }
 
 // ../server/dist/auth-oauth-invite-reservation.js
-import { createHash as createHash18, randomUUID as randomUUID12 } from "crypto";
+import { createHash as createHash19, randomUUID as randomUUID12 } from "crypto";
 import { sql as sql33 } from "drizzle-orm";
 
 // ../server/dist/invite-code.js
@@ -191809,7 +191992,7 @@ function rowsFromResult21(result2) {
   return [];
 }
 function hashReservationToken(token) {
-  return createHash18("sha256").update(token).digest("hex");
+  return createHash19("sha256").update(token).digest("hex");
 }
 function normalizeProvider2(provider) {
   return typeof provider === "string" && provider.trim() ? provider.trim().toLowerCase() : "unknown";
@@ -191915,7 +192098,7 @@ async function finalizeOAuthInviteReservation(db, token, user2) {
 }
 
 // ../server/dist/auth-oauth-signup-handoff.js
-import { createHash as createHash19, randomUUID as randomUUID13 } from "crypto";
+import { createHash as createHash20, randomUUID as randomUUID13 } from "crypto";
 import { sql as sql34 } from "drizzle-orm";
 init_postgres_temporal();
 var OAUTH_SIGNUP_HANDOFF_TTL_MS = BETTER_AUTH_OAUTH_STATE_TTL_MS + 5 * 60 * 1e3;
@@ -191928,7 +192111,7 @@ function rowsFromResult22(result2) {
   return [];
 }
 function hashHandoffToken(token) {
-  return createHash19("sha256").update(token).digest("hex");
+  return createHash20("sha256").update(token).digest("hex");
 }
 function normalizeProvider3(provider) {
   return typeof provider === "string" && provider.trim() ? provider.trim().toLowerCase() : "unknown";
@@ -192201,7 +192384,7 @@ function enforceAuthRateLimit(c, pathname, options = {}) {
 }
 
 // ../server/dist/server-node-proxy-signature.js
-import { createHash as createHash20, createHmac as createHmac8, timingSafeEqual as timingSafeEqual10 } from "crypto";
+import { createHash as createHash21, createHmac as createHmac8, timingSafeEqual as timingSafeEqual10 } from "crypto";
 var NODE_PROXY_VERSION = "v1";
 var DEFAULT_MAX_SKEW_MS = 3e4;
 var NODE_PROXY_CONTROL_PLANE_APP_HEADER = "X-Gencow-Control-Plane-App";
@@ -192225,7 +192408,7 @@ function canonicalSignedHeaderHash(headers) {
     const value = headers?.get(name) ?? "";
     return `${name}:${value.trim()}`;
   }).join("\n");
-  return createHash20("sha256").update(canonical).digest("base64url");
+  return createHash21("sha256").update(canonical).digest("base64url");
 }
 function buildSignaturePayload(input) {
   return [
@@ -193827,6 +194010,20 @@ var SYSTEM_ENV_KEYS = [
   // local/test-only release storage override
   "GENCOW_RELEASE_OBJECTS_DIR",
   // local/test-only release storage path
+  "GENCOW_SERVING_ROUTE_PUBLICATION_MODE",
+  // Platform serving route rollout policy
+  "GENCOW_SERVING_ROUTE_PUBLICATION_ENVIRONMENT",
+  // Platform serving route environment fence
+  "GENCOW_SERVING_ROUTE_PUBLICATION_APP_ALLOWLIST",
+  // Platform serving route app policy
+  "GENCOW_SERVING_ROUTE_TOPOLOGY_REVISION",
+  // Platform serving route topology fence
+  "GENCOW_SERVING_ROUTE_UPSERT_TARGET_SET_ID",
+  // Platform serving route target set
+  "GENCOW_SERVING_ROUTE_TOMBSTONE_TARGET_SET_ID",
+  // Platform serving route tombstone target set
+  "GENCOW_SERVING_ROUTE_ARTIFACT_QUOTA_BYTES",
+  // serving artifact quota
   "GENCOW_BACKUP_RETENTION_V2_MODE",
   // platform backup retention rollout mode
   "GENCOW_BACKUP_RETENTION_V2_EFFECTIVE_AT",