gencow 0.1.214 → 0.1.216

package/lib/app-delete-operation.mjs

@@ -16,7 +16,14 @@ const APP_DELETE_ACTIVE_STATES = new Set([
   "failed_tenant_db_cleanup",
   "failed_catalog_cleanup",
 ]);
-const APP_DELETE_TERMINAL_FAILURE_STATES = new Set(["blocked_catalog_outcome_unknown"]);
+const APP_DELETE_TERMINAL_FAILURE_STATES = new Set([
+  "blocked_catalog_outcome_unknown",
+  "failed_delete_recovery_required",
+]);
+const APP_DELETE_RESPONSE_STATES = new Set([
+  ...APP_DELETE_ACTIVE_STATES,
+  ...APP_DELETE_TERMINAL_FAILURE_STATES,
+]);
 const CORRELATION_ID_PATTERN = /^[A-Za-z0-9_-]{8,128}$/u;
 const DIAGNOSTIC_CODE_PATTERN = /^[A-Za-z][A-Za-z0-9_]{2,127}$/u;
 const APP_ID_PATTERN = /^(?=.{3,63}$)[a-z][a-z0-9]*(?:-[a-z0-9]+){2,7}$/u;
@@ -44,7 +51,7 @@ export function parseAppDeleteOperationEnvelope(value, expectedAppId) {
   if (!value || typeof value !== "object" || Array.isArray(value)) return null;
   if (value.success !== undefined && value.success !== false) return null;
   if (!APP_DELETE_OPERATION_PATTERN.test(value.operationId ?? "")) return null;
-  if (!APP_DELETE_ACTIVE_STATES.has(value.state)) return null;
+  if (!APP_DELETE_RESPONSE_STATES.has(value.state)) return null;
   if (
     !CORRELATION_ID_PATTERN.test(value.correlationId ?? "") ||
     (value.deleted !== undefined && value.deleted !== expectedAppId) ||

package/lib/deploy-static-fullstack-runtime.mjs

@@ -164,6 +164,19 @@ export async function deployBackendPackage({
   }
 
   if (!deployed) return null;
+  // Persist the production target before finalize. Finalize is a separate durable
+  // release effect; if it fails, the next customer retry must reuse this exact
+  // app instead of creating a second production child.
+  if (envTarget === "prod") {
+    writeProjectMetadata({
+      appId: deployed.appId,
+      displayName,
+      envTarget,
+      platformUrl: creds.platformUrl,
+      gencowJsonPath,
+      writeFileSyncImpl,
+    });
+  }
   if (releaseAttempt && finalizeReleaseAttempt) {
     try {
       await finalizeReleaseAttempt({ appId: deployed.appId, attemptId: releaseAttempt.attemptId });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gencow",
-  "version": "0.1.214",
+  "version": "0.1.216",
   "description": "Gencow — AI Backend Engine",
   "type": "module",
   "bin": {
@@ -33,8 +33,8 @@
   "devDependencies": {
     "@types/node": "^25.9.5",
     "better-auth": "^1.6.23",
-    "@gencow/client": "0.2.7",
     "@gencow/core": "0.1.43",
+    "@gencow/client": "0.2.7",
     "@gencow/react": "0.2.7"
   },
   "scripts": {

package/runtime/server.mjs

@@ -110824,9 +110824,14 @@ function registerPlatformSlotPromotionRoute(params) {
       }
       state2 = "promoting";
       const approval = routeProjectionDegradedApprovalFromRequest(c);
-      const currentEvidence = await params.assertTenantRouteParity?.(approval);
-      if (!currentEvidence || currentEvidence.bindingCount !== preparedEvidence.bindingCount || currentEvidence.evidenceDigest !== preparedEvidence.evidenceDigest)
-        throw new Error("PLATFORM_TENANT_ROUTE_PARITY_EVIDENCE_CHANGED");
+      let currentEvidence = await params.assertTenantRouteParity?.(approval);
+      if (!currentEvidence || currentEvidence.bindingCount !== preparedEvidence.bindingCount || currentEvidence.evidenceDigest !== preparedEvidence.evidenceDigest) {
+        await params.autoRecoveryRuntime.promote();
+        currentEvidence = await params.assertTenantRouteParity?.(approval);
+        if (!currentEvidence || currentEvidence.bindingCount !== preparedEvidence.bindingCount || currentEvidence.evidenceDigest !== preparedEvidence.evidenceDigest) {
+          throw new Error("PLATFORM_TENANT_ROUTE_PARITY_EVIDENCE_CHANGED");
+        }
+      }
       params.setCandidateMode(false);
       state2 = "active";
       logger3.log("[platform] standby candidate promoted to active runtime mode");
@@ -149200,6 +149205,7 @@ async function bootstrapRuntimeRouteProjectionForPlatform(input) {
   return {
     convergeServingFleet: runtime.convergeServingFleet,
     isAppProjectionBlocked: runtime.isAppProjectionBlocked,
+    resolveExistingOnlyRoute: runtime.resolveExistingOnlyRoute,
     setAbsentRouteConverger: runtime.setAbsentRouteConverger,
     startAfter,
     startAfterRecovery
@@ -152045,13 +152051,55 @@ function rejectNodeMismatch(c, enabled, currentNodeId, homeNodeId) {
     code: "APP_RUNTIME_EXISTING_ROUTE_NODE_MISMATCH"
   }, 409);
 }
-function routeUnavailable(c) {
+function routeResolutionUnavailable(c, result2) {
   return c.json({
-    error: "Exact serving runtime is not mapped on this platform candidate",
-    code: "APP_RUNTIME_EXISTING_ROUTE_UNAVAILABLE",
-    status: "unavailable"
+    error: "Exact serving runtime is not available on this platform slot",
+    code: result2.code,
+    status: result2.status
   }, 503);
 }
+async function proxyResolvedExistingOnlyRoute(input) {
+  const route = await input.resolveRoute?.(input.appId);
+  if (!route || route.kind !== "ready") {
+    return routeResolutionUnavailable(input.c, route ?? { code: "APP_RUNTIME_EXISTING_ROUTE_UNAVAILABLE", status: "unavailable" });
+  }
+  try {
+    return await input.proxyRequest({
+      c: input.c,
+      host: input.host,
+      appName: input.appName,
+      port: route.port,
+      url: input.url,
+      requestBody: input.requestBody,
+      addProxyMetric: input.addProxyMetric
+    });
+  } catch {
+    return routeResolutionUnavailable(input.c, {
+      code: "APP_RUNTIME_EXISTING_ROUTE_UNAVAILABLE",
+      status: "unavailable"
+    });
+  }
+}
+async function resolveExistingOnlyRoutePort(input) {
+  const route = await input.resolveRoute?.(input.appId);
+  if (!route || route.kind !== "ready") {
+    return routeResolutionUnavailable(input.c, route ?? { code: "APP_RUNTIME_EXISTING_ROUTE_UNAVAILABLE", status: "unavailable" });
+  }
+  return route.port;
+}
+async function resolveWebsocketPort(input) {
+  if (input.existingOnly) {
+    return resolveExistingOnlyRoutePort({
+      c: input.c,
+      appId: input.appRow.id,
+      resolveRoute: input.resolveRoute
+    });
+  }
+  const runtime = await input.recover();
+  if (runtime.kind === "unavailable")
+    return input.c.json(runtime, runtime.httpStatus);
+  return runtime.port;
+}
 async function acquireRuntimeActivity(params) {
   if (params.existingOnlyRuntimeProbe)
     return void 0;
@@ -153315,12 +153363,12 @@ var init_server_serving_plane_dependency_metrics = __esm({
   }
 });
 
-// ../server/dist/server-platform-app-proxy.js
+// ../server/dist/server-platform-app-proxy-routing.js
 function isPublicAppBackendRoute(pathname) {
   return /^\/(api|platform|internal|ws)\b/.test(pathname);
 }
 function isDeniedPublicAppSystemPath(pathname) {
-  return /^\/(_admin|_dashboard)\b/.test(pathname);
+  return /^\/(?:_admin|_dashboard)\b/.test(pathname);
 }
 function classifyServingPlaneRequest(pathname, upgradeHeader) {
   if (pathname.startsWith("/_gencow/analytics"))
@@ -153334,6 +153382,13 @@ function classifyServingPlaneRequest(pathname, upgradeHeader) {
     return "backend_http";
   return "static_candidate";
 }
+var init_server_platform_app_proxy_routing = __esm({
+  "../server/dist/server-platform-app-proxy-routing.js"() {
+    "use strict";
+  }
+});
+
+// ../server/dist/server-platform-app-proxy.js
 async function proxyToBackendWithRecoveryInternal(params) {
   if (params.appRow.status === "memory_exceeded") {
     const unavailable2 = params.buildAppRuntimeUnavailableResult(params.appRow.name, params.appRow.status, params.appRow.statusReason);
@@ -153364,7 +153419,17 @@ async function proxyToBackendWithRecoveryInternal(params) {
     }
   }
   if (params.existingOnlyRuntimeProbe)
-    return routeUnavailable(params.c);
+    return proxyResolvedExistingOnlyRoute({
+      c: params.c,
+      host: params.host,
+      appName: params.appRow.name,
+      appId: params.appRow.id,
+      url: params.url,
+      requestBody: replayPlan.initialBody,
+      addProxyMetric: params.addProxyMetric,
+      proxyRequest,
+      resolveRoute: params.resolveExistingOnlyRoute
+    });
   let runtime;
   try {
     runtime = await params.ensureAppRuntimeReady(params.appRow, buildPlatformRuntimeRecoveryDeps({
@@ -153374,6 +153439,7 @@ async function proxyToBackendWithRecoveryInternal(params) {
   } catch (error51) {
     const message = error51 instanceof Error ? error51.message : String(error51);
     params.logger.error(`[platform] ${params.appRow.name}: runtime recovery failed: ${message}`);
+    params.c.header("Retry-After", "2");
     return params.c.json(buildTransientProxyFailureBody(), 503);
   }
   if (runtime.kind === "unavailable") {
@@ -153400,6 +153466,7 @@ async function proxyToBackendWithRecoveryInternal(params) {
   } catch (error51) {
     const message = error51 instanceof Error ? error51.message : String(error51);
     params.logger.error(`[platform] ${params.appRow.name}: retry proxy failed on :${runtime.port} ${requestLabel}: ${message}`);
+    params.c.header("Retry-After", "2");
     return params.c.json(buildTransientProxyFailureBody(), 502);
   }
 }
@@ -153622,18 +153689,23 @@ function registerPlatformAppProxy(params) {
             });
             let websocketPort = params.getLivePort(appName) || appRow.port;
             if (appRow.runtimeAuthorityMode === "enforce") {
-              const runtime = await params.ensureAppRuntimeReady(appRow, buildPlatformRuntimeRecoveryDeps({
-                ...params,
-                logger: logger3,
-                probePort: async (_appName, port) => probeAppPort(port)
-              }), {
-                stalePort: params.getLivePort(appName) ?? null,
-                currentNodeId: params.currentNodeId
+              const resolvedPort = await resolveWebsocketPort({
+                c,
+                appRow,
+                existingOnly: existingOnlyRuntimeProbe,
+                resolveRoute: params.resolveExistingOnlyRoute,
+                recover: async () => params.ensureAppRuntimeReady(appRow, buildPlatformRuntimeRecoveryDeps({
+                  ...params,
+                  logger: logger3,
+                  probePort: async (_appName, port) => probeAppPort(port)
+                }), {
+                  stalePort: params.getLivePort(appName) ?? null,
+                  currentNodeId: params.currentNodeId
+                })
               });
-              if (runtime.kind === "unavailable") {
-                return c.json(buildRuntimeUnavailableBody(runtime), runtime.httpStatus);
-              }
-              websocketPort = runtime.port;
+              if (resolvedPort instanceof Response)
+                return resolvedPort;
+              websocketPort = resolvedPort;
             }
             let runtimeActivityLease;
             try {
@@ -153720,6 +153792,7 @@ function registerPlatformAppProxy(params) {
                 addProxyMetric: params.addProxyMetric,
                 beginRuntimeActivity: params.beginRuntimeActivity,
                 admitRuntimeActivity: params.admitRuntimeActivity,
+                resolveExistingOnlyRoute: params.resolveExistingOnlyRoute,
                 existingOnlyRuntimeProbe
               });
             }
@@ -153761,6 +153834,7 @@ function registerPlatformAppProxy(params) {
               addProxyMetric: params.addProxyMetric,
               beginRuntimeActivity: params.beginRuntimeActivity,
               admitRuntimeActivity: params.admitRuntimeActivity,
+              resolveExistingOnlyRoute: params.resolveExistingOnlyRoute,
               existingOnlyRuntimeProbe
             });
           }
@@ -153797,6 +153871,8 @@ var init_server_platform_app_proxy = __esm({
     init_server_platform_app_proxy_responses();
     init_server_platform_hosted_analytics();
     init_server_serving_plane_dependency_metrics();
+    init_server_platform_app_proxy_routing();
+    init_server_platform_app_proxy_routing();
     init_static_file_routing();
     init_server_platform_app_proxy_domain();
     init_server_realtime_connection_limit();
@@ -170732,7 +170808,8 @@ var init_schema_app_lifecycle_operation = __esm({
         'accepted', 'stopping_children', 'draining_runtime', 'stop_dispatched',
         'runtime_absent', 'tenant_db_drop_pending', 'catalog_delete_pending', 'completed',
         'blocked_ownership_ambiguous', 'blocked_active_work', 'blocked_runtime_outcome_unknown',
-        'blocked_catalog_outcome_unknown', 'failed_tenant_db_cleanup', 'failed_catalog_cleanup'
+        'blocked_catalog_outcome_unknown', 'failed_tenant_db_cleanup', 'failed_catalog_cleanup',
+        'failed_delete_recovery_required'
       )`
         )
       ]
@@ -184114,6 +184191,12 @@ function clearCandidateCleanupFailureCode(error51) {
   } catch {
   }
 }
+function getDeploymentFailureCause(error51) {
+  if (!error51 || typeof error51 !== "object" && typeof error51 !== "function") return null;
+  const direct = error51.deploymentFailureCause;
+  if (typeof direct === "string" && DEPLOYMENT_FAILURE_CAUSE_PATTERN.test(direct)) return direct;
+  return DEPLOYMENT_FAILURE_CAUSE_BY_ERROR.get(error51) ?? null;
+}
 function attachDeploymentFailureStage(error51, stage) {
   if (!error51 || typeof error51 !== "object" && typeof error51 !== "function") return error51;
   const existing = error51.deploymentFailureStage;
@@ -184199,7 +184282,8 @@ function retry(code, message) {
 }
 function resolveGenericDeploymentFailurePolicy(input) {
   const explicitCode = getStableDeploymentFailureCode(input.error);
-  const code = explicitCode ?? (input.stage === "candidate" ? classifyCandidateFailure(input.error, input.crashLogs) : defaultFailureCode(input.stage));
+  const typedArtifactCause = getDeploymentFailureCause(input.error)?.startsWith("DEPLOY_ARTIFACT_");
+  const code = (typedArtifactCause ? "DEPLOY_ARTIFACT_MATERIALIZE_FAILED" : explicitCode) ?? (input.stage === "candidate" ? classifyCandidateFailure(input.error, input.crashLogs) : defaultFailureCode(input.stage));
   if (getCandidateCleanupFailureCode(input.error)) {
     return {
       status: 503,
@@ -184281,7 +184365,7 @@ function resolveStoredDeploymentFailurePolicy(stage, code) {
     error: Object.assign(new Error(code), { code })
   });
 }
-var DEPLOYMENT_FAILURE_STAGE_FIELD, STABLE_CONTROL_PLANE_CODE, DEPLOYMENT_FAILURE_STAGE_SET, DEPLOYMENT_FAILURE_STAGE_BY_ERROR, CANDIDATE_CLEANUP_FAILURES, RETRYABLE_CODES, PLATFORM_OWNED_CODES, PLATFORM_OWNED_PREFIXES;
+var DEPLOYMENT_FAILURE_STAGE_FIELD, STABLE_CONTROL_PLANE_CODE, DEPLOYMENT_FAILURE_STAGE_SET, DEPLOYMENT_FAILURE_STAGE_BY_ERROR, CANDIDATE_CLEANUP_FAILURES, DEPLOYMENT_FAILURE_CAUSE_BY_ERROR, DEPLOYMENT_FAILURE_CAUSE_PATTERN, RETRYABLE_CODES, PLATFORM_OWNED_CODES, PLATFORM_OWNED_PREFIXES;
 var init_deployment_failure_contract = __esm({
   "../platform/src/deployment-failure-contract.ts"() {
     "use strict";
@@ -184292,6 +184376,8 @@ var init_deployment_failure_contract = __esm({
     DEPLOYMENT_FAILURE_STAGE_SET = new Set(DEPLOYMENT_FAILURE_STAGES);
     DEPLOYMENT_FAILURE_STAGE_BY_ERROR = /* @__PURE__ */ new WeakMap();
     CANDIDATE_CLEANUP_FAILURES = /* @__PURE__ */ new WeakSet();
+    DEPLOYMENT_FAILURE_CAUSE_BY_ERROR = /* @__PURE__ */ new WeakMap();
+    DEPLOYMENT_FAILURE_CAUSE_PATTERN = /^[A-Z][A-Z0-9_]{2,99}$/u;
     RETRYABLE_CODES = /* @__PURE__ */ new Set([
       "APP_DEPENDENCY_INSTALL_FAILED",
       "ACTIVATION_EPOCH_CAS_FAILED",
@@ -184756,6 +184842,9 @@ function getDeploymentAttemptFailureCode(error51, stage, crashLogs) {
   if (guardrailCode) return guardrailCode;
   if (isMigrationDeployError(error51)) return String(error51.code);
   if (isDeployDependencyVersionError(error51) || isCronManifestDeployError(error51)) return error51.code;
+  if (getDeploymentFailureCause(error51)?.startsWith("DEPLOY_ARTIFACT_")) {
+    return "DEPLOY_ARTIFACT_MATERIALIZE_FAILED";
+  }
   const stableCode = getStableDeploymentFailureCode(error51);
   if (stableCode) return stableCode;
   if (stage === "admission") return "DEPLOY_CONTRACT_INVALID";
@@ -185369,7 +185458,8 @@ function fleetSelection() {
     ...bindingSelection(),
     authorityMode: appRuntimeDesiredState.authorityMode,
     desiredRuntimeState: appRuntimeDesiredState.desiredRuntimeState,
-    hasBackend: apps.hasBackend
+    hasBackend: apps.hasBackend,
+    appStatus: apps.status
   };
 }
 function queryBindingRows(db) {
@@ -185488,6 +185578,10 @@ async function loadCanonicalTenantRouteFleetSnapshot(input) {
     const desired = appRows[0];
     if (desired.hasBackend && desired.desiredRuntimeState === "serving" && desired.desiredNodeId === input.nodeId) {
       const joinedRows = appRows.filter((row) => row.instanceId !== null);
+      if (joinedRows.length === 0 && desired.appStatus === "stopped") {
+        removalTargets.push({ appId: desired.appId, appName: desired.appName });
+        continue;
+      }
       try {
         bindings.push(bindingFromRows(joinedRows));
       } catch (error51) {
@@ -186001,6 +186095,16 @@ function appDeleteOperationResult(operation, options = {}) {
     recoveryActivityEpoch: options.recovery.activityEpoch
   } : {};
   const blockedWithoutRecovery = operation.state === "blocked_active_work" && !options.recovery;
+  const terminalRecoveryRequired = operation.state === "failed_delete_recovery_required";
+  const recoveryDeadlineAnchor = operation.createdAt instanceof Date ? operation.createdAt : /* @__PURE__ */ new Date();
+  const recoveryReconcileAnchor = operation.updatedAt instanceof Date ? operation.updatedAt : recoveryDeadlineAnchor;
+  const recoveryEvidence = !isAppDeleteTerminalState(operation.state) ? {
+    recoveryOwnerId: operation.ownerId,
+    recoveryDeadlineAt: new Date(recoveryDeadlineAnchor.getTime() + RECOVERY_DEADLINE_MS).toISOString(),
+    nextReconcileAt: new Date(
+      recoveryReconcileAnchor.getTime() + RECOVERY_RECONCILE_INTERVAL_MS
+    ).toISOString()
+  } : {};
   return {
     success: false,
     deleted: operation.appName,
@@ -186013,6 +186117,13 @@ function appDeleteOperationResult(operation, options = {}) {
     ...operation.state === "completed" ? { success: true } : {},
     ...options.executorAvailable === void 0 ? {} : { executorAvailable: options.executorAvailable },
     ...operation.lastErrorCode ? { code: operation.lastErrorCode } : executorUnavailable ? { code: "PLATFORM_LIFECYCLE_EXECUTOR_UNAVAILABLE" } : {},
+    ...terminalRecoveryRequired ? {
+      code: "APP_DELETE_RECOVERY_REQUIRED",
+      failureClass: "PLATFORM_ACTION_REQUIRED",
+      action: "CONTACT_SUPPORT",
+      recoveryCauseCode: operation.lastErrorCode
+    } : {},
+    ...recoveryEvidence,
     ...blockedRecovery,
     ...blockedWithoutRecovery ? {
       code: "PLATFORM_ACTION_REQUIRED",
@@ -186021,7 +186132,7 @@ function appDeleteOperationResult(operation, options = {}) {
     } : {}
   };
 }
-var APP_DELETE_ACTIVE_STATES2, APP_DELETE_TERMINAL_FAILURE_STATES2, APP_DELETE_TERMINAL_STATES, APP_DELETE_ADMISSION_BLOCKING_STATES, RETENTION_MS;
+var APP_DELETE_ACTIVE_STATES2, APP_DELETE_TERMINAL_FAILURE_STATES2, APP_DELETE_TERMINAL_STATES, APP_DELETE_ADMISSION_BLOCKING_STATES, RETENTION_MS, RECOVERY_DEADLINE_MS, RECOVERY_RECONCILE_INTERVAL_MS;
 var init_app_delete_operation_store = __esm({
   "../platform/src/app-delete-operation-store.ts"() {
     "use strict";
@@ -186040,13 +186151,18 @@ var init_app_delete_operation_store = __esm({
       "failed_tenant_db_cleanup",
       "failed_catalog_cleanup"
     ];
-    APP_DELETE_TERMINAL_FAILURE_STATES2 = ["blocked_catalog_outcome_unknown"];
+    APP_DELETE_TERMINAL_FAILURE_STATES2 = [
+      "blocked_catalog_outcome_unknown",
+      "failed_delete_recovery_required"
+    ];
     APP_DELETE_TERMINAL_STATES = ["completed", ...APP_DELETE_TERMINAL_FAILURE_STATES2];
     APP_DELETE_ADMISSION_BLOCKING_STATES = [
       ...APP_DELETE_ACTIVE_STATES2,
       ...APP_DELETE_TERMINAL_FAILURE_STATES2
     ];
     RETENTION_MS = 7 * 24 * 60 * 6e4;
+    RECOVERY_DEADLINE_MS = 15 * 6e4;
+    RECOVERY_RECONCILE_INTERVAL_MS = 3e4;
   }
 });
 
@@ -401444,7 +401560,11 @@ async function waitForDirectRunnerHealth(port, timeoutMs, dependencies = {}) {
   while (now() - startedAt < timeoutMs) {
     if (dependencies.shouldContinue && !await dependencies.shouldContinue()) return false;
     const remainingMs = Math.max(1, timeoutMs - (now() - startedAt));
-    if (await probe(port, Math.min(2e3, remainingMs), dependencies.healthPath)) return true;
+    const probeTimeoutMs = Math.min(2e3, remainingMs);
+    if (await probe(port, probeTimeoutMs, dependencies.healthPath)) return true;
+    if (dependencies.fallbackHealthPath && dependencies.fallbackHealthPath !== dependencies.healthPath && await probe(port, probeTimeoutMs, dependencies.fallbackHealthPath)) {
+      return true;
+    }
     const remainingAfterProbeMs = timeoutMs - (now() - startedAt);
     if (remainingAfterProbeMs <= 0) break;
     await sleep6(Math.min(DIRECT_RUNNER_HEALTH_POLL_INTERVAL_MS, remainingAfterProbeMs));
@@ -403162,7 +403282,8 @@ var init_direct_runner = __esm({
         const healthy = await spawnLifecycle.waitFor(
           this.waitForHealth(port, healthTimeoutMs, {
             ...options?.lifecycleRole === "candidate" ? { shouldContinue: () => this.isAlive(appName) } : {},
-            ...options?.lifecycleRole === "candidate" ? { healthPath: "/health/core-ready" } : {}
+            ...options?.lifecycleRole === "candidate" ? { healthPath: "/health/core-ready" } : {},
+            ...options?.lifecycleRole === "candidate" ? { fallbackHealthPath: "/health/live" } : {}
           })
         );
         if (!healthy) {
@@ -424339,10 +424460,21 @@ async function publishIdleRouteAndAwaitAck(input) {
      WHERE app_id = ${input.appId}
   `);
   const authority = rowsFromResult34(result2)[0];
-  if (!authority || authority.app_instance_id !== input.appInstanceId) {
+  if (!authority) {
     fail31("RUNTIME_IDLE_ROUTE_AUTHORITY_MISSING");
   }
   const current = decodeAuthority(authority);
+  if (current.mode === "tombstoned" && current.disposition === "tombstone") {
+    const convergence = await assessAppServingRouteConvergence({
+      db: input.db,
+      eventId: authority.last_event_id
+    });
+    if (convergence.status !== "converged") fail31("RUNTIME_IDLE_ROUTE_ACK_PENDING");
+    return;
+  }
+  if (authority.app_instance_id !== input.appInstanceId) {
+    fail31("RUNTIME_IDLE_ROUTE_AUTHORITY_MISSING");
+  }
   if (current.appId !== input.appId || current.appInstanceId !== input.appInstanceId || current.mode !== "backend_only" && current.mode !== "fullstack" || !current.backend || current.backend.activationId !== input.activationId || current.backend.activationEpoch !== input.activationEpoch) {
     fail31("RUNTIME_IDLE_ROUTE_IDENTITY_MISMATCH");
   }
@@ -425511,6 +425643,7 @@ async function armExplicitRuntimeStop(input) {
     ]);
     const drain = drainRows[0];
     const activity = activityRows[0];
+    const appDeleteWithoutActivityObservation = drain?.reason === "app_delete" && !activity;
     if (!drain) throw new Error("RUNTIME_EXPLICIT_DRAIN_EXPIRED");
     if (drain.state === "dispatched" || drain.state === "completed") {
       const intents = await tx.select().from(runtimeStopIntents).where(eq60(runtimeStopIntents.drainRequestId, drain.id));
@@ -425523,7 +425656,7 @@ async function armExplicitRuntimeStop(input) {
       if (!intents[0]) throw new Error("RUNTIME_EXPLICIT_STOP_REPLAY_INVALID");
       return { intentId: intents[0].id, drainId: drain.id, replayed: true };
     }
-    if (drain.state !== "planned" || !activity || !activity.observationComplete || activity.activationId !== drain.activationId || activity.instanceId !== drain.instanceId || activity.nodeId !== drain.nodeId || activity.nodeEpoch !== drain.nodeEpoch || activity.httpActive !== 0 || activity.streamActive !== 0 || activity.websocketActive !== 0 || activity.workloadActive !== 0) {
+    if (drain.state !== "planned" || !appDeleteWithoutActivityObservation && (!activity || !activity.observationComplete || activity.activationId !== drain.activationId || activity.instanceId !== drain.instanceId || activity.nodeId !== drain.nodeId || activity.nodeEpoch !== drain.nodeEpoch || activity.httpActive !== 0 || activity.streamActive !== 0 || activity.websocketActive !== 0 || activity.workloadActive !== 0)) {
       throw new Error("RUNTIME_EXPLICIT_DRAIN_NOT_QUIESCENT");
     }
     const intent = await createRuntimeStopIntentInTransaction({
@@ -425539,7 +425672,11 @@ async function armExplicitRuntimeStop(input) {
       now,
       drainRequestId: drain.id
     });
-    const armed = await tx.update(runtimeDrainRequests).set({ state: "armed", activityEpoch: activity.activityEpoch, updatedAt: now }).where(and56(eq60(runtimeDrainRequests.id, drain.id), eq60(runtimeDrainRequests.state, "planned"))).returning({ id: runtimeDrainRequests.id });
+    const armed = await tx.update(runtimeDrainRequests).set({
+      state: "armed",
+      activityEpoch: activity?.activityEpoch ?? drain.activityEpoch,
+      updatedAt: now
+    }).where(and56(eq60(runtimeDrainRequests.id, drain.id), eq60(runtimeDrainRequests.state, "planned"))).returning({ id: runtimeDrainRequests.id });
     if (!armed[0]) throw new Error("RUNTIME_EXPLICIT_DRAIN_ARM_CAS_FAILED");
     await tx.update(runtimeLifecycleRequests).set({ state: "executing", updatedAt: now }).where(eq60(runtimeLifecycleRequests.id, drain.lifecycleRequestId));
     return { intentId: intent.id, drainId: drain.id, replayed: false };
@@ -428050,9 +428187,12 @@ function deriveRuntimeStopOperationState(input) {
   if (input.drainState === "blocked" && input.terminalResultCode === "RUNTIME_STOP_OWNERSHIP_AMBIGUOUS") {
     return "blocked_ambiguous_binding";
   }
-  if (input.drainState === "blocked" && ["RUNTIME_EXPLICIT_DRAIN_TIMEOUT", "RUNTIME_EXPLICIT_DRAIN_EXPIRED"].includes(
-    input.terminalResultCode ?? ""
-  )) {
+  if (input.drainState === "blocked" && [
+    "RUNTIME_EXPLICIT_DRAIN_TIMEOUT",
+    "RUNTIME_EXPLICIT_DRAIN_EXPIRED",
+    "RUNTIME_EXPLICIT_DRAIN_NOT_QUIESCENT",
+    "RUNTIME_STOP_INTENT_ACTIVITY_CHANGED"
+  ].includes(input.terminalResultCode ?? "")) {
     return "blocked_active_work";
   }
   if (input.requestState === "failed" || input.drainState === "cancelled") return "failed";
@@ -428074,10 +428214,7 @@ async function loadAppDeleteRuntimeRecoveryReceipt(input) {
     drainReason: runtimeDrainRequests.reason,
     intentReason: runtimeStopIntents.reason,
     intentCorrelationId: runtimeStopIntents.correlationId
-  }).from(runtimeLifecycleRequests).leftJoin(
-    runtimeDrainRequests,
-    eq70(runtimeDrainRequests.lifecycleRequestId, runtimeLifecycleRequests.id)
-  ).leftJoin(runtimeStopIntents, eq70(runtimeStopIntents.drainRequestId, runtimeDrainRequests.id)).where(
+  }).from(runtimeLifecycleRequests).leftJoin(runtimeDrainRequests, eq70(runtimeDrainRequests.lifecycleRequestId, runtimeLifecycleRequests.id)).leftJoin(runtimeStopIntents, eq70(runtimeStopIntents.drainRequestId, runtimeDrainRequests.id)).where(
     and66(
       eq70(runtimeLifecycleRequests.appId, input.appId),
       eq70(runtimeLifecycleRequests.action, "app_delete"),
@@ -428092,7 +428229,8 @@ async function loadAppDeleteRuntimeRecoveryReceipt(input) {
     requestState: row.requestState,
     drainState: row.drainState,
     terminalResultCode: row.terminalResultCode
-  }) !== "blocked_active_work") return null;
+  }) !== "blocked_active_work")
+    return null;
   const ownerId = row.leaseOwner;
   if (!ownerId || !row.deadlineAt || !Number.isSafeInteger(Number(row.activityEpoch))) return null;
   return {
@@ -428184,10 +428322,12 @@ var init_app_delete_catalog_finalizer = __esm({
 });
 
 // ../platform/src/app-delete-operation-controller.ts
+var APP_DELETE_RECOVERY_DEADLINE_MS;
 var init_app_delete_operation_controller = __esm({
   "../platform/src/app-delete-operation-controller.ts"() {
     "use strict";
     init_app_delete_operation_store();
+    APP_DELETE_RECOVERY_DEADLINE_MS = 15 * 6e4;
   }
 });
 
@@ -428338,7 +428478,7 @@ function publicAppDeleteErrorCode(error51) {
   return PUBLICLY_NORMALIZED_APP_DELETE_CODES.has(code) ? "APP_DELETE_OPERATION_PENDING" : code;
 }
 function appDeleteFailureWithReceipt(error51, receipt) {
-  const code = publicAppDeleteErrorCode(error51);
+  const code = receipt.state === "failed_delete_recovery_required" ? "APP_DELETE_RECOVERY_REQUIRED" : publicAppDeleteErrorCode(error51);
   return Object.assign(new Error(code, { cause: error51 }), {
     code,
     operationId: receipt.id,
@@ -428348,7 +428488,7 @@ function appDeleteFailureWithReceipt(error51, receipt) {
   });
 }
 function appDeleteFailureHttpStatus(state2) {
-  return state2.startsWith("blocked_") ? 409 : 503;
+  return state2.startsWith("blocked_") || state2 === "failed_delete_recovery_required" ? 409 : 503;
 }
 var PUBLICLY_NORMALIZED_APP_DELETE_CODES;
 var init_app_delete_diagnostic = __esm({
@@ -429573,6 +429713,7 @@ async function bootstrapPlatformRuntime(params) {
     subdomainRegex,
     getBunServer: params.getBunServer,
     domainLookupDb: params.db,
+    resolveExistingOnlyRoute: routeProjectionRuntime.resolveExistingOnlyRoute,
     async findActiveDomainApp(host) {
       const rows4 = await params.db.select({
         name: appsTable.name,
@@ -429916,8 +430057,7 @@ async function bootstrapPlatformRuntime(params) {
     logger3.error("[ws-gateway] realtime usage metering bootstrap failed:", message);
   }
   startHeartbeat();
-  const limits = getGatewayLimits();
-  logger3.log(`[ws-gateway] WS Gateway active (max ${limits.maxConnectionsPerApp}/app, ${limits.maxTotalConnections} total)`);
+  logger3.log(`[ws-gateway] WS Gateway active (max ${getGatewayLimits().maxConnectionsPerApp}/app, ${getGatewayLimits().maxTotalConnections} total)`);
   const startAfterListen = async () => {
     await routeProjectionRuntime.startAfterRecovery(autoRecoveryRuntime);
     explicitStopReconciler.start();
@@ -448229,7 +448369,10 @@ var APP_DELETE_ACTIVE_STATES = /* @__PURE__ */ new Set([
   "failed_tenant_db_cleanup",
   "failed_catalog_cleanup"
 ]);
-var APP_DELETE_TERMINAL_FAILURE_STATES = /* @__PURE__ */ new Set(["blocked_catalog_outcome_unknown"]);
+var APP_DELETE_TERMINAL_FAILURE_STATES = /* @__PURE__ */ new Set([
+  "blocked_catalog_outcome_unknown",
+  "failed_delete_recovery_required"
+]);
 var CAPABILITY_PATTERN = /^[a-z0-9][a-z0-9-]{0,63}$/u;
 function createAppDeleteStatusLoader(getQueryDef2) {
   return async ({ ctx, args, result: result2 }) => {
@@ -448331,7 +448474,7 @@ import { randomUUID as randomUUID7 } from "node:crypto";
 var POSTGRES_CODE_PATTERN = /^[0-9A-Z]{5}$/u;
 var SAFE_CORRELATION_PATTERN = /^[A-Za-z0-9_-]{8,128}$/u;
 var SAFE_APP_DELETE_OPERATION_PATTERN = /^app_delete_(?:[a-f0-9]{32}|[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12})$/u;
-var SAFE_APP_DELETE_STATE_PATTERN = /^(?:accepted|stopping_children|draining_runtime|stop_dispatched|runtime_absent|tenant_db_drop_pending|catalog_delete_pending|completed|blocked_ownership_ambiguous|blocked_active_work|blocked_runtime_outcome_unknown|failed_tenant_db_cleanup|failed_catalog_cleanup)$/u;
+var SAFE_APP_DELETE_STATE_PATTERN = /^(?:accepted|stopping_children|draining_runtime|stop_dispatched|runtime_absent|tenant_db_drop_pending|catalog_delete_pending|completed|blocked_ownership_ambiguous|blocked_active_work|blocked_runtime_outcome_unknown|blocked_catalog_outcome_unknown|failed_tenant_db_cleanup|failed_catalog_cleanup|failed_delete_recovery_required)$/u;
 var SAFE_PUBLIC_ERROR_CODE_PATTERN = /^[A-Z][A-Z0-9_]{2,127}$/u;
 var SAFE_RETRY_CLASSES = /* @__PURE__ */ new Set([
   "operator-action",