@moretea-labs/forge 1.7.0 → 1.7.2

package/src/runtime/standalone-recovery/core.ts

@@ -513,8 +513,34 @@ function recoveryLockOwnerAlive(lock: RecoveryLock): boolean {
   return observed === undefined || observed === lock.processStartTime;
 }
 
+function recoveryLockOwnerAttributable(owner: RecoveryLock): boolean {
+  return Boolean(owner.action && owner.requestId?.trim());
+}
+
+function recoveryLockIdentityFailure(owner: RecoveryLock): string {
+  return `RECOVERY_OPERATION_LOCK_IDENTITY_UNCERTAIN: live mutation lock pid=${owner.pid} instance=${owner.instanceId} is missing action/request identity`;
+}
+
+function assertRecoveryLockOwnerAttributable(config: RecoveryConfig, owner: RecoveryLock): void {
+  if (recoveryLockOwnerAttributable(owner)) return;
+  audit(config, 'recovery_operation_lock_identity_uncertain', {
+    pid: owner.pid,
+    instanceId: owner.instanceId,
+    action: owner.action ?? null,
+    requestId: owner.requestId ?? null,
+    acquiredAt: owner.acquiredAt,
+  });
+  throw new Error(recoveryLockIdentityFailure(owner));
+}
+
 function recoveryBusyDetail(owner: RecoveryLock): string {
-  return `Recovery mutation already in progress: action=${owner.action ?? 'unknown'} request=${owner.requestId ?? 'unknown'} pid=${owner.pid}`;
+  return `Recovery mutation already in progress: action=${owner.action ?? 'invalid'} request=${owner.requestId ?? 'invalid'} pid=${owner.pid} instance=${owner.instanceId}`;
+}
+
+function liveRecoveryMutationLock(config: RecoveryConfig): RecoveryLock | undefined {
+  const owner = json<RecoveryLock>(lockPath(config));
+  if (!owner || !Number.isInteger(owner.pid) || owner.pid <= 0 || typeof owner.instanceId !== 'string' || !owner.instanceId.trim()) return undefined;
+  return recoveryLockOwnerAlive(owner) ? owner : undefined;
 }
 
 async function withLock<T>(
@@ -524,14 +550,16 @@ async function withLock<T>(
 ): Promise<RecoveryLockAttempt<T>> {
   const path = lockPath(config);
   mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
+  const instanceId = randomUUID();
+  const requestId = intent.requestId?.trim() || `internal:${intent.action}:${instanceId}`;
   const lock: RecoveryLock = {
     schemaVersion: 1,
     pid: process.pid,
-    instanceId: randomUUID(),
+    instanceId,
     processStartTime: processStartTime(process.pid),
     acquiredAt: new Date().toISOString(),
     action: intent.action,
-    ...(intent.requestId ? { requestId: intent.requestId } : {}),
+    requestId,
   };
 
   for (let attempt = 0; attempt < 2; attempt += 1) {
@@ -548,13 +576,31 @@ async function withLock<T>(
         if (!existing || !Number.isInteger(existing.pid) || typeof existing.instanceId !== 'string') {
           throw new Error('RECOVERY_OPERATION_LOCK_UNCERTAIN');
         }
-        if (recoveryLockOwnerAlive(existing)) return { acquired: false, owner: existing };
+        if (recoveryLockOwnerAlive(existing)) {
+          assertRecoveryLockOwnerAttributable(config, existing);
+          return { acquired: false, owner: existing };
+        }
         const latest = json<RecoveryLock>(path);
         if (!latest || latest.instanceId !== existing.instanceId) {
-          if (attempt === 1) return { acquired: false, owner: latest ?? existing };
+          if (attempt === 1) {
+            if (latest && recoveryLockOwnerAlive(latest)) {
+              assertRecoveryLockOwnerAttributable(config, latest);
+              return { acquired: false, owner: latest };
+            }
+            audit(config, 'recovery_operation_lock_race', {
+              existingInstanceId: existing.instanceId,
+              existingPid: existing.pid,
+              latestInstanceId: latest?.instanceId ?? null,
+              latestPid: latest?.pid ?? null,
+            });
+            throw new Error('RECOVERY_OPERATION_LOCK_RACE');
+          }
           continue;
         }
-        if (recoveryLockOwnerAlive(latest)) return { acquired: false, owner: latest };
+        if (recoveryLockOwnerAlive(latest)) {
+          assertRecoveryLockOwnerAttributable(config, latest);
+          return { acquired: false, owner: latest };
+        }
         try { writeFileSync(`${path}.stale-${Date.now()}-${existing.instanceId}`, readFileSync(path)); } catch { /* evidence best effort */ }
         rmSync(path, { force: true });
         continue;
@@ -632,7 +678,7 @@ async function probeExternalMcp(transport: RecoveryHttpTransport, url: string):
 
 function publicGatewayReadinessEndpoint(endpoint: string): string {
   const url = new URL(endpoint);
-  url.pathname = '/ready';
+  url.pathname = '/transport-ready';
   url.search = '';
   url.hash = '';
   return url.toString();
@@ -644,6 +690,7 @@ async function probePublicGatewayReadiness(
 ): Promise<{ ok: boolean; detail: string; status?: number; value?: unknown }> {
   const controller = new AbortController();
   const timer = setTimeout(() => controller.abort('RECOVERY_HTTP_TIMEOUT'), 4_000);
+  const surface = 'transport-ready';
   try {
     const response = await transport.request({
       url: publicGatewayReadinessEndpoint(endpoint),
@@ -651,6 +698,18 @@ async function probePublicGatewayReadiness(
       timeoutMs: 4_000,
       signal: controller.signal,
     });
+    if (response.status === 404 || response.status === 405) {
+      // Older Connectors predate the transport-scoped capacity surface. Their
+      // whole-control-plane /ready route is intentionally not a transport
+      // liveness contract and can depend on the Runtime that Recovery is in the
+      // middle of restarting. Falling back to it creates a circular cutover
+      // dependency. Verify the actual MCP transport instead: a successful
+      // initialize response or the expected OAuth Bearer challenge proves the
+      // legacy Connector transport is reachable without weakening 5xx/timeout
+      // failure handling.
+      const legacy = await probeExternalMcp(transport, endpoint);
+      return { ...legacy, detail: `legacy-mcp ${legacy.detail}` };
+    }
     let sessionCapacity: unknown;
     try {
       const payload = JSON.parse(response.body) as { sessionCapacity?: unknown };
@@ -663,12 +722,12 @@ async function probePublicGatewayReadiness(
     );
     return {
       ok: response.ok && !recoveryRecommended,
-      detail: `HTTP ${response.status}${recoveryRecommended ? '; session capacity recommends Connector recovery' : ''}`,
+      detail: `${surface} HTTP ${response.status}${recoveryRecommended ? '; session capacity recommends Connector recovery' : ''}`,
       status: response.status,
       ...(sessionCapacity === undefined ? {} : { value: sessionCapacity }),
     };
   } catch (error) {
-    return { ok: false, detail: error instanceof Error ? error.message.slice(0, 180) : 'request failed' };
+    return { ok: false, detail: `${surface} ${error instanceof Error ? error.message.slice(0, 180) : 'request failed'}` };
   } finally {
     clearTimeout(timer);
   }
@@ -855,8 +914,11 @@ export async function verifyStableRuntime(
     : { ok: false, detail: 'canonical Runtime endpoint is unavailable' };
   if (config.publicMcpUrl) {
     probes.external_mcp_http = await probeExternalMcp(transport, config.publicMcpUrl);
-    probes.primary_connector_ready = await probePublicGatewayReadiness(transport, config.publicMcpUrl);
+    const connectorReadinessEndpoint = config.primaryConnectorService?.localMcpUrl?.trim() || config.publicMcpUrl;
+    probes.primary_connector_ready = await probePublicGatewayReadiness(transport, connectorReadinessEndpoint);
   }
+  const primaryConnectorLocal = await probePrimaryConnectorLocal(config, transport);
+  if (primaryConnectorLocal) probes.primary_connector_local = primaryConnectorLocal;
   if (config.gateway) probes.recovery_gateway = await probe(transport, `http://${config.gateway.host}:${config.gateway.port}/health`);
   const watchdogHealth = observeRecoveryWatchdogHealth(config.controllerHome);
   probes.recovery_watchdog = {
@@ -1369,6 +1431,7 @@ export interface PrimaryRuntimeRecoveryDependencies {
   runCommand?: CommandRunner;
   verifyLocal?: (config: RecoveryConfig) => Promise<VerifyResult>;
   runtimeRunning?: (config: RecoveryConfig) => boolean;
+  ensureRuntimeLaunchContract?: (controllerHome: string) => void;
   now?: () => number;
   sleep?: (ms: number) => Promise<void>;
 }
@@ -1402,6 +1465,8 @@ function primaryRuntimeLaunchdService(config: RecoveryConfig, uid: number): Laun
 }
 
 export interface PrimaryConnectorRecoveryDependencies {
+  /** Durable caller identity propagated into the Recovery mutation lock and audit trail. */
+  requestId?: string;
   platform?: NodeJS.Platform;
   currentUid?: () => Promise<number | undefined>;
   runCommand?: CommandRunner;
@@ -1424,10 +1489,13 @@ function primaryConnectorLaunchdService(config: RecoveryConfig, uid: number): La
   return { uid, domain, target: `${domain}/${label}`, label, plistPath };
 }
 
-async function probePrimaryConnectorLocal(config: RecoveryConfig): Promise<{ ok: boolean; detail: string; status?: number } | undefined> {
+async function probePrimaryConnectorLocal(
+  config: RecoveryConfig,
+  transport = createRecoveryHttpTransport(config.controllerHome),
+): Promise<{ ok: boolean; detail: string; status?: number } | undefined> {
   const localMcpUrl = config.primaryConnectorService?.localMcpUrl?.trim();
   if (!localMcpUrl) return undefined;
-  return probeExternalMcp(createRecoveryHttpTransport(config.controllerHome), localMcpUrl);
+  return probeExternalMcp(transport, localMcpUrl);
 }
 
 function activePackageConnectorReleaseBinding(config: RecoveryConfig): PackageConnectorReleaseBinding | undefined {
@@ -1503,7 +1571,16 @@ export async function restartPrimaryConnector(
 ): Promise<PrimaryConnectorRestartResult> {
   const verifyLocal = dependencies.verifyLocal ?? verifyLocalRuntime;
   const initialLocal = await verifyLocal(config);
-  if (!initialLocal.ok) {
+  // `verifyLocalRuntime` also reports the Connector's own loopback probe.
+  // That probe is expected to be false when this recovery action is needed,
+  // so it must not prevent a healthy canonical Runtime from repairing the
+  // Connector service.
+  const canonicalRuntimeHealthy = initialLocal.runtime.ok
+    && initialLocal.runtime.running
+    && initialLocal.runtime.ready
+    && !initialLocal.runtime.stale
+    && initialLocal.probes.active_gateway?.ok === true;
+  if (!canonicalRuntimeHealthy) {
     return {
       ok: false,
       attempted: false,
@@ -1528,7 +1605,7 @@ export async function restartPrimaryConnector(
   const now = dependencies.now ?? Date.now;
   const wait = dependencies.sleep ?? sleep;
   const timeoutMs = config.primaryConnectorService?.postRestartVerifyTimeoutMs ?? 30_000;
-  const locked = await withLock(config, { action: 'restart_primary_connector' }, async () => {
+  const locked = await withLock(config, { action: 'restart_primary_connector', requestId: dependencies.requestId }, async () => {
     const tunnelConfigured = Boolean(configuredPrimaryPublicTunnel(config));
     const repairConnectorBinding = dependencies.repairConnectorBinding ?? repairPrimaryConnectorBinding;
     const bindingRepair = await repairConnectorBinding(config);
@@ -1818,6 +1895,77 @@ async function cleanupVerifiedStalePrimaryRuntimeListener(input: {
     : { released: false, cleaned: false, detail: 'verified stale Forge Runtime listener remained on the port after SIGKILL', pid };
 }
 
+interface PrimaryRuntimeStoppedTransition {
+  ok: boolean;
+  detail: string;
+  staleListenerCleanup?: PrimaryRuntimePortCleanupResult;
+}
+
+/**
+ * Stop the one canonical Runtime service and prove that every execution owner
+ * boundary is gone before release authority may change. This is the shared
+ * transition primitive for activation and recovery; callers must not publish
+ * or roll back a release after a partial stop.
+ */
+async function stopPrimaryRuntimeForReleaseTransition(input: {
+  config: RecoveryConfig;
+  service: LaunchdService;
+  now: () => number;
+  wait: (ms: number) => Promise<void>;
+  runCommand: CommandRunner;
+  runtimeRunning: (config: RecoveryConfig) => boolean;
+}): Promise<PrimaryRuntimeStoppedTransition> {
+  const stopped = await input.runCommand('launchctl', ['bootout', input.service.target], 15_000);
+  const stoppedCleanly = stopped.ok || /not found|no such process|could not find service|service is not loaded/i.test(`${stopped.stderr}\n${stopped.stdout}`);
+  if (!stoppedCleanly) {
+    return { ok: false, detail: `primary Runtime bootout failed: ${stopped.stderr || stopped.stdout || stopped.status}` };
+  }
+  const runtimeStopped = await waitForPrimaryRuntimeState({
+    config: input.config,
+    expectedRunning: false,
+    timeoutMs: 20_000,
+    now: input.now,
+    wait: input.wait,
+    runtimeRunning: input.runtimeRunning,
+  });
+  if (!runtimeStopped) return { ok: false, detail: 'primary Runtime owner remained live after bounded launchd bootout' };
+  const serviceUnloaded = await waitForLaunchdServiceUnloaded({
+    service: input.service,
+    timeoutMs: 20_000,
+    now: input.now,
+    wait: input.wait,
+    runCommand: input.runCommand,
+  });
+  if (!serviceUnloaded) return { ok: false, detail: 'primary Runtime launchd service remained loaded after bounded bootout' };
+
+  let portReleased = await waitForPrimaryRuntimePortReleased({
+    config: input.config,
+    timeoutMs: 20_000,
+    now: input.now,
+    wait: input.wait,
+    runCommand: input.runCommand,
+  });
+  let staleListenerCleanup: PrimaryRuntimePortCleanupResult | undefined;
+  if (!portReleased) {
+    staleListenerCleanup = await cleanupVerifiedStalePrimaryRuntimeListener({
+      config: input.config,
+      uid: input.service.uid,
+      now: input.now,
+      wait: input.wait,
+      runCommand: input.runCommand,
+    });
+    portReleased = staleListenerCleanup.released;
+  }
+  if (!portReleased) {
+    return {
+      ok: false,
+      detail: `primary Runtime TCP port remained occupied after bounded launchd bootout: ${staleListenerCleanup?.detail ?? 'listener did not release'}`,
+      staleListenerCleanup,
+    };
+  }
+  return { ok: true, detail: 'primary Runtime service, owner, and TCP listener are stopped', staleListenerCleanup };
+}
+
 async function verifyPrimaryRuntimeAfterStart(input: {
   config: RecoveryConfig;
   timeoutMs: number;
@@ -1834,6 +1982,73 @@ async function verifyPrimaryRuntimeAfterStart(input: {
   return observed;
 }
 
+interface PrimaryRuntimeRebindStartResult {
+  ok: boolean;
+  detail: string;
+  verify: VerifyResult;
+}
+
+/**
+ * Complete the only safe restart sequence after release authority changes:
+ * rebind the release-pinned launchd contract, start the sole Runtime service,
+ * and require whole-Runtime verification. A started-but-unverified Runtime is
+ * stopped again so callers never leave a restart loop behind.
+ */
+async function rebindStartAndVerifyPrimaryRuntime(input: {
+  config: RecoveryConfig;
+  service: LaunchdService;
+  runCommand: CommandRunner;
+  now: () => number;
+  wait: (ms: number) => Promise<void>;
+  verifyLocal: (config: RecoveryConfig) => Promise<VerifyResult>;
+  ensureRuntimeLaunchContract?: (controllerHome: string) => void;
+  beforeStart?: () => void;
+  contractFailureContext?: string;
+  timeoutMs: number;
+  successDetail: string;
+}): Promise<PrimaryRuntimeRebindStartResult> {
+  try {
+    const ensureRuntimeLaunchContract = input.ensureRuntimeLaunchContract
+      ?? ((controllerHome: string) => { ensureForgeRuntimeLaunchAgentContract({ controllerHome, installUserLaunchAgent: true }); });
+    ensureRuntimeLaunchContract(input.config.controllerHome);
+  } catch (error) {
+    return {
+      ok: false,
+      detail: `primary Runtime launchd contract rebuild failed ${input.contractFailureContext ?? 'after release transition'}: ${error instanceof Error ? error.message : String(error)}`,
+      verify: await input.verifyLocal(input.config),
+    };
+  }
+
+  try {
+    input.beforeStart?.();
+  } catch (error) {
+    return {
+      ok: false,
+      detail: `release transition pre-start preparation failed: ${error instanceof Error ? error.message : String(error)}`,
+      verify: await input.verifyLocal(input.config),
+    };
+  }
+
+  const started = await ensureLaunchdServiceStarted(input.service, input.runCommand);
+  if (!started.ok) {
+    return { ok: false, detail: started.detail, verify: await input.verifyLocal(input.config) };
+  }
+  const verify = await verifyPrimaryRuntimeAfterStart({
+    config: input.config,
+    timeoutMs: input.timeoutMs,
+    now: input.now,
+    wait: input.wait,
+    verifyLocal: input.verifyLocal,
+  });
+  if (verify.ok) return { ok: true, detail: input.successDetail, verify };
+  await input.runCommand('launchctl', ['bootout', input.service.target], 15_000);
+  return {
+    ok: false,
+    detail: 'release transition started a Runtime that failed whole-Runtime verification; service was stopped to prevent a restart loop',
+    verify,
+  };
+}
+
 export async function restartPrimaryRuntime(
   config: RecoveryConfig,
   dependencies: PrimaryRuntimeRecoveryDependencies = {},
@@ -1897,44 +2112,21 @@ export async function recoverPrimaryRuntime(
     const before = await verifyLocal(config);
     if (before.ok) return { ok: true, attempted: false, noOp: true, detail: 'Canonical Forge Runtime recovered before rollback', serviceTarget: service.target, verify: before } satisfies PrimaryRuntimeRecoveryResult;
 
-    const stopped = await runCommand('launchctl', ['bootout', service.target], 15_000);
-    const stoppedCleanly = stopped.ok || /not found|no such process|could not find service|service is not loaded/i.test(`${stopped.stderr}\n${stopped.stdout}`);
-    if (!stoppedCleanly) {
-      const detail = `primary Runtime bootout failed: ${stopped.stderr || stopped.stdout || stopped.status}`;
-      audit(config, 'primary_runtime_recovery_stop_failed', { serviceTarget: service.target, detail });
-      return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: before } satisfies PrimaryRuntimeRecoveryResult;
-    }
-    const runtimeStopped = await waitForPrimaryRuntimeState({ config, expectedRunning: false, timeoutMs: 20_000, now, wait, runtimeRunning });
-    if (!runtimeStopped) {
-      const detail = 'primary Runtime owner remained live after bounded launchd bootout';
-      audit(config, 'primary_runtime_recovery_stop_unverified', { serviceTarget: service.target });
-      return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies PrimaryRuntimeRecoveryResult;
-    }
-    const serviceUnloaded = await waitForLaunchdServiceUnloaded({ service, timeoutMs: 20_000, now, wait, runCommand });
-    if (!serviceUnloaded) {
-      const detail = 'primary Runtime launchd service remained loaded after bounded bootout';
-      audit(config, 'primary_runtime_recovery_stop_unverified', { serviceTarget: service.target, detail });
-      return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies PrimaryRuntimeRecoveryResult;
-    }
-
-    let portReleased = await waitForPrimaryRuntimePortReleased({ config, timeoutMs: 20_000, now, wait, runCommand });
-    let staleListenerCleanup: PrimaryRuntimePortCleanupResult | undefined;
-    if (!portReleased) {
-      staleListenerCleanup = await cleanupVerifiedStalePrimaryRuntimeListener({ config, uid: service.uid, now, wait, runCommand });
-      portReleased = staleListenerCleanup.released;
-      if (staleListenerCleanup.cleaned) {
-        audit(config, 'primary_runtime_recovery_stale_listener_cleaned', {
-          serviceTarget: service.target,
-          pid: staleListenerCleanup.pid,
-          signal: staleListenerCleanup.signal,
-          detail: staleListenerCleanup.detail,
-        });
-      }
+    const stopped = await stopPrimaryRuntimeForReleaseTransition({ config, service, now, wait, runCommand, runtimeRunning });
+    if (!stopped.ok) {
+      const action = stopped.detail.startsWith('primary Runtime bootout failed')
+        ? 'primary_runtime_recovery_stop_failed'
+        : 'primary_runtime_recovery_stop_unverified';
+      audit(config, action, { serviceTarget: service.target, detail: stopped.detail, pid: stopped.staleListenerCleanup?.pid });
+      return { ok: false, attempted: true, detail: stopped.detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies PrimaryRuntimeRecoveryResult;
     }
-    if (!portReleased) {
-      const detail = `primary Runtime TCP port remained occupied after bounded launchd bootout: ${staleListenerCleanup?.detail ?? 'listener did not release'}`;
-      audit(config, 'primary_runtime_recovery_stop_unverified', { serviceTarget: service.target, detail, pid: staleListenerCleanup?.pid });
-      return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies PrimaryRuntimeRecoveryResult;
+    if (stopped.staleListenerCleanup?.cleaned) {
+      audit(config, 'primary_runtime_recovery_stale_listener_cleaned', {
+        serviceTarget: service.target,
+        pid: stopped.staleListenerCleanup.pid,
+        signal: stopped.staleListenerCleanup.signal,
+        detail: stopped.staleListenerCleanup.detail,
+      });
     }
 
     const rollback = await rollbackPreviousLocked(config, reason);
@@ -1943,26 +2135,30 @@ export async function recoverPrimaryRuntime(
       return { ok: false, attempted: true, detail: rollback.detail, serviceTarget: service.target, rollback, verify: rollback.verify ?? await verifyLocal(config) } satisfies PrimaryRuntimeRecoveryResult;
     }
 
-    const started = await ensureLaunchdServiceStarted(service, runCommand);
-    if (!started.ok) {
-      audit(config, 'primary_runtime_recovery_start_failed', { serviceTarget: service.target, detail: started.detail, rollbackOperationId: rollback.operationId });
-      return { ok: false, attempted: true, detail: started.detail, serviceTarget: service.target, rollback, verify: await verifyLocal(config) } satisfies PrimaryRuntimeRecoveryResult;
-    }
-    const after = await verifyPrimaryRuntimeAfterStart({
+    const restarted = await rebindStartAndVerifyPrimaryRuntime({
       config,
-      timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 45_000,
+      service,
+      runCommand,
       now,
       wait,
       verifyLocal,
+      ensureRuntimeLaunchContract: dependencies.ensureRuntimeLaunchContract,
+      contractFailureContext: 'after rollback',
+      timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 45_000,
+      successDetail: 'previous whole-Runtime release and SQLite backup restored, restarted, and verified',
     });
-    if (after.ok) {
-      audit(config, 'primary_runtime_recovery_succeeded', { serviceTarget: service.target, rollbackOperationId: rollback.operationId, restoredRelease: after.releases.active?.revision });
-      return { ok: true, attempted: true, detail: 'previous whole-Runtime release and SQLite backup restored, restarted, and verified', serviceTarget: service.target, rollback, verify: after } satisfies PrimaryRuntimeRecoveryResult;
+    if (restarted.ok) {
+      audit(config, 'primary_runtime_recovery_succeeded', { serviceTarget: service.target, rollbackOperationId: rollback.operationId, restoredRelease: restarted.verify.releases.active?.revision });
+      return { ok: true, attempted: true, detail: restarted.detail, serviceTarget: service.target, rollback, verify: restarted.verify } satisfies PrimaryRuntimeRecoveryResult;
     }
-
-    await runCommand('launchctl', ['bootout', service.target], 15_000);
-    audit(config, 'primary_runtime_recovery_unverified', { serviceTarget: service.target, rollbackOperationId: rollback.operationId, reasonCodes: after.runtime.reasonCodes });
-    return { ok: false, attempted: true, detail: 'previous whole-Runtime release started but failed verification; service was stopped to prevent a restart loop', serviceTarget: service.target, rollback, verify: after } satisfies PrimaryRuntimeRecoveryResult;
+    const contractFailure = restarted.detail.startsWith('primary Runtime launchd contract rebuild failed');
+    audit(config, contractFailure ? 'primary_runtime_recovery_launchd_contract_failed' : 'primary_runtime_recovery_unverified', {
+      serviceTarget: service.target,
+      rollbackOperationId: rollback.operationId,
+      detail: restarted.detail,
+      reasonCodes: restarted.verify.runtime.reasonCodes,
+    });
+    return { ok: false, attempted: true, detail: restarted.detail, serviceTarget: service.target, rollback, verify: restarted.verify } satisfies PrimaryRuntimeRecoveryResult;
   });
   if (!locked.acquired) return { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(locked.owner), serviceTarget: service.target, verify: initial };
   return locked.value;
@@ -2160,44 +2356,22 @@ export async function activateRuntimeRelease(
       return { ok: false, attempted: false, noOp: true, detail, operationId } satisfies RuntimeReleaseActivationResult;
     }
     const before = await verifyLocal(config);
-    const stopped = await runCommand('launchctl', ['bootout', service.target], 15_000);
-    const stoppedCleanly = stopped.ok || /not found|no such process|could not find service|service is not loaded/i.test(`${stopped.stderr}\n${stopped.stdout}`);
-    if (!stoppedCleanly) {
-      const detail = `primary Runtime bootout failed: ${stopped.stderr || stopped.stdout || stopped.status}`;
-      audit(config, 'runtime_release_activation_stop_failed', { serviceTarget: service.target, operationId, detail });
-      return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies RuntimeReleaseActivationResult;
-    }
-    const runtimeStopped = await waitForPrimaryRuntimeState({ config, expectedRunning: false, timeoutMs: 20_000, now, wait, runtimeRunning });
-    if (!runtimeStopped) {
-      const detail = 'primary Runtime owner remained live after bounded launchd bootout';
-      audit(config, 'runtime_release_activation_stop_unverified', { serviceTarget: service.target, operationId });
-      return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies RuntimeReleaseActivationResult;
-    }
-    const serviceUnloaded = await waitForLaunchdServiceUnloaded({ service, timeoutMs: 20_000, now, wait, runCommand });
-    if (!serviceUnloaded) {
-      const detail = 'primary Runtime launchd service remained loaded after bounded bootout';
-      audit(config, 'runtime_release_activation_stop_unverified', { serviceTarget: service.target, operationId, detail });
-      return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies RuntimeReleaseActivationResult;
-    }
-    let portReleased = await waitForPrimaryRuntimePortReleased({ config, timeoutMs: 20_000, now, wait, runCommand });
-    let staleListenerCleanup: PrimaryRuntimePortCleanupResult | undefined;
-    if (!portReleased) {
-      staleListenerCleanup = await cleanupVerifiedStalePrimaryRuntimeListener({ config, uid: service.uid, now, wait, runCommand });
-      portReleased = staleListenerCleanup.released;
-      if (staleListenerCleanup.cleaned) {
-        audit(config, 'runtime_release_activation_stale_listener_cleaned', {
-          serviceTarget: service.target,
-          operationId,
-          pid: staleListenerCleanup.pid,
-          signal: staleListenerCleanup.signal,
-          detail: staleListenerCleanup.detail,
-        });
-      }
+    const stopped = await stopPrimaryRuntimeForReleaseTransition({ config, service, now, wait, runCommand, runtimeRunning });
+    if (!stopped.ok) {
+      const action = stopped.detail.startsWith('primary Runtime bootout failed')
+        ? 'runtime_release_activation_stop_failed'
+        : 'runtime_release_activation_stop_unverified';
+      audit(config, action, { serviceTarget: service.target, operationId, detail: stopped.detail, pid: stopped.staleListenerCleanup?.pid });
+      return { ok: false, attempted: true, detail: stopped.detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies RuntimeReleaseActivationResult;
     }
-    if (!portReleased) {
-      const detail = `primary Runtime TCP port remained occupied after bounded launchd bootout: ${staleListenerCleanup?.detail ?? 'listener did not release'}`;
-      audit(config, 'runtime_release_activation_stop_unverified', { serviceTarget: service.target, operationId, detail, pid: staleListenerCleanup?.pid });
-      return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies RuntimeReleaseActivationResult;
+    if (stopped.staleListenerCleanup?.cleaned) {
+      audit(config, 'runtime_release_activation_stale_listener_cleaned', {
+        serviceTarget: service.target,
+        operationId,
+        pid: stopped.staleListenerCleanup.pid,
+        signal: stopped.staleListenerCleanup.signal,
+        detail: stopped.staleListenerCleanup.detail,
+      });
     }
     let committed: RuntimeReleaseAuthority;
     try {
@@ -2212,73 +2386,59 @@ export async function activateRuntimeRelease(
       audit(config, 'runtime_release_activation_commit_mismatch', { serviceTarget: service.target, operationId });
       return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies RuntimeReleaseActivationResult;
     }
-    try {
-      ensureForgeRuntimeLaunchAgentContract({ controllerHome: config.controllerHome, installUserLaunchAgent: true });
-    } catch (error) {
-      const detail = `runtime launchd stable-release contract update failed: ${error instanceof Error ? error.message : String(error)}`;
-      audit(config, 'runtime_release_activation_launchd_contract_failed', { serviceTarget: service.target, operationId, detail });
-      return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies RuntimeReleaseActivationResult;
-    }
     let storageMigration: ControllerHomeStorageMigration | undefined;
-    let after: VerifyResult;
+    const activated = await rebindStartAndVerifyPrimaryRuntime({
+      config,
+      service,
+      runCommand,
+      now,
+      wait,
+      verifyLocal,
+      timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 60_000,
+      successDetail: 'requested Runtime release started and passed whole-Runtime verification',
+      beforeStart: () => {
+        storageMigration = migrateStoppedRepoLocalControllerHomeStorage(config.controllerHome, platform);
+        if (storageMigration.migrated) {
+          audit(config, 'runtime_controller_home_noindex_migrated', {
+            serviceTarget: service.target,
+            operationId,
+            logicalHome: storageMigration.logicalHome,
+            physicalHome: storageMigration.physicalHome,
+          });
+        }
+      },
+    });
+    let after = activated.verify;
     let activationFailureDetail: string | undefined;
-    try {
-      storageMigration = migrateStoppedRepoLocalControllerHomeStorage(config.controllerHome, platform);
-      if (storageMigration.migrated) {
-        audit(config, 'runtime_controller_home_noindex_migrated', {
-          serviceTarget: service.target,
-          operationId,
-          logicalHome: storageMigration.logicalHome,
-          physicalHome: storageMigration.physicalHome,
-        });
-      }
-    } catch (error) {
-      activationFailureDetail = `Controller Home .noindex migration failed: ${error instanceof Error ? error.message : String(error)}`;
-      audit(config, 'runtime_controller_home_noindex_migration_failed', { serviceTarget: service.target, operationId, detail: activationFailureDetail });
+    if (activated.ok && after.releases.active?.revision === candidate.manifest.releaseId) {
+      audit(config, 'runtime_release_activation_succeeded', { serviceTarget: service.target, operationId, requestId: lockRequestId, activeRevision: after.releases.active?.revision, expectedAuthorityRevision: guard.expectedAuthorityRevision, expectedActiveReleaseId: guard.expectedActiveReleaseId, controllerHomeStorageMigrated: storageMigration?.migrated === true });
+      return { ok: true, attempted: true, detail: storageMigration?.migrated ? 'requested Runtime release activated, Controller Home migrated to .noindex storage, and whole-Runtime verification passed' : 'requested Runtime release activated and passed whole-Runtime verification', serviceTarget: service.target, operationId, verify: after } satisfies RuntimeReleaseActivationResult;
     }
-
-    if (activationFailureDetail) {
-      after = await verifyLocal(config);
+    if (!activated.ok) {
+      activationFailureDetail = activated.detail;
+      const auditAction = activated.detail.startsWith('primary Runtime launchd contract rebuild failed')
+        ? 'runtime_release_activation_launchd_contract_failed'
+        : activated.detail.startsWith('release transition pre-start preparation failed')
+          ? 'runtime_controller_home_noindex_migration_failed'
+          : activated.detail.includes('start')
+            ? 'runtime_release_activation_start_failed'
+            : 'runtime_release_activation_unverified';
+      audit(config, auditAction, { serviceTarget: service.target, operationId, detail: activated.detail });
     } else {
-      const started = await ensureLaunchdServiceStarted(service, runCommand);
-      if (!started.ok) {
-        activationFailureDetail = started.detail;
-        audit(config, 'runtime_release_activation_start_failed', { serviceTarget: service.target, operationId, detail: started.detail });
-        after = await verifyLocal(config);
-      } else {
-        after = await verifyPrimaryRuntimeAfterStart({
-          config,
-          timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 60_000,
-          now,
-          wait,
-          verifyLocal,
-        });
-        if (after.ok && after.releases.active?.revision === candidate.manifest.releaseId) {
-          audit(config, 'runtime_release_activation_succeeded', { serviceTarget: service.target, operationId, requestId: lockRequestId, activeRevision: after.releases.active?.revision, expectedAuthorityRevision: guard.expectedAuthorityRevision, expectedActiveReleaseId: guard.expectedActiveReleaseId, controllerHomeStorageMigrated: storageMigration?.migrated === true });
-          return { ok: true, attempted: true, detail: storageMigration?.migrated ? 'requested Runtime release activated, Controller Home migrated to .noindex storage, and whole-Runtime verification passed' : 'requested Runtime release activated and passed whole-Runtime verification', serviceTarget: service.target, operationId, verify: after } satisfies RuntimeReleaseActivationResult;
-        }
-      }
+      activationFailureDetail = `activated Runtime release identity mismatch: expected ${candidate.manifest.releaseId}, observed ${after.releases.active?.revision ?? 'none'}`;
+      audit(config, 'runtime_release_activation_commit_mismatch', { serviceTarget: service.target, operationId, detail: activationFailureDetail });
     }
 
     // Activation failed: stop, restore the previous whole release and its SQLite
     // backup, restart the one service, and require verification again.
-    await runCommand('launchctl', ['bootout', service.target], 15_000);
     let rollback: RollbackResult;
-    const rollbackRuntimeStopped = await waitForPrimaryRuntimeState({ config, expectedRunning: false, timeoutMs: 20_000, now, wait, runtimeRunning });
-    const rollbackServiceUnloaded = rollbackRuntimeStopped
-      ? await waitForLaunchdServiceUnloaded({ service, timeoutMs: 20_000, now, wait, runCommand })
-      : false;
-    const rollbackPortReleased = rollbackRuntimeStopped && rollbackServiceUnloaded
-      ? await waitForPrimaryRuntimePortReleased({ config, timeoutMs: 20_000, now, wait, runCommand })
-      : false;
-    if (!rollbackRuntimeStopped || !rollbackServiceUnloaded || !rollbackPortReleased) {
+    const rollbackStop = await stopPrimaryRuntimeForReleaseTransition({ config, service, now, wait, runCommand, runtimeRunning });
+    if (!rollbackStop.ok) {
       rollback = {
         ok: false,
-        detail: !rollbackRuntimeStopped
-          ? 'candidate Runtime owner remained live after bounded rollback bootout'
-          : !rollbackServiceUnloaded
-            ? 'candidate Runtime launchd service remained loaded after bounded rollback bootout'
-            : 'candidate Runtime TCP port remained occupied after bounded rollback bootout',
+        detail: rollbackStop.detail
+          .replace('primary Runtime', 'candidate Runtime')
+          .replace('bounded launchd bootout', 'bounded rollback bootout'),
       };
     } else {
       try {
@@ -2294,7 +2454,6 @@ export async function activateRuntimeRelease(
             physicalHome: storageMigration.physicalHome,
           });
         }
-        ensureForgeRuntimeLaunchAgentContract({ controllerHome: config.controllerHome, installUserLaunchAgent: true });
         const previousRevision = previousActive?.releaseId ?? '';
         const previousIdentity = previousActive?.artifactIdentity;
         if (
@@ -2303,24 +2462,18 @@ export async function activateRuntimeRelease(
         ) {
           throw new Error('RECOVERY_RUNTIME_RELEASE_ROLLBACK_AUTHORITY_MISMATCH');
         }
-        const restarted = await ensureLaunchdServiceStarted(service, runCommand);
-        if (!restarted.ok) {
-          rollback = { ok: false, operationId: rollbackOperationId, detail: `previous release authority restored but service start failed: ${restarted.detail}` };
-        } else {
-          const rolledBack = await verifyPrimaryRuntimeAfterStart({
-            config,
-            timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 60_000,
-            now,
-            wait,
-            verifyLocal,
-          });
-          if (rolledBack.ok) {
-            rollback = { ok: true, operationId: rollbackOperationId, detail: 'previous whole-Runtime release and SQLite backup restored, restarted, and verified', verify: rolledBack };
-          } else {
-            await runCommand('launchctl', ['bootout', service.target], 15_000);
-            rollback = { ok: false, operationId: rollbackOperationId, detail: 'previous whole-Runtime release started but failed verification; service was stopped to prevent a restart loop', verify: rolledBack };
-          }
-        }
+        const restarted = await rebindStartAndVerifyPrimaryRuntime({
+          config,
+          service,
+          runCommand,
+          now,
+          wait,
+          verifyLocal,
+          contractFailureContext: 'after rollback',
+          timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 60_000,
+          successDetail: 'previous whole-Runtime release and SQLite backup restored, restarted, and verified',
+        });
+        rollback = { ok: restarted.ok, operationId: rollbackOperationId, detail: restarted.detail, verify: restarted.verify };
       } catch (error) {
         rollback = { ok: false, detail: error instanceof Error ? error.message : 'previous whole-Runtime release rollback failed' };
       }
@@ -2369,15 +2522,26 @@ export async function stageAndActivateConfiguredRuntimeRelease(
   // Capture the base before staging. Staging can take long enough for another
   // activation to win; the later publish must prove this base is still current.
   const expectedAuthority = releaseAuthority(config);
-  let staged: StagedRuntimeRelease;
-  try {
-    staged = (dependencies.stage ?? stageRuntimeReleaseFromCandidateSource)({ controllerHome: config.controllerHome, sourceRoot });
-    assertRuntimeReleaseFiles(staged);
-  } catch (error) {
-    const detail = error instanceof Error ? error.message : 'Runtime release staging failed';
-    audit(config, 'runtime_release_stage_failed', { sourceRoot, detail });
-    return { ok: false, attempted: false, noOp: true, detail };
+  const stagedAttempt = await withLock(config, {
+    action: 'stage_and_activate_runtime_release',
+    ...(requestId?.trim() ? { requestId: requestId.trim() } : {}),
+  }, async () => {
+    try {
+      const staged = (dependencies.stage ?? stageRuntimeReleaseFromCandidateSource)({ controllerHome: config.controllerHome, sourceRoot });
+      assertRuntimeReleaseFiles(staged);
+      return { ok: true as const, staged };
+    } catch (error) {
+      return { ok: false as const, detail: error instanceof Error ? error.message : 'Runtime release staging failed' };
+    }
+  });
+  if (!stagedAttempt.acquired) {
+    return { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(stagedAttempt.owner) };
   }
+  if (!stagedAttempt.value.ok) {
+    audit(config, 'runtime_release_stage_failed', { sourceRoot, detail: stagedAttempt.value.detail });
+    return { ok: false, attempted: false, noOp: true, detail: stagedAttempt.value.detail };
+  }
+  const staged: StagedRuntimeRelease = stagedAttempt.value.staged;
   const activationGuard: RuntimeReleaseActivationGuard = {
     ...(requestId?.trim() ? { requestId: requestId.trim() } : {}),
     expectedAuthorityRevision: expectedAuthority?.revision ?? null,
@@ -2691,6 +2855,35 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
     fullVerificationPerformed = true;
   }
   const lastFullVerifyAt = fullVerificationPerformed ? now : scopedPrior.lastFullVerifyAt;
+  const activeMutation = liveRecoveryMutationLock(config);
+  if (activeMutation) {
+    const attributable = recoveryLockOwnerAttributable(activeMutation);
+    if (!attributable) {
+      audit(config, 'recovery_operation_lock_identity_uncertain', {
+        pid: activeMutation.pid,
+        instanceId: activeMutation.instanceId,
+        action: activeMutation.action ?? null,
+        requestId: activeMutation.requestId ?? null,
+        acquiredAt: activeMutation.acquiredAt,
+      });
+    }
+    const reason = attributable
+      ? `Recovery mutation in progress: action=${activeMutation.action} request=${activeMutation.requestId}; watchdog repair escalation suppressed`
+      : 'Recovery mutation lock identity is uncertain; watchdog repair escalation suppressed';
+    const state: WatchdogState = {
+      ...scopedPrior,
+      failures: 0,
+      firstFailureAt: undefined,
+      publicTunnelFailures: 0,
+      publicTunnelFirstFailureAt: undefined,
+      primaryConnectorFailures: 0,
+      primaryConnectorFirstFailureAt: undefined,
+      lastFullVerifyAt,
+      lastDecision: 'degraded',
+      lastReason: reason,
+    };
+    return { state, decision: { action: 'degraded', reason }, verify: verified };
+  }
   if (runtimeStartupGrace && !localVerify.ok) {
     const state: WatchdogState = {
       ...scopedPrior,
@@ -2709,18 +2902,34 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
   }
   const recoveryHealthy = verified.probes.recovery_gateway?.ok !== false
     && verified.probes.recovery_external_http?.ok !== false;
-  const primaryRuntimeHealthy = localVerify.ok;
-  const primaryConnectorConfigured = Boolean(config.primaryConnectorService && config.publicMcpUrl);
-  const primaryConnectorFailed = Boolean(
-    primaryConnectorConfigured
+  const primaryRuntimeHealthy = localVerify.runtime.ok
+    && localVerify.runtime.running
+    && localVerify.runtime.ready
+    && !localVerify.runtime.stale
+    && localVerify.probes.active_gateway?.ok === true;
+  const primaryConnectorConfigured = Boolean(config.primaryConnectorService);
+  const primaryConnectorLocalFailed = verified.probes.primary_connector_local?.ok === false;
+  const primaryConnectorCapacityFailed = connectorCapacityRecoveryRecommended(verified);
+  const primaryPublicTransportManaged = Boolean(configuredPrimaryPublicTunnel(config));
+  const primaryPublicTransportFailed = Boolean(
+    config.publicMcpUrl
     && primaryRuntimeHealthy
+    && !primaryConnectorCapacityFailed
     && (
       verified.probes.external_mcp_http?.ok === false
       || verified.probes.primary_connector_ready?.ok === false
-      || connectorCapacityRecoveryRecommended(verified)
     ),
   );
-  if (primaryRuntimeHealthy && !primaryConnectorFailed && recoveryHealthy) {
+  const primaryConnectorFailed = Boolean(
+    primaryConnectorConfigured
+    && primaryRuntimeHealthy
+    && (
+      primaryConnectorLocalFailed
+      || primaryConnectorCapacityFailed
+      || (primaryPublicTransportManaged && primaryPublicTransportFailed)
+    ),
+  );
+  if (primaryRuntimeHealthy && !primaryConnectorFailed && !primaryPublicTransportFailed && recoveryHealthy) {
     if (fullVerificationPerformed && verified.ok && !matchingKnownGood(config, verified.releases.active)) {
       try {
         const attestation = await withLock(config, { action: 'attest_known_good' }, async () => persistVerifiedKnownGood(config, verified));
@@ -2759,6 +2968,26 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
     };
     return { state, decision: { action: 'healthy', reason: 'primary runtime and standalone Recovery verification passed' }, verify: verified };
   }
+  if (primaryRuntimeHealthy && !primaryConnectorFailed && primaryPublicTransportFailed && recoveryHealthy) {
+    const reason = primaryPublicTransportManaged
+      ? 'primary public transport is unavailable but no safely repairable primary Connector path was established'
+      : 'primary public transport is unavailable while the local Connector is healthy, but no managed primary public tunnel is configured; external transport repair is required';
+    const state: WatchdogState = {
+      ...recordWatchdogRuntimeHealthy(
+        scopeWatchdogStateToRuntimeRelease(scopedPrior, verified.releases.active),
+        now,
+        watchdogRuntimeRestartBudgetStableMs(config),
+      ),
+      failures: scopedPrior.failures + 1,
+      firstFailureAt: scopedPrior.firstFailureAt ?? now,
+      primaryConnectorFailures: 0,
+      primaryConnectorFirstFailureAt: undefined,
+      lastFullVerifyAt,
+      lastDecision: 'degraded',
+      lastReason: reason,
+    };
+    return { state, decision: { action: 'degraded', reason }, verify: verified };
+  }
   const publicTunnelFailed = isExternalTunnelFailure(config, verified, localVerify);
   const evidenceClasses = Object.entries(localVerify.probes)
     .filter(([name, value]) => !name.startsWith('recovery_') && !value.ok)
@@ -2805,7 +3034,7 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
         primaryConnectorFailures: 0,
         primaryConnectorFirstFailureAt: undefined,
       };
-  if (!localVerify.ok) state.runtimeHealthySince = undefined;
+  if (!primaryRuntimeHealthy) state.runtimeHealthySince = undefined;
   const primaryConfig = configuredPrimaryRuntimeService(config);
   const decision = decideWatchdog({
     ...state,
@@ -2813,7 +3042,7 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
     evidenceClasses,
     activeKnownGood,
     previousKnownGood,
-    primaryRuntimeFailed: !localVerify.ok,
+    primaryRuntimeFailed: !primaryRuntimeHealthy,
     runtimeMaximumRestartAttempts: primaryConfig.maximumRestartAttempts,
     runtimeRestartCooldownMs: primaryConfig.restartCooldownMs,
     runtimeMinimumFailures: primaryConfig.minimumFailures,
@@ -2850,7 +3079,9 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
     return { state: { ...state, recoveryGatewayRestartUsed: true }, decision, verify: verified, recoveryGatewayRestart };
   }
   if (decision.action === 'restart_primary_connector') {
-    const primaryConnectorRestart = await restartPrimaryConnector(config);
+    const primaryConnectorRestart = await restartPrimaryConnector(config, {
+      requestId: `watchdog:restart_primary_connector:${now}`,
+    });
     const attempts = (state.primaryConnectorRestartAttempts ?? 0) + (primaryConnectorRestart.attempted ? 1 : 0);
     const nextState: WatchdogState = primaryConnectorRestart.ok
       ? {