@moretea-labs/forge 1.5.1 → 1.7.0

package/src/runtime/standalone-recovery/core.ts

@@ -4,7 +4,14 @@ import { chmodSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, re
 import { homedir } from 'os';
 import { basename, dirname, isAbsolute, join, resolve } from 'path';
 import { observeRuntimeStatus } from '../root/status';
-import { ensureForgeRuntimeLaunchAgentContract, forgeRuntimeServicePaths, readForgeRuntimeServiceConfig } from '../root/service';
+import {
+  activeRuntimeEntrypoint,
+  activeRuntimeLaunchSpec,
+  ensureForgeRuntimeLaunchAgentContract,
+  forgeRuntimeServicePaths,
+  inspectForgeRuntimeLaunchAgentContract,
+  readForgeRuntimeServiceConfig,
+} from '../root/service';
 import { loadRuntimeReleaseManifest } from '../root/release-manifest';
 import { assertRuntimeReleaseFiles, stageRuntimeReleaseFromCandidateSource, type StagedRuntimeRelease } from '../root/release-materialize';
 import {
@@ -15,7 +22,10 @@ import {
   type RuntimeReleaseAuthority,
 } from '../root/release-store';
 import type { RuntimeReleaseManifest } from '../root/types';
+import { ensurePackageConnectorService, packageConnectorServicePaths, type PackageConnectorReleaseBinding } from '../root/package-connector-service';
 import { createRecoveryHttpTransport, type RecoveryHttpTransport } from './http-transport';
+import { observeRecoveryWatchdogHealth } from './watchdog-heartbeat';
+import { RECOVERY_GATEWAY_LABEL, RECOVERY_WATCHDOG_LABEL } from './service-labels';
 import {
   migrateStoppedRepoLocalControllerHomeStorage,
   repoLocalControllerHomeStorageNeedsMigration,
@@ -50,6 +60,12 @@ export interface PrimaryConnectorServiceConfig {
   platform: 'launchd';
   label: string;
   plistPath?: string;
+  /** Local OAuth MCP endpoint used to distinguish Connector failure from tunnel failure. */
+  localMcpUrl?: string;
+  minimumFailures?: number;
+  minimumFailureDurationMs?: number;
+  restartCooldownMs?: number;
+  maximumRestartAttempts?: number;
   postRestartVerifyTimeoutMs?: number;
 }
 
@@ -59,6 +75,8 @@ export interface RecoveryConfig {
   publicMcpUrl?: string;
   recoveryPublicUrl?: string;
   recoveryTunnelService?: PublicTunnelServiceConfig;
+  /** Optional public tunnel serving publicMcpUrl; independent from the OAuth Connector process. */
+  primaryPublicTunnelService?: PublicTunnelServiceConfig;
   primaryRuntimeService?: PrimaryRuntimeServiceConfig;
   primaryRuntimeSourceRoot?: string;
   primaryConnectorService?: PrimaryConnectorServiceConfig;
@@ -95,6 +113,7 @@ export type RecoveryMutationAction =
   | 'stage_and_activate_runtime_release'
   | 'restart_primary_connector'
   | 'restart_recovery_gateway'
+  | 'restart_recovery_watchdog'
   | 'repair_public_tunnel';
 
 interface RecoveryLock {
@@ -132,8 +151,8 @@ export interface VerifyStableRuntimeOptions {
 export interface RecoveryWatchdogDiagnosticEvidence {
   fingerprint: string;
   releaseIdentity: string;
-  components: Array<'runtime' | 'gateway' | 'public_mcp' | 'recovery_gateway' | 'recovery_tunnel'>;
-  failedProbes: Array<{ name: string; component: 'runtime' | 'gateway' | 'public_mcp' | 'recovery_gateway' | 'recovery_tunnel'; detail: string; status?: number }>;
+  components: Array<'runtime' | 'gateway' | 'public_mcp' | 'recovery_gateway' | 'recovery_watchdog' | 'recovery_tunnel'>;
+  failedProbes: Array<{ name: string; component: 'runtime' | 'gateway' | 'public_mcp' | 'recovery_gateway' | 'recovery_watchdog' | 'recovery_tunnel'; detail: string; status?: number }>;
   firstObservedAt: string;
   lastObservedAt: string;
   occurrences: number;
@@ -153,7 +172,7 @@ export interface RollbackResult {
 }
 
 export interface WatchdogDecision {
-  action: 'healthy' | 'degraded' | 'repair_public_tunnel' | 'restart_recovery_gateway' | 'restart_primary_runtime' | 'rollback' | 'recovery_exhausted';
+  action: 'healthy' | 'degraded' | 'repair_public_tunnel' | 'restart_recovery_gateway' | 'restart_primary_connector' | 'restart_primary_runtime' | 'rollback' | 'recovery_exhausted';
   reason: string;
 }
 
@@ -175,10 +194,16 @@ export interface WatchdogState {
   publicTunnelFailures?: number;
   publicTunnelFirstFailureAt?: number;
   publicTunnelRepairFailures?: number;
+  primaryConnectorFailures?: number;
+  primaryConnectorFirstFailureAt?: number;
+  primaryConnectorRestartAttempts?: number;
+  primaryConnectorRestartFailures?: number;
+  primaryConnectorRestartLastAttemptAt?: number;
   recoveryGatewayRestartUsed?: boolean;
   recoveryReleaseRevision?: string;
   lastFullVerifyAt?: number;
   lastDecision?: WatchdogDecision['action'];
+  lastReason?: string;
   updatedAt?: string;
 }
 
@@ -300,6 +325,10 @@ function configuredRecoveryTunnel(config: RecoveryConfig): PublicTunnelServiceCo
   return config.recoveryTunnelService;
 }
 
+function configuredPrimaryPublicTunnel(config: RecoveryConfig): PublicTunnelServiceConfig | undefined {
+  return config.primaryPublicTunnelService;
+}
+
 function configuredRecoveryPublicUrl(config: RecoveryConfig): string | undefined {
   return config.recoveryPublicUrl;
 }
@@ -324,6 +353,7 @@ export function loadRecoveryConfig(controllerHome: string, explicit?: string): R
     ...(typeof loaded.publicMcpUrl === 'string' ? { publicMcpUrl: loaded.publicMcpUrl } : {}),
     ...(typeof loaded.recoveryPublicUrl === 'string' ? { recoveryPublicUrl: loaded.recoveryPublicUrl } : {}),
     ...(loaded.recoveryTunnelService ? { recoveryTunnelService: loaded.recoveryTunnelService } : {}),
+    ...(loaded.primaryPublicTunnelService ? { primaryPublicTunnelService: loaded.primaryPublicTunnelService } : {}),
     ...(loaded.primaryRuntimeService ? { primaryRuntimeService: loaded.primaryRuntimeService } : {}),
     ...(typeof loaded.primaryRuntimeSourceRoot === 'string' ? { primaryRuntimeSourceRoot: resolve(loaded.primaryRuntimeSourceRoot) } : {}),
     ...(loaded.primaryConnectorService ? { primaryConnectorService: loaded.primaryConnectorService } : {}),
@@ -423,6 +453,7 @@ function watchdogProbeComponent(name: string): RecoveryWatchdogDiagnosticEvidenc
   if (name === 'runtime_status') return 'runtime';
   if (name === 'active_gateway') return 'gateway';
   if (name === 'recovery_gateway') return 'recovery_gateway';
+  if (name === 'recovery_watchdog') return 'recovery_watchdog';
   if (name === 'recovery_external_http') return 'recovery_tunnel';
   return 'public_mcp';
 }
@@ -541,7 +572,23 @@ async function withLock<T>(
 }
 
 export async function runtimeStatus(config: RecoveryConfig) {
-  return observeRuntimeStatus(config.controllerHome);
+  const observation = observeRuntimeStatus(config.controllerHome);
+  const watchdog = loadWatchdogState(config);
+  return {
+    ...observation,
+    recoveryWatchdog: {
+      lastDecision: watchdog.lastDecision,
+      lastReason: watchdog.lastReason,
+      updatedAt: watchdog.updatedAt,
+      failures: watchdog.failures,
+      rollbackUsed: watchdog.rollbackUsed,
+      runtimeRestartAttempts: watchdog.runtimeRestartAttempts ?? 0,
+      runtimeRestartFailures: watchdog.runtimeRestartFailures ?? 0,
+      primaryConnectorFailures: watchdog.primaryConnectorFailures ?? 0,
+      primaryConnectorRestartAttempts: watchdog.primaryConnectorRestartAttempts ?? 0,
+      primaryConnectorRestartFailures: watchdog.primaryConnectorRestartFailures ?? 0,
+    },
+  };
 }
 
 async function probe(transport: RecoveryHttpTransport, url: string, timeoutMs = 4_000): Promise<{ ok: boolean; detail: string; status?: number }> {
@@ -583,6 +630,59 @@ async function probeExternalMcp(transport: RecoveryHttpTransport, url: string):
   } finally { clearTimeout(timer); }
 }
 
+function publicGatewayReadinessEndpoint(endpoint: string): string {
+  const url = new URL(endpoint);
+  url.pathname = '/ready';
+  url.search = '';
+  url.hash = '';
+  return url.toString();
+}
+
+async function probePublicGatewayReadiness(
+  transport: RecoveryHttpTransport,
+  endpoint: string,
+): Promise<{ ok: boolean; detail: string; status?: number; value?: unknown }> {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort('RECOVERY_HTTP_TIMEOUT'), 4_000);
+  try {
+    const response = await transport.request({
+      url: publicGatewayReadinessEndpoint(endpoint),
+      headers: { accept: 'application/json' },
+      timeoutMs: 4_000,
+      signal: controller.signal,
+    });
+    let sessionCapacity: unknown;
+    try {
+      const payload = JSON.parse(response.body) as { sessionCapacity?: unknown };
+      sessionCapacity = payload.sessionCapacity;
+    } catch { /* malformed readiness payload remains a failed/opaque probe */ }
+    const recoveryRecommended = Boolean(
+      sessionCapacity
+      && typeof sessionCapacity === 'object'
+      && (sessionCapacity as { recoveryRecommended?: unknown }).recoveryRecommended === true,
+    );
+    return {
+      ok: response.ok && !recoveryRecommended,
+      detail: `HTTP ${response.status}${recoveryRecommended ? '; session capacity recommends Connector recovery' : ''}`,
+      status: response.status,
+      ...(sessionCapacity === undefined ? {} : { value: sessionCapacity }),
+    };
+  } catch (error) {
+    return { ok: false, detail: error instanceof Error ? error.message.slice(0, 180) : 'request failed' };
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+function connectorCapacityRecoveryRecommended(verified: VerifyResult): boolean {
+  const value = verified.probes.primary_connector_ready?.value;
+  return Boolean(
+    value
+    && typeof value === 'object'
+    && (value as { recoveryRecommended?: unknown }).recoveryRecommended === true,
+  );
+}
+
 function mainToken(config: RecoveryConfig): string | undefined {
   const candidate = config.mainMcpTokenFile ?? join(config.controllerHome, 'mcp', 'mcp.tokens.json');
   const parsed = json<{ bearerToken?: unknown }>(candidate);
@@ -753,8 +853,23 @@ export async function verifyStableRuntime(
   probes.active_gateway = endpoint
     ? await probe(transport, runtimeHealthEndpoint(endpoint))
     : { ok: false, detail: 'canonical Runtime endpoint is unavailable' };
-  if (config.publicMcpUrl) probes.external_mcp_http = await probeExternalMcp(transport, config.publicMcpUrl);
+  if (config.publicMcpUrl) {
+    probes.external_mcp_http = await probeExternalMcp(transport, config.publicMcpUrl);
+    probes.primary_connector_ready = await probePublicGatewayReadiness(transport, config.publicMcpUrl);
+  }
   if (config.gateway) probes.recovery_gateway = await probe(transport, `http://${config.gateway.host}:${config.gateway.port}/health`);
+  const watchdogHealth = observeRecoveryWatchdogHealth(config.controllerHome);
+  probes.recovery_watchdog = {
+    ok: watchdogHealth.ok,
+    detail: watchdogHealth.detail,
+    value: {
+      pulseAgeMs: watchdogHealth.pulseAgeMs,
+      tickAgeMs: watchdogHealth.tickAgeMs,
+      currentReleaseRevision: watchdogHealth.currentReleaseRevision,
+      watchdogReleaseRevision: watchdogHealth.runtimeIdentity?.releaseRevision,
+      watchdogPid: watchdogHealth.runtimeIdentity?.pid,
+    },
+  };
   const recoveryPublicUrl = configuredRecoveryPublicUrl(config);
   if (recoveryPublicUrl) probes.recovery_external_http = await probeExternalMcp(transport, recoveryPublicUrl);
   if (options.probeMcpProtocol !== false) Object.assign(probes, await probeMcp(config, transport));
@@ -801,10 +916,8 @@ export async function verifyStableRuntime(
   return result;
 }
 /** Explicitly records evidence only after the full independent verification passed. */
-export async function attestKnownGood(config: RecoveryConfig): Promise<ReleaseEvidence> {
-  const locked = await withLock(config, { action: 'attest_known_good' }, async () => {
-    const verified = await verifyStableRuntime(config);
-    const authority = releaseAuthority(config);
+function persistVerifiedKnownGood(config: RecoveryConfig, verified: VerifyResult): ReleaseEvidence {
+  const authority = releaseAuthority(config);
     const active = verified.releases.active;
     if (
       !verified.ok
@@ -825,7 +938,10 @@ export async function attestKnownGood(config: RecoveryConfig): Promise<ReleaseEv
       attestedAt: new Date().toISOString(),
     };
     const store = knownGood(config);
-    const releases = [attested, ...store.releases.filter((entry) => entry.path !== active.path)].slice(0, 8);
+    const releases = [
+      attested,
+      ...store.releases.filter((entry) => entry.path !== active.path && existsSync(entry.path)),
+    ].slice(0, 8);
     writeJson(statePath(config), { schemaVersion: 1, releases, updatedAt: new Date().toISOString() } satisfies KnownGoodStore);
     audit(config, 'known_good_attested', {
       revision: active.revision,
@@ -833,8 +949,13 @@ export async function attestKnownGood(config: RecoveryConfig): Promise<ReleaseEv
       manifestSha256: active.manifestSha256,
       releaseAuthorityRevision: authority.revision,
     });
-    return attested;
-  });
+  return attested;
+}
+
+export async function attestKnownGood(config: RecoveryConfig): Promise<ReleaseEvidence> {
+  const locked = await withLock(config, { action: 'attest_known_good' }, async () => (
+    persistVerifiedKnownGood(config, await verifyStableRuntime(config))
+  ));
   if (!locked.acquired) throw new Error(recoveryBusyDetail(locked.owner));
   return locked.value;
 }
@@ -849,8 +970,8 @@ function quarantine(config: RecoveryConfig, release: ReleaseEvidence | undefined
 async function rollbackPreviousLocked(config: RecoveryConfig, reason: string): Promise<RollbackResult> {
   const before = await verifyStableRuntime(config);
   const active = before.releases.active;
-  if (matchingKnownGood(config, active)) {
-    return { ok: true, noOp: true, detail: 'active whole-Runtime release is already independently attested known-good', verify: before };
+  if (before.ok && matchingKnownGood(config, active)) {
+    return { ok: true, noOp: true, detail: 'active whole-Runtime release is currently healthy and independently attested known-good', verify: before };
   }
   if (before.runtime.running) {
     return {
@@ -911,7 +1032,9 @@ export async function reconnectMain(config: RecoveryConfig): Promise<{ ok: boole
   // intentionally a bounded health/reconnect observation, never a rollout.
   const verified = await verifyStableRuntime(config);
   const publicProbe = verified.probes.external_mcp_http;
-  const ok = verified.probes.active_gateway?.ok === true && (publicProbe?.ok === true || publicProbe?.status === 401);
+  const ok = verified.probes.active_gateway?.ok === true
+    && verified.probes.primary_connector_ready?.ok !== false
+    && (publicProbe?.ok === true || publicProbe?.status === 401);
   audit(config, 'reconnect_main', { ok, externalStatus: publicProbe?.status });
   return { ok, detail: ok ? 'canonical Runtime Gateway and primary endpoint are reachable; client session may refresh' : 'primary endpoint remains unavailable; recovery channel remains independent', verify: verified };
 }
@@ -1037,6 +1160,16 @@ export function decideWatchdog(input: {
   rollbackUsed: boolean;
   recoveryGatewayFailed?: boolean;
   recoveryGatewayRestartUsed?: boolean;
+  primaryConnectorConfigured?: boolean;
+  primaryConnectorFailed?: boolean;
+  primaryConnectorFailures?: number;
+  primaryConnectorFirstFailureAt?: number;
+  primaryConnectorRestartAttempts?: number;
+  primaryConnectorMaximumRestartAttempts?: number;
+  primaryConnectorRestartLastAttemptAt?: number;
+  primaryConnectorRestartCooldownMs?: number;
+  primaryConnectorMinimumFailures?: number;
+  primaryConnectorMinimumFailureDurationMs?: number;
   primaryRuntimeFailed?: boolean;
   runtimeRestartAttempts?: number;
   runtimeMaximumRestartAttempts?: number;
@@ -1064,6 +1197,22 @@ export function decideWatchdog(input: {
     if (failures >= minimumFailures && sustained) return { action: 'repair_public_tunnel', reason: 'local runtime and Recovery Gateway are healthy while the dedicated Recovery public endpoint is unavailable' };
     return { action: 'degraded', reason: 'Recovery tunnel failure has not yet met the bounded repair threshold' };
   }
+  if (input.primaryConnectorConfigured && input.primaryConnectorFailed) {
+    const failures = input.primaryConnectorFailures ?? 0;
+    const minimumFailures = Math.max(1, input.primaryConnectorMinimumFailures ?? 2);
+    const minimumDurationMs = Math.max(0, input.primaryConnectorMinimumFailureDurationMs ?? 5_000);
+    const sustained = input.primaryConnectorFirstFailureAt !== undefined && now - input.primaryConnectorFirstFailureAt >= minimumDurationMs;
+    const attempts = input.primaryConnectorRestartAttempts ?? 0;
+    const maximumAttempts = Math.max(1, input.primaryConnectorMaximumRestartAttempts ?? 3);
+    const cooldownMs = Math.max(0, input.primaryConnectorRestartCooldownMs ?? 30_000);
+    const cooldownElapsed = input.primaryConnectorRestartLastAttemptAt === undefined || now - input.primaryConnectorRestartLastAttemptAt >= cooldownMs;
+    if (failures >= minimumFailures && sustained && attempts < maximumAttempts && cooldownElapsed) {
+      return { action: 'restart_primary_connector', reason: `local Runtime is healthy but the primary Connector endpoint is unavailable; restart attempt ${attempts + 1}/${maximumAttempts}` };
+    }
+    return { action: 'degraded', reason: attempts >= maximumAttempts
+      ? `primary Connector restart budget exhausted (${attempts}/${maximumAttempts}); keeping the local Runtime available for independent Recovery`
+      : 'primary Connector failure has not yet met its bounded restart threshold or cooldown' };
+  }
   if (input.failures === 0) return { action: 'healthy', reason: 'all recovery probes healthy' };
   const recoveryRestartSustained = input.firstFailureAt !== undefined && now - input.firstFailureAt >= 5_000;
   if (input.failures >= 2 && recoveryRestartSustained && input.recoveryGatewayFailed && !input.recoveryGatewayRestartUsed) {
@@ -1258,6 +1407,9 @@ export interface PrimaryConnectorRecoveryDependencies {
   runCommand?: CommandRunner;
   verifyLocal?: (config: RecoveryConfig) => Promise<VerifyResult>;
   reconnect?: (config: RecoveryConfig) => Promise<{ ok: boolean; detail: string; verify: VerifyResult }>;
+  probeConnectorLocal?: (config: RecoveryConfig) => Promise<{ ok: boolean; detail: string; status?: number } | undefined>;
+  probeConnectorOwnership?: (config: RecoveryConfig) => Promise<{ ok: boolean; detail: string }>;
+  repairConnectorBinding?: (config: RecoveryConfig) => Promise<{ ok: boolean; attempted: boolean; noOp?: boolean; detail: string }>;
   now?: () => number;
   sleep?: (ms: number) => Promise<void>;
 }
@@ -1272,6 +1424,79 @@ function primaryConnectorLaunchdService(config: RecoveryConfig, uid: number): La
   return { uid, domain, target: `${domain}/${label}`, label, plistPath };
 }
 
+async function probePrimaryConnectorLocal(config: RecoveryConfig): Promise<{ ok: boolean; detail: string; status?: number } | undefined> {
+  const localMcpUrl = config.primaryConnectorService?.localMcpUrl?.trim();
+  if (!localMcpUrl) return undefined;
+  return probeExternalMcp(createRecoveryHttpTransport(config.controllerHome), localMcpUrl);
+}
+
+function activePackageConnectorReleaseBinding(config: RecoveryConfig): PackageConnectorReleaseBinding | undefined {
+  const configured = config.primaryConnectorService;
+  if (!configured || configured.platform !== 'launchd' || !configured.localMcpUrl?.trim()) return undefined;
+  const authority = readRuntimeReleaseAuthority(config.controllerHome);
+  if (!authority) return undefined;
+  const releaseRoot = dirname(resolve(authority.active.manifestPath));
+  const packageRoot = join(releaseRoot, 'package');
+  const paths = packageConnectorServicePaths(config.controllerHome);
+  const configuredPlist = resolve(configured.plistPath ?? paths.installedPlistPath);
+  if (configured.label !== paths.label || configuredPlist !== resolve(paths.installedPlistPath)) return undefined;
+  if (!existsSync(join(packageRoot, 'src', 'cli', 'index.ts')) || !existsSync(join(packageRoot, 'src', 'runtime', 'shared', 'node-ts-loader.mjs'))) return undefined;
+  return { releaseId: authority.active.releaseId, releaseRoot, packageRoot };
+}
+
+async function repairPrimaryConnectorBinding(config: RecoveryConfig): Promise<{ ok: boolean; attempted: boolean; noOp?: boolean; detail: string }> {
+  const release = activePackageConnectorReleaseBinding(config);
+  const endpoint = config.primaryConnectorService?.localMcpUrl?.trim();
+  if (!release || !endpoint) return { ok: true, attempted: false, noOp: true, detail: 'primary Connector does not use the canonical package-release binding contract' };
+  try {
+    const result = await ensurePackageConnectorService({ release, controllerHome: config.controllerHome, endpoint, platform: 'darwin' });
+    const attempted = result.reused !== true;
+    return {
+      ok: true,
+      attempted,
+      ...(attempted ? {} : { noOp: true }),
+      detail: attempted
+        ? `primary Connector binding repaired to active immutable Runtime release ${release.releaseId}`
+        : `primary Connector binding already matches active immutable Runtime release ${release.releaseId}`,
+    };
+  } catch (error) {
+    return { ok: false, attempted: true, detail: error instanceof Error ? error.message : 'primary Connector binding repair failed' };
+  }
+}
+
+async function probePrimaryConnectorOwnership(
+  config: RecoveryConfig,
+  service: LaunchdService,
+  runCommand: CommandRunner = command,
+): Promise<{ ok: boolean; detail: string }> {
+  const localMcpUrl = config.primaryConnectorService?.localMcpUrl?.trim();
+  if (!localMcpUrl) return { ok: false, detail: 'primary Connector local MCP endpoint is not configured' };
+  let port: number;
+  try {
+    const parsed = new URL(localMcpUrl);
+    const rawPort = parsed.port || (parsed.protocol === 'https:' ? '443' : parsed.protocol === 'http:' ? '80' : '');
+    port = Number(rawPort);
+    if (!Number.isInteger(port) || port < 1 || port > 65_535) throw new Error('invalid port');
+  } catch {
+    return { ok: false, detail: `primary Connector local MCP endpoint has no verifiable TCP port: ${localMcpUrl}` };
+  }
+  const printed = await runCommand('launchctl', ['print', service.target], 5_000);
+  const pidMatch = printed.ok ? printed.stdout.match(/\bpid\s*=\s*(\d+)/) : null;
+  const pid = pidMatch ? Number(pidMatch[1]) : undefined;
+  if (!pid || !Number.isInteger(pid)) {
+    return { ok: false, detail: `configured primary Connector launchd service has no live pid: ${service.target}` };
+  }
+  const listening = await runCommand('lsof', ['-nP', '-a', '-p', String(pid), `-iTCP:${port}`, '-sTCP:LISTEN', '-t'], 5_000);
+  const listenerPids = listening.stdout.split(/\s+/).map((value) => Number(value)).filter(Number.isInteger);
+  const ok = listening.ok && listenerPids.includes(pid);
+  return {
+    ok,
+    detail: ok
+      ? `configured primary Connector pid ${pid} owns TCP ${port}`
+      : `configured primary Connector pid ${pid} does not own TCP ${port}`,
+  };
+}
+
 export async function restartPrimaryConnector(
   config: RecoveryConfig,
   dependencies: PrimaryConnectorRecoveryDependencies = {},
@@ -1296,21 +1521,107 @@ export async function restartPrimaryConnector(
     return { ok: false, attempted: false, noOp: true, detail: 'primary Connector launchd service is not configured or installed', verify: initialLocal };
   }
   const reconnect = dependencies.reconnect ?? reconnectMain;
+  const probeConnectorLocal = dependencies.probeConnectorLocal ?? probePrimaryConnectorLocal;
+  const runCommand = dependencies.runCommand ?? command;
+  const probeConnectorOwnership = dependencies.probeConnectorOwnership
+    ?? ((candidateConfig: RecoveryConfig) => probePrimaryConnectorOwnership(candidateConfig, service, runCommand));
   const now = dependencies.now ?? Date.now;
   const wait = dependencies.sleep ?? sleep;
   const timeoutMs = config.primaryConnectorService?.postRestartVerifyTimeoutMs ?? 30_000;
   const locked = await withLock(config, { action: 'restart_primary_connector' }, async () => {
-    const restarted = await ensureLaunchdServiceStarted(service, dependencies.runCommand ?? command);
-    if (!restarted.ok) {
-      audit(config, 'primary_connector_restart_failed', { serviceTarget: service.target, detail: restarted.detail });
-      return { ok: false, attempted: true, detail: restarted.detail, serviceTarget: service.target, verify: initialLocal } satisfies PrimaryConnectorRestartResult;
+    const tunnelConfigured = Boolean(configuredPrimaryPublicTunnel(config));
+    const repairConnectorBinding = dependencies.repairConnectorBinding ?? repairPrimaryConnectorBinding;
+    const bindingRepair = await repairConnectorBinding(config);
+    if (bindingRepair.attempted) {
+      audit(config, bindingRepair.ok ? 'primary_connector_binding_repaired' : 'primary_connector_binding_repair_failed', { detail: bindingRepair.detail });
     }
-    const deadline = now() + timeoutMs;
-    let observed = await reconnect(config);
-    while (!observed.ok && now() < deadline) {
-      await wait(1_000);
-      observed = await reconnect(config);
+    if (!bindingRepair.ok) {
+      return { ok: false, attempted: true, detail: `primary Connector immutable release binding repair failed: ${bindingRepair.detail}`, serviceTarget: service.target, verify: initialLocal } satisfies PrimaryConnectorRestartResult;
+    }
+    let localConnector = await probeConnectorLocal(config);
+    let connectorOwnership = localConnector?.ok ? await probeConnectorOwnership(config) : undefined;
+    let observed = localConnector?.ok && connectorOwnership?.ok ? await reconnect(config) : undefined;
+    if (observed?.ok) {
+      return {
+        ok: true,
+        attempted: bindingRepair.attempted,
+        ...(bindingRepair.attempted ? {} : { noOp: true }),
+        detail: bindingRepair.attempted
+          ? 'primary Connector immutable release binding was repaired and the public MCP endpoint recovered'
+          : 'primary Connector and public MCP endpoint recovered before restart',
+        serviceTarget: service.target,
+        verify: observed.verify,
+      } satisfies PrimaryConnectorRestartResult;
+    }
+
+    // A healthy local OAuth endpoint proves the Connector is not the broken hop.
+    // If a distinct primary tunnel is configured, skip the pointless Gateway
+    // restart and repair the public hop directly.
+    const restartConnector = !localConnector?.ok || !connectorOwnership?.ok || !tunnelConfigured;
+    if (restartConnector) {
+      const restarted = await ensureLaunchdServiceStarted(service, runCommand);
+      if (!restarted.ok) {
+        audit(config, 'primary_connector_restart_failed', { serviceTarget: service.target, detail: restarted.detail });
+        return { ok: false, attempted: true, detail: restarted.detail, serviceTarget: service.target, verify: initialLocal } satisfies PrimaryConnectorRestartResult;
+      }
+
+      const localDeadline = now() + timeoutMs;
+      localConnector = await probeConnectorLocal(config);
+      while (localConnector && !localConnector.ok && now() < localDeadline) {
+        await wait(1_000);
+        localConnector = await probeConnectorLocal(config);
+      }
+      connectorOwnership = localConnector?.ok ? await probeConnectorOwnership(config) : undefined;
+      if (localConnector && (!localConnector.ok || !connectorOwnership?.ok)) {
+        const detail = localConnector.ok
+          ? (connectorOwnership?.detail ?? 'configured primary Connector listener ownership could not be verified')
+          : localConnector.detail;
+        audit(config, 'primary_connector_restart_local_unverified', { serviceTarget: service.target, detail });
+        return {
+          ok: false,
+          attempted: true,
+          detail: localConnector.ok
+            ? 'primary Connector restarted but the configured launchd service does not own its local MCP listener'
+            : 'primary Connector restarted but its local MCP endpoint did not recover before timeout',
+          serviceTarget: service.target,
+          verify: initialLocal,
+        } satisfies PrimaryConnectorRestartResult;
+      }
     }
+
+    observed ??= await reconnect(config);
+    if (!observed.ok && localConnector?.ok && connectorOwnership?.ok && configuredPrimaryPublicTunnel(config)) {
+      const tunnel = primaryPublicTunnelService(config, service.uid);
+      if (tunnel) {
+        const tunnelRestarted = await ensureLaunchdServiceStarted(tunnel, dependencies.runCommand ?? command);
+        if (!tunnelRestarted.ok) {
+          audit(config, 'primary_public_tunnel_restart_failed', { serviceTarget: tunnel.target, detail: tunnelRestarted.detail });
+          return {
+            ok: false,
+            attempted: true,
+            detail: `primary Connector is locally healthy but public tunnel restart failed: ${tunnelRestarted.detail}`,
+            serviceTarget: service.target,
+            verify: observed.verify,
+          } satisfies PrimaryConnectorRestartResult;
+        }
+        const tunnelDeadline = now() + (configuredPrimaryPublicTunnel(config)?.postRestartVerifyTimeoutMs ?? 20_000);
+        while (!observed.ok && now() < tunnelDeadline) {
+          await wait(1_000);
+          observed = await reconnect(config);
+        }
+        audit(config, observed.ok ? 'primary_public_tunnel_restart_succeeded' : 'primary_public_tunnel_restart_unverified', {
+          serviceTarget: tunnel.target,
+          detail: observed.detail,
+        });
+      }
+    } else {
+      const deadline = now() + timeoutMs;
+      while (!observed.ok && now() < deadline) {
+        await wait(1_000);
+        observed = await reconnect(config);
+      }
+    }
+
     audit(config, observed.ok ? 'primary_connector_restart_succeeded' : 'primary_connector_restart_unverified', {
       serviceTarget: service.target,
       detail: observed.detail,
@@ -1318,7 +1629,9 @@ export async function restartPrimaryConnector(
     return {
       ok: observed.ok,
       attempted: true,
-      detail: observed.ok ? 'primary Connector restarted and the public MCP endpoint is reachable' : 'primary Connector restarted but the public MCP endpoint did not recover before timeout',
+      detail: observed.ok
+        ? 'primary Connector recovery completed and the public MCP endpoint is reachable'
+        : 'primary Connector recovery completed but the public MCP endpoint did not recover before timeout',
       serviceTarget: service.target,
       verify: observed.verify,
     } satisfies PrimaryConnectorRestartResult;
@@ -1362,6 +1675,44 @@ async function waitForLaunchdServiceUnloaded(input: {
   return !printed.ok;
 }
 
+interface PrimaryRuntimePortObservation {
+  port: number;
+  pids: number[];
+  uncertain?: string;
+}
+
+interface PrimaryRuntimePortCleanupResult {
+  released: boolean;
+  cleaned: boolean;
+  detail: string;
+  pid?: number;
+  signal?: 'SIGTERM' | 'SIGKILL';
+}
+
+async function observePrimaryRuntimePort(config: RecoveryConfig, runCommand: CommandRunner): Promise<PrimaryRuntimePortObservation> {
+  const paths = forgeRuntimeServicePaths(config.controllerHome);
+  let port: number;
+  try {
+    port = readForgeRuntimeServiceConfig(paths.configPath).port;
+  } catch {
+    return { port: -1, pids: [], uncertain: 'primary Runtime service port could not be read' };
+  }
+  const result = await runCommand('lsof', ['-nP', '-Fp', `-iTCP:${port}`, '-sTCP:LISTEN'], 3_000);
+  if (result.status === 1 && result.stderr.trim().length === 0) return { port, pids: [] };
+  if (!result.ok) {
+    return { port, pids: [], uncertain: `primary Runtime listener observation failed: ${result.stderr || result.stdout || result.status}` };
+  }
+  const pids = [...new Set(result.stdout.split(/\r?\n/)
+    .map((line) => /^p(\d+)$/.exec(line.trim())?.[1])
+    .filter((value): value is string => Boolean(value))
+    .map(Number)
+    .filter((value) => Number.isInteger(value) && value > 0))];
+  if (result.stdout.trim().length > 0 && pids.length === 0) {
+    return { port, pids: [], uncertain: 'primary Runtime listener identity was not machine-readable' };
+  }
+  return { port, pids };
+}
+
 async function waitForPrimaryRuntimePortReleased(input: {
   config: RecoveryConfig;
   timeoutMs: number;
@@ -1369,25 +1720,102 @@ async function waitForPrimaryRuntimePortReleased(input: {
   wait: (ms: number) => Promise<void>;
   runCommand: CommandRunner;
 }): Promise<boolean> {
-  const paths = forgeRuntimeServicePaths(input.config.controllerHome);
-  let port: number;
-  try {
-    port = readForgeRuntimeServiceConfig(paths.configPath).port;
-  } catch {
-    return false;
-  }
-  const isListening = async (): Promise<boolean> => {
-    const result = await input.runCommand('lsof', ['-nP', `-iTCP:${port}`, '-sTCP:LISTEN'], 3_000);
-    if (result.ok) return result.stdout.trim().length > 0;
-    if (result.status === 1 && result.stderr.trim().length === 0) return false;
-    return true;
-  };
   const deadline = input.now() + input.timeoutMs;
   while (input.now() < deadline) {
-    if (!(await isListening())) return true;
+    const observation = await observePrimaryRuntimePort(input.config, input.runCommand);
+    if (!observation.uncertain && observation.pids.length === 0) return true;
     await input.wait(250);
   }
-  return !(await isListening());
+  const observation = await observePrimaryRuntimePort(input.config, input.runCommand);
+  return !observation.uncertain && observation.pids.length === 0;
+}
+
+function commandLineHasArgument(commandLine: string, flag: string, value: string): boolean {
+  const candidates = [
+    `${flag} ${value}`,
+    `${flag} "${value}"`,
+    `${flag} '${value}'`,
+  ];
+  return candidates.some((candidate) => {
+    const index = commandLine.indexOf(candidate);
+    if (index < 0) return false;
+    const before = index === 0 ? '' : commandLine[index - 1];
+    const afterIndex = index + candidate.length;
+    const after = afterIndex >= commandLine.length ? '' : commandLine[afterIndex];
+    return (!before || /\s/.test(before)) && (!after || /\s/.test(after));
+  });
+}
+
+async function verifyPrimaryRuntimeListenerIdentity(input: {
+  config: RecoveryConfig;
+  pid: number;
+  uid: number;
+  port: number;
+  runCommand: CommandRunner;
+}): Promise<{ ok: boolean; detail: string }> {
+  const paths = forgeRuntimeServicePaths(input.config.controllerHome);
+  const physicalEntrypoint = activeRuntimeEntrypoint(input.config.controllerHome);
+  const launchSpec = activeRuntimeLaunchSpec(input.config.controllerHome);
+  if (!physicalEntrypoint || !launchSpec) return { ok: false, detail: 'active Runtime release identity is unavailable' };
+  const manifestIndex = launchSpec.args.indexOf('--release-manifest');
+  const manifestPath = manifestIndex >= 0 ? launchSpec.args[manifestIndex + 1] : undefined;
+  if (!manifestPath) return { ok: false, detail: 'active Runtime release manifest identity is unavailable' };
+
+  const result = await input.runCommand('ps', ['-ww', '-p', String(input.pid), '-o', 'uid=', '-o', 'command='], 3_000);
+  if (!result.ok) return { ok: false, detail: `listener process identity could not be read: ${result.stderr || result.stdout || result.status}` };
+  const line = result.stdout.trim().split(/\r?\n/).find(Boolean) ?? '';
+  const match = /^\s*(\d+)\s+(.+)$/.exec(line);
+  if (!match) return { ok: false, detail: 'listener process identity was not machine-readable' };
+  const observedUid = Number(match[1]);
+  const commandLine = match[2]!.trim();
+  if (observedUid !== input.uid) return { ok: false, detail: `listener uid ${observedUid} does not match Recovery uid ${input.uid}` };
+  const executableMatches = [resolve(paths.activeEntrypointPath), resolve(physicalEntrypoint)]
+    .some((candidate) => commandLine === candidate || commandLine.startsWith(`${candidate} `));
+  if (!executableMatches) return { ok: false, detail: 'listener executable is not the active Forge Runtime entrypoint' };
+  if (!commandLineHasArgument(commandLine, '--controller-home', resolve(input.config.controllerHome))) {
+    return { ok: false, detail: 'listener controller-home identity does not match' };
+  }
+  if (!commandLineHasArgument(commandLine, '--port', String(input.port))) {
+    return { ok: false, detail: 'listener port identity does not match' };
+  }
+  if (!commandLineHasArgument(commandLine, '--release-manifest', resolve(manifestPath))) {
+    return { ok: false, detail: 'listener release-manifest identity does not match' };
+  }
+  return { ok: true, detail: 'listener matches the current Forge Runtime release and controller identity' };
+}
+
+async function cleanupVerifiedStalePrimaryRuntimeListener(input: {
+  config: RecoveryConfig;
+  uid: number;
+  now: () => number;
+  wait: (ms: number) => Promise<void>;
+  runCommand: CommandRunner;
+}): Promise<PrimaryRuntimePortCleanupResult> {
+  const initial = await observePrimaryRuntimePort(input.config, input.runCommand);
+  if (initial.uncertain) return { released: false, cleaned: false, detail: initial.uncertain };
+  if (initial.pids.length === 0) return { released: true, cleaned: false, detail: 'primary Runtime port is already released' };
+  if (initial.pids.length !== 1) return { released: false, cleaned: false, detail: `primary Runtime port has ${initial.pids.length} listeners; cleanup requires exactly one` };
+  const pid = initial.pids[0]!;
+  const identity = await verifyPrimaryRuntimeListenerIdentity({ config: input.config, pid, uid: input.uid, port: initial.port, runCommand: input.runCommand });
+  if (!identity.ok) return { released: false, cleaned: false, detail: identity.detail, pid };
+
+  const term = await input.runCommand('kill', ['-TERM', String(pid)], 3_000);
+  const releasedAfterTerm = await waitForPrimaryRuntimePortReleased({ ...input, timeoutMs: 5_000 });
+  if (releasedAfterTerm) return { released: true, cleaned: true, detail: 'verified stale Forge Runtime listener released after SIGTERM', pid, signal: 'SIGTERM' };
+  if (!term.ok) return { released: false, cleaned: false, detail: `verified listener SIGTERM failed: ${term.stderr || term.stdout || term.status}`, pid };
+
+  const beforeKill = await observePrimaryRuntimePort(input.config, input.runCommand);
+  if (beforeKill.uncertain || beforeKill.pids.length !== 1 || beforeKill.pids[0] !== pid) {
+    return { released: false, cleaned: false, detail: 'listener identity changed after SIGTERM; refusing SIGKILL', pid };
+  }
+  const identityBeforeKill = await verifyPrimaryRuntimeListenerIdentity({ config: input.config, pid, uid: input.uid, port: beforeKill.port, runCommand: input.runCommand });
+  if (!identityBeforeKill.ok) return { released: false, cleaned: false, detail: `listener identity changed after SIGTERM: ${identityBeforeKill.detail}`, pid };
+  const killed = await input.runCommand('kill', ['-KILL', String(pid)], 3_000);
+  if (!killed.ok) return { released: false, cleaned: false, detail: `verified listener SIGKILL failed: ${killed.stderr || killed.stdout || killed.status}`, pid };
+  const releasedAfterKill = await waitForPrimaryRuntimePortReleased({ ...input, timeoutMs: 5_000 });
+  return releasedAfterKill
+    ? { released: true, cleaned: true, detail: 'verified stale Forge Runtime listener released after SIGKILL', pid, signal: 'SIGKILL' }
+    : { released: false, cleaned: false, detail: 'verified stale Forge Runtime listener remained on the port after SIGKILL', pid };
 }
 
 async function verifyPrimaryRuntimeAfterStart(input: {
@@ -1489,10 +1917,23 @@ export async function recoverPrimaryRuntime(
       return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies PrimaryRuntimeRecoveryResult;
     }
 
-    const portReleased = await waitForPrimaryRuntimePortReleased({ config, timeoutMs: 20_000, now, wait, runCommand });
+    let portReleased = await waitForPrimaryRuntimePortReleased({ config, timeoutMs: 20_000, now, wait, runCommand });
+    let staleListenerCleanup: PrimaryRuntimePortCleanupResult | undefined;
     if (!portReleased) {
-      const detail = 'primary Runtime TCP port remained occupied after bounded launchd bootout';
-      audit(config, 'primary_runtime_recovery_stop_unverified', { serviceTarget: service.target, detail });
+      staleListenerCleanup = await cleanupVerifiedStalePrimaryRuntimeListener({ config, uid: service.uid, now, wait, runCommand });
+      portReleased = staleListenerCleanup.released;
+      if (staleListenerCleanup.cleaned) {
+        audit(config, 'primary_runtime_recovery_stale_listener_cleaned', {
+          serviceTarget: service.target,
+          pid: staleListenerCleanup.pid,
+          signal: staleListenerCleanup.signal,
+          detail: staleListenerCleanup.detail,
+        });
+      }
+    }
+    if (!portReleased) {
+      const detail = `primary Runtime TCP port remained occupied after bounded launchd bootout: ${staleListenerCleanup?.detail ?? 'listener did not release'}`;
+      audit(config, 'primary_runtime_recovery_stop_unverified', { serviceTarget: service.target, detail, pid: staleListenerCleanup?.pid });
       return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies PrimaryRuntimeRecoveryResult;
     }
 
@@ -1668,7 +2109,25 @@ export async function activateRuntimeRelease(
       } satisfies RuntimeReleaseActivationResult;
     }
     if (current) {
-      if (runtimeBehaviorEquivalent(current.active.manifestPath, candidate.manifestPath) && !storageMigrationRequired) {
+      const behaviorEquivalent = runtimeBehaviorEquivalent(current.active.manifestPath, candidate.manifestPath);
+      let serviceContractMatches = false;
+      if (behaviorEquivalent && !storageMigrationRequired) {
+        try {
+          serviceContractMatches = inspectForgeRuntimeLaunchAgentContract({
+            controllerHome: config.controllerHome,
+            inspectUserLaunchAgent: true,
+          }).matches;
+        } catch (error) {
+          audit(config, 'runtime_release_activation_service_contract_inspection_failed', {
+            operationId,
+            requestId: lockRequestId,
+            activeRevision: current.active.releaseId,
+            candidateRevision: candidate.manifest.releaseId,
+            detail: error instanceof Error ? error.message : String(error),
+          });
+        }
+      }
+      if (behaviorEquivalent && !storageMigrationRequired && serviceContractMatches) {
         const verify = await verifyLocal(config);
         audit(config, 'runtime_release_activation_behavior_identical', {
           operationId,
@@ -1682,8 +2141,8 @@ export async function activateRuntimeRelease(
           attempted: false,
           noOp: true,
           detail: verify.ok
-            ? 'candidate Runtime behavior is identical to the active release; restart skipped'
-            : 'candidate Runtime behavior is identical to the active release, but the active Runtime is unhealthy',
+            ? 'candidate Runtime behavior and launchd service contract are identical to the active release; restart skipped'
+            : 'candidate Runtime behavior and launchd service contract are identical to the active release, but the active Runtime is unhealthy',
           operationId,
           verify,
         } satisfies RuntimeReleaseActivationResult;
@@ -1720,10 +2179,24 @@ export async function activateRuntimeRelease(
       audit(config, 'runtime_release_activation_stop_unverified', { serviceTarget: service.target, operationId, detail });
       return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies RuntimeReleaseActivationResult;
     }
-    const portReleased = await waitForPrimaryRuntimePortReleased({ config, timeoutMs: 20_000, now, wait, runCommand });
+    let portReleased = await waitForPrimaryRuntimePortReleased({ config, timeoutMs: 20_000, now, wait, runCommand });
+    let staleListenerCleanup: PrimaryRuntimePortCleanupResult | undefined;
     if (!portReleased) {
-      const detail = 'primary Runtime TCP port remained occupied after bounded launchd bootout';
-      audit(config, 'runtime_release_activation_stop_unverified', { serviceTarget: service.target, operationId, detail });
+      staleListenerCleanup = await cleanupVerifiedStalePrimaryRuntimeListener({ config, uid: service.uid, now, wait, runCommand });
+      portReleased = staleListenerCleanup.released;
+      if (staleListenerCleanup.cleaned) {
+        audit(config, 'runtime_release_activation_stale_listener_cleaned', {
+          serviceTarget: service.target,
+          operationId,
+          pid: staleListenerCleanup.pid,
+          signal: staleListenerCleanup.signal,
+          detail: staleListenerCleanup.detail,
+        });
+      }
+    }
+    if (!portReleased) {
+      const detail = `primary Runtime TCP port remained occupied after bounded launchd bootout: ${staleListenerCleanup?.detail ?? 'listener did not release'}`;
+      audit(config, 'runtime_release_activation_stop_unverified', { serviceTarget: service.target, operationId, detail, pid: staleListenerCleanup?.pid });
       return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies RuntimeReleaseActivationResult;
     }
     let committed: RuntimeReleaseAuthority;
@@ -1740,7 +2213,7 @@ export async function activateRuntimeRelease(
       return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies RuntimeReleaseActivationResult;
     }
     try {
-      ensureForgeRuntimeLaunchAgentContract({ controllerHome: config.controllerHome });
+      ensureForgeRuntimeLaunchAgentContract({ controllerHome: config.controllerHome, installUserLaunchAgent: true });
     } catch (error) {
       const detail = `runtime launchd stable-release contract update failed: ${error instanceof Error ? error.message : String(error)}`;
       audit(config, 'runtime_release_activation_launchd_contract_failed', { serviceTarget: service.target, operationId, detail });
@@ -1821,7 +2294,7 @@ export async function activateRuntimeRelease(
             physicalHome: storageMigration.physicalHome,
           });
         }
-        ensureForgeRuntimeLaunchAgentContract({ controllerHome: config.controllerHome });
+        ensureForgeRuntimeLaunchAgentContract({ controllerHome: config.controllerHome, installUserLaunchAgent: true });
         const previousRevision = previousActive?.releaseId ?? '';
         const previousIdentity = previousActive?.artifactIdentity;
         if (
@@ -1933,8 +2406,7 @@ export async function stageAndActivateConfiguredRuntimeRelease(
   };
 }
 
-function recoveryTunnelService(config: RecoveryConfig, uid: number): LaunchdService | undefined {
-  const configured = configuredRecoveryTunnel(config);
+function tunnelLaunchdService(configured: PublicTunnelServiceConfig | undefined, uid: number): LaunchdService | undefined {
   if (!configured || configured.platform !== 'launchd') return undefined;
   if (!/^com\.[A-Za-z0-9._-]{1,180}$/.test(configured.label)) return undefined;
   const plistPath = configured.plistPath;
@@ -1948,6 +2420,14 @@ function recoveryTunnelService(config: RecoveryConfig, uid: number): LaunchdServ
   };
 }
 
+function recoveryTunnelService(config: RecoveryConfig, uid: number): LaunchdService | undefined {
+  return tunnelLaunchdService(configuredRecoveryTunnel(config), uid);
+}
+
+function primaryPublicTunnelService(config: RecoveryConfig, uid: number): LaunchdService | undefined {
+  return tunnelLaunchdService(configuredPrimaryPublicTunnel(config), uid);
+}
+
 function tunnelRepairAllowed(config: RecoveryConfig, now: number): boolean {
   const cooldownMs = configuredRecoveryTunnel(config)?.cooldownMs ?? 60_000;
   const prior = json<{ lastAttemptAt?: unknown }>(publicTunnelRepairStatePath(config));
@@ -2015,17 +2495,32 @@ export interface RecoveryGatewayRestartResult {
   serviceTarget?: string;
 }
 
-export interface RecoveryGatewayRestartDependencies {
+export interface RecoveryWatchdogRestartResult {
+  ok: boolean;
+  attempted: boolean;
+  noOp?: boolean;
+  detail: string;
+  serviceTarget?: string;
+}
+
+interface RecoveryRoleRestartDependencies {
   platform?: NodeJS.Platform;
   currentUid?: () => Promise<number | undefined>;
   runCommand?: CommandRunner;
-  probeGateway?: (config: RecoveryConfig) => Promise<{ ok: boolean; detail: string }>;
   now?: () => number;
   sleep?: (ms: number) => Promise<void>;
 }
 
-function recoveryGatewayLaunchdService(config: RecoveryConfig, uid: number): LaunchdService {
-  const label = 'com.moretea.forge-recovery-gateway';
+export interface RecoveryGatewayRestartDependencies extends RecoveryRoleRestartDependencies {
+  probeGateway?: (config: RecoveryConfig) => Promise<{ ok: boolean; detail: string }>;
+}
+
+export interface RecoveryWatchdogRestartDependencies extends RecoveryRoleRestartDependencies {
+  probeWatchdog?: (config: RecoveryConfig) => Promise<{ ok: boolean; detail: string }>;
+}
+
+function recoveryRoleLaunchdService(config: RecoveryConfig, uid: number, role: 'gateway' | 'watchdog'): LaunchdService {
+  const label = role === 'gateway' ? RECOVERY_GATEWAY_LABEL : RECOVERY_WATCHDOG_LABEL;
   const generated = join(recoveryRoot(config), 'launchd', `${label}.plist`);
   const installed = join(homedir(), 'Library', 'LaunchAgents', `${label}.plist`);
   return {
@@ -2042,46 +2537,81 @@ async function probeRecoveryGateway(config: RecoveryConfig): Promise<{ ok: boole
   return probe(createRecoveryHttpTransport(config.controllerHome), `http://${config.gateway.host}:${config.gateway.port}/health`);
 }
 
-export async function restartRecoveryGateway(
-  config: RecoveryConfig,
-  dependencies: RecoveryGatewayRestartDependencies = {},
-): Promise<RecoveryGatewayRestartResult> {
-  if (!config.gateway) return { ok: false, attempted: false, noOp: true, detail: 'Recovery Gateway is not configured' };
-  if ((dependencies.platform ?? process.platform) !== 'darwin') return { ok: false, attempted: false, noOp: true, detail: 'Recovery Gateway launchd restart is only supported on macOS' };
-  const check = dependencies.probeGateway ?? probeRecoveryGateway;
-  const initial = await check(config);
-  if (initial.ok) return { ok: true, attempted: false, noOp: true, detail: 'Recovery Gateway is already healthy' };
-  const uid = await (dependencies.currentUid ?? currentUid)();
-  if (uid === undefined) return { ok: false, attempted: false, noOp: true, detail: 'Recovery Gateway launchd UID is unavailable' };
-  const service = recoveryGatewayLaunchdService(config, uid);
-  if (!existsSync(service.plistPath)) return { ok: false, attempted: false, noOp: true, detail: `Recovery Gateway launchd plist is missing: ${service.plistPath}`, serviceTarget: service.target };
-  const locked = await withLock(config, { action: 'restart_recovery_gateway' }, async () => {
-    const before = await check(config);
-    if (before.ok) return { ok: true, attempted: false, noOp: true, detail: 'Recovery Gateway recovered before restart', serviceTarget: service.target } satisfies RecoveryGatewayRestartResult;
-    const started = await ensureLaunchdServiceStarted(service, dependencies.runCommand ?? command);
+async function probeRecoveryWatchdog(config: RecoveryConfig): Promise<{ ok: boolean; detail: string }> {
+  const health = observeRecoveryWatchdogHealth(config.controllerHome);
+  return { ok: health.ok, detail: health.detail };
+}
+
+async function restartRecoveryRole(input: {
+  config: RecoveryConfig;
+  role: 'gateway' | 'watchdog';
+  action: 'restart_recovery_gateway' | 'restart_recovery_watchdog';
+  check: (config: RecoveryConfig) => Promise<{ ok: boolean; detail: string }>;
+  dependencies: RecoveryRoleRestartDependencies;
+}): Promise<RecoveryGatewayRestartResult> {
+  const display = input.role === 'gateway' ? 'Recovery Gateway' : 'Recovery Watchdog';
+  if ((input.dependencies.platform ?? process.platform) !== 'darwin') return { ok: false, attempted: false, noOp: true, detail: `${display} launchd restart is only supported on macOS` };
+  const initial = await input.check(input.config);
+  if (initial.ok) return { ok: true, attempted: false, noOp: true, detail: `${display} is already healthy` };
+  const uid = await (input.dependencies.currentUid ?? currentUid)();
+  if (uid === undefined) return { ok: false, attempted: false, noOp: true, detail: `${display} launchd UID is unavailable` };
+  const service = recoveryRoleLaunchdService(input.config, uid, input.role);
+  if (!existsSync(service.plistPath)) return { ok: false, attempted: false, noOp: true, detail: `${display} launchd plist is missing: ${service.plistPath}`, serviceTarget: service.target };
+  const eventPrefix = input.role === 'gateway' ? 'recovery_gateway' : 'recovery_watchdog';
+  const locked = await withLock(input.config, { action: input.action }, async () => {
+    const before = await input.check(input.config);
+    if (before.ok) return { ok: true, attempted: false, noOp: true, detail: `${display} recovered before restart`, serviceTarget: service.target } satisfies RecoveryGatewayRestartResult;
+    const started = await ensureLaunchdServiceStarted(service, input.dependencies.runCommand ?? command);
     if (!started.ok) {
-      audit(config, 'recovery_gateway_restart_failed', { serviceTarget: service.target, detail: started.detail });
+      audit(input.config, `${eventPrefix}_restart_failed`, { serviceTarget: service.target, detail: started.detail });
       return { ok: false, attempted: true, detail: started.detail, serviceTarget: service.target } satisfies RecoveryGatewayRestartResult;
     }
-    const now = dependencies.now ?? Date.now;
-    const wait = dependencies.sleep ?? sleep;
+    const now = input.dependencies.now ?? Date.now;
+    const wait = input.dependencies.sleep ?? sleep;
     const deadline = now() + 20_000;
     let observed = before;
     while (now() < deadline) {
       await wait(1_000);
-      observed = await check(config);
+      observed = await input.check(input.config);
       if (observed.ok) {
-        audit(config, 'recovery_gateway_restart_succeeded', { serviceTarget: service.target });
-        return { ok: true, attempted: true, detail: 'Recovery Gateway restarted and its local health endpoint recovered', serviceTarget: service.target } satisfies RecoveryGatewayRestartResult;
+        audit(input.config, `${eventPrefix}_restart_succeeded`, { serviceTarget: service.target });
+        return { ok: true, attempted: true, detail: `${display} restarted and passed local health verification`, serviceTarget: service.target } satisfies RecoveryGatewayRestartResult;
       }
     }
-    audit(config, 'recovery_gateway_restart_unverified', { serviceTarget: service.target, detail: observed.detail });
-    return { ok: false, attempted: true, detail: 'Recovery Gateway restarted but did not pass local health verification before timeout', serviceTarget: service.target } satisfies RecoveryGatewayRestartResult;
+    audit(input.config, `${eventPrefix}_restart_unverified`, { serviceTarget: service.target, detail: observed.detail });
+    return { ok: false, attempted: true, detail: `${display} restarted but did not pass local health verification before timeout`, serviceTarget: service.target } satisfies RecoveryGatewayRestartResult;
   });
   if (!locked.acquired) return { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(locked.owner), serviceTarget: service.target };
   return locked.value;
 }
 
+export async function restartRecoveryGateway(
+  config: RecoveryConfig,
+  dependencies: RecoveryGatewayRestartDependencies = {},
+): Promise<RecoveryGatewayRestartResult> {
+  if (!config.gateway) return { ok: false, attempted: false, noOp: true, detail: 'Recovery Gateway is not configured' };
+  return restartRecoveryRole({
+    config,
+    role: 'gateway',
+    action: 'restart_recovery_gateway',
+    check: dependencies.probeGateway ?? probeRecoveryGateway,
+    dependencies,
+  });
+}
+
+export async function restartRecoveryWatchdog(
+  config: RecoveryConfig,
+  dependencies: RecoveryWatchdogRestartDependencies = {},
+): Promise<RecoveryWatchdogRestartResult> {
+  return restartRecoveryRole({
+    config,
+    role: 'watchdog',
+    action: 'restart_recovery_watchdog',
+    check: dependencies.probeWatchdog ?? probeRecoveryWatchdog,
+    dependencies,
+  });
+}
+
 function pidAlive(pid: number | undefined): boolean {
   if (typeof pid !== 'number' || !Number.isInteger(pid) || pid <= 0) return false;
   try { process.kill(pid, 0); return true; } catch { return false; }
@@ -2101,7 +2631,7 @@ export function gatewayToken(config: RecoveryConfig): string | undefined {
 export function initializeStandaloneRecovery(
   controllerHome: string,
   port = 8787,
-  extensions: Partial<Pick<RecoveryConfig, 'publicMcpUrl' | 'recoveryPublicUrl' | 'recoveryTunnelService' | 'primaryRuntimeService' | 'primaryRuntimeSourceRoot' | 'primaryConnectorService' | 'readOnlyTool'>> = {},
+  extensions: Partial<Pick<RecoveryConfig, 'publicMcpUrl' | 'recoveryPublicUrl' | 'recoveryTunnelService' | 'primaryPublicTunnelService' | 'primaryRuntimeService' | 'primaryRuntimeSourceRoot' | 'primaryConnectorService' | 'readOnlyTool'>> = {},
 ): RecoveryConfig {
   const root = resolve(controllerHome);
   const tokenPath = join(root, 'recovery', 'config', 'gateway-token.json');
@@ -2128,6 +2658,7 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
   rollback?: RollbackResult;
   publicTunnelRepair?: PublicTunnelRepairResult;
   recoveryGatewayRestart?: RecoveryGatewayRestartResult;
+  primaryConnectorRestart?: PrimaryConnectorRestartResult;
   primaryRuntimeRestart?: PrimaryRuntimeRestartResult;
   primaryRuntimeRecovery?: PrimaryRuntimeRecoveryResult;
 }> {
@@ -2168,6 +2699,7 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
       runtimeHealthySince: undefined,
       lastFullVerifyAt,
       lastDecision: 'degraded',
+      lastReason: 'canonical Runtime is within startup grace; restart escalation suppressed',
     };
     return {
       state,
@@ -2177,8 +2709,30 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
   }
   const recoveryHealthy = verified.probes.recovery_gateway?.ok !== false
     && verified.probes.recovery_external_http?.ok !== false;
-  const primaryRuntimeHealthy = verified.ok && localVerify.ok;
-  if (primaryRuntimeHealthy && recoveryHealthy) {
+  const primaryRuntimeHealthy = localVerify.ok;
+  const primaryConnectorConfigured = Boolean(config.primaryConnectorService && config.publicMcpUrl);
+  const primaryConnectorFailed = Boolean(
+    primaryConnectorConfigured
+    && primaryRuntimeHealthy
+    && (
+      verified.probes.external_mcp_http?.ok === false
+      || verified.probes.primary_connector_ready?.ok === false
+      || connectorCapacityRecoveryRecommended(verified)
+    ),
+  );
+  if (primaryRuntimeHealthy && !primaryConnectorFailed && recoveryHealthy) {
+    if (fullVerificationPerformed && verified.ok && !matchingKnownGood(config, verified.releases.active)) {
+      try {
+        const attestation = await withLock(config, { action: 'attest_known_good' }, async () => persistVerifiedKnownGood(config, verified));
+        if (!attestation.acquired) {
+          audit(config, 'known_good_attestation_deferred', { reason: recoveryBusyDetail(attestation.owner) });
+        }
+      } catch (error) {
+        audit(config, 'known_good_attestation_deferred', {
+          reason: error instanceof Error ? error.message : 'watchdog known-good attestation failed',
+        });
+      }
+    }
     const stable = recordWatchdogRuntimeHealthy(
       scopeWatchdogStateToRuntimeRelease(scopedPrior, verified.releases.active),
       now,
@@ -2193,9 +2747,15 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
       publicTunnelFailures: 0,
       publicTunnelFirstFailureAt: undefined,
       publicTunnelRepairFailures: 0,
+      primaryConnectorFailures: 0,
+      primaryConnectorFirstFailureAt: undefined,
+      primaryConnectorRestartAttempts: 0,
+      primaryConnectorRestartFailures: 0,
+      primaryConnectorRestartLastAttemptAt: undefined,
       recoveryGatewayRestartUsed: false,
       lastFullVerifyAt,
       lastDecision: 'healthy',
+      lastReason: 'primary runtime and standalone Recovery verification passed',
     };
     return { state, decision: { action: 'healthy', reason: 'primary runtime and standalone Recovery verification passed' }, verify: verified };
   }
@@ -2221,15 +2781,30 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
       firstFailureAt: budgetPrior.firstFailureAt ?? now,
       publicTunnelFailures: (budgetPrior.publicTunnelFailures ?? 0) + 1,
       publicTunnelFirstFailureAt: budgetPrior.publicTunnelFirstFailureAt ?? now,
+      primaryConnectorFailures: 0,
+      primaryConnectorFirstFailureAt: undefined,
     }
-    : {
-      ...budgetPrior,
-      lastFullVerifyAt,
-      failures: budgetPrior.failures + 1,
-      firstFailureAt: budgetPrior.firstFailureAt ?? now,
-      publicTunnelFailures: 0,
-      publicTunnelFirstFailureAt: undefined,
-    };
+    : primaryConnectorFailed
+      ? {
+        ...budgetPrior,
+        lastFullVerifyAt,
+        failures: budgetPrior.failures + 1,
+        firstFailureAt: budgetPrior.firstFailureAt ?? now,
+        publicTunnelFailures: 0,
+        publicTunnelFirstFailureAt: undefined,
+        primaryConnectorFailures: (budgetPrior.primaryConnectorFailures ?? 0) + 1,
+        primaryConnectorFirstFailureAt: budgetPrior.primaryConnectorFirstFailureAt ?? now,
+      }
+      : {
+        ...budgetPrior,
+        lastFullVerifyAt,
+        failures: budgetPrior.failures + 1,
+        firstFailureAt: budgetPrior.firstFailureAt ?? now,
+        publicTunnelFailures: 0,
+        publicTunnelFirstFailureAt: undefined,
+        primaryConnectorFailures: 0,
+        primaryConnectorFirstFailureAt: undefined,
+      };
   if (!localVerify.ok) state.runtimeHealthySince = undefined;
   const primaryConfig = configuredPrimaryRuntimeService(config);
   const decision = decideWatchdog({
@@ -2246,12 +2821,23 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
     runtimeRecoveryLastAttemptAt: state.runtimeRecoveryLastAttemptAt,
     runtimeRecoveryCooldownMs: primaryConfig.recoveryCooldownMs,
     recoveryGatewayFailed: verified.probes.recovery_gateway?.ok === false,
+    primaryConnectorConfigured,
+    primaryConnectorFailed,
+    primaryConnectorFailures: state.primaryConnectorFailures,
+    primaryConnectorFirstFailureAt: state.primaryConnectorFirstFailureAt,
+    primaryConnectorRestartAttempts: state.primaryConnectorRestartAttempts,
+    primaryConnectorMaximumRestartAttempts: config.primaryConnectorService?.maximumRestartAttempts,
+    primaryConnectorRestartLastAttemptAt: state.primaryConnectorRestartLastAttemptAt,
+    primaryConnectorRestartCooldownMs: config.primaryConnectorService?.restartCooldownMs,
+    primaryConnectorMinimumFailures: config.primaryConnectorService?.minimumFailures,
+    primaryConnectorMinimumFailureDurationMs: config.primaryConnectorService?.minimumFailureDurationMs,
     publicTunnelConfigured: Boolean(configuredRecoveryTunnel(config)),
     publicTunnelFailed,
     publicTunnelMinimumFailures: configuredRecoveryTunnel(config)?.minimumFailures,
     publicTunnelMinimumFailureDurationMs: configuredRecoveryTunnel(config)?.minimumFailureDurationMs,
   });
   state.lastDecision = decision.action;
+  state.lastReason = decision.reason;
   if (decision.action === 'repair_public_tunnel') {
     const publicTunnelRepair = await repairPublicTunnel(config);
     const nextState = publicTunnelRepair.ok
@@ -2263,6 +2849,28 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
     const recoveryGatewayRestart = await restartRecoveryGateway(config);
     return { state: { ...state, recoveryGatewayRestartUsed: true }, decision, verify: verified, recoveryGatewayRestart };
   }
+  if (decision.action === 'restart_primary_connector') {
+    const primaryConnectorRestart = await restartPrimaryConnector(config);
+    const attempts = (state.primaryConnectorRestartAttempts ?? 0) + (primaryConnectorRestart.attempted ? 1 : 0);
+    const nextState: WatchdogState = primaryConnectorRestart.ok
+      ? {
+        ...state,
+        failures: 0,
+        firstFailureAt: undefined,
+        primaryConnectorFailures: 0,
+        primaryConnectorFirstFailureAt: undefined,
+        primaryConnectorRestartAttempts: 0,
+        primaryConnectorRestartFailures: 0,
+        primaryConnectorRestartLastAttemptAt: now,
+      }
+      : {
+        ...state,
+        primaryConnectorRestartAttempts: attempts,
+        primaryConnectorRestartFailures: (state.primaryConnectorRestartFailures ?? 0) + (primaryConnectorRestart.attempted ? 1 : 0),
+        primaryConnectorRestartLastAttemptAt: now,
+      };
+    return { state: nextState, decision, verify: verified, primaryConnectorRestart };
+  }
   if (decision.action === 'restart_primary_runtime') {
     const primaryRuntimeRestart = await restartPrimaryRuntime(config);
     const attempts = (state.runtimeRestartAttempts ?? 0) + (primaryRuntimeRestart.attempted ? 1 : 0);