@agent-vm/agent-vm 0.0.92 → 0.0.93

package/dist/controller/zone-runtimes/openclaw-zone-runtime.js

@@ -1,13 +1,26 @@
+import { randomUUID } from 'node:crypto';
 import { resolveZoneSecrets } from '../../gateway/credential-manager.js';
 import { runGatewayHealthCheck } from '../../gateway/gateway-health-check.js';
+import { GatewayOwnershipUnsafeError } from '../../gateway/gateway-ownership-evidence.js';
 import { deleteGatewayRuntimeRecord as deleteGatewayRuntimeRecordDefault } from '../../gateway/gateway-runtime-record.js';
 import { startGatewayZone } from '../../gateway/gateway-zone-orchestrator.js';
 import { runControllerCredentialsRefresh as runControllerCredentialsRefreshDefault } from '../../operations/credentials-refresh.js';
 import { runControllerDestroy as runControllerDestroyDefault } from '../../operations/destroy-zone.js';
 import { runControllerUpgrade as runControllerUpgradeDefault } from '../../operations/upgrade-zone.js';
 import { runControllerLogs as runControllerLogsDefault } from '../../operations/zone-logs.js';
+import { isProcessAlive as defaultIsProcessAlive } from '../../shared/managed-vm-process.js';
+import { appendGatewayLifecycleOperationRecord as appendGatewayLifecycleOperationRecordDefault, } from './gateway-lifecycle-operation-record.js';
+import { classifyGatewayStartError, deriveGatewayDiagnosisSnapshot, } from './gateway-zone-state-machine.js';
 import { ControllerZoneRuntimeStartError, ControllerZoneRuntimeUnavailableError, } from './zone-runtime-errors.js';
 const defaultGatewayCloseTimeoutMs = 60_000;
+function isLifecycleOperationExecutionWithLock(execution) {
+    return typeof execution === 'object' && execution !== null && 'lock' in execution;
+}
+function isRecoverySecretResolutionFailure(record) {
+    return (record.errorCode === 'secret-resolution-failed' &&
+        (record.operationTrigger === 'auto-recovery' ||
+            record.operationTrigger === 'credentials-refresh'));
+}
 class OpenClawZoneRestartTimeoutError extends Error {
     code = 'OPENCLAW_GATEWAY_RESTART_TIMEOUT';
     constructor(zoneId, timeoutMs) {
@@ -33,32 +46,280 @@ function buildOpenClawCombinedLogsCommand(logPath) {
 function writeOpenClawZoneRuntimeLog(message) {
     process.stderr.write(`[openclaw-zone-runtime] ${message}\n`);
 }
+function unavailableReasonForState(state) {
+    switch (state.kind) {
+        case 'failed':
+            return state.error.message;
+        case 'owner-unsafe':
+            return `Gateway runtime ownership is unsafe: ${state.evidence.kind}.`;
+        case 'restarting':
+        case 'starting':
+        case 'stopping':
+            return `Gateway runtime is ${state.kind}.`;
+        case 'running':
+        case 'running-degraded':
+        case 'stopped':
+            return undefined;
+    }
+    return assertNeverGatewayZoneLifecycleState(state);
+}
+function assertNeverGatewayZoneLifecycleState(state) {
+    throw new Error(`Unhandled gateway zone lifecycle state: ${JSON.stringify(state)}`);
+}
+function assertNeverGatewayLifecycleOperationRecordKind(kind) {
+    throw new Error(`Unhandled gateway lifecycle operation record kind: ${String(kind)}`);
+}
+function gatewayIdentityFor(runtimeGateway) {
+    if (!runtimeGateway) {
+        return undefined;
+    }
+    const hostPid = runtimeGateway.vm.getHostPid();
+    return {
+        ...(typeof hostPid === 'number' && hostPid > 0 ? { hostPid } : {}),
+        vmId: runtimeGateway.vm.id,
+    };
+}
+async function executeGatewayCommand(runtimeGateway, command) {
+    const result = await runtimeGateway.vm.exec(command);
+    return {
+        exitCode: result.exitCode,
+        stderr: result.stderr,
+        stdout: result.stdout,
+    };
+}
 export function createOpenClawZoneRuntime(options) {
     const clearTimeoutImpl = options.clearTimeoutImpl ?? clearTimeout;
     const closeGatewayTimeoutMs = options.closeGatewayTimeoutMs ?? defaultGatewayCloseTimeoutMs;
+    const isProcessAlive = options.isProcessAlive ?? defaultIsProcessAlive;
     const setTimeoutImpl = options.setTimeoutImpl ?? setTimeout;
+    const appendGatewayLifecycleOperationRecord = options.appendGatewayLifecycleOperationRecord;
     let gateway;
     let bootedAt;
     let lastError;
+    let lastOperation = 'none';
+    let originalOutageCause = { kind: 'unknown' };
+    let lifecycleState = { kind: 'stopped' };
     let lifecycleOperation = Promise.resolve();
     let lifecycleGeneration = 0;
-    const startGateway = async () => options.restartGatewayZone
-        ? await options.restartGatewayZone(options.zone.id)
+    let staleGatewayPendingClose;
+    const startGateway = async (startOptions = {}) => options.restartGatewayZone
+        ? await options.restartGatewayZone(options.zone.id, startOptions)
         : await startGatewayZone({
-            secretResolver: options.secretResolver,
+            secretResolver: startOptions.secretResolver ?? options.secretResolver,
             systemConfig: options.systemConfig,
             zoneId: options.zone.id,
         });
     const requireGateway = () => {
-        if (!gateway) {
-            throw new ControllerZoneRuntimeUnavailableError(options.zone.id, lastError);
+        const currentState = getLifecycleState();
+        if (currentState.kind !== 'running' && currentState.kind !== 'running-degraded') {
+            throw new ControllerZoneRuntimeUnavailableError(options.zone.id, lastError ?? unavailableReasonForState(currentState));
+        }
+        return currentState.gateway;
+    };
+    const createOperationId = (operationName) => `${options.zone.id}-${operationName}-${randomUUID()}`;
+    const operationForRecordKind = (kind) => {
+        switch (kind) {
+            case 'cold-start-requested':
+                return 'cold-start';
+            case 'credentials-refresh-requested':
+                return 'credentials-refresh';
+            case 'restart-requested':
+                return 'restart';
+            case 'start-requested':
+                return 'start';
+            case 'stop-requested':
+                return 'stop';
+            case 'operation-failed':
+            case 'operation-finished':
+            case 'runtime-record-deleted':
+            case 'runtime-record-written':
+            case 'vm-close-finished':
+            case 'vm-close-started':
+                return undefined;
+        }
+        return assertNeverGatewayLifecycleOperationRecordKind(kind);
+    };
+    const setOriginalOutageCauseIfUnknown = (errorCode) => {
+        if (originalOutageCause.kind !== 'unknown') {
+            return;
+        }
+        originalOutageCause = {
+            ...(errorCode === undefined ? {} : { errorCode }),
+            eventKind: 'gateway-lifecycle-operation',
+            kind: 'proven',
+        };
+    };
+    const recordLifecycleOperation = async (record) => {
+        const operation = operationForRecordKind(record.kind);
+        if (operation !== undefined) {
+            lastOperation = operation;
+        }
+        if (record.kind === 'operation-failed' && !isRecoverySecretResolutionFailure(record)) {
+            setOriginalOutageCauseIfUnknown(record.errorCode);
+        }
+        const operationRecord = {
+            controllerPid: process.pid,
+            gatewayType: 'openclaw',
+            observedAtMs: options.now(),
+            zoneId: options.zone.id,
+            ...record,
+        };
+        try {
+            if (appendGatewayLifecycleOperationRecord) {
+                await appendGatewayLifecycleOperationRecord(operationRecord);
+                return;
+            }
+            await appendGatewayLifecycleOperationRecordDefault({
+                record: operationRecord,
+                runtimeDir: options.systemConfig.runtimeDir,
+                zoneId: options.zone.id,
+            });
+        }
+        catch (error) {
+            writeOpenClawZoneRuntimeLog(`failed to append gateway lifecycle operation record for zone '${options.zone.id}': ${formatUnknownError(error)}`);
+        }
+    };
+    const markGatewayHostPidMissing = (message) => {
+        if (staleGatewayPendingClose === undefined &&
+            (lifecycleState.kind === 'running' || lifecycleState.kind === 'running-degraded')) {
+            staleGatewayPendingClose = lifecycleState.gateway;
+        }
+        const errorMessage = `vm-process-missing: ${message}`;
+        setOriginalOutageCauseIfUnknown('vm-process-missing');
+        gateway = undefined;
+        bootedAt = undefined;
+        lastError = errorMessage;
+        lifecycleState = {
+            coldStartEligible: true,
+            error: { code: 'vm-process-missing', message: errorMessage },
+            kind: 'failed',
+        };
+        return lifecycleState;
+    };
+    const closeStaleGatewayBeforeColdStart = async (operationContext) => {
+        const staleGateway = staleGatewayPendingClose;
+        if (!staleGateway) {
+            return;
+        }
+        staleGatewayPendingClose = undefined;
+        try {
+            await recordLifecycleOperation({
+                kind: 'vm-close-started',
+                operationId: operationContext.operationId,
+                operationTrigger: operationContext.operationTrigger,
+                previousGateway: gatewayIdentityFor(staleGateway),
+            });
+            await closeGatewayWithDeadline(staleGateway);
+            await recordLifecycleOperation({
+                kind: 'vm-close-finished',
+                operationId: operationContext.operationId,
+                operationTrigger: operationContext.operationTrigger,
+                previousGateway: gatewayIdentityFor(staleGateway),
+            });
+        }
+        catch (error) {
+            staleGatewayPendingClose = staleGateway;
+            lastError = formatUnknownError(error);
+            lifecycleState = {
+                coldStartEligible: false,
+                error: {
+                    code: 'owner-unsafe',
+                    message: lastError,
+                },
+                kind: 'failed',
+            };
+            await recordLifecycleOperation({
+                errorCode: 'owner-unsafe',
+                errorMessage: lastError,
+                kind: 'operation-failed',
+                operationId: operationContext.operationId,
+                operationTrigger: operationContext.operationTrigger,
+                previousGateway: gatewayIdentityFor(staleGateway),
+            });
+            throw error;
+        }
+    };
+    const classifyLastError = (message) => {
+        if (message.startsWith('vm-process-missing:')) {
+            return {
+                coldStartEligible: true,
+                error: { code: 'vm-process-missing', message },
+                kind: 'failed',
+            };
         }
-        return gateway;
+        const error = classifyGatewayStartError(new Error(message));
+        return {
+            coldStartEligible: error.code !== 'owner-unsafe',
+            error,
+            kind: 'failed',
+        };
+    };
+    const getLifecycleState = () => {
+        if (lifecycleState.kind === 'running' || lifecycleState.kind === 'running-degraded') {
+            const hostPid = lifecycleState.gateway.vm.getHostPid();
+            if (hostPid === undefined || hostPid === null) {
+                return markGatewayHostPidMissing(`Gateway VM host pid is unavailable for zone '${options.zone.id}'.`);
+            }
+            if (!isProcessAlive(hostPid)) {
+                return markGatewayHostPidMissing(`Gateway VM host pid ${String(hostPid)} is not alive for zone '${options.zone.id}'.`);
+            }
+            return lifecycleState;
+        }
+        if (lifecycleState.kind === 'failed' || lifecycleState.kind === 'owner-unsafe') {
+            return lifecycleState;
+        }
+        if (lifecycleState.kind === 'starting' ||
+            lifecycleState.kind === 'stopping' ||
+            lifecycleState.kind === 'restarting') {
+            return lifecycleState;
+        }
+        if (lastError) {
+            lifecycleState = classifyLastError(lastError);
+            return lifecycleState;
+        }
+        return lifecycleState;
     };
     const runLifecycleOperation = async (operation) => {
-        const operationPromise = lifecycleOperation.then(operation, operation);
-        lifecycleOperation = operationPromise.then(() => undefined, () => undefined);
-        return await operationPromise;
+        const runAfterPrevious = async () => {
+            await lifecycleOperation.catch(() => undefined);
+            return await operation();
+        };
+        const executionPromise = runAfterPrevious();
+        const operationResultPromise = executionPromise.then(async (execution) => {
+            if (isLifecycleOperationExecutionWithLock(execution)) {
+                return await execution.publicResult;
+            }
+            return await execution;
+        });
+        lifecycleOperation = executionPromise
+            .then(async (execution) => {
+            if (isLifecycleOperationExecutionWithLock(execution)) {
+                await execution.lock;
+                return;
+            }
+            await execution;
+        })
+            .then(() => undefined, () => undefined);
+        return await operationResultPromise;
+    };
+    const withLifecycleTimeout = (props) => {
+        let timeout;
+        const timeoutPromise = new Promise((_resolve, reject) => {
+            timeout = setTimeoutImpl(() => {
+                lifecycleGeneration += 1;
+                reject(new OpenClawZoneRestartTimeoutError(options.zone.id, props.timeoutMs));
+            }, props.timeoutMs);
+            timeout.unref?.();
+        });
+        const publicResult = Promise.race([props.operation, timeoutPromise]).finally(() => {
+            if (timeout) {
+                clearTimeoutImpl(timeout);
+            }
+        });
+        return {
+            lock: props.operation.then(() => undefined, () => undefined),
+            publicResult,
+        };
     };
     const releaseZoneLeases = async (zoneId) => {
         const leases = options.leaseManager
@@ -95,24 +356,125 @@ export function createOpenClawZoneRuntime(options) {
             }
         }
     };
-    const stopNow = async () => {
+    const stopNow = async (next = 'stopped', operationContext) => {
         const activeGateway = gateway;
-        gateway = undefined;
-        bootedAt = undefined;
-        lastError = undefined;
-        if (activeGateway) {
-            await closeGatewayWithDeadline(activeGateway);
+        const operationId = operationContext?.operationId ?? createOperationId('stop');
+        const operationTrigger = operationContext?.operationTrigger ?? 'operator-stop';
+        const previousGateway = operationContext?.previousGateway ?? activeGateway;
+        lifecycleState = {
+            kind: 'stopping',
+            next,
+            operationId,
+            previousGateway,
+        };
+        try {
+            await recordLifecycleOperation({
+                kind: 'stop-requested',
+                operationId,
+                operationTrigger,
+                previousGateway: gatewayIdentityFor(previousGateway),
+            });
+            if (activeGateway) {
+                await recordLifecycleOperation({
+                    kind: 'vm-close-started',
+                    operationId,
+                    operationTrigger,
+                    previousGateway: gatewayIdentityFor(previousGateway),
+                });
+                await closeGatewayWithDeadline(activeGateway);
+                await recordLifecycleOperation({
+                    kind: 'vm-close-finished',
+                    operationId,
+                    operationTrigger,
+                    previousGateway: gatewayIdentityFor(previousGateway),
+                });
+            }
+            await (options.deleteGatewayRuntimeRecord ?? deleteGatewayRuntimeRecordDefault)(options.zone.gateway.stateDir);
+            await recordLifecycleOperation({
+                kind: 'runtime-record-deleted',
+                operationId,
+                operationTrigger,
+                previousGateway: gatewayIdentityFor(previousGateway),
+            });
+            gateway = undefined;
+            bootedAt = undefined;
+            lastError = undefined;
+            lifecycleState = { kind: 'stopped' };
+        }
+        catch (error) {
+            lastError = formatUnknownError(error);
+            lifecycleState = {
+                coldStartEligible: false,
+                error: {
+                    code: 'owner-unsafe',
+                    message: lastError,
+                },
+                kind: 'failed',
+            };
+            await recordLifecycleOperation({
+                errorCode: 'owner-unsafe',
+                errorMessage: lastError,
+                kind: 'operation-failed',
+                operationId,
+                operationTrigger,
+                previousGateway: gatewayIdentityFor(previousGateway),
+            });
+            throw error;
         }
-        await (options.deleteGatewayRuntimeRecord ?? deleteGatewayRuntimeRecordDefault)(options.zone.gateway.stateDir);
     };
-    const startNow = async (expectedGeneration) => {
+    const startNow = async (expectedGeneration, startOptions = {}, operationContext) => {
+        const operationId = operationContext?.operationId ?? createOperationId('start');
+        const operationTrigger = operationContext?.operationTrigger ?? 'operator-start';
+        lifecycleState = {
+            kind: 'starting',
+            operationId,
+            startedAtMs: options.now(),
+        };
         try {
-            const startedGateway = await startGateway();
+            await recordLifecycleOperation({
+                kind: 'start-requested',
+                operationId,
+                operationTrigger,
+                previousGateway: gatewayIdentityFor(operationContext?.previousGateway),
+            });
+            const startedGateway = await startGateway(startOptions);
             if (expectedGeneration !== undefined && expectedGeneration !== lifecycleGeneration) {
                 try {
                     await closeGatewayWithDeadline(startedGateway);
+                    if (lifecycleGeneration === expectedGeneration + 1) {
+                        await (options.deleteGatewayRuntimeRecord ?? deleteGatewayRuntimeRecordDefault)(options.zone.gateway.stateDir);
+                        await recordLifecycleOperation({
+                            currentGateway: gatewayIdentityFor(startedGateway),
+                            kind: 'runtime-record-deleted',
+                            operationId,
+                            operationTrigger,
+                            previousGateway: gatewayIdentityFor(operationContext?.previousGateway),
+                        });
+                    }
+                    lastError = `stale-generation-closed: Closed stale gateway start for zone '${options.zone.id}'.`;
+                    lifecycleState = classifyLastError(lastError);
+                    await recordLifecycleOperation({
+                        currentGateway: gatewayIdentityFor(startedGateway),
+                        errorCode: 'stale-generation-closed',
+                        errorMessage: lastError,
+                        kind: 'operation-failed',
+                        operationId,
+                        operationTrigger,
+                        previousGateway: gatewayIdentityFor(operationContext?.previousGateway),
+                    });
                 }
                 catch (error) {
+                    lastError = `stale-generation-closed: Failed to close stale gateway start for zone '${options.zone.id}': ${formatUnknownError(error)}`;
+                    lifecycleState = classifyLastError(lastError);
+                    await recordLifecycleOperation({
+                        currentGateway: gatewayIdentityFor(startedGateway),
+                        errorCode: 'stale-generation-closed',
+                        errorMessage: lastError,
+                        kind: 'operation-failed',
+                        operationId,
+                        operationTrigger,
+                        previousGateway: gatewayIdentityFor(operationContext?.previousGateway),
+                    });
                     writeOpenClawZoneRuntimeLog(`stale gateway start cleanup failed for zone '${options.zone.id}': ${formatUnknownError(error)}`);
                 }
                 return;
@@ -120,54 +482,171 @@ export function createOpenClawZoneRuntime(options) {
             gateway = startedGateway;
             bootedAt = new Date(options.now()).toISOString();
             lastError = undefined;
+            lifecycleState = { gateway: startedGateway, kind: 'running' };
+            await recordLifecycleOperation({
+                currentGateway: gatewayIdentityFor(startedGateway),
+                kind: 'operation-finished',
+                operationId,
+                operationTrigger,
+                previousGateway: gatewayIdentityFor(operationContext?.previousGateway),
+            });
         }
         catch (error) {
+            if (error instanceof GatewayOwnershipUnsafeError) {
+                gateway = undefined;
+                bootedAt = undefined;
+                lastError = error.message;
+                lifecycleState = {
+                    evidence: error.evidence,
+                    kind: 'owner-unsafe',
+                };
+                await recordLifecycleOperation({
+                    errorCode: 'owner-unsafe',
+                    errorMessage: error.message,
+                    kind: 'operation-failed',
+                    operationId,
+                    operationTrigger,
+                    previousGateway: gatewayIdentityFor(operationContext?.previousGateway),
+                });
+                throw new ControllerZoneRuntimeStartError(options.zone.id, error, {
+                    gatewayLifecycleErrorCode: 'owner-unsafe',
+                    operationId,
+                });
+            }
+            const classifiedError = classifyGatewayStartError(error);
             gateway = undefined;
             bootedAt = undefined;
             lastError = formatUnknownError(error);
-            throw new ControllerZoneRuntimeStartError(options.zone.id, error);
+            lifecycleState = {
+                coldStartEligible: true,
+                error: classifiedError,
+                kind: 'failed',
+            };
+            await recordLifecycleOperation({
+                errorCode: classifiedError.code,
+                errorMessage: classifiedError.message,
+                kind: 'operation-failed',
+                operationId,
+                operationTrigger,
+                previousGateway: gatewayIdentityFor(operationContext?.previousGateway),
+            });
+            throw new ControllerZoneRuntimeStartError(options.zone.id, error, {
+                gatewayLifecycleErrorCode: classifiedError.code,
+                operationId,
+            });
         }
     };
     const stop = async () => await runLifecycleOperation(async () => await stopNow());
-    const start = async () => await runLifecycleOperation(async () => await startNow());
-    const restart = async (restartOptions = {}) => {
+    const start = async () => await runLifecycleOperation(async () => await startNow(undefined, {}, {
+        operationId: createOperationId('start'),
+        operationTrigger: 'controller-start',
+    }));
+    const restartWithStartOptions = async (restartOptions = {}, startOptions = {}, operationMetadata = {}) => {
         return await runLifecycleOperation(async () => {
             lifecycleGeneration += 1;
             const operationGeneration = lifecycleGeneration;
-            const restartOperation = (async () => {
+            const currentState = getLifecycleState();
+            const operationId = operationMetadata.operationId ?? createOperationId('restart');
+            const operationContext = {
+                operationId,
+                operationTrigger: operationMetadata.operationTrigger ??
+                    restartOptions.operationTrigger ??
+                    'operator-restart',
+                previousGateway: currentState.kind === 'running' || currentState.kind === 'running-degraded'
+                    ? currentState.gateway
+                    : undefined,
+            };
+            if (currentState.kind === 'running' || currentState.kind === 'running-degraded') {
+                lifecycleState = {
+                    kind: 'restarting',
+                    operationId,
+                    previousGateway: currentState.gateway,
+                };
+            }
+            const restartOperation = currentState.kind === 'running' || currentState.kind === 'running-degraded'
+                ? (async () => {
+                    await recordLifecycleOperation({
+                        kind: 'restart-requested',
+                        operationId,
+                        operationTrigger: operationContext.operationTrigger,
+                        previousGateway: gatewayIdentityFor(operationContext.previousGateway),
+                    });
+                    const leaseReleaseResult = await releaseZoneLeases(options.zone.id);
+                    await stopNow('starting', operationContext);
+                    await startNow(operationGeneration, startOptions, operationContext);
+                    if (operationGeneration !== lifecycleGeneration) {
+                        throw new OpenClawZoneRestartTimeoutError(options.zone.id, restartOptions.timeoutMs ?? 0);
+                    }
+                    return {
+                        leaseReleaseFailureCount: leaseReleaseResult.failedLeaseIds.length,
+                        operationId,
+                    };
+                })()
+                : (async () => {
+                    await recordLifecycleOperation({
+                        kind: 'cold-start-requested',
+                        operationId,
+                        operationTrigger: operationContext.operationTrigger,
+                    });
+                    const leaseReleaseResult = await releaseZoneLeases(options.zone.id);
+                    await closeStaleGatewayBeforeColdStart(operationContext);
+                    await startNow(operationGeneration, startOptions, operationContext);
+                    if (operationGeneration !== lifecycleGeneration) {
+                        throw new OpenClawZoneRestartTimeoutError(options.zone.id, restartOptions.timeoutMs ?? 0);
+                    }
+                    return {
+                        leaseReleaseFailureCount: leaseReleaseResult.failedLeaseIds.length,
+                        operationId,
+                    };
+                })();
+            if (restartOptions.timeoutMs === undefined) {
+                return await restartOperation;
+            }
+            return withLifecycleTimeout({
+                operation: restartOperation,
+                timeoutMs: restartOptions.timeoutMs,
+            });
+        });
+    };
+    const restart = async (restartOptions = {}) => await restartWithStartOptions(restartOptions);
+    const coldStartWithStartOptions = async (restartOptions = {}, startOptions = {}, operationMetadata = {}) => {
+        return await runLifecycleOperation(async () => {
+            lifecycleGeneration += 1;
+            const operationGeneration = lifecycleGeneration;
+            const operationContext = {
+                operationId: operationMetadata.operationId ?? createOperationId('cold-start'),
+                operationTrigger: operationMetadata.operationTrigger ?? restartOptions.operationTrigger ?? 'auto-recovery',
+            };
+            const coldStartOperation = (async () => {
+                getLifecycleState();
+                await recordLifecycleOperation({
+                    kind: 'cold-start-requested',
+                    operationId: operationContext.operationId,
+                    operationTrigger: operationContext.operationTrigger,
+                });
                 const leaseReleaseResult = await releaseZoneLeases(options.zone.id);
-                await stopNow();
-                await startNow(operationGeneration);
+                await closeStaleGatewayBeforeColdStart(operationContext);
+                await startNow(operationGeneration, startOptions, operationContext);
                 if (operationGeneration !== lifecycleGeneration) {
                     throw new OpenClawZoneRestartTimeoutError(options.zone.id, restartOptions.timeoutMs ?? 0);
                 }
-                return { leaseReleaseFailureCount: leaseReleaseResult.failedLeaseIds.length };
+                return {
+                    leaseReleaseFailureCount: leaseReleaseResult.failedLeaseIds.length,
+                    operationId: operationContext.operationId,
+                };
             })();
             if (restartOptions.timeoutMs === undefined) {
-                return await restartOperation;
-            }
-            const restartTimeoutMs = restartOptions.timeoutMs;
-            let timeout;
-            try {
-                return await Promise.race([
-                    restartOperation,
-                    new Promise((_resolve, reject) => {
-                        timeout = setTimeoutImpl(() => {
-                            lifecycleGeneration += 1;
-                            reject(new OpenClawZoneRestartTimeoutError(options.zone.id, restartTimeoutMs));
-                        }, restartTimeoutMs);
-                        timeout.unref?.();
-                    }),
-                ]);
-            }
-            finally {
-                if (timeout) {
-                    clearTimeoutImpl(timeout);
-                }
+                return await coldStartOperation;
             }
+            return withLifecycleTimeout({
+                operation: coldStartOperation,
+                timeoutMs: restartOptions.timeoutMs,
+            });
         });
     };
+    const coldStart = async (restartOptions = {}) => await coldStartWithStartOptions(restartOptions);
     return {
+        coldStart,
         destroy: async (purge) => await (options.runControllerDestroy ?? runControllerDestroyDefault)({ purge, systemConfig: options.systemConfig, zoneId: options.zone.id }, {
             releaseZoneLeases: async (zoneId) => {
                 await releaseZoneLeases(zoneId);
@@ -175,12 +654,13 @@ export function createOpenClawZoneRuntime(options) {
             stopGatewayZone: async () => await stop(),
         }),
         enableSsh: async () => await requireGateway().vm.enableSsh(),
-        exec: async (command) => await requireGateway().vm.exec(command),
+        exec: async (command) => await executeGatewayCommand(requireGateway(), command),
         gatewayType: 'openclaw',
         getHealth: async () => {
+            getLifecycleState();
             const activeGateway = requireGateway();
             const result = await runGatewayHealthCheck({
-                exec: async (command) => await activeGateway.vm.exec(command),
+                exec: async (command) => await executeGatewayCommand(activeGateway, command),
                 healthCheck: activeGateway.processSpec.healthCheck,
             });
             return {
@@ -192,42 +672,118 @@ export function createOpenClawZoneRuntime(options) {
                 zoneId: options.zone.id,
             };
         },
+        getDiagnosis: () => deriveGatewayDiagnosisSnapshot({
+            channelProviderPlane: 'unknown',
+            controllerLiveness: 'ok',
+            lastOperation,
+            originalOutageCause,
+            state: getLifecycleState(),
+            toolVmPlane: 'unknown',
+        }),
         getLogs: async () => {
             const activeGateway = requireGateway();
             return await (options.runControllerLogs ?? runControllerLogsDefault)({ zoneId: options.zone.id }, {
                 readGatewayLogs: async () => (await activeGateway.vm.exec(buildOpenClawCombinedLogsCommand(activeGateway.processSpec.logPath))).stdout,
             });
         },
+        getLifecycleState,
         getSnapshot: () => {
-            if (gateway) {
-                const hostPid = gateway.vm.getHostPid();
+            const currentLifecycleState = getLifecycleState();
+            if (currentLifecycleState.kind === 'running') {
+                const hostPid = currentLifecycleState.gateway.vm.getHostPid();
+                if (hostPid === undefined || hostPid === null) {
+                    const missingHostPidState = markGatewayHostPidMissing(`Gateway VM host pid is unavailable for zone '${options.zone.id}'.`);
+                    return {
+                        lastError: missingHostPidState.error.message,
+                        lifecycleState: 'failed',
+                    };
+                }
                 return {
                     ...(bootedAt ? { bootedAt } : {}),
                     gateway: {
-                        ingress: gateway.ingress,
+                        ingress: currentLifecycleState.gateway.ingress,
                         vm: {
-                            ...(hostPid === undefined || hostPid === null ? {} : { hostPid }),
-                            id: gateway.vm.id,
+                            hostPid,
+                            id: currentLifecycleState.gateway.vm.id,
                         },
                     },
+                    ...(lastError ? { lastError } : {}),
                     lifecycleState: 'running',
                 };
             }
             return lastError ? { lastError, lifecycleState: 'failed' } : { lifecycleState: 'stopped' };
         },
-        refreshCredentials: async () => await (options.runControllerCredentialsRefresh ?? runControllerCredentialsRefreshDefault)({ zoneId: options.zone.id }, {
-            refreshZoneSecrets: async (zoneId) => {
-                await resolveZoneSecrets({
-                    audience: 'gateway',
-                    secretResolver: options.secretResolver,
-                    systemConfig: options.systemConfig,
-                    zoneId,
+        refreshCredentials: async () => await (async () => {
+            const operationId = createOperationId('credentials-refresh');
+            const operationTrigger = 'credentials-refresh';
+            await recordLifecycleOperation({
+                kind: 'credentials-refresh-requested',
+                operationId,
+                operationTrigger,
+                previousGateway: gatewayIdentityFor(gateway),
+            });
+            const failCredentialsRefreshSecretResolution = async (error) => {
+                const classifiedError = {
+                    code: 'secret-resolution-failed',
+                    message: formatUnknownError(error),
+                };
+                const currentLifecycleState = getLifecycleState();
+                lastError = classifiedError.message;
+                if (currentLifecycleState.kind !== 'running' &&
+                    currentLifecycleState.kind !== 'running-degraded') {
+                    lifecycleState = {
+                        coldStartEligible: true,
+                        error: classifiedError,
+                        kind: 'failed',
+                    };
+                }
+                await recordLifecycleOperation({
+                    errorCode: classifiedError.code,
+                    errorMessage: classifiedError.message,
+                    kind: 'operation-failed',
+                    operationId,
+                    operationTrigger,
+                    previousGateway: gatewayIdentityFor(gateway),
                 });
-            },
-            restartGatewayZone: async () => {
-                await restart();
-            },
-        }),
+                throw new ControllerZoneRuntimeStartError(options.zone.id, error, {
+                    gatewayLifecycleErrorCode: classifiedError.code,
+                    operationId,
+                });
+            };
+            let refreshedSecretResolver;
+            try {
+                refreshedSecretResolver = options.createFreshSecretResolver
+                    ? await options.createFreshSecretResolver()
+                    : options.secretResolver;
+            }
+            catch (error) {
+                await failCredentialsRefreshSecretResolution(error);
+            }
+            return await (options.runControllerCredentialsRefresh ?? runControllerCredentialsRefreshDefault)({ zoneId: options.zone.id }, {
+                refreshZoneSecrets: async (zoneId) => {
+                    try {
+                        await resolveZoneSecrets({
+                            audience: 'gateway',
+                            secretResolver: refreshedSecretResolver,
+                            systemConfig: options.systemConfig,
+                            zoneId,
+                        });
+                    }
+                    catch (error) {
+                        await failCredentialsRefreshSecretResolution(error);
+                    }
+                },
+                restartGatewayZone: async () => {
+                    const currentLifecycleState = getLifecycleState();
+                    if (currentLifecycleState.kind === 'running' ||
+                        currentLifecycleState.kind === 'running-degraded') {
+                        await restartWithStartOptions({}, { secretResolver: refreshedSecretResolver }, { operationId, operationTrigger });
+                        return;
+                    }
+                    await coldStartWithStartOptions({}, { secretResolver: refreshedSecretResolver }, { operationId, operationTrigger });
+                },
+            });
+        })(),
         restart,
         shutdown: stop,
         start,
@@ -235,7 +791,7 @@ export function createOpenClawZoneRuntime(options) {
         upgrade: async () => await (options.runControllerUpgrade ?? runControllerUpgradeDefault)({ systemConfig: options.systemConfig, zoneId: options.zone.id }, {
             rebuildGatewayImage: async () => { },
             restartGatewayZone: async () => {
-                await restart();
+                await restart({ operationTrigger: 'upgrade' });
             },
         }),
         zoneId: options.zone.id,