@agent-vm/agent-vm 0.0.92 → 0.0.94

package/dist/controller/health/gateway-vm-recovery-policy.js

@@ -1,32 +1,54 @@
+export const defaultGatewayVmChannelProviderRecoveryPolicy = {
+    consecutiveFailureThreshold: 3,
+    enabled: true,
+    restartGatewayOnRecoverable: true,
+    restartGatewayOnUnrecoverable: false,
+    transitioningTimeoutMs: 120_000,
+};
+function createInitialGatewayVmRecoveryBudgetState() {
+    return {
+        consecutiveFailedRecoveries: 0,
+        lastRecoveryAttemptAtMs: undefined,
+    };
+}
 const createInitialGatewayVmRecoveryTrackerState = () => ({
-    consecutiveFailedRecoveries: 0,
+    agentChannelProviderConsecutiveFailuresById: new Map(),
     gatewayControlLinkConsecutiveFailures: 0,
     gatewayServiceConsecutiveFailures: 0,
-    lastRecoveryAttemptAtMs: undefined,
+    recoveryBudgets: {
+        'gateway-vm-cold-start': createInitialGatewayVmRecoveryBudgetState(),
+        'gateway-vm-restart': createInitialGatewayVmRecoveryBudgetState(),
+    },
     recoveryInFlight: false,
 });
+function recoveryBudgetClassForEvent(event) {
+    return event.recoveryBudgetClass ?? 'gateway-vm-restart';
+}
 function isHealthyObservation(result) {
     return result === 'ok';
 }
 function isDegradedObservation(result) {
     return result === 'failed' || result === 'stale' || result === 'timeout';
 }
-function isWithinCooldown(state, policy, observedAtMs) {
-    return (state.lastRecoveryAttemptAtMs !== undefined &&
-        observedAtMs - state.lastRecoveryAttemptAtMs < policy.cooldownMs);
+function isWithinCooldown(budgetState, policy, observedAtMs) {
+    return (budgetState.lastRecoveryAttemptAtMs !== undefined &&
+        observedAtMs - budgetState.lastRecoveryAttemptAtMs < policy.cooldownMs);
 }
-function hasActiveRecoverySuspension(state, policy, observedAtMs) {
-    return (state.consecutiveFailedRecoveries >= policy.maxConsecutiveFailedRecoveries &&
-        state.lastRecoveryAttemptAtMs !== undefined &&
-        observedAtMs - state.lastRecoveryAttemptAtMs < policy.failedRecoveryResetMs);
+function hasActiveRecoverySuspension(budgetState, policy, observedAtMs) {
+    return (budgetState.consecutiveFailedRecoveries >= policy.maxConsecutiveFailedRecoveries &&
+        budgetState.lastRecoveryAttemptAtMs !== undefined &&
+        observedAtMs - budgetState.lastRecoveryAttemptAtMs < policy.failedRecoveryResetMs);
 }
-function resetFailedRecoveriesIfSuspensionExpired(state, policy, observedAtMs) {
-    if (state.consecutiveFailedRecoveries >= policy.maxConsecutiveFailedRecoveries &&
-        state.lastRecoveryAttemptAtMs !== undefined &&
-        observedAtMs - state.lastRecoveryAttemptAtMs >= policy.failedRecoveryResetMs) {
-        state.consecutiveFailedRecoveries = 0;
+function resetFailedRecoveriesIfSuspensionExpired(budgetState, policy, observedAtMs) {
+    if (budgetState.consecutiveFailedRecoveries >= policy.maxConsecutiveFailedRecoveries &&
+        budgetState.lastRecoveryAttemptAtMs !== undefined &&
+        observedAtMs - budgetState.lastRecoveryAttemptAtMs >= policy.failedRecoveryResetMs) {
+        budgetState.consecutiveFailedRecoveries = 0;
     }
 }
+function channelProviderCounterKey(observation) {
+    return observation.channelProviderId ?? '(unknown-channel-provider)';
+}
 function decideRecovery(props) {
     if (!props.policy.enabled) {
         return { consecutiveFailures: props.consecutiveFailures, kind: 'none', reason: 'disabled' };
@@ -37,13 +59,14 @@ function decideRecovery(props) {
     if (props.consecutiveFailures < props.policy.consecutiveFailureThreshold) {
         return { consecutiveFailures: props.consecutiveFailures, kind: 'none' };
     }
-    if (isWithinCooldown(props.state, props.policy, props.observedAtMs)) {
+    const budgetState = props.state.recoveryBudgets[props.recoveryBudgetClass];
+    if (isWithinCooldown(budgetState, props.policy, props.observedAtMs)) {
         return { consecutiveFailures: props.consecutiveFailures, kind: 'none', reason: 'cooldown' };
     }
-    resetFailedRecoveriesIfSuspensionExpired(props.state, props.policy, props.observedAtMs);
-    if (hasActiveRecoverySuspension(props.state, props.policy, props.observedAtMs)) {
+    resetFailedRecoveriesIfSuspensionExpired(budgetState, props.policy, props.observedAtMs);
+    if (hasActiveRecoverySuspension(budgetState, props.policy, props.observedAtMs)) {
         return {
-            consecutiveFailedRecoveries: props.state.consecutiveFailedRecoveries,
+            consecutiveFailedRecoveries: budgetState.consecutiveFailedRecoveries,
             consecutiveFailures: props.consecutiveFailures,
             kind: 'suspended',
             reason: 'max-failed-recoveries',
@@ -73,20 +96,59 @@ export function createGatewayVmRecoveryTracker(options) {
             const state = getStateForZone(event.zoneId);
             state.recoveryInFlight = false;
             if (event.result === 'ok') {
-                state.consecutiveFailedRecoveries = 0;
+                state.agentChannelProviderConsecutiveFailuresById.clear();
                 state.gatewayControlLinkConsecutiveFailures = 0;
                 state.gatewayServiceConsecutiveFailures = 0;
+                for (const budgetState of Object.values(state.recoveryBudgets)) {
+                    budgetState.consecutiveFailedRecoveries = 0;
+                }
                 return;
             }
             if (event.result === 'failed') {
-                state.consecutiveFailedRecoveries += 1;
+                state.recoveryBudgets[recoveryBudgetClassForEvent(event)].consecutiveFailedRecoveries += 1;
             }
         },
         markRecoveryStarted(event) {
             const state = getStateForZone(event.zoneId);
-            state.lastRecoveryAttemptAtMs = event.observedAtMs;
+            state.recoveryBudgets[recoveryBudgetClassForEvent(event)].lastRecoveryAttemptAtMs =
+                event.observedAtMs;
             state.recoveryInFlight = true;
         },
+        recordAgentChannelProviderObservation(observation) {
+            const channelProviderPolicy = options.policy.channelProviderHealth ?? defaultGatewayVmChannelProviderRecoveryPolicy;
+            const state = getStateForZone(observation.zoneId);
+            const counterKey = channelProviderCounterKey(observation);
+            const currentConsecutiveFailures = state.agentChannelProviderConsecutiveFailuresById.get(counterKey) ?? 0;
+            if (observation.result === 'unobserved') {
+                return {
+                    consecutiveFailures: currentConsecutiveFailures,
+                    kind: 'none',
+                    reason: 'unobserved',
+                };
+            }
+            if (isHealthyObservation(observation.result)) {
+                state.agentChannelProviderConsecutiveFailuresById.set(counterKey, 0);
+                return { consecutiveFailures: 0, kind: 'none' };
+            }
+            let consecutiveFailures = currentConsecutiveFailures;
+            if (isDegradedObservation(observation.result)) {
+                consecutiveFailures += 1;
+                state.agentChannelProviderConsecutiveFailuresById.set(counterKey, consecutiveFailures);
+            }
+            return decideRecovery({
+                consecutiveFailures,
+                observedAtMs: observation.observedAtMs,
+                policy: {
+                    ...options.policy,
+                    consecutiveFailureThreshold: channelProviderPolicy.consecutiveFailureThreshold,
+                    enabled: options.policy.enabled && channelProviderPolicy.enabled,
+                },
+                reason: 'agent-channel-provider-unhealthy',
+                recoveryBudgetClass: observation.recoveryBudgetClass ?? 'gateway-vm-restart',
+                state,
+                zoneId: observation.zoneId,
+            });
+        },
         recordGatewayControlLinkObservation(observation) {
             const state = getStateForZone(observation.zoneId);
             if (observation.result === 'unobserved') {
@@ -108,6 +170,7 @@ export function createGatewayVmRecoveryTracker(options) {
                 observedAtMs: observation.observedAtMs,
                 policy: options.policy,
                 reason: 'gateway-control-link-unhealthy',
+                recoveryBudgetClass: observation.recoveryBudgetClass ?? 'gateway-vm-restart',
                 state,
                 zoneId: observation.zoneId,
             });
@@ -126,6 +189,7 @@ export function createGatewayVmRecoveryTracker(options) {
                 observedAtMs: observation.observedAtMs,
                 policy: options.policy,
                 reason: 'gateway-service-unhealthy',
+                recoveryBudgetClass: observation.recoveryBudgetClass ?? 'gateway-vm-restart',
                 state,
                 zoneId: observation.zoneId,
             });

package/dist/controller/health/gateway-vm-recovery-policy.js.map

@@ -1 +1 @@
-{"version":3,"file":"gateway-vm-recovery-policy.js","sourceRoot":"","sources":["../../../src/controller/health/gateway-vm-recovery-policy.ts"],"names":[],"mappings":"AA2EA,MAAM,0CAA0C,GAAG,GAAkC,EAAE,CAAC,CAAC;IACxF,2BAA2B,EAAE,CAAC;IAC9B,qCAAqC,EAAE,CAAC;IACxC,iCAAiC,EAAE,CAAC;IACpC,uBAAuB,EAAE,SAAS;IAClC,gBAAgB,EAAE,KAAK;CACvB,CAAC,CAAC;AAEH,SAAS,oBAAoB,CAAC,MAA0C;IACvE,OAAO,MAAM,KAAK,IAAI,CAAC;AACxB,CAAC;AAED,SAAS,qBAAqB,CAAC,MAA0C;IACxE,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,OAAO,IAAI,MAAM,KAAK,SAAS,CAAC;AAC1E,CAAC;AAED,SAAS,gBAAgB,CACxB,KAAoC,EACpC,MAAmC,EACnC,YAAoB;IAEpB,OAAO,CACN,KAAK,CAAC,uBAAuB,KAAK,SAAS;QAC3C,YAAY,GAAG,KAAK,CAAC,uBAAuB,GAAG,MAAM,CAAC,UAAU,CAChE,CAAC;AACH,CAAC;AAED,SAAS,2BAA2B,CACnC,KAAoC,EACpC,MAAmC,EACnC,YAAoB;IAEpB,OAAO,CACN,KAAK,CAAC,2BAA2B,IAAI,MAAM,CAAC,8BAA8B;QAC1E,KAAK,CAAC,uBAAuB,KAAK,SAAS;QAC3C,YAAY,GAAG,KAAK,CAAC,uBAAuB,GAAG,MAAM,CAAC,qBAAqB,CAC3E,CAAC;AACH,CAAC;AAED,SAAS,wCAAwC,CAChD,KAAoC,EACpC,MAAmC,EACnC,YAAoB;IAEpB,IACC,KAAK,CAAC,2BAA2B,IAAI,MAAM,CAAC,8BAA8B;QAC1E,KAAK,CAAC,uBAAuB,KAAK,SAAS;QAC3C,YAAY,GAAG,KAAK,CAAC,uBAAuB,IAAI,MAAM,CAAC,qBAAqB,EAC3E,CAAC;QACF,KAAK,CAAC,2BAA2B,GAAG,CAAC,CAAC;IACvC,CAAC;AACF,CAAC;AAED,SAAS,cAAc,CAAC,KAOvB;IACA,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QAC3B,OAAO,EAAE,mBAAmB,EAAE,KAAK,CAAC,mBAAmB,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IAC7F,CAAC;IACD,IAAI,KAAK,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAClC,OAAO,EAAE,mBAAmB,EAAE,KAAK,CAAC,mBAAmB,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IAC9F,CAAC;IACD,IAAI,KAAK,CAAC,mBAAmB,GAAG,KAAK,CAAC,MAAM,CAAC,2BAA2B,EAAE,CAAC;QAC1E,OAAO,EAAE,mBAAmB,EAAE,KAAK,CAAC,mBAAmB,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;IACzE,CAAC;IACD,IAAI,gBAAgB,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC;QACrE,OAAO,EAAE,mBAAmB,EAAE,KAAK,CAAC,mBAAmB,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IAC7F,CAAC;IACD,wCAAwC,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IACxF,IAAI,2BAA2B,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC;QAChF,OAAO;YACN,2BAA2B,EAAE,KAAK,CAAC,KAAK,CAAC,2BAA2B;YACpE,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;YAC9C,IAAI,EAAE,WAAW;YACjB,MAAM,EAAE,uBAAuB;YAC/B,MAAM,EAAE,KAAK,CAAC,MAAM;SACpB,CAAC;IACH,CAAC;IACD,OAAO;QACN,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;QAC9C,IAAI,EAAE,SAAS;QACf,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,MAAM,EAAE,KAAK,CAAC,MAAM;KACpB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,8BAA8B,CAC7C,OAA8C;IAE9C,MAAM,aAAa,GAAG,IAAI,GAAG,EAAyC,CAAC;IAEvE,MAAM,eAAe,GAAG,CAAC,MAAc,EAAiC,EAAE;QACzE,MAAM,aAAa,GAAG,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAChD,IAAI,aAAa,EAAE,CAAC;YACnB,OAAO,aAAa,CAAC;QACtB,CAAC;QACD,MAAM,KAAK,GAAG,0CAA0C,EAAE,CAAC;QAC3D,aAAa,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACjC,OAAO,KAAK,CAAC;IACd,CAAC,CAAC;IAEF,OAAO;QACN,oBAAoB,CAAC,KAAK;YACzB,MAAM,KAAK,GAAG,eAAe,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC5C,KAAK,CAAC,gBAAgB,GAAG,KAAK,CAAC;YAC/B,IAAI,KAAK,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;gBAC3B,KAAK,CAAC,2BAA2B,GAAG,CAAC,CAAC;gBACtC,KAAK,CAAC,qCAAqC,GAAG,CAAC,CAAC;gBAChD,KAAK,CAAC,iCAAiC,GAAG,CAAC,CAAC;gBAC5C,OAAO;YACR,CAAC;YACD,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAC/B,KAAK,CAAC,2BAA2B,IAAI,CAAC,CAAC;YACxC,CAAC;QACF,CAAC;QACD,mBAAmB,CAAC,KAAK;YACxB,MAAM,KAAK,GAAG,eAAe,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC5C,KAAK,CAAC,uBAAuB,GAAG,KAAK,CAAC,YAAY,CAAC;YACnD,KAAK,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAC/B,CAAC;QACD,mCAAmC,CAAC,WAAW;YAC9C,MAAM,KAAK,GAAG,eAAe,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YAClD,IAAI,WAAW,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;gBACzC,OAAO;oBACN,mBAAmB,EAAE,KAAK,CAAC,qCAAqC;oBAChE,IAAI,EAAE,MAAM;oBACZ,MAAM,EAAE,YAAY;iBACpB,CAAC;YACH,CAAC;YACD,IAAI,oBAAoB,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9C,KAAK,CAAC,qCAAqC,GAAG,CAAC,CAAC;gBAChD,OAAO,EAAE,mBAAmB,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;YACjD,CAAC;YACD,IAAI,qBAAqB,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC/C,KAAK,CAAC,qCAAqC,IAAI,CAAC,CAAC;YAClD,CAAC;YACD,OAAO,cAAc,CAAC;gBACrB,mBAAmB,EAAE,KAAK,CAAC,qCAAqC;gBAChE,YAAY,EAAE,WAAW,CAAC,YAAY;gBACtC,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,MAAM,EAAE,gCAAgC;gBACxC,KAAK;gBACL,MAAM,EAAE,WAAW,CAAC,MAAM;aAC1B,CAAC,CAAC;QACJ,CAAC;QACD,yBAAyB,CAAC,WAAW;YACpC,MAAM,KAAK,GAAG,eAAe,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YAClD,IAAI,oBAAoB,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9C,KAAK,CAAC,iCAAiC,GAAG,CAAC,CAAC;gBAC5C,OAAO,EAAE,mBAAmB,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;YACjD,CAAC;YACD,IAAI,qBAAqB,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC/C,KAAK,CAAC,iCAAiC,IAAI,CAAC,CAAC;YAC9C,CAAC;YACD,OAAO,cAAc,CAAC;gBACrB,mBAAmB,EAAE,KAAK,CAAC,iCAAiC;gBAC5D,YAAY,EAAE,WAAW,CAAC,YAAY;gBACtC,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,MAAM,EAAE,2BAA2B;gBACnC,KAAK;gBACL,MAAM,EAAE,WAAW,CAAC,MAAM;aAC1B,CAAC,CAAC;QACJ,CAAC;KACD,CAAC;AACH,CAAC"}
+{"version":3,"file":"gateway-vm-recovery-policy.js","sourceRoot":"","sources":["../../../src/controller/health/gateway-vm-recovery-policy.ts"],"names":[],"mappings":"AA8BA,MAAM,CAAC,MAAM,6CAA6C,GAAG;IAC5D,2BAA2B,EAAE,CAAC;IAC9B,OAAO,EAAE,IAAI;IACb,2BAA2B,EAAE,IAAI;IACjC,6BAA6B,EAAE,KAAK;IACpC,sBAAsB,EAAE,OAAO;CACkB,CAAC;AAoEnD,SAAS,yCAAyC;IACjD,OAAO;QACN,2BAA2B,EAAE,CAAC;QAC9B,uBAAuB,EAAE,SAAS;KAClC,CAAC;AACH,CAAC;AAED,MAAM,0CAA0C,GAAG,GAAkC,EAAE,CAAC,CAAC;IACxF,2CAA2C,EAAE,IAAI,GAAG,EAAE;IACtD,qCAAqC,EAAE,CAAC;IACxC,iCAAiC,EAAE,CAAC;IACpC,eAAe,EAAE;QAChB,uBAAuB,EAAE,yCAAyC,EAAE;QACpE,oBAAoB,EAAE,yCAAyC,EAAE;KACjE;IACD,gBAAgB,EAAE,KAAK;CACvB,CAAC,CAAC;AAEH,SAAS,2BAA2B,CACnC,KAAmE;IAEnE,OAAO,KAAK,CAAC,mBAAmB,IAAI,oBAAoB,CAAC;AAC1D,CAAC;AAED,SAAS,oBAAoB,CAAC,MAA0C;IACvE,OAAO,MAAM,KAAK,IAAI,CAAC;AACxB,CAAC;AAED,SAAS,qBAAqB,CAAC,MAA0C;IACxE,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,OAAO,IAAI,MAAM,KAAK,SAAS,CAAC;AAC1E,CAAC;AAED,SAAS,gBAAgB,CACxB,WAAyC,EACzC,MAAmC,EACnC,YAAoB;IAEpB,OAAO,CACN,WAAW,CAAC,uBAAuB,KAAK,SAAS;QACjD,YAAY,GAAG,WAAW,CAAC,uBAAuB,GAAG,MAAM,CAAC,UAAU,CACtE,CAAC;AACH,CAAC;AAED,SAAS,2BAA2B,CACnC,WAAyC,EACzC,MAAmC,EACnC,YAAoB;IAEpB,OAAO,CACN,WAAW,CAAC,2BAA2B,IAAI,MAAM,CAAC,8BAA8B;QAChF,WAAW,CAAC,uBAAuB,KAAK,SAAS;QACjD,YAAY,GAAG,WAAW,CAAC,uBAAuB,GAAG,MAAM,CAAC,qBAAqB,CACjF,CAAC;AACH,CAAC;AAED,SAAS,wCAAwC,CAChD,WAAyC,EACzC,MAAmC,EACnC,YAAoB;IAEpB,IACC,WAAW,CAAC,2BAA2B,IAAI,MAAM,CAAC,8BAA8B;QAChF,WAAW,CAAC,uBAAuB,KAAK,SAAS;QACjD,YAAY,GAAG,WAAW,CAAC,uBAAuB,IAAI,MAAM,CAAC,qBAAqB,EACjF,CAAC;QACF,WAAW,CAAC,2BAA2B,GAAG,CAAC,CAAC;IAC7C,CAAC;AACF,CAAC;AAED,SAAS,yBAAyB,CAAC,WAAyC;IAC3E,OAAO,WAAW,CAAC,iBAAiB,IAAI,4BAA4B,CAAC;AACtE,CAAC;AAED,SAAS,cAAc,CAAC,KAQvB;IACA,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QAC3B,OAAO,EAAE,mBAAmB,EAAE,KAAK,CAAC,mBAAmB,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IAC7F,CAAC;IACD,IAAI,KAAK,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAClC,OAAO,EAAE,mBAAmB,EAAE,KAAK,CAAC,mBAAmB,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IAC9F,CAAC;IACD,IAAI,KAAK,CAAC,mBAAmB,GAAG,KAAK,CAAC,MAAM,CAAC,2BAA2B,EAAE,CAAC;QAC1E,OAAO,EAAE,mBAAmB,EAAE,KAAK,CAAC,mBAAmB,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;IACzE,CAAC;IACD,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,eAAe,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;IAC3E,IAAI,gBAAgB,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC;QACrE,OAAO,EAAE,mBAAmB,EAAE,KAAK,CAAC,mBAAmB,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IAC7F,CAAC;IACD,wCAAwC,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IACxF,IAAI,2BAA2B,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC;QAChF,OAAO;YACN,2BAA2B,EAAE,WAAW,CAAC,2BAA2B;YACpE,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;YAC9C,IAAI,EAAE,WAAW;YACjB,MAAM,EAAE,uBAAuB;YAC/B,MAAM,EAAE,KAAK,CAAC,MAAM;SACpB,CAAC;IACH,CAAC;IACD,OAAO;QACN,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;QAC9C,IAAI,EAAE,SAAS;QACf,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,MAAM,EAAE,KAAK,CAAC,MAAM;KACpB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,8BAA8B,CAC7C,OAA8C;IAE9C,MAAM,aAAa,GAAG,IAAI,GAAG,EAAyC,CAAC;IAEvE,MAAM,eAAe,GAAG,CAAC,MAAc,EAAiC,EAAE;QACzE,MAAM,aAAa,GAAG,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAChD,IAAI,aAAa,EAAE,CAAC;YACnB,OAAO,aAAa,CAAC;QACtB,CAAC;QACD,MAAM,KAAK,GAAG,0CAA0C,EAAE,CAAC;QAC3D,aAAa,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACjC,OAAO,KAAK,CAAC;IACd,CAAC,CAAC;IAEF,OAAO;QACN,oBAAoB,CAAC,KAAK;YACzB,MAAM,KAAK,GAAG,eAAe,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC5C,KAAK,CAAC,gBAAgB,GAAG,KAAK,CAAC;YAC/B,IAAI,KAAK,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;gBAC3B,KAAK,CAAC,2CAA2C,CAAC,KAAK,EAAE,CAAC;gBAC1D,KAAK,CAAC,qCAAqC,GAAG,CAAC,CAAC;gBAChD,KAAK,CAAC,iCAAiC,GAAG,CAAC,CAAC;gBAC5C,KAAK,MAAM,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,EAAE,CAAC;oBAChE,WAAW,CAAC,2BAA2B,GAAG,CAAC,CAAC;gBAC7C,CAAC;gBACD,OAAO;YACR,CAAC;YACD,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAC/B,KAAK,CAAC,eAAe,CAAC,2BAA2B,CAAC,KAAK,CAAC,CAAC,CAAC,2BAA2B,IAAI,CAAC,CAAC;YAC5F,CAAC;QACF,CAAC;QACD,mBAAmB,CAAC,KAAK;YACxB,MAAM,KAAK,GAAG,eAAe,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC5C,KAAK,CAAC,eAAe,CAAC,2BAA2B,CAAC,KAAK,CAAC,CAAC,CAAC,uBAAuB;gBAChF,KAAK,CAAC,YAAY,CAAC;YACpB,KAAK,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAC/B,CAAC;QACD,qCAAqC,CAAC,WAAW;YAChD,MAAM,qBAAqB,GAC1B,OAAO,CAAC,MAAM,CAAC,qBAAqB,IAAI,6CAA6C,CAAC;YACvF,MAAM,KAAK,GAAG,eAAe,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YAClD,MAAM,UAAU,GAAG,yBAAyB,CAAC,WAAW,CAAC,CAAC;YAC1D,MAAM,0BAA0B,GAC/B,KAAK,CAAC,2CAA2C,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YACxE,IAAI,WAAW,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;gBACzC,OAAO;oBACN,mBAAmB,EAAE,0BAA0B;oBAC/C,IAAI,EAAE,MAAM;oBACZ,MAAM,EAAE,YAAY;iBACpB,CAAC;YACH,CAAC;YACD,IAAI,oBAAoB,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9C,KAAK,CAAC,2CAA2C,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;gBACrE,OAAO,EAAE,mBAAmB,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;YACjD,CAAC;YACD,IAAI,mBAAmB,GAAG,0BAA0B,CAAC;YACrD,IAAI,qBAAqB,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC/C,mBAAmB,IAAI,CAAC,CAAC;gBACzB,KAAK,CAAC,2CAA2C,CAAC,GAAG,CAAC,UAAU,EAAE,mBAAmB,CAAC,CAAC;YACxF,CAAC;YACD,OAAO,cAAc,CAAC;gBACrB,mBAAmB;gBACnB,YAAY,EAAE,WAAW,CAAC,YAAY;gBACtC,MAAM,EAAE;oBACP,GAAG,OAAO,CAAC,MAAM;oBACjB,2BAA2B,EAAE,qBAAqB,CAAC,2BAA2B;oBAC9E,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,OAAO,IAAI,qBAAqB,CAAC,OAAO;iBAChE;gBACD,MAAM,EAAE,kCAAkC;gBAC1C,mBAAmB,EAAE,WAAW,CAAC,mBAAmB,IAAI,oBAAoB;gBAC5E,KAAK;gBACL,MAAM,EAAE,WAAW,CAAC,MAAM;aAC1B,CAAC,CAAC;QACJ,CAAC;QACD,mCAAmC,CAAC,WAAW;YAC9C,MAAM,KAAK,GAAG,eAAe,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YAClD,IAAI,WAAW,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;gBACzC,OAAO;oBACN,mBAAmB,EAAE,KAAK,CAAC,qCAAqC;oBAChE,IAAI,EAAE,MAAM;oBACZ,MAAM,EAAE,YAAY;iBACpB,CAAC;YACH,CAAC;YACD,IAAI,oBAAoB,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9C,KAAK,CAAC,qCAAqC,GAAG,CAAC,CAAC;gBAChD,OAAO,EAAE,mBAAmB,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;YACjD,CAAC;YACD,IAAI,qBAAqB,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC/C,KAAK,CAAC,qCAAqC,IAAI,CAAC,CAAC;YAClD,CAAC;YACD,OAAO,cAAc,CAAC;gBACrB,mBAAmB,EAAE,KAAK,CAAC,qCAAqC;gBAChE,YAAY,EAAE,WAAW,CAAC,YAAY;gBACtC,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,MAAM,EAAE,gCAAgC;gBACxC,mBAAmB,EAAE,WAAW,CAAC,mBAAmB,IAAI,oBAAoB;gBAC5E,KAAK;gBACL,MAAM,EAAE,WAAW,CAAC,MAAM;aAC1B,CAAC,CAAC;QACJ,CAAC;QACD,yBAAyB,CAAC,WAAW;YACpC,MAAM,KAAK,GAAG,eAAe,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YAClD,IAAI,oBAAoB,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9C,KAAK,CAAC,iCAAiC,GAAG,CAAC,CAAC;gBAC5C,OAAO,EAAE,mBAAmB,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;YACjD,CAAC;YACD,IAAI,qBAAqB,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC/C,KAAK,CAAC,iCAAiC,IAAI,CAAC,CAAC;YAC9C,CAAC;YACD,OAAO,cAAc,CAAC;gBACrB,mBAAmB,EAAE,KAAK,CAAC,iCAAiC;gBAC5D,YAAY,EAAE,WAAW,CAAC,YAAY;gBACtC,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,MAAM,EAAE,2BAA2B;gBACnC,mBAAmB,EAAE,WAAW,CAAC,mBAAmB,IAAI,oBAAoB;gBAC5E,KAAK;gBACL,MAAM,EAAE,WAAW,CAAC,MAAM;aAC1B,CAAC,CAAC;QACJ,CAAC;KACD,CAAC;AACH,CAAC"}

package/dist/controller/health/gateway-vm-recovery-runner.d.ts

@@ -0,0 +1,39 @@
+import type { ControllerRuntimeReadiness } from '../http/controller-http-route-support.js';
+import type { GatewayZoneLifecycleState } from '../zone-runtimes/gateway-zone-state-machine.js';
+import type { OpenClawZoneRestartResult } from '../zone-runtimes/zone-runtime-types.js';
+import type { GatewayVmRecoveryRequest, GatewayVmRecoveryResult } from './gateway-service-health-monitor.js';
+export interface CreateGatewayVmRecoveryRunnerOptions {
+    readonly getRecoverableGatewayRuntime: (zoneId: string) => RecoverableGatewayRuntime;
+    readonly getRuntimeReadiness: () => ControllerRuntimeReadiness;
+    readonly now: () => number;
+    readonly restartTimeoutMs: number;
+    readonly writeLog: (message: string) => void;
+}
+export interface RecoverableGatewayRuntime {
+    readonly coldStart: (options: {
+        readonly operationTrigger: 'auto-recovery';
+        readonly timeoutMs: number;
+    }) => Promise<OpenClawZoneRestartResult>;
+    readonly getLifecycleState: () => GatewayZoneLifecycleState;
+    readonly getSnapshot: () => {
+        readonly bootedAt?: string | undefined;
+        readonly gateway?: {
+            readonly vm: {
+                readonly hostPid?: number | undefined;
+                readonly id: string;
+            };
+        } | undefined;
+        readonly lifecycleState: 'failed' | 'running' | 'stopped';
+    };
+    readonly refreshCredentials: () => Promise<{
+        readonly ok: true;
+        readonly zoneId: string;
+    }>;
+    readonly restart: (options: {
+        readonly operationTrigger: 'auto-recovery';
+        readonly timeoutMs: number;
+    }) => Promise<OpenClawZoneRestartResult>;
+}
+export declare function classifyGatewayRecoveryRestartError(error: unknown): string;
+export declare function createGatewayVmRecoveryRunner(options: CreateGatewayVmRecoveryRunnerOptions): (request: GatewayVmRecoveryRequest) => Promise<GatewayVmRecoveryResult>;
+//# sourceMappingURL=gateway-vm-recovery-runner.d.ts.map

package/dist/controller/health/gateway-vm-recovery-runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gateway-vm-recovery-runner.d.ts","sourceRoot":"","sources":["../../../src/controller/health/gateway-vm-recovery-runner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,0CAA0C,CAAC;AAC3F,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,gDAAgD,CAAC;AAGhG,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,wCAAwC,CAAC;AAExF,OAAO,KAAK,EACX,wBAAwB,EACxB,uBAAuB,EACvB,MAAM,qCAAqC,CAAC;AAE7C,MAAM,WAAW,oCAAoC;IACpD,QAAQ,CAAC,4BAA4B,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,yBAAyB,CAAC;IACrF,QAAQ,CAAC,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;IAC/D,QAAQ,CAAC,GAAG,EAAE,MAAM,MAAM,CAAC;IAC3B,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CAC7C;AAED,MAAM,WAAW,yBAAyB;IACzC,QAAQ,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE;QAC7B,QAAQ,CAAC,gBAAgB,EAAE,eAAe,CAAC;QAC3C,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;KAC3B,KAAK,OAAO,CAAC,yBAAyB,CAAC,CAAC;IACzC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;IAC5D,QAAQ,CAAC,WAAW,EAAE,MAAM;QAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;QACvC,QAAQ,CAAC,OAAO,CAAC,EACd;YACA,QAAQ,CAAC,EAAE,EAAE;gBAAE,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;gBAAC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;aAAE,CAAC;SAC3E,GACD,SAAS,CAAC;QACb,QAAQ,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,SAAS,CAAC;KAC1D,CAAC;IACF,QAAQ,CAAC,kBAAkB,EAAE,MAAM,OAAO,CAAC;QAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;QAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC3F,QAAQ,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE;QAC3B,QAAQ,CAAC,gBAAgB,EAAE,eAAe,CAAC;QAC3C,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;KAC3B,KAAK,OAAO,CAAC,yBAAyB,CAAC,CAAC;CACzC;AAgBD,wBAAgB,mCAAmC,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAiC1E;AAoBD,wBAAgB,6BAA6B,CAC5C,OAAO,EAAE,oCAAoC,GAC3C,CAAC,OAAO,EAAE,wBAAwB,KAAK,OAAO,CAAC,uBAAuB,CAAC,CAyLzE"}

package/dist/controller/health/gateway-vm-recovery-runner.js

@@ -0,0 +1,251 @@
+import { isOpenClawZoneRestartTimeoutError } from '../zone-runtimes/openclaw-zone-runtime.js';
+import { ControllerZoneRuntimeStartError } from '../zone-runtimes/zone-runtime-errors.js';
+import { classifyGatewayRecoveryAction } from './gateway-recovery-actions.js';
+function formatUnknownError(error) {
+    if (error instanceof Error) {
+        return error.message;
+    }
+    return typeof error === 'string' ? error : JSON.stringify(error);
+}
+function getUnknownErrorCode(error) {
+    if (typeof error !== 'object' || error === null || !('code' in error)) {
+        return undefined;
+    }
+    return typeof error.code === 'string' ? error.code : undefined;
+}
+export function classifyGatewayRecoveryRestartError(error) {
+    if (error instanceof ControllerZoneRuntimeStartError &&
+        error.gatewayLifecycleErrorCode !== undefined) {
+        return error.gatewayLifecycleErrorCode;
+    }
+    if (isOpenClawZoneRestartTimeoutError(error)) {
+        return 'recovery-timeout';
+    }
+    const code = getUnknownErrorCode(error);
+    if (code === 'EACCES' ||
+        code === 'EIO' ||
+        code === 'ENOENT' ||
+        code === 'ENOSPC' ||
+        code === 'EROFS') {
+        return 'restart-disk-failure';
+    }
+    const message = formatUnknownError(error).toLowerCase();
+    if (message.includes('1password') ||
+        message.includes('credential') ||
+        message.includes('op://') ||
+        message.includes('secret')) {
+        return 'restart-secret-failure';
+    }
+    if (message.includes('gondolin') || message.includes('qemu') || message.includes('vm.create')) {
+        return 'restart-vm-create-failed';
+    }
+    return 'restart-threw';
+}
+function operationIdForRecoveryError(error) {
+    return error instanceof ControllerZoneRuntimeStartError ? error.operationId : undefined;
+}
+function classifyGatewayRecoveryRuntimeError(runtime, error) {
+    const lifecycleState = runtime.getLifecycleState();
+    if (lifecycleState.kind === 'failed' && lifecycleState.error.code === 'owner-unsafe') {
+        return 'owner-unsafe';
+    }
+    if (lifecycleState.kind === 'owner-unsafe') {
+        return 'owner-unsafe';
+    }
+    return classifyGatewayRecoveryRestartError(error);
+}
+export function createGatewayVmRecoveryRunner(options) {
+    return async (request) => {
+        const startedAtMs = options.now();
+        const elapsedMs = () => options.now() - startedAtMs;
+        if (options.getRuntimeReadiness().state === 'stopping') {
+            return {
+                action: 'observe-only',
+                elapsedMs: elapsedMs(),
+                errorCode: 'controller-stopping',
+                result: 'failed',
+            };
+        }
+        let runtime;
+        try {
+            runtime = options.getRecoverableGatewayRuntime(request.zoneId);
+        }
+        catch (error) {
+            options.writeLog(`Gateway VM recovery failed to find OpenClaw runtime for zone '${request.zoneId}': ${formatUnknownError(error)}`);
+            return {
+                action: 'observe-only',
+                elapsedMs: elapsedMs(),
+                errorCode: 'runtime-unavailable',
+                result: 'failed',
+            };
+        }
+        const oldSnapshot = runtime.getSnapshot();
+        const recoveryAction = classifyGatewayRecoveryAction({
+            lifecycleState: runtime.getLifecycleState(),
+            recoveryDecision: {
+                consecutiveFailures: request.consecutiveFailures,
+                kind: 'restart',
+                reason: request.reason,
+                zoneId: request.zoneId,
+            },
+        });
+        if (recoveryAction.kind === 'refresh-secret-resolver') {
+            options.writeLog(`Refreshing gateway secret resolver for zone '${request.zoneId}' after ${request.consecutiveFailures} consecutive ${request.reason} observations.`);
+            try {
+                await runtime.refreshCredentials();
+            }
+            catch (error) {
+                options.writeLog(`Gateway VM recovery credential refresh failed for zone '${request.zoneId}': ${formatUnknownError(error)}`);
+                return {
+                    action: 'gateway-vm-cold-start',
+                    elapsedMs: elapsedMs(),
+                    errorCode: classifyGatewayRecoveryRuntimeError(runtime, error),
+                    ...(operationIdForRecoveryError(error) === undefined
+                        ? {}
+                        : { operationId: operationIdForRecoveryError(error) }),
+                    result: 'failed',
+                };
+            }
+            return verifyColdStartRecovery({
+                elapsedMs,
+                leaseReleaseFailureCount: 0,
+                operationId: undefined,
+                runtime,
+            });
+        }
+        if (recoveryAction.kind === 'cold-start-gateway') {
+            options.writeLog(`Auto cold-starting gateway VM for zone '${request.zoneId}' after ${request.consecutiveFailures} consecutive ${request.reason} observations.`);
+            let coldStartResult;
+            try {
+                coldStartResult = await runtime.coldStart({
+                    operationTrigger: 'auto-recovery',
+                    timeoutMs: options.restartTimeoutMs,
+                });
+            }
+            catch (error) {
+                options.writeLog(`Gateway VM recovery cold-start failed for zone '${request.zoneId}': ${formatUnknownError(error)}`);
+                return {
+                    action: 'gateway-vm-cold-start',
+                    elapsedMs: elapsedMs(),
+                    errorCode: classifyGatewayRecoveryRuntimeError(runtime, error),
+                    ...(operationIdForRecoveryError(error) === undefined
+                        ? {}
+                        : { operationId: operationIdForRecoveryError(error) }),
+                    result: 'failed',
+                };
+            }
+            return verifyColdStartRecovery({
+                elapsedMs,
+                leaseReleaseFailureCount: coldStartResult.leaseReleaseFailureCount,
+                operationId: coldStartResult.operationId,
+                runtime,
+            });
+        }
+        if (oldSnapshot.lifecycleState !== 'running' || !oldSnapshot.gateway) {
+            const failedAction = recoveryAction.kind === 'operator-required'
+                ? 'operator-required'
+                : recoveryAction.kind === 'observe-only'
+                    ? 'observe-only'
+                    : 'operator-required';
+            return {
+                action: failedAction,
+                elapsedMs: elapsedMs(),
+                errorCode: recoveryAction.kind === 'operator-required'
+                    ? recoveryAction.reason
+                    : recoveryAction.kind === 'observe-only'
+                        ? recoveryAction.reason
+                        : 'old-gateway-not-running',
+                result: 'failed',
+            };
+        }
+        const oldGateway = oldSnapshot.gateway;
+        const oldBootedAt = oldSnapshot.bootedAt;
+        const oldHostPid = oldGateway.vm.hostPid;
+        options.writeLog(`Auto-restarting gateway VM for zone '${request.zoneId}' after ${request.consecutiveFailures} consecutive ${request.reason} observations.`);
+        let restartResult;
+        try {
+            restartResult = await runtime.restart({
+                operationTrigger: 'auto-recovery',
+                timeoutMs: options.restartTimeoutMs,
+            });
+        }
+        catch (error) {
+            options.writeLog(`Gateway VM recovery restart failed for zone '${request.zoneId}': ${formatUnknownError(error)}`);
+            return {
+                action: 'gateway-vm-restart',
+                elapsedMs: elapsedMs(),
+                errorCode: classifyGatewayRecoveryRuntimeError(runtime, error),
+                ...(oldBootedAt === undefined ? {} : { oldBootedAt }),
+                ...(oldHostPid === undefined ? {} : { oldHostPid }),
+                oldVmId: oldGateway.vm.id,
+                ...(operationIdForRecoveryError(error) === undefined
+                    ? {}
+                    : { operationId: operationIdForRecoveryError(error) }),
+                result: 'failed',
+            };
+        }
+        const newSnapshot = runtime.getSnapshot();
+        const newBootedAt = newSnapshot.bootedAt;
+        const newGateway = newSnapshot.gateway;
+        if (newSnapshot.lifecycleState !== 'running' ||
+            !newGateway ||
+            oldBootedAt === undefined ||
+            newBootedAt === undefined ||
+            oldHostPid === undefined ||
+            newGateway.vm.hostPid === undefined ||
+            newGateway.vm.id === oldGateway.vm.id ||
+            newGateway.vm.hostPid === oldHostPid ||
+            newBootedAt === oldBootedAt) {
+            return {
+                action: 'gateway-vm-restart',
+                elapsedMs: elapsedMs(),
+                errorCode: 'restart-verification-failed',
+                ...(oldBootedAt === undefined ? {} : { oldBootedAt }),
+                ...(oldHostPid === undefined ? {} : { oldHostPid }),
+                oldVmId: oldGateway.vm.id,
+                result: 'failed',
+            };
+        }
+        return {
+            elapsedMs: elapsedMs(),
+            leaseReleaseFailureCount: restartResult.leaseReleaseFailureCount,
+            newBootedAt,
+            newHostPid: newGateway.vm.hostPid,
+            newVmId: newGateway.vm.id,
+            oldBootedAt,
+            oldHostPid,
+            oldVmId: oldGateway.vm.id,
+            ...(restartResult.operationId === undefined
+                ? {}
+                : { operationId: restartResult.operationId }),
+            result: 'ok',
+        };
+    };
+}
+function verifyColdStartRecovery(options) {
+    const newSnapshot = options.runtime.getSnapshot();
+    const newBootedAt = newSnapshot.bootedAt;
+    const newGateway = newSnapshot.gateway;
+    if (newSnapshot.lifecycleState !== 'running' ||
+        !newGateway ||
+        newBootedAt === undefined ||
+        newGateway.vm.hostPid === undefined) {
+        return {
+            action: 'gateway-vm-cold-start',
+            elapsedMs: options.elapsedMs(),
+            errorCode: 'cold-start-verification-failed',
+            result: 'failed',
+        };
+    }
+    return {
+        action: 'gateway-vm-cold-start',
+        elapsedMs: options.elapsedMs(),
+        leaseReleaseFailureCount: options.leaseReleaseFailureCount,
+        newBootedAt,
+        newHostPid: newGateway.vm.hostPid,
+        newVmId: newGateway.vm.id,
+        ...(options.operationId === undefined ? {} : { operationId: options.operationId }),
+        result: 'ok',
+    };
+}
+//# sourceMappingURL=gateway-vm-recovery-runner.js.map

package/dist/controller/health/gateway-vm-recovery-runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gateway-vm-recovery-runner.js","sourceRoot":"","sources":["../../../src/controller/health/gateway-vm-recovery-runner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iCAAiC,EAAE,MAAM,2CAA2C,CAAC;AAC9F,OAAO,EAAE,+BAA+B,EAAE,MAAM,yCAAyC,CAAC;AAE1F,OAAO,EAAE,6BAA6B,EAAE,MAAM,+BAA+B,CAAC;AAoC9E,SAAS,kBAAkB,CAAC,KAAc;IACzC,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;QAC5B,OAAO,KAAK,CAAC,OAAO,CAAC;IACtB,CAAC;IACD,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAc;IAC1C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,EAAE,CAAC;QACvE,OAAO,SAAS,CAAC;IAClB,CAAC;IACD,OAAO,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;AAChE,CAAC;AAED,MAAM,UAAU,mCAAmC,CAAC,KAAc;IACjE,IACC,KAAK,YAAY,+BAA+B;QAChD,KAAK,CAAC,yBAAyB,KAAK,SAAS,EAC5C,CAAC;QACF,OAAO,KAAK,CAAC,yBAAyB,CAAC;IACxC,CAAC;IACD,IAAI,iCAAiC,CAAC,KAAK,CAAC,EAAE,CAAC;QAC9C,OAAO,kBAAkB,CAAC;IAC3B,CAAC;IACD,MAAM,IAAI,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;IACxC,IACC,IAAI,KAAK,QAAQ;QACjB,IAAI,KAAK,KAAK;QACd,IAAI,KAAK,QAAQ;QACjB,IAAI,KAAK,QAAQ;QACjB,IAAI,KAAK,OAAO,EACf,CAAC;QACF,OAAO,sBAAsB,CAAC;IAC/B,CAAC;IACD,MAAM,OAAO,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;IACxD,IACC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC;QAC7B,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;QAC9B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;QACzB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EACzB,CAAC;QACF,OAAO,wBAAwB,CAAC;IACjC,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAC/F,OAAO,0BAA0B,CAAC;IACnC,CAAC;IACD,OAAO,eAAe,CAAC;AACxB,CAAC;AAED,SAAS,2BAA2B,CAAC,KAAc;IAClD,OAAO,KAAK,YAAY,+BAA+B,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;AACzF,CAAC;AAED,SAAS,mCAAmC,CAC3C,OAAkC,EAClC,KAAc;IAEd,MAAM,cAAc,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;IACnD,IAAI,cAAc,CAAC,IAAI,KAAK,QAAQ,IAAI,cAAc,CAAC,KAAK,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;QACtF,OAAO,cAAc,CAAC;IACvB,CAAC;IACD,IAAI,cAAc,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;QAC5C,OAAO,cAAc,CAAC;IACvB,CAAC;IACD,OAAO,mCAAmC,CAAC,KAAK,CAAC,CAAC;AACnD,CAAC;AAED,MAAM,UAAU,6BAA6B,CAC5C,OAA6C;IAE7C,OAAO,KAAK,EAAE,OAAO,EAAoC,EAAE;QAC1D,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QAClC,MAAM,SAAS,GAAG,GAAW,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC;QAC5D,IAAI,OAAO,CAAC,mBAAmB,EAAE,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;YACxD,OAAO;gBACN,MAAM,EAAE,cAAc;gBACtB,SAAS,EAAE,SAAS,EAAE;gBACtB,SAAS,EAAE,qBAAqB;gBAChC,MAAM,EAAE,QAAQ;aAChB,CAAC;QACH,CAAC;QAED,IAAI,OAAkC,CAAC;QACvC,IAAI,CAAC;YACJ,OAAO,GAAG,OAAO,CAAC,4BAA4B,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAChE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,OAAO,CAAC,QAAQ,CACf,iEAAiE,OAAO,CAAC,MAAM,MAAM,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAChH,CAAC;YACF,OAAO;gBACN,MAAM,EAAE,cAAc;gBACtB,SAAS,EAAE,SAAS,EAAE;gBACtB,SAAS,EAAE,qBAAqB;gBAChC,MAAM,EAAE,QAAQ;aAChB,CAAC;QACH,CAAC;QAED,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;QAC1C,MAAM,cAAc,GAAG,6BAA6B,CAAC;YACpD,cAAc,EAAE,OAAO,CAAC,iBAAiB,EAAE;YAC3C,gBAAgB,EAAE;gBACjB,mBAAmB,EAAE,OAAO,CAAC,mBAAmB;gBAChD,IAAI,EAAE,SAAS;gBACf,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,MAAM,EAAE,OAAO,CAAC,MAAM;aACtB;SACD,CAAC,CAAC;QACH,IAAI,cAAc,CAAC,IAAI,KAAK,yBAAyB,EAAE,CAAC;YACvD,OAAO,CAAC,QAAQ,CACf,gDAAgD,OAAO,CAAC,MAAM,WAAW,OAAO,CAAC,mBAAmB,gBAAgB,OAAO,CAAC,MAAM,gBAAgB,CAClJ,CAAC;YACF,IAAI,CAAC;gBACJ,MAAM,OAAO,CAAC,kBAAkB,EAAE,CAAC;YACpC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,OAAO,CAAC,QAAQ,CACf,2DAA2D,OAAO,CAAC,MAAM,MAAM,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAC1G,CAAC;gBACF,OAAO;oBACN,MAAM,EAAE,uBAAuB;oBAC/B,SAAS,EAAE,SAAS,EAAE;oBACtB,SAAS,EAAE,mCAAmC,CAAC,OAAO,EAAE,KAAK,CAAC;oBAC9D,GAAG,CAAC,2BAA2B,CAAC,KAAK,CAAC,KAAK,SAAS;wBACnD,CAAC,CAAC,EAAE;wBACJ,CAAC,CAAC,EAAE,WAAW,EAAE,2BAA2B,CAAC,KAAK,CAAC,EAAE,CAAC;oBACvD,MAAM,EAAE,QAAQ;iBAChB,CAAC;YACH,CAAC;YACD,OAAO,uBAAuB,CAAC;gBAC9B,SAAS;gBACT,wBAAwB,EAAE,CAAC;gBAC3B,WAAW,EAAE,SAAS;gBACtB,OAAO;aACP,CAAC,CAAC;QACJ,CAAC;QACD,IAAI,cAAc,CAAC,IAAI,KAAK,oBAAoB,EAAE,CAAC;YAClD,OAAO,CAAC,QAAQ,CACf,2CAA2C,OAAO,CAAC,MAAM,WAAW,OAAO,CAAC,mBAAmB,gBAAgB,OAAO,CAAC,MAAM,gBAAgB,CAC7I,CAAC;YACF,IAAI,eAA0C,CAAC;YAC/C,IAAI,CAAC;gBACJ,eAAe,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC;oBACzC,gBAAgB,EAAE,eAAe;oBACjC,SAAS,EAAE,OAAO,CAAC,gBAAgB;iBACnC,CAAC,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,OAAO,CAAC,QAAQ,CACf,mDAAmD,OAAO,CAAC,MAAM,MAAM,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAClG,CAAC;gBACF,OAAO;oBACN,MAAM,EAAE,uBAAuB;oBAC/B,SAAS,EAAE,SAAS,EAAE;oBACtB,SAAS,EAAE,mCAAmC,CAAC,OAAO,EAAE,KAAK,CAAC;oBAC9D,GAAG,CAAC,2BAA2B,CAAC,KAAK,CAAC,KAAK,SAAS;wBACnD,CAAC,CAAC,EAAE;wBACJ,CAAC,CAAC,EAAE,WAAW,EAAE,2BAA2B,CAAC,KAAK,CAAC,EAAE,CAAC;oBACvD,MAAM,EAAE,QAAQ;iBAChB,CAAC;YACH,CAAC;YACD,OAAO,uBAAuB,CAAC;gBAC9B,SAAS;gBACT,wBAAwB,EAAE,eAAe,CAAC,wBAAwB;gBAClE,WAAW,EAAE,eAAe,CAAC,WAAW;gBACxC,OAAO;aACP,CAAC,CAAC;QACJ,CAAC;QACD,IAAI,WAAW,CAAC,cAAc,KAAK,SAAS,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;YACtE,MAAM,YAAY,GACjB,cAAc,CAAC,IAAI,KAAK,mBAAmB;gBAC1C,CAAC,CAAC,mBAAmB;gBACrB,CAAC,CAAC,cAAc,CAAC,IAAI,KAAK,cAAc;oBACvC,CAAC,CAAC,cAAc;oBAChB,CAAC,CAAC,mBAAmB,CAAC;YACzB,OAAO;gBACN,MAAM,EAAE,YAAY;gBACpB,SAAS,EAAE,SAAS,EAAE;gBACtB,SAAS,EACR,cAAc,CAAC,IAAI,KAAK,mBAAmB;oBAC1C,CAAC,CAAC,cAAc,CAAC,MAAM;oBACvB,CAAC,CAAC,cAAc,CAAC,IAAI,KAAK,cAAc;wBACvC,CAAC,CAAC,cAAc,CAAC,MAAM;wBACvB,CAAC,CAAC,yBAAyB;gBAC9B,MAAM,EAAE,QAAQ;aAChB,CAAC;QACH,CAAC;QACD,MAAM,UAAU,GAAG,WAAW,CAAC,OAAO,CAAC;QACvC,MAAM,WAAW,GAAG,WAAW,CAAC,QAAQ,CAAC;QACzC,MAAM,UAAU,GAAG,UAAU,CAAC,EAAE,CAAC,OAAO,CAAC;QACzC,OAAO,CAAC,QAAQ,CACf,wCAAwC,OAAO,CAAC,MAAM,WAAW,OAAO,CAAC,mBAAmB,gBAAgB,OAAO,CAAC,MAAM,gBAAgB,CAC1I,CAAC;QAEF,IAAI,aAAwC,CAAC;QAC7C,IAAI,CAAC;YACJ,aAAa,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC;gBACrC,gBAAgB,EAAE,eAAe;gBACjC,SAAS,EAAE,OAAO,CAAC,gBAAgB;aACnC,CAAC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,OAAO,CAAC,QAAQ,CACf,gDAAgD,OAAO,CAAC,MAAM,MAAM,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAC/F,CAAC;YACF,OAAO;gBACN,MAAM,EAAE,oBAAoB;gBAC5B,SAAS,EAAE,SAAS,EAAE;gBACtB,SAAS,EAAE,mCAAmC,CAAC,OAAO,EAAE,KAAK,CAAC;gBAC9D,GAAG,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC;gBACrD,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC;gBACnD,OAAO,EAAE,UAAU,CAAC,EAAE,CAAC,EAAE;gBACzB,GAAG,CAAC,2BAA2B,CAAC,KAAK,CAAC,KAAK,SAAS;oBACnD,CAAC,CAAC,EAAE;oBACJ,CAAC,CAAC,EAAE,WAAW,EAAE,2BAA2B,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvD,MAAM,EAAE,QAAQ;aAChB,CAAC;QACH,CAAC;QAED,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;QAC1C,MAAM,WAAW,GAAG,WAAW,CAAC,QAAQ,CAAC;QACzC,MAAM,UAAU,GAAG,WAAW,CAAC,OAAO,CAAC;QACvC,IACC,WAAW,CAAC,cAAc,KAAK,SAAS;YACxC,CAAC,UAAU;YACX,WAAW,KAAK,SAAS;YACzB,WAAW,KAAK,SAAS;YACzB,UAAU,KAAK,SAAS;YACxB,UAAU,CAAC,EAAE,CAAC,OAAO,KAAK,SAAS;YACnC,UAAU,CAAC,EAAE,CAAC,EAAE,KAAK,UAAU,CAAC,EAAE,CAAC,EAAE;YACrC,UAAU,CAAC,EAAE,CAAC,OAAO,KAAK,UAAU;YACpC,WAAW,KAAK,WAAW,EAC1B,CAAC;YACF,OAAO;gBACN,MAAM,EAAE,oBAAoB;gBAC5B,SAAS,EAAE,SAAS,EAAE;gBACtB,SAAS,EAAE,6BAA6B;gBACxC,GAAG,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC;gBACrD,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC;gBACnD,OAAO,EAAE,UAAU,CAAC,EAAE,CAAC,EAAE;gBACzB,MAAM,EAAE,QAAQ;aAChB,CAAC;QACH,CAAC;QACD,OAAO;YACN,SAAS,EAAE,SAAS,EAAE;YACtB,wBAAwB,EAAE,aAAa,CAAC,wBAAwB;YAChE,WAAW;YACX,UAAU,EAAE,UAAU,CAAC,EAAE,CAAC,OAAO;YACjC,OAAO,EAAE,UAAU,CAAC,EAAE,CAAC,EAAE;YACzB,WAAW;YACX,UAAU;YACV,OAAO,EAAE,UAAU,CAAC,EAAE,CAAC,EAAE;YACzB,GAAG,CAAC,aAAa,CAAC,WAAW,KAAK,SAAS;gBAC1C,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,WAAW,EAAE,aAAa,CAAC,WAAW,EAAE,CAAC;YAC9C,MAAM,EAAE,IAAI;SACZ,CAAC;IACH,CAAC,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,OAKhC;IACA,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IAClD,MAAM,WAAW,GAAG,WAAW,CAAC,QAAQ,CAAC;IACzC,MAAM,UAAU,GAAG,WAAW,CAAC,OAAO,CAAC;IACvC,IACC,WAAW,CAAC,cAAc,KAAK,SAAS;QACxC,CAAC,UAAU;QACX,WAAW,KAAK,SAAS;QACzB,UAAU,CAAC,EAAE,CAAC,OAAO,KAAK,SAAS,EAClC,CAAC;QACF,OAAO;YACN,MAAM,EAAE,uBAAuB;YAC/B,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE;YAC9B,SAAS,EAAE,gCAAgC;YAC3C,MAAM,EAAE,QAAQ;SAChB,CAAC;IACH,CAAC;IACD,OAAO;QACN,MAAM,EAAE,uBAAuB;QAC/B,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE;QAC9B,wBAAwB,EAAE,OAAO,CAAC,wBAAwB;QAC1D,WAAW;QACX,UAAU,EAAE,UAAU,CAAC,EAAE,CAAC,OAAO;QACjC,OAAO,EAAE,UAAU,CAAC,EAAE,CAAC,EAAE;QACzB,GAAG,CAAC,OAAO,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC;QAClF,MAAM,EAAE,IAAI;KACZ,CAAC;AACH,CAAC"}

package/dist/controller/health/health-event-store.d.ts

@@ -1,5 +1,8 @@
 import { type AgentVmHealthEvent, type ZoneHealthSnapshot } from '@agent-vm/gateway-interface';
 export interface HealthEventStoreOptions {
+    readonly durableEventLog?: {
+        readonly append: (event: AgentVmHealthEvent) => Promise<void>;
+    } | undefined;
     readonly eventHistoryLimit: number;
     readonly latestBucketLimit?: number | undefined;
     readonly staleAfterMs: number;
@@ -13,6 +16,7 @@ export declare class HealthEventStore {
     constructor(options: HealthEventStoreOptions);
     record(event: AgentVmHealthEvent): void;
     listHistory(): readonly AgentVmHealthEvent[];
+    flushDurableWrites(): Promise<void>;
     listLatestEventsForZone(zoneId: string): readonly AgentVmHealthEvent[];
     deriveSnapshot(options: DeriveHealthSnapshotOptions): ZoneHealthSnapshot;
 }

package/dist/controller/health/health-event-store.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"health-event-store.d.ts","sourceRoot":"","sources":["../../../src/controller/health/health-event-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAGN,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EACvB,MAAM,6BAA6B,CAAC;AAErC,MAAM,WAAW,uBAAuB;IACvC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,2BAA2B;IAC3C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACxB;AAED,qBAAa,gBAAgB;;gBAOhB,OAAO,EAAE,uBAAuB;IAM5C,MAAM,CAAC,KAAK,EAAE,kBAAkB,GAAG,IAAI;IA4BvC,WAAW,IAAI,SAAS,kBAAkB,EAAE;IAI5C,uBAAuB,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,kBAAkB,EAAE;IAMtE,cAAc,CAAC,OAAO,EAAE,2BAA2B,GAAG,kBAAkB;CAOxE"}
+{"version":3,"file":"health-event-store.d.ts","sourceRoot":"","sources":["../../../src/controller/health/health-event-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAGN,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EACvB,MAAM,6BAA6B,CAAC;AAErC,MAAM,WAAW,uBAAuB;IACvC,QAAQ,CAAC,eAAe,CAAC,EACtB;QAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,kBAAkB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;KAAE,GACjE,SAAS,CAAC;IACb,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,2BAA2B;IAC3C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACxB;AAED,qBAAa,gBAAgB;;gBAShB,OAAO,EAAE,uBAAuB;IAO5C,MAAM,CAAC,KAAK,EAAE,kBAAkB,GAAG,IAAI;IA6BvC,WAAW,IAAI,SAAS,kBAAkB,EAAE;IAItC,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC;IAIzC,uBAAuB,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,kBAAkB,EAAE;IAMtE,cAAc,CAAC,OAAO,EAAE,2BAA2B,GAAG,kBAAkB;CAoBxE"}

package/dist/controller/health/health-event-store.js

@@ -4,8 +4,11 @@ export class HealthEventStore {
     #latestBucketLimit;
     #latestByBucket = new Map();
     #history = [];
+    #durableEventLog;
+    #durableWriteQueue = Promise.resolve();
     #staleAfterMs;
     constructor(options) {
+        this.#durableEventLog = options.durableEventLog;
         this.#eventHistoryLimit = options.eventHistoryLimit;
         this.#latestBucketLimit = options.latestBucketLimit ?? 1_000;
         this.#staleAfterMs = options.staleAfterMs;
@@ -21,6 +24,7 @@ export class HealthEventStore {
             this.#latestByBucket.set(key, event);
         }
         this.#evictOldestLatestBuckets();
+        this.#queueDurableWrite(event);
     }
     #evictOldestLatestBuckets() {
         if (this.#latestByBucket.size <= this.#latestBucketLimit) {
@@ -38,6 +42,9 @@ export class HealthEventStore {
     listHistory() {
         return [...this.#history];
     }
+    async flushDurableWrites() {
+        await this.#durableWriteQueue;
+    }
     listLatestEventsForZone(zoneId) {
         return [...this.#latestByBucket.values()]
             .filter((event) => event.zoneId === zoneId)
@@ -50,5 +57,17 @@ export class HealthEventStore {
             zoneId: options.zoneId,
         });
     }
+    #queueDurableWrite(event) {
+        if (!this.#durableEventLog) {
+            return;
+        }
+        this.#durableWriteQueue = this.#durableWriteQueue
+            .then(async () => {
+            await this.#durableEventLog?.append(event);
+        })
+            .catch(() => {
+            // Durable logs are evidence. In-memory health remains the serving path.
+        });
+    }
 }
 //# sourceMappingURL=health-event-store.js.map

package/dist/controller/health/health-event-store.js.map

@@ -1 +1 @@
-{"version":3,"file":"health-event-store.js","sourceRoot":"","sources":["../../../src/controller/health/health-event-store.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,wBAAwB,EACxB,oBAAoB,GAGpB,MAAM,6BAA6B,CAAC;AAarC,MAAM,OAAO,gBAAgB;IACnB,kBAAkB,CAAS;IAC3B,kBAAkB,CAAS;IAC3B,eAAe,GAAG,IAAI,GAAG,EAA8B,CAAC;IACxD,QAAQ,GAAyB,EAAE,CAAC;IACpC,aAAa,CAAS;IAE/B,YAAY,OAAgC;QAC3C,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,iBAAiB,CAAC;QACpD,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,iBAAiB,IAAI,KAAK,CAAC;QAC7D,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IAC3C,CAAC;IAED,MAAM,CAAC,KAAyB;QAC/B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACvD,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;QAED,MAAM,GAAG,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;QACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/C,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,YAAY,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;YAC9D,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,CAAC,yBAAyB,EAAE,CAAC;IAClC,CAAC;IAED,yBAAyB;QACxB,IAAI,IAAI,CAAC,eAAe,CAAC,IAAI,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC1D,OAAO;QACR,CAAC;QACD,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC;QACvE,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC;aACrD,QAAQ,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC;aAC3E,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC;aACpB,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;QACtB,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YAC/B,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAClC,CAAC;IACF,CAAC;IAED,WAAW;QACV,OAAO,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC3B,CAAC;IAED,uBAAuB,CAAC,MAAc;QACrC,OAAO,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC;aACvC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC;aAC1C,QAAQ,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC;IACzE,CAAC;IAED,cAAc,CAAC,OAAoC;QAClD,OAAO,wBAAwB,CAAC,IAAI,CAAC,uBAAuB,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;YAC7E,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,YAAY,EAAE,IAAI,CAAC,aAAa;YAChC,MAAM,EAAE,OAAO,CAAC,MAAM;SACtB,CAAC,CAAC;IACJ,CAAC;CACD"}
+{"version":3,"file":"health-event-store.js","sourceRoot":"","sources":["../../../src/controller/health/health-event-store.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,wBAAwB,EACxB,oBAAoB,GAGpB,MAAM,6BAA6B,CAAC;AAgBrC,MAAM,OAAO,gBAAgB;IACnB,kBAAkB,CAAS;IAC3B,kBAAkB,CAAS;IAC3B,eAAe,GAAG,IAAI,GAAG,EAA8B,CAAC;IACxD,QAAQ,GAAyB,EAAE,CAAC;IACpC,gBAAgB,CAA6C;IACtE,kBAAkB,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAC7C,aAAa,CAAS;IAE/B,YAAY,OAAgC;QAC3C,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC;QAChD,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,iBAAiB,CAAC;QACpD,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,iBAAiB,IAAI,KAAK,CAAC;QAC7D,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IAC3C,CAAC;IAED,MAAM,CAAC,KAAyB;QAC/B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACvD,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;QAED,MAAM,GAAG,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;QACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/C,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,YAAY,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;YAC9D,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,CAAC,yBAAyB,EAAE,CAAC;QACjC,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,yBAAyB;QACxB,IAAI,IAAI,CAAC,eAAe,CAAC,IAAI,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC1D,OAAO;QACR,CAAC;QACD,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC;QACvE,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC;aACrD,QAAQ,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC;aAC3E,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC;aACpB,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;QACtB,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YAC/B,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAClC,CAAC;IACF,CAAC;IAED,WAAW;QACV,OAAO,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,kBAAkB;QACvB,MAAM,IAAI,CAAC,kBAAkB,CAAC;IAC/B,CAAC;IAED,uBAAuB,CAAC,MAAc;QACrC,OAAO,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC;aACvC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC;aAC1C,QAAQ,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC;IACzE,CAAC;IAED,cAAc,CAAC,OAAoC;QAClD,OAAO,wBAAwB,CAAC,IAAI,CAAC,uBAAuB,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;YAC7E,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,YAAY,EAAE,IAAI,CAAC,aAAa;YAChC,MAAM,EAAE,OAAO,CAAC,MAAM;SACtB,CAAC,CAAC;IACJ,CAAC;IAED,kBAAkB,CAAC,KAAyB;QAC3C,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO;QACR,CAAC;QACD,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB;aAC/C,IAAI,CAAC,KAAK,IAAI,EAAE;YAChB,MAAM,IAAI,CAAC,gBAAgB,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QAC5C,CAAC,CAAC;aACD,KAAK,CAAC,GAAG,EAAE;YACX,wEAAwE;QACzE,CAAC,CAAC,CAAC;IACL,CAAC;CACD"}

package/dist/controller/http/controller-health-event-routes.d.ts

@@ -1,6 +1,12 @@
 import { type Hono } from 'hono';
 import type { HealthEventStore } from '../health/health-event-store.js';
 export interface RegisterControllerHealthEventRoutesOptions {
+    readonly leaseRemediation?: {
+        readonly getLeaseOwner: ((leaseId: string) => {
+            readonly agentId: string;
+            readonly zoneId: string;
+        } | undefined) | undefined;
+    } | undefined;
     readonly now: () => number;
     readonly store: HealthEventStore;
     readonly zoneIds?: ReadonlySet<string> | undefined;

package/dist/controller/http/controller-health-event-routes.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"controller-health-event-routes.d.ts","sourceRoot":"","sources":["../../../src/controller/http/controller-health-event-routes.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,IAAI,EAAE,MAAM,MAAM,CAAC;AAEjC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AAExE,MAAM,WAAW,0CAA0C;IAC1D,QAAQ,CAAC,GAAG,EAAE,MAAM,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,gBAAgB,CAAC;IACjC,QAAQ,CAAC,OAAO,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC;CACnD;AAED,wBAAgB,mCAAmC,CAClD,GAAG,EAAE,IAAI,EACT,OAAO,EAAE,0CAA0C,GACjD,IAAI,CA0DN"}
+{"version":3,"file":"controller-health-event-routes.d.ts","sourceRoot":"","sources":["../../../src/controller/http/controller-health-event-routes.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,IAAI,EAAE,MAAM,MAAM,CAAC;AAEjC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AAcxE,MAAM,WAAW,0CAA0C;IAC1D,QAAQ,CAAC,gBAAgB,CAAC,EACvB;QACA,QAAQ,CAAC,aAAa,EACnB,CAAC,CAAC,OAAO,EAAE,MAAM,KAAK;YAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;SAAE,GAAG,SAAS,CAAC,GACxF,SAAS,CAAC;KACZ,GACD,SAAS,CAAC;IACb,QAAQ,CAAC,GAAG,EAAE,MAAM,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,gBAAgB,CAAC;IACjC,QAAQ,CAAC,OAAO,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC;CACnD;AAuDD,wBAAgB,mCAAmC,CAClD,GAAG,EAAE,IAAI,EACT,OAAO,EAAE,0CAA0C,GACjD,IAAI,CAsEN"}