@agent-vm/agent-vm 0.0.91 → 0.0.93

package/dist/controller/health/gateway-service-health-monitor.js

@@ -1,3 +1,5 @@
+import { deriveChannelProviderRecoveryObservation, } from './channel-provider-recovery-observation.js';
+import { createGatewayVmRecoveryTracker, defaultGatewayVmChannelProviderRecoveryPolicy, } from './gateway-vm-recovery-policy.js';
 function writeGatewayServiceHealthMonitorLog(message) {
     process.stderr.write(`[gateway-service-health-monitor] ${message}\n`);
 }
@@ -5,11 +7,346 @@ const unknownGatewayServiceHealthTarget = {
     path: '(unknown)',
     port: 0,
 };
+function channelProviderEventKey(event) {
+    return `${event.zoneId}\0${event.channelProviderId}\0${String(event.observedAtMs)}`;
+}
+function recoveryActionForBudgetClass(recoveryBudgetClass) {
+    return recoveryBudgetClass === 'gateway-vm-cold-start'
+        ? 'gateway-vm-cold-start'
+        : 'gateway-vm-restart';
+}
+function budgetResultForRecoveryResult(result) {
+    return result.action === 'observe-only' || result.action === 'operator-required'
+        ? undefined
+        : result.result;
+}
+function isGatewayRecoveryTimeoutErrorCodeValue(value) {
+    return value === 'recovery-callback-unconfigured' || value === 'recovery-timeout';
+}
 export function createGatewayServiceHealthMonitor(options) {
     const setIntervalImpl = options.setIntervalImpl ?? setInterval;
     const clearIntervalImpl = options.clearIntervalImpl ?? clearInterval;
+    const recoveryPolicy = options.gatewayServiceAutoRestart;
+    const channelProviderPolicy = recoveryPolicy.channelProviderHealth ?? defaultGatewayVmChannelProviderRecoveryPolicy;
+    const recoveryTracker = createGatewayVmRecoveryTracker({ policy: recoveryPolicy });
     let timer;
     let runningTick;
+    const recoveredChannelProviderEventKeys = new Set();
+    let stopped = false;
+    const classifyRecoveryBudgetClass = (request) => {
+        try {
+            return options.classifyRecoveryBudgetClass?.(request) ?? 'gateway-vm-restart';
+        }
+        catch (error) {
+            writeGatewayServiceHealthMonitorLog(`recovery budget classification failed for zone '${request.zoneId}': ${error instanceof Error ? error.message : String(error)}`);
+            return 'gateway-vm-restart';
+        }
+    };
+    const classifyGatewayControlLinkObservation = (props) => {
+        const controlLinkEvent = options.healthEventStore
+            .listLatestEventsForZone(props.zoneId)
+            .find((event) => {
+            return event.kind === 'gateway-control-link';
+        });
+        if (!controlLinkEvent) {
+            return 'unobserved';
+        }
+        if (props.nowMs - controlLinkEvent.observedAtMs > options.staleAfterMs) {
+            return 'stale';
+        }
+        return controlLinkEvent.result;
+    };
+    const latestChannelProviderHealthEvents = (zoneId) => options.healthEventStore
+        .listLatestEventsForZone(zoneId)
+        .filter((event) => event.kind === 'agent-channel-provider-health');
+    const runRecovery = async (request, intendedAction) => {
+        if (!options.recoverGatewayVm) {
+            return intendedAction === 'gateway-vm-restart'
+                ? {
+                    action: 'gateway-vm-restart',
+                    elapsedMs: 0,
+                    errorCode: 'recovery-callback-unconfigured',
+                    result: 'failed',
+                }
+                : {
+                    action: 'gateway-vm-cold-start',
+                    elapsedMs: 0,
+                    errorCode: 'recovery-callback-unconfigured',
+                    result: 'failed',
+                };
+        }
+        return await options.recoverGatewayVm(request);
+    };
+    const recordGatewayRecoveryEvent = (props) => {
+        const recoveryAction = props.result.action ?? 'gateway-vm-restart';
+        if (props.result.result === 'ok') {
+            const event = props.result.action !== 'gateway-vm-cold-start'
+                ? {
+                    action: 'gateway-vm-restart',
+                    consecutiveFailures: props.consecutiveFailures,
+                    cooldownMs: recoveryPolicy.cooldownMs,
+                    elapsedMs: props.result.elapsedMs,
+                    kind: 'gateway-recovery',
+                    leaseReleaseFailureCount: props.result.leaseReleaseFailureCount,
+                    newBootedAt: props.result.newBootedAt,
+                    newHostPid: props.result.newHostPid,
+                    newVmId: props.result.newVmId,
+                    observedAtMs: props.observedAtMs,
+                    oldBootedAt: props.result.oldBootedAt,
+                    oldHostPid: props.result.oldHostPid,
+                    oldVmId: props.result.oldVmId,
+                    ...(props.result.operationId === undefined
+                        ? {}
+                        : { operationId: props.result.operationId }),
+                    reason: props.reason,
+                    result: 'ok',
+                    zoneId: props.zoneId,
+                }
+                : {
+                    action: 'gateway-vm-cold-start',
+                    consecutiveFailures: props.consecutiveFailures,
+                    cooldownMs: recoveryPolicy.cooldownMs,
+                    elapsedMs: props.result.elapsedMs,
+                    kind: 'gateway-recovery',
+                    leaseReleaseFailureCount: props.result.leaseReleaseFailureCount,
+                    newBootedAt: props.result.newBootedAt,
+                    newHostPid: props.result.newHostPid,
+                    newVmId: props.result.newVmId,
+                    observedAtMs: props.observedAtMs,
+                    ...(props.result.operationId === undefined
+                        ? {}
+                        : { operationId: props.result.operationId }),
+                    reason: props.reason,
+                    result: 'ok',
+                    zoneId: props.zoneId,
+                };
+            options.healthEventStore.record(event);
+            return;
+        }
+        const event = {
+            consecutiveFailures: props.consecutiveFailures,
+            cooldownMs: recoveryPolicy.cooldownMs,
+            elapsedMs: props.result.elapsedMs,
+            errorCode: props.result.errorCode,
+            kind: 'gateway-recovery',
+            ...(props.result.leaseReleaseFailureCount === undefined
+                ? {}
+                : { leaseReleaseFailureCount: props.result.leaseReleaseFailureCount }),
+            observedAtMs: props.observedAtMs,
+            ...(props.result.operationId === undefined ? {} : { operationId: props.result.operationId }),
+            reason: props.reason,
+            result: 'failed',
+            zoneId: props.zoneId,
+        };
+        if (recoveryAction === 'gateway-vm-restart') {
+            if (props.result.oldVmId === undefined) {
+                if (!isGatewayRecoveryTimeoutErrorCodeValue(props.result.errorCode)) {
+                    const malformedRestartEvent = {
+                        ...event,
+                        action: 'operator-required',
+                        errorCode: props.result.errorCode,
+                    };
+                    options.healthEventStore.record(malformedRestartEvent);
+                    return;
+                }
+                const timeoutEvent = {
+                    ...event,
+                    action: 'gateway-vm-restart',
+                    errorCode: props.result.errorCode,
+                    oldBootedAt: props.result.oldBootedAt,
+                    oldHostPid: props.result.oldHostPid,
+                    oldVmId: props.result.oldVmId,
+                };
+                options.healthEventStore.record(timeoutEvent);
+                return;
+            }
+            const restartEvent = {
+                ...event,
+                action: 'gateway-vm-restart',
+                oldBootedAt: props.result.oldBootedAt,
+                oldHostPid: props.result.oldHostPid,
+                oldVmId: props.result.oldVmId,
+            };
+            options.healthEventStore.record(restartEvent);
+            return;
+        }
+        const nonRestartEvent = {
+            ...event,
+            action: recoveryAction,
+        };
+        options.healthEventStore.record(nonRestartEvent);
+    };
+    const recordGatewayRecoverySuspendedEvent = (props) => {
+        const event = {
+            action: props.action,
+            consecutiveFailedRecoveries: props.consecutiveFailedRecoveries,
+            consecutiveFailures: props.consecutiveFailures,
+            cooldownMs: recoveryPolicy.cooldownMs,
+            errorCode: 'max-failed-recoveries',
+            failedRecoveryResetMs: recoveryPolicy.failedRecoveryResetMs,
+            kind: 'gateway-recovery-suspended',
+            observedAtMs: props.observedAtMs,
+            reason: props.reason,
+            result: 'failed',
+            zoneId: props.zoneId,
+        };
+        options.healthEventStore.record(event);
+    };
+    const maybeRecoverGatewayVm = async (props) => {
+        const recoveryBudgetClass = classifyRecoveryBudgetClass({
+            consecutiveFailures: 0,
+            reason: props.reason,
+            zoneId: props.zoneId,
+        });
+        const decision = props.reason === 'gateway-service-unhealthy'
+            ? recoveryTracker.recordGatewayServiceProbe({
+                observedAtMs: props.observedAtMs,
+                recoveryBudgetClass,
+                result: props.serviceProbeResult,
+                zoneId: props.zoneId,
+            })
+            : recoveryTracker.recordGatewayControlLinkObservation({
+                observedAtMs: props.observedAtMs,
+                recoveryBudgetClass,
+                result: props.serviceProbeResult === 'ok'
+                    ? classifyGatewayControlLinkObservation({
+                        nowMs: props.observedAtMs,
+                        zoneId: props.zoneId,
+                    })
+                    : 'unobserved',
+                zoneId: props.zoneId,
+            });
+        if (decision.kind === 'suspended') {
+            recordGatewayRecoverySuspendedEvent({
+                consecutiveFailedRecoveries: decision.consecutiveFailedRecoveries,
+                consecutiveFailures: decision.consecutiveFailures,
+                observedAtMs: props.observedAtMs,
+                reason: props.reason,
+                action: recoveryActionForBudgetClass(recoveryBudgetClass),
+                zoneId: decision.zoneId,
+            });
+            return;
+        }
+        if (decision.kind !== 'restart') {
+            return;
+        }
+        if (stopped) {
+            writeGatewayServiceHealthMonitorLog(`recovery requested for zone '${decision.zoneId}' but monitor is stopped`);
+            return;
+        }
+        recoveryTracker.markRecoveryStarted({
+            observedAtMs: props.observedAtMs,
+            recoveryBudgetClass,
+            zoneId: decision.zoneId,
+        });
+        const recoveryResult = await runRecovery({
+            consecutiveFailures: decision.consecutiveFailures,
+            reason: decision.reason,
+            zoneId: decision.zoneId,
+        }, recoveryActionForBudgetClass(recoveryBudgetClass));
+        const observedAtMs = options.now();
+        recoveryTracker.markRecoveryFinished({
+            observedAtMs,
+            recoveryBudgetClass,
+            result: budgetResultForRecoveryResult(recoveryResult),
+            zoneId: decision.zoneId,
+        });
+        recordGatewayRecoveryEvent({
+            consecutiveFailures: decision.consecutiveFailures,
+            observedAtMs,
+            reason: decision.reason,
+            result: recoveryResult,
+            zoneId: decision.zoneId,
+        });
+    };
+    const maybeRecoverAgentChannelProvider = async (props) => {
+        const latestEvents = latestChannelProviderHealthEvents(props.zoneId);
+        if (latestEvents.length === 0) {
+            return;
+        }
+        let selectedDecision;
+        for (const latestEvent of latestEvents) {
+            const eventKey = channelProviderEventKey(latestEvent);
+            if (recoveredChannelProviderEventKeys.has(eventKey)) {
+                continue;
+            }
+            const recoveryObservation = deriveChannelProviderRecoveryObservation({
+                allowRestartWhenUnrecoverable: channelProviderPolicy.restartGatewayOnUnrecoverable,
+                event: latestEvent,
+                nowMs: props.observedAtMs,
+                restartGatewayOnRecoverable: channelProviderPolicy.restartGatewayOnRecoverable,
+                staleAfterMs: options.staleAfterMs,
+                transitioningTimeoutMs: channelProviderPolicy.transitioningTimeoutMs,
+            });
+            if (recoveryObservation.kind === 'observe-only') {
+                continue;
+            }
+            const recoveryBudgetClass = classifyRecoveryBudgetClass({
+                consecutiveFailures: 0,
+                reason: recoveryObservation.reason ?? 'agent-channel-provider-unhealthy',
+                zoneId: props.zoneId,
+            });
+            const decision = recoveryTracker.recordAgentChannelProviderObservation({
+                ...recoveryObservation.observation,
+                recoveryBudgetClass,
+            });
+            if (decision.kind === 'restart' || decision.kind === 'suspended') {
+                selectedDecision = {
+                    decision,
+                    eventKey,
+                    reason: recoveryObservation.reason ?? 'agent-channel-provider-unhealthy',
+                    recoveryBudgetClass,
+                };
+                break;
+            }
+        }
+        if (!selectedDecision) {
+            return;
+        }
+        const { decision, reason } = selectedDecision;
+        if (decision.kind === 'suspended') {
+            recordGatewayRecoverySuspendedEvent({
+                consecutiveFailedRecoveries: decision.consecutiveFailedRecoveries,
+                consecutiveFailures: decision.consecutiveFailures,
+                observedAtMs: props.observedAtMs,
+                reason,
+                action: recoveryActionForBudgetClass(selectedDecision.recoveryBudgetClass),
+                zoneId: decision.zoneId,
+            });
+            return;
+        }
+        if (stopped) {
+            writeGatewayServiceHealthMonitorLog(`recovery requested for zone '${decision.zoneId}' but monitor is stopped`);
+            return;
+        }
+        recoveryTracker.markRecoveryStarted({
+            observedAtMs: props.observedAtMs,
+            recoveryBudgetClass: selectedDecision.recoveryBudgetClass,
+            zoneId: decision.zoneId,
+        });
+        const recoveryResult = await runRecovery({
+            consecutiveFailures: decision.consecutiveFailures,
+            reason: decision.reason,
+            zoneId: decision.zoneId,
+        }, recoveryActionForBudgetClass(selectedDecision.recoveryBudgetClass));
+        const observedAtMs = options.now();
+        recoveryTracker.markRecoveryFinished({
+            observedAtMs,
+            recoveryBudgetClass: selectedDecision.recoveryBudgetClass,
+            result: budgetResultForRecoveryResult(recoveryResult),
+            zoneId: decision.zoneId,
+        });
+        recordGatewayRecoveryEvent({
+            consecutiveFailures: decision.consecutiveFailures,
+            observedAtMs,
+            reason: decision.reason,
+            result: recoveryResult,
+            zoneId: decision.zoneId,
+        });
+        if (recoveryResult.result === 'ok') {
+            recoveredChannelProviderEventKeys.add(selectedDecision.eventKey);
+        }
+    };
     const tick = async () => {
         if (runningTick) {
             return await runningTick;
@@ -18,26 +355,53 @@ export function createGatewayServiceHealthMonitor(options) {
             await Promise.all(options.zoneIds.map(async (zoneId) => {
                 try {
                     const result = await options.probeZoneHealth(zoneId);
+                    const observedAtMs = options.now();
+                    const serviceProbeResult = result.ok ? 'ok' : 'failed';
                     options.healthEventStore.record({
                         kind: 'gateway-service-health',
-                        observedAtMs: options.now(),
+                        observedAtMs,
                         path: result.path,
                         port: result.port,
-                        result: result.ok ? 'ok' : 'failed',
+                        result: serviceProbeResult,
                         ...(result.statusCode === undefined ? {} : { statusCode: result.statusCode }),
                         zoneId: result.zoneId,
                     });
+                    await maybeRecoverGatewayVm({
+                        observedAtMs,
+                        reason: 'gateway-service-unhealthy',
+                        serviceProbeResult,
+                        zoneId: result.zoneId,
+                    });
+                    if (serviceProbeResult === 'ok') {
+                        await maybeRecoverGatewayVm({
+                            observedAtMs,
+                            reason: 'gateway-control-link-unhealthy',
+                            serviceProbeResult,
+                            zoneId: result.zoneId,
+                        });
+                        await maybeRecoverAgentChannelProvider({
+                            observedAtMs,
+                            zoneId: result.zoneId,
+                        });
+                    }
                 }
                 catch (error) {
                     writeGatewayServiceHealthMonitorLog(`probe failed for zone '${zoneId}': ${error instanceof Error ? error.message : String(error)}`);
+                    const observedAtMs = options.now();
                     options.healthEventStore.record({
                         kind: 'gateway-service-health',
-                        observedAtMs: options.now(),
+                        observedAtMs,
                         path: unknownGatewayServiceHealthTarget.path,
                         port: unknownGatewayServiceHealthTarget.port,
                         result: 'failed',
                         zoneId,
                     });
+                    await maybeRecoverGatewayVm({
+                        observedAtMs,
+                        reason: 'gateway-service-unhealthy',
+                        serviceProbeResult: 'failed',
+                        zoneId,
+                    });
                 }
             }));
         })().finally(() => {
@@ -50,17 +414,26 @@ export function createGatewayServiceHealthMonitor(options) {
             if (timer) {
                 return;
             }
-            timer = setIntervalImpl(() => {
-                void tick();
+            stopped = false;
+            timer = setIntervalImpl(async () => {
+                try {
+                    await tick();
+                }
+                catch (error) {
+                    writeGatewayServiceHealthMonitorLog(`scheduled tick failed: ${error instanceof Error ? error.message : String(error)}`);
+                }
             }, options.intervalMs);
             timer.unref?.();
         },
-        stop: () => {
-            if (!timer) {
-                return;
+        stop: async () => {
+            stopped = true;
+            if (timer) {
+                clearIntervalImpl(timer);
+                timer = undefined;
+            }
+            if (runningTick) {
+                await runningTick;
             }
-            clearIntervalImpl(timer);
-            timer = undefined;
         },
         tick,
     };

package/dist/controller/health/gateway-service-health-monitor.js.map

@@ -1 +1 @@
-{"version":3,"file":"gateway-service-health-monitor.js","sourceRoot":"","sources":["../../../src/controller/health/gateway-service-health-monitor.ts"],"names":[],"mappings":"AA6BA,SAAS,mCAAmC,CAAC,OAAe;IAC3D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,oCAAoC,OAAO,IAAI,CAAC,CAAC;AACvE,CAAC;AAED,MAAM,iCAAiC,GAAG;IACzC,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,CAAC;CACE,CAAC;AAEX,MAAM,UAAU,iCAAiC,CAChD,OAAiD;IAEjD,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,WAAW,CAAC;IAC/D,MAAM,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,IAAI,aAAa,CAAC;IACrE,IAAI,KAAiC,CAAC;IACtC,IAAI,WAAsC,CAAC;IAE3C,MAAM,IAAI,GAAG,KAAK,IAAmB,EAAE;QACtC,IAAI,WAAW,EAAE,CAAC;YACjB,OAAO,MAAM,WAAW,CAAC;QAC1B,CAAC;QACD,WAAW,GAAG,CAAC,KAAK,IAAI,EAAE;YACzB,MAAM,OAAO,CAAC,GAAG,CAChB,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;gBACpC,IAAI,CAAC;oBACJ,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;oBACrD,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC;wBAC/B,IAAI,EAAE,wBAAwB;wBAC9B,YAAY,EAAE,OAAO,CAAC,GAAG,EAAE;wBAC3B,IAAI,EAAE,MAAM,CAAC,IAAI;wBACjB,IAAI,EAAE,MAAM,CAAC,IAAI;wBACjB,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ;wBACnC,GAAG,CAAC,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC;wBAC7E,MAAM,EAAE,MAAM,CAAC,MAAM;qBACrB,CAAC,CAAC;gBACJ,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,mCAAmC,CAClC,0BAA0B,MAAM,MAAM,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC9F,CAAC;oBACF,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC;wBAC/B,IAAI,EAAE,wBAAwB;wBAC9B,YAAY,EAAE,OAAO,CAAC,GAAG,EAAE;wBAC3B,IAAI,EAAE,iCAAiC,CAAC,IAAI;wBAC5C,IAAI,EAAE,iCAAiC,CAAC,IAAI;wBAC5C,MAAM,EAAE,QAAQ;wBAChB,MAAM;qBACN,CAAC,CAAC;gBACJ,CAAC;YACF,CAAC,CAAC,CACF,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;YACjB,WAAW,GAAG,SAAS,CAAC;QACzB,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,WAAW,CAAC;IAC1B,CAAC,CAAC;IAEF,OAAO;QACN,KAAK,EAAE,GAAG,EAAE;YACX,IAAI,KAAK,EAAE,CAAC;gBACX,OAAO;YACR,CAAC;YACD,KAAK,GAAG,eAAe,CAAC,GAAG,EAAE;gBAC5B,KAAK,IAAI,EAAE,CAAC;YACb,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;YACvB,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;QACjB,CAAC;QACD,IAAI,EAAE,GAAG,EAAE;YACV,IAAI,CAAC,KAAK,EAAE,CAAC;gBACZ,OAAO;YACR,CAAC;YACD,iBAAiB,CAAC,KAAK,CAAC,CAAC;YACzB,KAAK,GAAG,SAAS,CAAC;QACnB,CAAC;QACD,IAAI;KACJ,CAAC;AACH,CAAC"}
+{"version":3,"file":"gateway-service-health-monitor.js","sourceRoot":"","sources":["../../../src/controller/health/gateway-service-health-monitor.ts"],"names":[],"mappings":"AAMA,OAAO,EACN,wCAAwC,GAExC,MAAM,4CAA4C,CAAC;AACpD,OAAO,EACN,8BAA8B,EAC9B,6CAA6C,GAM7C,MAAM,iCAAiC,CAAC;AA2GzC,SAAS,mCAAmC,CAAC,OAAe;IAC3D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,oCAAoC,OAAO,IAAI,CAAC,CAAC;AACvE,CAAC;AAED,MAAM,iCAAiC,GAAG;IACzC,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,CAAC;CACE,CAAC;AAEX,SAAS,uBAAuB,CAAC,KAAsC;IACtE,OAAO,GAAG,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,iBAAiB,KAAK,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC;AACrF,CAAC;AAED,SAAS,4BAA4B,CACpC,mBAAiD;IAEjD,OAAO,mBAAmB,KAAK,uBAAuB;QACrD,CAAC,CAAC,uBAAuB;QACzB,CAAC,CAAC,oBAAoB,CAAC;AACzB,CAAC;AAED,SAAS,6BAA6B,CACrC,MAA+B;IAE/B,OAAO,MAAM,CAAC,MAAM,KAAK,cAAc,IAAI,MAAM,CAAC,MAAM,KAAK,mBAAmB;QAC/E,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;AAClB,CAAC;AAED,SAAS,sCAAsC,CAC9C,KAAa;IAEb,OAAO,KAAK,KAAK,gCAAgC,IAAI,KAAK,KAAK,kBAAkB,CAAC;AACnF,CAAC;AAED,MAAM,UAAU,iCAAiC,CAChD,OAAiD;IAEjD,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,WAAW,CAAC;IAC/D,MAAM,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,IAAI,aAAa,CAAC;IACrE,MAAM,cAAc,GAAG,OAAO,CAAC,yBAAyB,CAAC;IACzD,MAAM,qBAAqB,GAC1B,cAAc,CAAC,qBAAqB,IAAI,6CAA6C,CAAC;IACvF,MAAM,eAAe,GAAG,8BAA8B,CAAC,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC,CAAC;IACnF,IAAI,KAAiC,CAAC;IACtC,IAAI,WAAsC,CAAC;IAC3C,MAAM,iCAAiC,GAAG,IAAI,GAAG,EAAU,CAAC;IAC5D,IAAI,OAAO,GAAG,KAAK,CAAC;IAEpB,MAAM,2BAA2B,GAAG,CACnC,OAA2C,EACZ,EAAE;QACjC,IAAI,CAAC;YACJ,OAAO,OAAO,CAAC,2BAA2B,EAAE,CAAC,OAAO,CAAC,IAAI,oBAAoB,CAAC;QAC/E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,mCAAmC,CAClC,mDAAmD,OAAO,CAAC,MAAM,MAChE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CACtD,EAAE,CACF,CAAC;YACF,OAAO,oBAAoB,CAAC;QAC7B,CAAC;IACF,CAAC,CAAC;IAEF,MAAM,qCAAqC,GAAG,CAAC,KAG9C,EAAsC,EAAE;QACxC,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB;aAC/C,uBAAuB,CAAC,KAAK,CAAC,MAAM,CAAC;aACrC,IAAI,CAAC,CAAC,KAAK,EAA2E,EAAE;YACxF,OAAO,KAAK,CAAC,IAAI,KAAK,sBAAsB,CAAC;QAC9C,CAAC,CAAC,CAAC;QACJ,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACvB,OAAO,YAAY,CAAC;QACrB,CAAC;QACD,IAAI,KAAK,CAAC,KAAK,GAAG,gBAAgB,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;YACxE,OAAO,OAAO,CAAC;QAChB,CAAC;QACD,OAAO,gBAAgB,CAAC,MAAM,CAAC;IAChC,CAAC,CAAC;IAEF,MAAM,iCAAiC,GAAG,CACzC,MAAc,EAC+B,EAAE,CAC/C,OAAO,CAAC,gBAAgB;SACtB,uBAAuB,CAAC,MAAM,CAAC;SAC/B,MAAM,CACN,CAAC,KAAK,EAA4C,EAAE,CACnD,KAAK,CAAC,IAAI,KAAK,+BAA+B,CAC/C,CAAC;IAEJ,MAAM,WAAW,GAAG,KAAK,EACxB,OAAiC,EACjC,cAGC,EACkC,EAAE;QACrC,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC;YAC/B,OAAO,cAAc,KAAK,oBAAoB;gBAC7C,CAAC,CAAC;oBACA,MAAM,EAAE,oBAAoB;oBAC5B,SAAS,EAAE,CAAC;oBACZ,SAAS,EAAE,gCAAgC;oBAC3C,MAAM,EAAE,QAAQ;iBAChB;gBACF,CAAC,CAAC;oBACA,MAAM,EAAE,uBAAuB;oBAC/B,SAAS,EAAE,CAAC;oBACZ,SAAS,EAAE,gCAAgC;oBAC3C,MAAM,EAAE,QAAQ;iBAChB,CAAC;QACL,CAAC;QACD,OAAO,MAAM,OAAO,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAChD,CAAC,CAAC;IAEF,MAAM,0BAA0B,GAAG,CAAC,KAMnC,EAAQ,EAAE;QACV,MAAM,cAAc,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,IAAI,oBAAoB,CAAC;QACnE,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;YAClC,MAAM,KAAK,GACV,KAAK,CAAC,MAAM,CAAC,MAAM,KAAK,uBAAuB;gBAC9C,CAAC,CAAE;oBACD,MAAM,EAAE,oBAAoB;oBAC5B,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;oBAC9C,UAAU,EAAE,cAAc,CAAC,UAAU;oBACrC,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;oBACjC,IAAI,EAAE,kBAAkB;oBACxB,wBAAwB,EAAE,KAAK,CAAC,MAAM,CAAC,wBAAwB;oBAC/D,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;oBACrC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU;oBACnC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO;oBAC7B,YAAY,EAAE,KAAK,CAAC,YAAY;oBAChC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;oBACrC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU;oBACnC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO;oBAC7B,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,KAAK,SAAS;wBACzC,CAAC,CAAC,EAAE;wBACJ,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;oBAC7C,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,MAAM,EAAE,IAAI;oBACZ,MAAM,EAAE,KAAK,CAAC,MAAM;iBACU;gBAChC,CAAC,CAAE;oBACD,MAAM,EAAE,uBAAuB;oBAC/B,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;oBAC9C,UAAU,EAAE,cAAc,CAAC,UAAU;oBACrC,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;oBACjC,IAAI,EAAE,kBAAkB;oBACxB,wBAAwB,EAAE,KAAK,CAAC,MAAM,CAAC,wBAAwB;oBAC/D,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;oBACrC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU;oBACnC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO;oBAC7B,YAAY,EAAE,KAAK,CAAC,YAAY;oBAChC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,KAAK,SAAS;wBACzC,CAAC,CAAC,EAAE;wBACJ,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;oBAC7C,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,MAAM,EAAE,IAAI;oBACZ,MAAM,EAAE,KAAK,CAAC,MAAM;iBACU,CAAC;YACnC,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACvC,OAAO;QACR,CAAC;QAED,MAAM,KAAK,GAAG;YACb,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;YAC9C,UAAU,EAAE,cAAc,CAAC,UAAU;YACrC,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;YACjC,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;YACjC,IAAI,EAAE,kBAAkB;YACxB,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,wBAAwB,KAAK,SAAS;gBACtD,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,wBAAwB,EAAE,KAAK,CAAC,MAAM,CAAC,wBAAwB,EAAE,CAAC;YACvE,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAC5F,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,MAAM,EAAE,QAAQ;YAChB,MAAM,EAAE,KAAK,CAAC,MAAM;SACX,CAAC;QACX,IAAI,cAAc,KAAK,oBAAoB,EAAE,CAAC;YAC7C,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;gBACxC,IAAI,CAAC,sCAAsC,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;oBACrE,MAAM,qBAAqB,GAAG;wBAC7B,GAAG,KAAK;wBACR,MAAM,EAAE,mBAAmB;wBAC3B,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;qBACJ,CAAC;oBAC/B,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;oBACvD,OAAO;gBACR,CAAC;gBACD,MAAM,YAAY,GAAG;oBACpB,GAAG,KAAK;oBACR,MAAM,EAAE,oBAAoB;oBAC5B,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;oBACjC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;oBACrC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU;oBACnC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO;iBACA,CAAC;gBAC/B,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;gBAC9C,OAAO;YACR,CAAC;YACD,MAAM,YAAY,GAAG;gBACpB,GAAG,KAAK;gBACR,MAAM,EAAE,oBAAoB;gBAC5B,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;gBACrC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU;gBACnC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO;aACA,CAAC;YAC/B,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YAC9C,OAAO;QACR,CAAC;QACD,MAAM,eAAe,GAAG;YACvB,GAAG,KAAK;YACR,MAAM,EAAE,cAAc;SACO,CAAC;QAC/B,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAClD,CAAC,CAAC;IAEF,MAAM,mCAAmC,GAAG,CAAC,KAO5C,EAAQ,EAAE;QACV,MAAM,KAAK,GAAG;YACb,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,2BAA2B,EAAE,KAAK,CAAC,2BAA2B;YAC9D,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;YAC9C,UAAU,EAAE,cAAc,CAAC,UAAU;YACrC,SAAS,EAAE,uBAAuB;YAClC,qBAAqB,EAAE,cAAc,CAAC,qBAAqB;YAC3D,IAAI,EAAE,4BAA4B;YAClC,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,MAAM,EAAE,QAAQ;YAChB,MAAM,EAAE,KAAK,CAAC,MAAM;SACS,CAAC;QAC/B,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACxC,CAAC,CAAC;IAEF,MAAM,qBAAqB,GAAG,KAAK,EAAE,KAKpC,EAAiB,EAAE;QACnB,MAAM,mBAAmB,GAAG,2BAA2B,CAAC;YACvD,mBAAmB,EAAE,CAAC;YACtB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,MAAM,EAAE,KAAK,CAAC,MAAM;SACpB,CAAC,CAAC;QACH,MAAM,QAAQ,GACb,KAAK,CAAC,MAAM,KAAK,2BAA2B;YAC3C,CAAC,CAAC,eAAe,CAAC,yBAAyB,CAAC;gBAC1C,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,mBAAmB;gBACnB,MAAM,EAAE,KAAK,CAAC,kBAAkB;gBAChC,MAAM,EAAE,KAAK,CAAC,MAAM;aACpB,CAAC;YACH,CAAC,CAAC,eAAe,CAAC,mCAAmC,CAAC;gBACpD,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,mBAAmB;gBACnB,MAAM,EACL,KAAK,CAAC,kBAAkB,KAAK,IAAI;oBAChC,CAAC,CAAC,qCAAqC,CAAC;wBACtC,KAAK,EAAE,KAAK,CAAC,YAAY;wBACzB,MAAM,EAAE,KAAK,CAAC,MAAM;qBACpB,CAAC;oBACH,CAAC,CAAC,YAAY;gBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;aACpB,CAAC,CAAC;QAEN,IAAI,QAAQ,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YACnC,mCAAmC,CAAC;gBACnC,2BAA2B,EAAE,QAAQ,CAAC,2BAA2B;gBACjE,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;gBACjD,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,MAAM,EAAE,4BAA4B,CAAC,mBAAmB,CAAC;gBACzD,MAAM,EAAE,QAAQ,CAAC,MAAM;aACvB,CAAC,CAAC;YACH,OAAO;QACR,CAAC;QACD,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YACjC,OAAO;QACR,CAAC;QACD,IAAI,OAAO,EAAE,CAAC;YACb,mCAAmC,CAClC,gCAAgC,QAAQ,CAAC,MAAM,0BAA0B,CACzE,CAAC;YACF,OAAO;QACR,CAAC;QAED,eAAe,CAAC,mBAAmB,CAAC;YACnC,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,mBAAmB;YACnB,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,CAAC,CAAC;QACH,MAAM,cAAc,GAAG,MAAM,WAAW,CACvC;YACC,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;YACjD,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,EACD,4BAA4B,CAAC,mBAAmB,CAAC,CACjD,CAAC;QACF,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QACnC,eAAe,CAAC,oBAAoB,CAAC;YACpC,YAAY;YACZ,mBAAmB;YACnB,MAAM,EAAE,6BAA6B,CAAC,cAAc,CAAC;YACrD,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,CAAC,CAAC;QACH,0BAA0B,CAAC;YAC1B,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;YACjD,YAAY;YACZ,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,cAAc;YACtB,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,CAAC,CAAC;IACJ,CAAC,CAAC;IAEF,MAAM,gCAAgC,GAAG,KAAK,EAAE,KAG/C,EAAiB,EAAE;QACnB,MAAM,YAAY,GAAG,iCAAiC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACrE,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO;QACR,CAAC;QACD,IAAI,gBAUQ,CAAC;QACb,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;YACxC,MAAM,QAAQ,GAAG,uBAAuB,CAAC,WAAW,CAAC,CAAC;YACtD,IAAI,iCAAiC,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACrD,SAAS;YACV,CAAC;YACD,MAAM,mBAAmB,GAAG,wCAAwC,CAAC;gBACpE,6BAA6B,EAAE,qBAAqB,CAAC,6BAA6B;gBAClF,KAAK,EAAE,WAAW;gBAClB,KAAK,EAAE,KAAK,CAAC,YAAY;gBACzB,2BAA2B,EAAE,qBAAqB,CAAC,2BAA2B;gBAC9E,YAAY,EAAE,OAAO,CAAC,YAAY;gBAClC,sBAAsB,EAAE,qBAAqB,CAAC,sBAAsB;aACpE,CAAC,CAAC;YACH,IAAI,mBAAmB,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;gBACjD,SAAS;YACV,CAAC;YACD,MAAM,mBAAmB,GAAG,2BAA2B,CAAC;gBACvD,mBAAmB,EAAE,CAAC;gBACtB,MAAM,EAAE,mBAAmB,CAAC,MAAM,IAAI,kCAAkC;gBACxE,MAAM,EAAE,KAAK,CAAC,MAAM;aACpB,CAAC,CAAC;YACH,MAAM,QAAQ,GAAG,eAAe,CAAC,qCAAqC,CAAC;gBACtE,GAAG,mBAAmB,CAAC,WAAW;gBAClC,mBAAmB;aACnB,CAAC,CAAC;YACH,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS,IAAI,QAAQ,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBAClE,gBAAgB,GAAG;oBAClB,QAAQ;oBACR,QAAQ;oBACR,MAAM,EAAE,mBAAmB,CAAC,MAAM,IAAI,kCAAkC;oBACxE,mBAAmB;iBACnB,CAAC;gBACF,MAAM;YACP,CAAC;QACF,CAAC;QACD,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACvB,OAAO;QACR,CAAC;QACD,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,gBAAgB,CAAC;QAC9C,IAAI,QAAQ,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YACnC,mCAAmC,CAAC;gBACnC,2BAA2B,EAAE,QAAQ,CAAC,2BAA2B;gBACjE,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;gBACjD,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,MAAM;gBACN,MAAM,EAAE,4BAA4B,CAAC,gBAAgB,CAAC,mBAAmB,CAAC;gBAC1E,MAAM,EAAE,QAAQ,CAAC,MAAM;aACvB,CAAC,CAAC;YACH,OAAO;QACR,CAAC;QACD,IAAI,OAAO,EAAE,CAAC;YACb,mCAAmC,CAClC,gCAAgC,QAAQ,CAAC,MAAM,0BAA0B,CACzE,CAAC;YACF,OAAO;QACR,CAAC;QAED,eAAe,CAAC,mBAAmB,CAAC;YACnC,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,mBAAmB,EAAE,gBAAgB,CAAC,mBAAmB;YACzD,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,CAAC,CAAC;QACH,MAAM,cAAc,GAAG,MAAM,WAAW,CACvC;YACC,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;YACjD,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,EACD,4BAA4B,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,CAClE,CAAC;QACF,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QACnC,eAAe,CAAC,oBAAoB,CAAC;YACpC,YAAY;YACZ,mBAAmB,EAAE,gBAAgB,CAAC,mBAAmB;YACzD,MAAM,EAAE,6BAA6B,CAAC,cAAc,CAAC;YACrD,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,CAAC,CAAC;QACH,0BAA0B,CAAC;YAC1B,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;YACjD,YAAY;YACZ,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,cAAc;YACtB,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,CAAC,CAAC;QACH,IAAI,cAAc,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;YACpC,iCAAiC,CAAC,GAAG,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAClE,CAAC;IACF,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,KAAK,IAAmB,EAAE;QACtC,IAAI,WAAW,EAAE,CAAC;YACjB,OAAO,MAAM,WAAW,CAAC;QAC1B,CAAC;QACD,WAAW,GAAG,CAAC,KAAK,IAAI,EAAE;YACzB,MAAM,OAAO,CAAC,GAAG,CAChB,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;gBACpC,IAAI,CAAC;oBACJ,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;oBACrD,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;oBACnC,MAAM,kBAAkB,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC;oBACvD,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC;wBAC/B,IAAI,EAAE,wBAAwB;wBAC9B,YAAY;wBACZ,IAAI,EAAE,MAAM,CAAC,IAAI;wBACjB,IAAI,EAAE,MAAM,CAAC,IAAI;wBACjB,MAAM,EAAE,kBAAkB;wBAC1B,GAAG,CAAC,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC;wBAC7E,MAAM,EAAE,MAAM,CAAC,MAAM;qBACrB,CAAC,CAAC;oBACH,MAAM,qBAAqB,CAAC;wBAC3B,YAAY;wBACZ,MAAM,EAAE,2BAA2B;wBACnC,kBAAkB;wBAClB,MAAM,EAAE,MAAM,CAAC,MAAM;qBACrB,CAAC,CAAC;oBACH,IAAI,kBAAkB,KAAK,IAAI,EAAE,CAAC;wBACjC,MAAM,qBAAqB,CAAC;4BAC3B,YAAY;4BACZ,MAAM,EAAE,gCAAgC;4BACxC,kBAAkB;4BAClB,MAAM,EAAE,MAAM,CAAC,MAAM;yBACrB,CAAC,CAAC;wBACH,MAAM,gCAAgC,CAAC;4BACtC,YAAY;4BACZ,MAAM,EAAE,MAAM,CAAC,MAAM;yBACrB,CAAC,CAAC;oBACJ,CAAC;gBACF,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,mCAAmC,CAClC,0BAA0B,MAAM,MAAM,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC9F,CAAC;oBACF,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;oBACnC,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC;wBAC/B,IAAI,EAAE,wBAAwB;wBAC9B,YAAY;wBACZ,IAAI,EAAE,iCAAiC,CAAC,IAAI;wBAC5C,IAAI,EAAE,iCAAiC,CAAC,IAAI;wBAC5C,MAAM,EAAE,QAAQ;wBAChB,MAAM;qBACN,CAAC,CAAC;oBACH,MAAM,qBAAqB,CAAC;wBAC3B,YAAY;wBACZ,MAAM,EAAE,2BAA2B;wBACnC,kBAAkB,EAAE,QAAQ;wBAC5B,MAAM;qBACN,CAAC,CAAC;gBACJ,CAAC;YACF,CAAC,CAAC,CACF,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;YACjB,WAAW,GAAG,SAAS,CAAC;QACzB,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,WAAW,CAAC;IAC1B,CAAC,CAAC;IAEF,OAAO;QACN,KAAK,EAAE,GAAG,EAAE;YACX,IAAI,KAAK,EAAE,CAAC;gBACX,OAAO;YACR,CAAC;YACD,OAAO,GAAG,KAAK,CAAC;YAChB,KAAK,GAAG,eAAe,CAAC,KAAK,IAAI,EAAE;gBAClC,IAAI,CAAC;oBACJ,MAAM,IAAI,EAAE,CAAC;gBACd,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,mCAAmC,CAClC,0BAA0B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAClF,CAAC;gBACH,CAAC;YACF,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;YACvB,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;QACjB,CAAC;QACD,IAAI,EAAE,KAAK,IAAI,EAAE;YAChB,OAAO,GAAG,IAAI,CAAC;YACf,IAAI,KAAK,EAAE,CAAC;gBACX,iBAAiB,CAAC,KAAK,CAAC,CAAC;gBACzB,KAAK,GAAG,SAAS,CAAC;YACnB,CAAC;YACD,IAAI,WAAW,EAAE,CAAC;gBACjB,MAAM,WAAW,CAAC;YACnB,CAAC;QACF,CAAC;QACD,IAAI;KACJ,CAAC;AACH,CAAC"}

package/dist/controller/health/gateway-vm-recovery-policy.d.ts

@@ -0,0 +1,68 @@
+import type { GatewayRecoveryHealthReason } from '@agent-vm/gateway-interface';
+export type GatewayVmRecoveryObservationResult = 'failed' | 'ok' | 'stale' | 'timeout' | 'unobserved';
+export type GatewayVmRecoveryReason = GatewayRecoveryHealthReason;
+export type GatewayVmRecoveryBudgetClass = 'gateway-vm-cold-start' | 'gateway-vm-restart';
+export interface GatewayVmAutoRecoveryPolicy {
+    readonly channelProviderHealth?: GatewayVmChannelProviderRecoveryPolicy | undefined;
+    readonly consecutiveFailureThreshold: number;
+    readonly cooldownMs: number;
+    readonly enabled: boolean;
+    readonly failedRecoveryResetMs: number;
+    readonly maxConsecutiveFailedRecoveries: number;
+    readonly restartTimeoutMs: number;
+}
+export interface GatewayVmChannelProviderRecoveryPolicy {
+    readonly consecutiveFailureThreshold: number;
+    readonly enabled: boolean;
+    readonly restartGatewayOnRecoverable: boolean;
+    readonly restartGatewayOnUnrecoverable: boolean;
+    readonly transitioningTimeoutMs: number;
+}
+export declare const defaultGatewayVmChannelProviderRecoveryPolicy: {
+    consecutiveFailureThreshold: number;
+    enabled: true;
+    restartGatewayOnRecoverable: true;
+    restartGatewayOnUnrecoverable: false;
+    transitioningTimeoutMs: number;
+};
+export interface CreateGatewayVmRecoveryTrackerOptions {
+    readonly policy: GatewayVmAutoRecoveryPolicy;
+}
+export interface GatewayVmRecoveryObservation {
+    readonly channelProviderId?: string | undefined;
+    readonly observedAtMs: number;
+    readonly recoveryBudgetClass?: GatewayVmRecoveryBudgetClass | undefined;
+    readonly result: GatewayVmRecoveryObservationResult;
+    readonly zoneId: string;
+}
+export interface GatewayVmRecoveryLifecycleEvent {
+    readonly observedAtMs: number;
+    readonly recoveryBudgetClass?: GatewayVmRecoveryBudgetClass | undefined;
+    readonly result?: 'failed' | 'ok' | undefined;
+    readonly zoneId: string;
+}
+export type GatewayVmRecoveryDecision = {
+    readonly consecutiveFailures: number;
+    readonly kind: 'none';
+    readonly reason?: 'cooldown' | 'disabled' | 'in-flight' | 'unobserved' | undefined;
+} | {
+    readonly consecutiveFailures: number;
+    readonly kind: 'restart';
+    readonly reason: GatewayVmRecoveryReason;
+    readonly zoneId: string;
+} | {
+    readonly consecutiveFailedRecoveries: number;
+    readonly consecutiveFailures: number;
+    readonly kind: 'suspended';
+    readonly reason: 'max-failed-recoveries';
+    readonly zoneId: string;
+};
+export interface GatewayVmRecoveryTracker {
+    readonly markRecoveryFinished: (event: GatewayVmRecoveryLifecycleEvent) => void;
+    readonly markRecoveryStarted: (event: GatewayVmRecoveryLifecycleEvent) => void;
+    readonly recordAgentChannelProviderObservation: (observation: GatewayVmRecoveryObservation) => GatewayVmRecoveryDecision;
+    readonly recordGatewayControlLinkObservation: (observation: GatewayVmRecoveryObservation) => GatewayVmRecoveryDecision;
+    readonly recordGatewayServiceProbe: (observation: GatewayVmRecoveryObservation) => GatewayVmRecoveryDecision;
+}
+export declare function createGatewayVmRecoveryTracker(options: CreateGatewayVmRecoveryTrackerOptions): GatewayVmRecoveryTracker;
+//# sourceMappingURL=gateway-vm-recovery-policy.d.ts.map

package/dist/controller/health/gateway-vm-recovery-policy.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gateway-vm-recovery-policy.d.ts","sourceRoot":"","sources":["../../../src/controller/health/gateway-vm-recovery-policy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,6BAA6B,CAAC;AAE/E,MAAM,MAAM,kCAAkC,GAC3C,QAAQ,GACR,IAAI,GACJ,OAAO,GACP,SAAS,GACT,YAAY,CAAC;AAEhB,MAAM,MAAM,uBAAuB,GAAG,2BAA2B,CAAC;AAClE,MAAM,MAAM,4BAA4B,GAAG,uBAAuB,GAAG,oBAAoB,CAAC;AAE1F,MAAM,WAAW,2BAA2B;IAC3C,QAAQ,CAAC,qBAAqB,CAAC,EAAE,sCAAsC,GAAG,SAAS,CAAC;IACpF,QAAQ,CAAC,2BAA2B,EAAE,MAAM,CAAC;IAC7C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC,QAAQ,CAAC,8BAA8B,EAAE,MAAM,CAAC;IAChD,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;CAClC;AAED,MAAM,WAAW,sCAAsC;IACtD,QAAQ,CAAC,2BAA2B,EAAE,MAAM,CAAC;IAC7C,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,2BAA2B,EAAE,OAAO,CAAC;IAC9C,QAAQ,CAAC,6BAA6B,EAAE,OAAO,CAAC;IAChD,QAAQ,CAAC,sBAAsB,EAAE,MAAM,CAAC;CACxC;AAED,eAAO,MAAM,6CAA6C;;;;;;CAMR,CAAC;AAEnD,MAAM,WAAW,qCAAqC;IACrD,QAAQ,CAAC,MAAM,EAAE,2BAA2B,CAAC;CAC7C;AAED,MAAM,WAAW,4BAA4B;IAC5C,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,mBAAmB,CAAC,EAAE,4BAA4B,GAAG,SAAS,CAAC;IACxE,QAAQ,CAAC,MAAM,EAAE,kCAAkC,CAAC;IACpD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,+BAA+B;IAC/C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,mBAAmB,CAAC,EAAE,4BAA4B,GAAG,SAAS,CAAC;IACxE,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,GAAG,IAAI,GAAG,SAAS,CAAC;IAC9C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,yBAAyB,GAClC;IACA,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,CAAC,EAAE,UAAU,GAAG,UAAU,GAAG,WAAW,GAAG,YAAY,GAAG,SAAS,CAAC;CAClF,GACD;IACA,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,uBAAuB,CAAC;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACvB,GACD;IACA,QAAQ,CAAC,2BAA2B,EAAE,MAAM,CAAC;IAC7C,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,uBAAuB,CAAC;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACvB,CAAC;AAEL,MAAM,WAAW,wBAAwB;IACxC,QAAQ,CAAC,oBAAoB,EAAE,CAAC,KAAK,EAAE,+BAA+B,KAAK,IAAI,CAAC;IAChF,QAAQ,CAAC,mBAAmB,EAAE,CAAC,KAAK,EAAE,+BAA+B,KAAK,IAAI,CAAC;IAC/E,QAAQ,CAAC,qCAAqC,EAAE,CAC/C,WAAW,EAAE,4BAA4B,KACrC,yBAAyB,CAAC;IAC/B,QAAQ,CAAC,mCAAmC,EAAE,CAC7C,WAAW,EAAE,4BAA4B,KACrC,yBAAyB,CAAC;IAC/B,QAAQ,CAAC,yBAAyB,EAAE,CACnC,WAAW,EAAE,4BAA4B,KACrC,yBAAyB,CAAC;CAC/B;AAgID,wBAAgB,8BAA8B,CAC7C,OAAO,EAAE,qCAAqC,GAC5C,wBAAwB,CAuH1B"}