@agent-vm/agent-vm 0.0.91 → 0.0.93

package/dist/controller/health/durable-health-event-log.js

@@ -0,0 +1,89 @@
+import { appendFile, mkdir, readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { isAgentVmHealthEvent } from '@agent-vm/gateway-interface';
+export function controllerHealthEventLogPath(runtimeDir) {
+    return path.join(runtimeDir, 'controller-health', 'events.jsonl');
+}
+export async function appendDurableHealthEvent(options) {
+    const logPath = controllerHealthEventLogPath(options.runtimeDir);
+    await mkdir(path.dirname(logPath), { recursive: true });
+    const operationId = options.operationId ?? operationIdForHealthEvent(options.event);
+    const record = {
+        body: options.event,
+        controllerPid: options.controllerPid,
+        controllerPort: options.controllerPort,
+        eventKind: options.event.kind,
+        observedAtMs: options.event.observedAtMs,
+        ...(operationId === undefined ? {} : { operationId }),
+        zoneId: options.event.zoneId,
+    };
+    await appendFile(logPath, `${JSON.stringify(record)}\n`, 'utf8');
+}
+export async function readDurableHealthEvents(options) {
+    const logPath = controllerHealthEventLogPath(options.runtimeDir);
+    let logText;
+    try {
+        logText = await readFile(logPath, 'utf8');
+    }
+    catch (error) {
+        if (isNodeFileSystemError(error) && error.code === 'ENOENT') {
+            return [];
+        }
+        throw error;
+    }
+    return logText
+        .split('\n')
+        .filter((line) => line.trim().length > 0)
+        .map((line) => parseDurableHealthEventRecord(line));
+}
+function parseDurableHealthEventRecord(line) {
+    const parsedRecord = JSON.parse(line);
+    if (!isDurableHealthEventRecord(parsedRecord)) {
+        throw new Error('Durable health event record does not match expected schema.');
+    }
+    return parsedRecord;
+}
+function isDurableHealthEventRecord(value) {
+    if (!isUnknownRecord(value)) {
+        return false;
+    }
+    return (isAgentVmHealthEvent(value.body) &&
+        typeof value.controllerPid === 'number' &&
+        Number.isInteger(value.controllerPid) &&
+        value.controllerPid > 0 &&
+        typeof value.controllerPort === 'number' &&
+        Number.isInteger(value.controllerPort) &&
+        value.controllerPort > 0 &&
+        value.eventKind === value.body.kind &&
+        typeof value.observedAtMs === 'number' &&
+        value.observedAtMs === value.body.observedAtMs &&
+        (value.operationId === undefined || typeof value.operationId === 'string') &&
+        value.zoneId === value.body.zoneId);
+}
+function isUnknownRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function isNodeFileSystemError(value) {
+    return (typeof value === 'object' && value !== null && 'code' in value && typeof value.code === 'string');
+}
+function operationIdForHealthEvent(event) {
+    switch (event.kind) {
+        case 'gateway-recovery':
+        case 'gateway-recovery-suspended':
+            return event.operationId;
+        case 'controller-request':
+        case 'gateway-control-link':
+        case 'gateway-plugin-health':
+        case 'gateway-service-health':
+        case 'lease-heartbeat':
+        case 'lease-renew':
+        case 'agent-channel-provider-health':
+        case 'tool-vm-ssh':
+            return undefined;
+    }
+    return assertNeverHealthEvent(event);
+}
+function assertNeverHealthEvent(event) {
+    throw new Error(`Unhandled health event kind for operation join key: ${JSON.stringify(event)}`);
+}
+//# sourceMappingURL=durable-health-event-log.js.map

package/dist/controller/health/durable-health-event-log.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"durable-health-event-log.js","sourceRoot":"","sources":["../../../src/controller/health/durable-health-event-log.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,oBAAoB,EAA2B,MAAM,6BAA6B,CAAC;AAwB5F,MAAM,UAAU,4BAA4B,CAAC,UAAkB;IAC9D,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,mBAAmB,EAAE,cAAc,CAAC,CAAC;AACnE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC7C,OAAwC;IAExC,MAAM,OAAO,GAAG,4BAA4B,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACjE,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACxD,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,yBAAyB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACpF,MAAM,MAAM,GAA6B;QACxC,IAAI,EAAE,OAAO,CAAC,KAAK;QACnB,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,IAAI;QAC7B,YAAY,EAAE,OAAO,CAAC,KAAK,CAAC,YAAY;QACxC,GAAG,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC;QACrD,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,MAAM;KAC5B,CAAC;IACF,MAAM,UAAU,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;AAClE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC5C,OAAuC;IAEvC,MAAM,OAAO,GAAG,4BAA4B,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACjE,IAAI,OAAe,CAAC;IACpB,IAAI,CAAC;QACJ,OAAO,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC3C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,IAAI,qBAAqB,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC7D,OAAO,EAAE,CAAC;QACX,CAAC;QACD,MAAM,KAAK,CAAC;IACb,CAAC;IACD,OAAO,OAAO;SACZ,KAAK,CAAC,IAAI,CAAC;SACX,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;SACxC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,6BAA6B,CAAC,IAAI,CAAC,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,6BAA6B,CAAC,IAAY;IAClD,MAAM,YAAY,GAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/C,IAAI,CAAC,0BAA0B,CAAC,YAAY,CAAC,EAAE,CAAC;QAC/C,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;IAChF,CAAC;IACD,OAAO,YAAY,CAAC;AACrB,CAAC;AAED,SAAS,0BAA0B,CAAC,KAAc;IACjD,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,KAAK,CAAC;IACd,CAAC;IACD,OAAO,CACN,oBAAoB,CAAC,KAAK,CAAC,IAAI,CAAC;QAChC,OAAO,KAAK,CAAC,aAAa,KAAK,QAAQ;QACvC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,aAAa,CAAC;QACrC,KAAK,CAAC,aAAa,GAAG,CAAC;QACvB,OAAO,KAAK,CAAC,cAAc,KAAK,QAAQ;QACxC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,cAAc,CAAC;QACtC,KAAK,CAAC,cAAc,GAAG,CAAC;QACxB,KAAK,CAAC,SAAS,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI;QACnC,OAAO,KAAK,CAAC,YAAY,KAAK,QAAQ;QACtC,KAAK,CAAC,YAAY,KAAK,KAAK,CAAC,IAAI,CAAC,YAAY;QAC9C,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,CAAC;QAC1E,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,IAAI,CAAC,MAAM,CAClC,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACtC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC7E,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAc;IAC5C,OAAO,CACN,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,MAAM,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,CAChG,CAAC;AACH,CAAC;AAED,SAAS,yBAAyB,CAAC,KAAyB;IAC3D,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,kBAAkB,CAAC;QACxB,KAAK,4BAA4B;YAChC,OAAO,KAAK,CAAC,WAAW,CAAC;QAC1B,KAAK,oBAAoB,CAAC;QAC1B,KAAK,sBAAsB,CAAC;QAC5B,KAAK,uBAAuB,CAAC;QAC7B,KAAK,wBAAwB,CAAC;QAC9B,KAAK,iBAAiB,CAAC;QACvB,KAAK,aAAa,CAAC;QACnB,KAAK,+BAA+B,CAAC;QACrC,KAAK,aAAa;YACjB,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,sBAAsB,CAAC,KAAK,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAY;IAC3C,MAAM,IAAI,KAAK,CAAC,uDAAuD,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;AACjG,CAAC"}

package/dist/controller/health/gateway-recovery-actions.d.ts

@@ -0,0 +1,27 @@
+import type { GatewayZoneLifecycleState } from '../zone-runtimes/gateway-zone-state-machine.js';
+import type { GatewayVmRecoveryDecision, GatewayVmRecoveryReason } from './gateway-vm-recovery-policy.js';
+export type GatewayRecoveryDecisionAction = {
+    readonly kind: 'restart-running-gateway';
+    readonly reason: GatewayVmRecoveryReason;
+} | {
+    readonly kind: 'cold-start-gateway';
+    readonly reason: GatewayVmRecoveryReason;
+} | {
+    readonly kind: 'refresh-secret-resolver';
+    readonly reason: 'secret-resolution-failed';
+} | {
+    readonly errorCode: 'max-failed-recoveries';
+    readonly kind: 'suspend-recovery';
+} | {
+    readonly kind: 'operator-required';
+    readonly reason: 'ambiguous-runtime-state' | 'owner-unsafe';
+} | {
+    readonly kind: 'observe-only';
+    readonly reason: 'recovery-disabled' | 'recovery-in-flight' | 'recovery-unobserved' | 'channel-provider-unrecoverable' | 'cooldown-active';
+};
+export interface ClassifyGatewayRecoveryActionOptions {
+    readonly lifecycleState: GatewayZoneLifecycleState;
+    readonly recoveryDecision: GatewayVmRecoveryDecision;
+}
+export declare function classifyGatewayRecoveryAction(options: ClassifyGatewayRecoveryActionOptions): GatewayRecoveryDecisionAction;
+//# sourceMappingURL=gateway-recovery-actions.d.ts.map

package/dist/controller/health/gateway-recovery-actions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gateway-recovery-actions.d.ts","sourceRoot":"","sources":["../../../src/controller/health/gateway-recovery-actions.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEX,yBAAyB,EACzB,MAAM,gDAAgD,CAAC;AACxD,OAAO,KAAK,EACX,yBAAyB,EACzB,uBAAuB,EACvB,MAAM,iCAAiC,CAAC;AAEzC,MAAM,MAAM,6BAA6B,GACtC;IAAE,QAAQ,CAAC,IAAI,EAAE,yBAAyB,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,uBAAuB,CAAA;CAAE,GACtF;IAAE,QAAQ,CAAC,IAAI,EAAE,oBAAoB,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,uBAAuB,CAAA;CAAE,GACjF;IAAE,QAAQ,CAAC,IAAI,EAAE,yBAAyB,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,0BAA0B,CAAA;CAAE,GACzF;IAAE,QAAQ,CAAC,SAAS,EAAE,uBAAuB,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,kBAAkB,CAAA;CAAE,GAClF;IACA,QAAQ,CAAC,IAAI,EAAE,mBAAmB,CAAC;IACnC,QAAQ,CAAC,MAAM,EAAE,yBAAyB,GAAG,cAAc,CAAC;CAC3D,GACD;IACA,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC;IAC9B,QAAQ,CAAC,MAAM,EACZ,mBAAmB,GACnB,oBAAoB,GACpB,qBAAqB,GACrB,gCAAgC,GAChC,iBAAiB,CAAC;CACpB,CAAC;AAEL,MAAM,WAAW,oCAAoC;IACpD,QAAQ,CAAC,cAAc,EAAE,yBAAyB,CAAC;IACnD,QAAQ,CAAC,gBAAgB,EAAE,yBAAyB,CAAC;CACrD;AAED,wBAAgB,6BAA6B,CAC5C,OAAO,EAAE,oCAAoC,GAC3C,6BAA6B,CA+B/B"}

package/dist/controller/health/gateway-recovery-actions.js

@@ -0,0 +1,71 @@
+export function classifyGatewayRecoveryAction(options) {
+    if (options.recoveryDecision.kind === 'suspended') {
+        return {
+            errorCode: options.recoveryDecision.reason,
+            kind: 'suspend-recovery',
+        };
+    }
+    if (options.recoveryDecision.kind === 'none') {
+        return {
+            kind: 'observe-only',
+            reason: classifyNoRecoveryReason(options.recoveryDecision.reason),
+        };
+    }
+    const recoveryReason = options.recoveryDecision.reason;
+    switch (options.lifecycleState.kind) {
+        case 'running':
+        case 'running-degraded':
+            return { kind: 'restart-running-gateway', reason: recoveryReason };
+        case 'stopped':
+            return { kind: 'cold-start-gateway', reason: recoveryReason };
+        case 'failed':
+            return classifyFailedGatewayRecoveryAction(options.lifecycleState, recoveryReason);
+        case 'owner-unsafe':
+            return { kind: 'operator-required', reason: 'owner-unsafe' };
+        case 'restarting':
+        case 'starting':
+        case 'stopping':
+            return { kind: 'observe-only', reason: 'recovery-in-flight' };
+    }
+    return assertNeverGatewayZoneLifecycleState(options.lifecycleState);
+}
+function classifyFailedGatewayRecoveryAction(state, recoveryReason) {
+    if (state.error.code === 'secret-resolution-failed') {
+        return { kind: 'refresh-secret-resolver', reason: state.error.code };
+    }
+    if (state.error.code === 'owner-unsafe') {
+        return { kind: 'operator-required', reason: 'owner-unsafe' };
+    }
+    if (state.coldStartEligible) {
+        return { kind: 'cold-start-gateway', reason: recoveryReason };
+    }
+    if (isUnrecoverableFailure(state.error.code)) {
+        return { kind: 'observe-only', reason: 'channel-provider-unrecoverable' };
+    }
+    return { kind: 'operator-required', reason: 'ambiguous-runtime-state' };
+}
+function isUnrecoverableFailure(errorCode) {
+    return errorCode === 'agent-channel-provider-unhealthy';
+}
+function classifyNoRecoveryReason(reason) {
+    switch (reason) {
+        case 'cooldown':
+            return 'cooldown-active';
+        case 'disabled':
+            return 'recovery-disabled';
+        case 'in-flight':
+            return 'recovery-in-flight';
+        case 'unobserved':
+            return 'recovery-unobserved';
+        case undefined:
+            return 'recovery-unobserved';
+    }
+    return assertNeverNoRecoveryReason(reason);
+}
+function assertNeverGatewayZoneLifecycleState(state) {
+    throw new Error(`Unhandled gateway zone lifecycle state: ${JSON.stringify(state)}`);
+}
+function assertNeverNoRecoveryReason(reason) {
+    throw new Error(`Unhandled gateway recovery none reason: ${String(reason)}`);
+}
+//# sourceMappingURL=gateway-recovery-actions.js.map

package/dist/controller/health/gateway-recovery-actions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gateway-recovery-actions.js","sourceRoot":"","sources":["../../../src/controller/health/gateway-recovery-actions.ts"],"names":[],"mappings":"AAiCA,MAAM,UAAU,6BAA6B,CAC5C,OAA6C;IAE7C,IAAI,OAAO,CAAC,gBAAgB,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QACnD,OAAO;YACN,SAAS,EAAE,OAAO,CAAC,gBAAgB,CAAC,MAAM;YAC1C,IAAI,EAAE,kBAAkB;SACxB,CAAC;IACH,CAAC;IACD,IAAI,OAAO,CAAC,gBAAgB,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC9C,OAAO;YACN,IAAI,EAAE,cAAc;YACpB,MAAM,EAAE,wBAAwB,CAAC,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC;SACjE,CAAC;IACH,CAAC;IAED,MAAM,cAAc,GAAG,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC;IACvD,QAAQ,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QACrC,KAAK,SAAS,CAAC;QACf,KAAK,kBAAkB;YACtB,OAAO,EAAE,IAAI,EAAE,yBAAyB,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC;QACpE,KAAK,SAAS;YACb,OAAO,EAAE,IAAI,EAAE,oBAAoB,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC;QAC/D,KAAK,QAAQ;YACZ,OAAO,mCAAmC,CAAC,OAAO,CAAC,cAAc,EAAE,cAAc,CAAC,CAAC;QACpF,KAAK,cAAc;YAClB,OAAO,EAAE,IAAI,EAAE,mBAAmB,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC;QAC9D,KAAK,YAAY,CAAC;QAClB,KAAK,UAAU,CAAC;QAChB,KAAK,UAAU;YACd,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC;IAChE,CAAC;IACD,OAAO,oCAAoC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;AACrE,CAAC;AAED,SAAS,mCAAmC,CAC3C,KAAsE,EACtE,cAAuC;IAEvC,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,0BAA0B,EAAE,CAAC;QACrD,OAAO,EAAE,IAAI,EAAE,yBAAyB,EAAE,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;IACtE,CAAC;IACD,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;QACzC,OAAO,EAAE,IAAI,EAAE,mBAAmB,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC;IAC9D,CAAC;IACD,IAAI,KAAK,CAAC,iBAAiB,EAAE,CAAC;QAC7B,OAAO,EAAE,IAAI,EAAE,oBAAoB,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC;IAC/D,CAAC;IACD,IAAI,sBAAsB,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9C,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,gCAAgC,EAAE,CAAC;IAC3E,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,mBAAmB,EAAE,MAAM,EAAE,yBAAyB,EAAE,CAAC;AACzE,CAAC;AAED,SAAS,sBAAsB,CAAC,SAAoC;IACnE,OAAO,SAAS,KAAK,kCAAkC,CAAC;AACzD,CAAC;AAED,SAAS,wBAAwB,CAChC,MAA+E;IAE/E,QAAQ,MAAM,EAAE,CAAC;QAChB,KAAK,UAAU;YACd,OAAO,iBAAiB,CAAC;QAC1B,KAAK,UAAU;YACd,OAAO,mBAAmB,CAAC;QAC5B,KAAK,WAAW;YACf,OAAO,oBAAoB,CAAC;QAC7B,KAAK,YAAY;YAChB,OAAO,qBAAqB,CAAC;QAC9B,KAAK,SAAS;YACb,OAAO,qBAAqB,CAAC;IAC/B,CAAC;IACD,OAAO,2BAA2B,CAAC,MAAM,CAAC,CAAC;AAC5C,CAAC;AAED,SAAS,oCAAoC,CAAC,KAAY;IACzD,MAAM,IAAI,KAAK,CAAC,2CAA2C,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;AACrF,CAAC;AAED,SAAS,2BAA2B,CAAC,MAAa;IACjD,MAAM,IAAI,KAAK,CAAC,2CAA2C,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;AAC9E,CAAC"}

package/dist/controller/health/gateway-service-health-monitor.d.ts

@@ -1,3 +1,5 @@
+import type { GatewayRecoveryTimeoutErrorCode } from '@agent-vm/gateway-interface';
+import { type GatewayVmAutoRecoveryPolicy, type GatewayVmRecoveryBudgetClass, type GatewayVmRecoveryReason } from './gateway-vm-recovery-policy.js';
 import type { HealthEventStore } from './health-event-store.js';
 export interface GatewayServiceHealthProbeResult {
     readonly ok: boolean;
@@ -8,16 +10,82 @@ export interface GatewayServiceHealthProbeResult {
 }
 export interface GatewayServiceHealthMonitor {
     start(): void;
-    stop(): void;
+    stop(): Promise<void>;
     tick(): Promise<void>;
 }
+export interface GatewayVmRecoveryRequest {
+    readonly consecutiveFailures: number;
+    readonly reason: GatewayVmRecoveryReason;
+    readonly zoneId: string;
+}
+export interface ClassifyRecoveryBudgetClassRequest {
+    readonly consecutiveFailures: number;
+    readonly reason: GatewayVmRecoveryReason;
+    readonly zoneId: string;
+}
+export type GatewayVmRecoveryResult = {
+    readonly action?: 'gateway-vm-restart' | undefined;
+    readonly elapsedMs: number;
+    readonly leaseReleaseFailureCount: number;
+    readonly newBootedAt: string;
+    readonly newHostPid: number;
+    readonly newVmId: string;
+    readonly oldBootedAt: string;
+    readonly oldHostPid: number;
+    readonly oldVmId: string;
+    readonly operationId?: string | undefined;
+    readonly result: 'ok';
+} | {
+    readonly action: 'gateway-vm-cold-start';
+    readonly elapsedMs: number;
+    readonly leaseReleaseFailureCount: number;
+    readonly newBootedAt: string;
+    readonly newHostPid: number;
+    readonly newVmId: string;
+    readonly operationId?: string | undefined;
+    readonly result: 'ok';
+} | {
+    readonly action: 'gateway-vm-restart';
+    readonly elapsedMs: number;
+    readonly errorCode: string;
+    readonly leaseReleaseFailureCount?: number | undefined;
+    readonly oldBootedAt?: string | undefined;
+    readonly oldHostPid?: number | undefined;
+    readonly oldVmId: string;
+    readonly operationId?: string | undefined;
+    readonly result: 'failed';
+} | {
+    readonly action: 'gateway-vm-restart';
+    readonly elapsedMs: number;
+    readonly errorCode: GatewayRecoveryTimeoutErrorCode;
+    readonly leaseReleaseFailureCount?: number | undefined;
+    readonly oldBootedAt?: string | undefined;
+    readonly oldHostPid?: number | undefined;
+    readonly oldVmId?: string | undefined;
+    readonly operationId?: string | undefined;
+    readonly result: 'failed';
+} | {
+    readonly action: 'gateway-vm-cold-start' | 'observe-only' | 'operator-required';
+    readonly elapsedMs: number;
+    readonly errorCode: string;
+    readonly leaseReleaseFailureCount?: number | undefined;
+    readonly oldBootedAt?: undefined;
+    readonly oldHostPid?: undefined;
+    readonly oldVmId?: undefined;
+    readonly operationId?: string | undefined;
+    readonly result: 'failed';
+};
 export interface CreateGatewayServiceHealthMonitorOptions {
-    readonly clearIntervalImpl?: (timer: NodeJS.Timeout) => void;
+    readonly clearIntervalImpl?: ((timer: NodeJS.Timeout) => void) | undefined;
+    readonly classifyRecoveryBudgetClass?: ((request: ClassifyRecoveryBudgetClassRequest) => GatewayVmRecoveryBudgetClass) | undefined;
+    readonly gatewayServiceAutoRestart: GatewayVmAutoRecoveryPolicy;
     readonly healthEventStore: HealthEventStore;
     readonly intervalMs: number;
     readonly now: () => number;
     readonly probeZoneHealth: (zoneId: string) => Promise<GatewayServiceHealthProbeResult>;
-    readonly setIntervalImpl?: (callback: () => void | Promise<void>, delayMs: number) => NodeJS.Timeout;
+    readonly recoverGatewayVm?: ((request: GatewayVmRecoveryRequest) => Promise<GatewayVmRecoveryResult>) | undefined;
+    readonly setIntervalImpl?: ((callback: () => void | Promise<void>, delayMs: number) => NodeJS.Timeout) | undefined;
+    readonly staleAfterMs: number;
     readonly zoneIds: readonly string[];
 }
 export declare function createGatewayServiceHealthMonitor(options: CreateGatewayServiceHealthMonitorOptions): GatewayServiceHealthMonitor;

package/dist/controller/health/gateway-service-health-monitor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"gateway-service-health-monitor.d.ts","sourceRoot":"","sources":["../../../src/controller/health/gateway-service-health-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAEhE,MAAM,WAAW,+BAA+B;IAC/C,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,2BAA2B;IAC3C,KAAK,IAAI,IAAI,CAAC;IACd,IAAI,IAAI,IAAI,CAAC;IACb,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACtB;AAED,MAAM,WAAW,wCAAwC;IACxD,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,OAAO,KAAK,IAAI,CAAC;IAC7D,QAAQ,CAAC,gBAAgB,EAAE,gBAAgB,CAAC;IAC5C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,MAAM,CAAC;IAC3B,QAAQ,CAAC,eAAe,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,+BAA+B,CAAC,CAAC;IACvF,QAAQ,CAAC,eAAe,CAAC,EAAE,CAC1B,QAAQ,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EACpC,OAAO,EAAE,MAAM,KACX,MAAM,CAAC,OAAO,CAAC;IACpB,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;CACpC;AAWD,wBAAgB,iCAAiC,CAChD,OAAO,EAAE,wCAAwC,GAC/C,2BAA2B,CAgE7B"}
+{"version":3,"file":"gateway-service-health-monitor.d.ts","sourceRoot":"","sources":["../../../src/controller/health/gateway-service-health-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAGX,+BAA+B,EAC/B,MAAM,6BAA6B,CAAC;AAMrC,OAAO,EAGN,KAAK,2BAA2B,EAChC,KAAK,4BAA4B,EAGjC,KAAK,uBAAuB,EAC5B,MAAM,iCAAiC,CAAC;AACzC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAEhE,MAAM,WAAW,+BAA+B;IAC/C,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,2BAA2B;IAC3C,KAAK,IAAI,IAAI,CAAC;IACd,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACtB;AAED,MAAM,WAAW,wBAAwB;IACxC,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,MAAM,EAAE,uBAAuB,CAAC;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,kCAAkC;IAClD,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,MAAM,EAAE,uBAAuB,CAAC;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,uBAAuB,GAChC;IACA,QAAQ,CAAC,MAAM,CAAC,EAAE,oBAAoB,GAAG,SAAS,CAAC;IACnD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,wBAAwB,EAAE,MAAM,CAAC;IAC1C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1C,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC;CACrB,GACD;IACA,QAAQ,CAAC,MAAM,EAAE,uBAAuB,CAAC;IACzC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,wBAAwB,EAAE,MAAM,CAAC;IAC1C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1C,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC;CACrB,GACD;IACA,QAAQ,CAAC,MAAM,EAAE,oBAAoB,CAAC;IACtC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,wBAAwB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvD,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1C,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1C,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC;CACzB,GACD;IACA,QAAQ,CAAC,MAAM,EAAE,oBAAoB,CAAC;IACtC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,+BAA+B,CAAC;IACpD,QAAQ,CAAC,wBAAwB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvD,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1C,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACtC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1C,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC;CACzB,GACD;IACA,QAAQ,CAAC,MAAM,EAAE,uBAAuB,GAAG,cAAc,GAAG,mBAAmB,CAAC;IAChF,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,wBAAwB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvD,QAAQ,CAAC,WAAW,CAAC,EAAE,SAAS,CAAC;IACjC,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,CAAC;IAChC,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,CAAC;IAC7B,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1C,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC;CACzB,CAAC;AAEL,MAAM,WAAW,wCAAwC;IACxD,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,OAAO,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;IAC3E,QAAQ,CAAC,2BAA2B,CAAC,EAClC,CAAC,CAAC,OAAO,EAAE,kCAAkC,KAAK,4BAA4B,CAAC,GAC/E,SAAS,CAAC;IACb,QAAQ,CAAC,yBAAyB,EAAE,2BAA2B,CAAC;IAChE,QAAQ,CAAC,gBAAgB,EAAE,gBAAgB,CAAC;IAC5C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,MAAM,CAAC;IAC3B,QAAQ,CAAC,eAAe,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,+BAA+B,CAAC,CAAC;IACvF,QAAQ,CAAC,gBAAgB,CAAC,EACvB,CAAC,CAAC,OAAO,EAAE,wBAAwB,KAAK,OAAO,CAAC,uBAAuB,CAAC,CAAC,GACzE,SAAS,CAAC;IACb,QAAQ,CAAC,eAAe,CAAC,EACtB,CAAC,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,KAAK,MAAM,CAAC,OAAO,CAAC,GAC3E,SAAS,CAAC;IACb,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;CACpC;AAqCD,wBAAgB,iCAAiC,CAChD,OAAO,EAAE,wCAAwC,GAC/C,2BAA2B,CAif7B"}