@agent-vm/agent-vm 0.0.92 → 0.0.94
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/build/managed-image-dockerfile.d.ts +2 -1
- package/dist/build/managed-image-dockerfile.d.ts.map +1 -1
- package/dist/build/managed-image-dockerfile.js +51 -27
- package/dist/build/managed-image-dockerfile.js.map +1 -1
- package/dist/cli/agent-vm-cli-support.d.ts +4 -1
- package/dist/cli/agent-vm-cli-support.d.ts.map +1 -1
- package/dist/cli/agent-vm-cli-support.js.map +1 -1
- package/dist/cli/commands/controller-definition.d.ts +42 -42
- package/dist/cli/commands/create-app.d.ts +42 -42
- package/dist/cli/commands/doctor-definition.d.ts.map +1 -1
- package/dist/cli/commands/doctor-definition.js +6 -0
- package/dist/cli/commands/doctor-definition.js.map +1 -1
- package/dist/cli/controller-operation-commands.d.ts +19 -0
- package/dist/cli/controller-operation-commands.d.ts.map +1 -1
- package/dist/cli/controller-operation-commands.js +63 -44
- package/dist/cli/controller-operation-commands.js.map +1 -1
- package/dist/cli/manual-templates.d.ts.map +1 -1
- package/dist/cli/manual-templates.js +11 -2
- package/dist/cli/manual-templates.js.map +1 -1
- package/dist/config/system-config.d.ts +7 -0
- package/dist/config/system-config.d.ts.map +1 -1
- package/dist/config/system-config.js +35 -0
- package/dist/config/system-config.js.map +1 -1
- package/dist/controller/controller-runtime-operations.d.ts +1 -0
- package/dist/controller/controller-runtime-operations.d.ts.map +1 -1
- package/dist/controller/controller-runtime-operations.js +2 -0
- package/dist/controller/controller-runtime-operations.js.map +1 -1
- package/dist/controller/controller-runtime-types.d.ts +3 -0
- package/dist/controller/controller-runtime-types.d.ts.map +1 -1
- package/dist/controller/controller-runtime.d.ts +1 -1
- package/dist/controller/controller-runtime.d.ts.map +1 -1
- package/dist/controller/controller-runtime.js +208 -117
- package/dist/controller/controller-runtime.js.map +1 -1
- package/dist/controller/health/channel-provider-recovery-observation.d.ts +23 -0
- package/dist/controller/health/channel-provider-recovery-observation.d.ts.map +1 -0
- package/dist/controller/health/channel-provider-recovery-observation.js +69 -0
- package/dist/controller/health/channel-provider-recovery-observation.js.map +1 -0
- package/dist/controller/health/durable-health-event-log.d.ts +24 -0
- package/dist/controller/health/durable-health-event-log.d.ts.map +1 -0
- package/dist/controller/health/durable-health-event-log.js +89 -0
- package/dist/controller/health/durable-health-event-log.js.map +1 -0
- package/dist/controller/health/gateway-recovery-actions.d.ts +27 -0
- package/dist/controller/health/gateway-recovery-actions.d.ts.map +1 -0
- package/dist/controller/health/gateway-recovery-actions.js +71 -0
- package/dist/controller/health/gateway-recovery-actions.js.map +1 -0
- package/dist/controller/health/gateway-service-health-monitor.d.ts +41 -3
- package/dist/controller/health/gateway-service-health-monitor.d.ts.map +1 -1
- package/dist/controller/health/gateway-service-health-monitor.js +231 -57
- package/dist/controller/health/gateway-service-health-monitor.js.map +1 -1
- package/dist/controller/health/gateway-vm-recovery-policy.d.ts +20 -0
- package/dist/controller/health/gateway-vm-recovery-policy.d.ts.map +1 -1
- package/dist/controller/health/gateway-vm-recovery-policy.js +85 -21
- package/dist/controller/health/gateway-vm-recovery-policy.js.map +1 -1
- package/dist/controller/health/gateway-vm-recovery-runner.d.ts +39 -0
- package/dist/controller/health/gateway-vm-recovery-runner.d.ts.map +1 -0
- package/dist/controller/health/gateway-vm-recovery-runner.js +251 -0
- package/dist/controller/health/gateway-vm-recovery-runner.js.map +1 -0
- package/dist/controller/health/health-event-store.d.ts +4 -0
- package/dist/controller/health/health-event-store.d.ts.map +1 -1
- package/dist/controller/health/health-event-store.js +19 -0
- package/dist/controller/health/health-event-store.js.map +1 -1
- package/dist/controller/http/controller-health-event-routes.d.ts +6 -0
- package/dist/controller/http/controller-health-event-routes.d.ts.map +1 -1
- package/dist/controller/http/controller-health-event-routes.js +49 -0
- package/dist/controller/http/controller-health-event-routes.js.map +1 -1
- package/dist/controller/http/controller-http-routes.d.ts.map +1 -1
- package/dist/controller/http/controller-http-routes.js +6 -0
- package/dist/controller/http/controller-http-routes.js.map +1 -1
- package/dist/controller/leases/lease-manager.d.ts.map +1 -1
- package/dist/controller/leases/lease-manager.js +37 -16
- package/dist/controller/leases/lease-manager.js.map +1 -1
- package/dist/controller/leases/tool-vm-lease-lifecycle.d.ts +44 -0
- package/dist/controller/leases/tool-vm-lease-lifecycle.d.ts.map +1 -0
- package/dist/controller/leases/tool-vm-lease-lifecycle.js +28 -0
- package/dist/controller/leases/tool-vm-lease-lifecycle.js.map +1 -0
- package/dist/controller/worker-task-runner.d.ts +6 -0
- package/dist/controller/worker-task-runner.d.ts.map +1 -1
- package/dist/controller/worker-task-runner.js +13 -4
- package/dist/controller/worker-task-runner.js.map +1 -1
- package/dist/controller/zone-runtimes/gateway-lifecycle-operation-record.d.ts +37 -0
- package/dist/controller/zone-runtimes/gateway-lifecycle-operation-record.d.ts.map +1 -0
- package/dist/controller/zone-runtimes/gateway-lifecycle-operation-record.js +133 -0
- package/dist/controller/zone-runtimes/gateway-lifecycle-operation-record.js.map +1 -0
- package/dist/controller/zone-runtimes/gateway-zone-state-machine.d.ts +101 -0
- package/dist/controller/zone-runtimes/gateway-zone-state-machine.d.ts.map +1 -0
- package/dist/controller/zone-runtimes/gateway-zone-state-machine.js +143 -0
- package/dist/controller/zone-runtimes/gateway-zone-state-machine.js.map +1 -0
- package/dist/controller/zone-runtimes/openclaw-zone-runtime.d.ts +8 -1
- package/dist/controller/zone-runtimes/openclaw-zone-runtime.d.ts.map +1 -1
- package/dist/controller/zone-runtimes/openclaw-zone-runtime.js +621 -65
- package/dist/controller/zone-runtimes/openclaw-zone-runtime.js.map +1 -1
- package/dist/controller/zone-runtimes/zone-runtime-errors.d.ts +7 -1
- package/dist/controller/zone-runtimes/zone-runtime-errors.d.ts.map +1 -1
- package/dist/controller/zone-runtimes/zone-runtime-errors.js +5 -1
- package/dist/controller/zone-runtimes/zone-runtime-errors.js.map +1 -1
- package/dist/controller/zone-runtimes/zone-runtime-registry.d.ts +2 -0
- package/dist/controller/zone-runtimes/zone-runtime-registry.d.ts.map +1 -1
- package/dist/controller/zone-runtimes/zone-runtime-registry.js +23 -0
- package/dist/controller/zone-runtimes/zone-runtime-registry.js.map +1 -1
- package/dist/controller/zone-runtimes/zone-runtime-types.d.ts +7 -0
- package/dist/controller/zone-runtimes/zone-runtime-types.d.ts.map +1 -1
- package/dist/gateway/gateway-ownership-evidence.d.ts +35 -0
- package/dist/gateway/gateway-ownership-evidence.d.ts.map +1 -0
- package/dist/gateway/gateway-ownership-evidence.js +10 -0
- package/dist/gateway/gateway-ownership-evidence.js.map +1 -0
- package/dist/gateway/gateway-recovery.d.ts +16 -0
- package/dist/gateway/gateway-recovery.d.ts.map +1 -1
- package/dist/gateway/gateway-recovery.js +105 -9
- package/dist/gateway/gateway-recovery.js.map +1 -1
- package/dist/gateway/gateway-zone-orchestrator.d.ts.map +1 -1
- package/dist/gateway/gateway-zone-orchestrator.js +50 -39
- package/dist/gateway/gateway-zone-orchestrator.js.map +1 -1
- package/dist/gateway/mcp-portal-effective-config.d.ts +11 -0
- package/dist/gateway/mcp-portal-effective-config.d.ts.map +1 -1
- package/dist/gateway/mcp-portal-effective-config.js +27 -8
- package/dist/gateway/mcp-portal-effective-config.js.map +1 -1
- package/dist/integration-tests/{smoke-harness.d.ts → e2e-harness.d.ts} +52 -37
- package/dist/integration-tests/e2e-harness.d.ts.map +1 -0
- package/dist/integration-tests/{smoke-harness.js → e2e-harness.js} +483 -131
- package/dist/integration-tests/e2e-harness.js.map +1 -0
- package/dist/integration-tests/e2e-protocol-wait.d.ts +2 -0
- package/dist/integration-tests/e2e-protocol-wait.d.ts.map +1 -0
- package/dist/integration-tests/e2e-protocol-wait.js +5 -0
- package/dist/integration-tests/e2e-protocol-wait.js.map +1 -0
- package/dist/integration-tests/e2e-workspace-build-global-setup.d.ts +27 -0
- package/dist/integration-tests/e2e-workspace-build-global-setup.d.ts.map +1 -0
- package/dist/integration-tests/e2e-workspace-build-global-setup.js +49 -0
- package/dist/integration-tests/e2e-workspace-build-global-setup.js.map +1 -0
- package/dist/integration-tests/live-agent-model-roundtrip-deployment.d.ts +11 -0
- package/dist/integration-tests/live-agent-model-roundtrip-deployment.d.ts.map +1 -0
- package/dist/integration-tests/live-agent-model-roundtrip-deployment.js +48 -0
- package/dist/integration-tests/live-agent-model-roundtrip-deployment.js.map +1 -0
- package/dist/integration-tests/live-agent-model-roundtrip-gates.d.ts +11 -0
- package/dist/integration-tests/live-agent-model-roundtrip-gates.d.ts.map +1 -0
- package/dist/integration-tests/live-agent-model-roundtrip-gates.js +21 -0
- package/dist/integration-tests/live-agent-model-roundtrip-gates.js.map +1 -0
- package/dist/integration-tests/live-vm-e2e-gates.d.ts +2 -0
- package/dist/integration-tests/live-vm-e2e-gates.d.ts.map +1 -0
- package/dist/integration-tests/live-vm-e2e-gates.js +4 -0
- package/dist/integration-tests/live-vm-e2e-gates.js.map +1 -0
- package/dist/operations/controller-status.d.ts +5 -0
- package/dist/operations/controller-status.d.ts.map +1 -1
- package/dist/operations/controller-status.js +42 -0
- package/dist/operations/controller-status.js.map +1 -1
- package/package.json +11 -11
- package/dist/integration-tests/live-integration-gates.d.ts +0 -2
- package/dist/integration-tests/live-integration-gates.d.ts.map +0 -1
- package/dist/integration-tests/live-integration-gates.js +0 -4
- package/dist/integration-tests/live-integration-gates.js.map +0 -1
- package/dist/integration-tests/smoke-harness.d.ts.map +0 -1
- package/dist/integration-tests/smoke-harness.js.map +0 -1
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { deriveChannelProviderRecoveryObservation, } from './channel-provider-recovery-observation.js';
|
|
2
|
+
import { createGatewayVmRecoveryTracker, defaultGatewayVmChannelProviderRecoveryPolicy, } from './gateway-vm-recovery-policy.js';
|
|
2
3
|
function writeGatewayServiceHealthMonitorLog(message) {
|
|
3
4
|
process.stderr.write(`[gateway-service-health-monitor] ${message}\n`);
|
|
4
5
|
}
|
|
@@ -6,16 +7,41 @@ const unknownGatewayServiceHealthTarget = {
|
|
|
6
7
|
path: '(unknown)',
|
|
7
8
|
port: 0,
|
|
8
9
|
};
|
|
10
|
+
function channelProviderEventKey(event) {
|
|
11
|
+
return `${event.zoneId}\0${event.channelProviderId}\0${String(event.observedAtMs)}`;
|
|
12
|
+
}
|
|
13
|
+
function recoveryActionForBudgetClass(recoveryBudgetClass) {
|
|
14
|
+
return recoveryBudgetClass === 'gateway-vm-cold-start'
|
|
15
|
+
? 'gateway-vm-cold-start'
|
|
16
|
+
: 'gateway-vm-restart';
|
|
17
|
+
}
|
|
18
|
+
function budgetResultForRecoveryResult(result) {
|
|
19
|
+
return result.action === 'observe-only' || result.action === 'operator-required'
|
|
20
|
+
? undefined
|
|
21
|
+
: result.result;
|
|
22
|
+
}
|
|
23
|
+
function isGatewayRecoveryTimeoutErrorCodeValue(value) {
|
|
24
|
+
return value === 'recovery-callback-unconfigured' || value === 'recovery-timeout';
|
|
25
|
+
}
|
|
9
26
|
export function createGatewayServiceHealthMonitor(options) {
|
|
10
27
|
const setIntervalImpl = options.setIntervalImpl ?? setInterval;
|
|
11
28
|
const clearIntervalImpl = options.clearIntervalImpl ?? clearInterval;
|
|
12
|
-
const setTimeoutImpl = options.setTimeoutImpl ?? setTimeout;
|
|
13
|
-
const clearTimeoutImpl = options.clearTimeoutImpl ?? clearTimeout;
|
|
14
29
|
const recoveryPolicy = options.gatewayServiceAutoRestart;
|
|
30
|
+
const channelProviderPolicy = recoveryPolicy.channelProviderHealth ?? defaultGatewayVmChannelProviderRecoveryPolicy;
|
|
15
31
|
const recoveryTracker = createGatewayVmRecoveryTracker({ policy: recoveryPolicy });
|
|
16
32
|
let timer;
|
|
17
33
|
let runningTick;
|
|
34
|
+
const recoveredChannelProviderEventKeys = new Set();
|
|
18
35
|
let stopped = false;
|
|
36
|
+
const classifyRecoveryBudgetClass = (request) => {
|
|
37
|
+
try {
|
|
38
|
+
return options.classifyRecoveryBudgetClass?.(request) ?? 'gateway-vm-restart';
|
|
39
|
+
}
|
|
40
|
+
catch (error) {
|
|
41
|
+
writeGatewayServiceHealthMonitorLog(`recovery budget classification failed for zone '${request.zoneId}': ${error instanceof Error ? error.message : String(error)}`);
|
|
42
|
+
return 'gateway-vm-restart';
|
|
43
|
+
}
|
|
44
|
+
};
|
|
19
45
|
const classifyGatewayControlLinkObservation = (props) => {
|
|
20
46
|
const controlLinkEvent = options.healthEventStore
|
|
21
47
|
.listLatestEventsForZone(props.zoneId)
|
|
@@ -30,62 +56,74 @@ export function createGatewayServiceHealthMonitor(options) {
|
|
|
30
56
|
}
|
|
31
57
|
return controlLinkEvent.result;
|
|
32
58
|
};
|
|
33
|
-
const
|
|
59
|
+
const latestChannelProviderHealthEvents = (zoneId) => options.healthEventStore
|
|
60
|
+
.listLatestEventsForZone(zoneId)
|
|
61
|
+
.filter((event) => event.kind === 'agent-channel-provider-health');
|
|
62
|
+
const runRecovery = async (request, intendedAction) => {
|
|
34
63
|
if (!options.recoverGatewayVm) {
|
|
35
|
-
return
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
resolve({
|
|
49
|
-
elapsedMs: options.now() - startedAtMs,
|
|
50
|
-
errorCode: 'recovery-timeout',
|
|
51
|
-
result: 'failed',
|
|
52
|
-
});
|
|
53
|
-
}, recoveryPolicy.restartTimeoutMs);
|
|
54
|
-
timeout.unref?.();
|
|
55
|
-
}),
|
|
56
|
-
]);
|
|
57
|
-
}
|
|
58
|
-
finally {
|
|
59
|
-
if (timeout) {
|
|
60
|
-
clearTimeoutImpl(timeout);
|
|
61
|
-
}
|
|
64
|
+
return intendedAction === 'gateway-vm-restart'
|
|
65
|
+
? {
|
|
66
|
+
action: 'gateway-vm-restart',
|
|
67
|
+
elapsedMs: 0,
|
|
68
|
+
errorCode: 'recovery-callback-unconfigured',
|
|
69
|
+
result: 'failed',
|
|
70
|
+
}
|
|
71
|
+
: {
|
|
72
|
+
action: 'gateway-vm-cold-start',
|
|
73
|
+
elapsedMs: 0,
|
|
74
|
+
errorCode: 'recovery-callback-unconfigured',
|
|
75
|
+
result: 'failed',
|
|
76
|
+
};
|
|
62
77
|
}
|
|
78
|
+
return await options.recoverGatewayVm(request);
|
|
63
79
|
};
|
|
64
80
|
const recordGatewayRecoveryEvent = (props) => {
|
|
81
|
+
const recoveryAction = props.result.action ?? 'gateway-vm-restart';
|
|
65
82
|
if (props.result.result === 'ok') {
|
|
66
|
-
const event =
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
83
|
+
const event = props.result.action !== 'gateway-vm-cold-start'
|
|
84
|
+
? {
|
|
85
|
+
action: 'gateway-vm-restart',
|
|
86
|
+
consecutiveFailures: props.consecutiveFailures,
|
|
87
|
+
cooldownMs: recoveryPolicy.cooldownMs,
|
|
88
|
+
elapsedMs: props.result.elapsedMs,
|
|
89
|
+
kind: 'gateway-recovery',
|
|
90
|
+
leaseReleaseFailureCount: props.result.leaseReleaseFailureCount,
|
|
91
|
+
newBootedAt: props.result.newBootedAt,
|
|
92
|
+
newHostPid: props.result.newHostPid,
|
|
93
|
+
newVmId: props.result.newVmId,
|
|
94
|
+
observedAtMs: props.observedAtMs,
|
|
95
|
+
oldBootedAt: props.result.oldBootedAt,
|
|
96
|
+
oldHostPid: props.result.oldHostPid,
|
|
97
|
+
oldVmId: props.result.oldVmId,
|
|
98
|
+
...(props.result.operationId === undefined
|
|
99
|
+
? {}
|
|
100
|
+
: { operationId: props.result.operationId }),
|
|
101
|
+
reason: props.reason,
|
|
102
|
+
result: 'ok',
|
|
103
|
+
zoneId: props.zoneId,
|
|
104
|
+
}
|
|
105
|
+
: {
|
|
106
|
+
action: 'gateway-vm-cold-start',
|
|
107
|
+
consecutiveFailures: props.consecutiveFailures,
|
|
108
|
+
cooldownMs: recoveryPolicy.cooldownMs,
|
|
109
|
+
elapsedMs: props.result.elapsedMs,
|
|
110
|
+
kind: 'gateway-recovery',
|
|
111
|
+
leaseReleaseFailureCount: props.result.leaseReleaseFailureCount,
|
|
112
|
+
newBootedAt: props.result.newBootedAt,
|
|
113
|
+
newHostPid: props.result.newHostPid,
|
|
114
|
+
newVmId: props.result.newVmId,
|
|
115
|
+
observedAtMs: props.observedAtMs,
|
|
116
|
+
...(props.result.operationId === undefined
|
|
117
|
+
? {}
|
|
118
|
+
: { operationId: props.result.operationId }),
|
|
119
|
+
reason: props.reason,
|
|
120
|
+
result: 'ok',
|
|
121
|
+
zoneId: props.zoneId,
|
|
122
|
+
};
|
|
84
123
|
options.healthEventStore.record(event);
|
|
85
124
|
return;
|
|
86
125
|
}
|
|
87
126
|
const event = {
|
|
88
|
-
action: 'gateway-vm-restart',
|
|
89
127
|
consecutiveFailures: props.consecutiveFailures,
|
|
90
128
|
cooldownMs: recoveryPolicy.cooldownMs,
|
|
91
129
|
elapsedMs: props.result.elapsedMs,
|
|
@@ -95,18 +133,52 @@ export function createGatewayServiceHealthMonitor(options) {
|
|
|
95
133
|
? {}
|
|
96
134
|
: { leaseReleaseFailureCount: props.result.leaseReleaseFailureCount }),
|
|
97
135
|
observedAtMs: props.observedAtMs,
|
|
98
|
-
|
|
99
|
-
oldHostPid: props.result.oldHostPid,
|
|
100
|
-
oldVmId: props.result.oldVmId,
|
|
136
|
+
...(props.result.operationId === undefined ? {} : { operationId: props.result.operationId }),
|
|
101
137
|
reason: props.reason,
|
|
102
138
|
result: 'failed',
|
|
103
139
|
zoneId: props.zoneId,
|
|
104
140
|
};
|
|
105
|
-
|
|
141
|
+
if (recoveryAction === 'gateway-vm-restart') {
|
|
142
|
+
if (props.result.oldVmId === undefined) {
|
|
143
|
+
if (!isGatewayRecoveryTimeoutErrorCodeValue(props.result.errorCode)) {
|
|
144
|
+
const malformedRestartEvent = {
|
|
145
|
+
...event,
|
|
146
|
+
action: 'operator-required',
|
|
147
|
+
errorCode: props.result.errorCode,
|
|
148
|
+
};
|
|
149
|
+
options.healthEventStore.record(malformedRestartEvent);
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
const timeoutEvent = {
|
|
153
|
+
...event,
|
|
154
|
+
action: 'gateway-vm-restart',
|
|
155
|
+
errorCode: props.result.errorCode,
|
|
156
|
+
oldBootedAt: props.result.oldBootedAt,
|
|
157
|
+
oldHostPid: props.result.oldHostPid,
|
|
158
|
+
oldVmId: props.result.oldVmId,
|
|
159
|
+
};
|
|
160
|
+
options.healthEventStore.record(timeoutEvent);
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
163
|
+
const restartEvent = {
|
|
164
|
+
...event,
|
|
165
|
+
action: 'gateway-vm-restart',
|
|
166
|
+
oldBootedAt: props.result.oldBootedAt,
|
|
167
|
+
oldHostPid: props.result.oldHostPid,
|
|
168
|
+
oldVmId: props.result.oldVmId,
|
|
169
|
+
};
|
|
170
|
+
options.healthEventStore.record(restartEvent);
|
|
171
|
+
return;
|
|
172
|
+
}
|
|
173
|
+
const nonRestartEvent = {
|
|
174
|
+
...event,
|
|
175
|
+
action: recoveryAction,
|
|
176
|
+
};
|
|
177
|
+
options.healthEventStore.record(nonRestartEvent);
|
|
106
178
|
};
|
|
107
179
|
const recordGatewayRecoverySuspendedEvent = (props) => {
|
|
108
180
|
const event = {
|
|
109
|
-
action:
|
|
181
|
+
action: props.action,
|
|
110
182
|
consecutiveFailedRecoveries: props.consecutiveFailedRecoveries,
|
|
111
183
|
consecutiveFailures: props.consecutiveFailures,
|
|
112
184
|
cooldownMs: recoveryPolicy.cooldownMs,
|
|
@@ -121,14 +193,21 @@ export function createGatewayServiceHealthMonitor(options) {
|
|
|
121
193
|
options.healthEventStore.record(event);
|
|
122
194
|
};
|
|
123
195
|
const maybeRecoverGatewayVm = async (props) => {
|
|
196
|
+
const recoveryBudgetClass = classifyRecoveryBudgetClass({
|
|
197
|
+
consecutiveFailures: 0,
|
|
198
|
+
reason: props.reason,
|
|
199
|
+
zoneId: props.zoneId,
|
|
200
|
+
});
|
|
124
201
|
const decision = props.reason === 'gateway-service-unhealthy'
|
|
125
202
|
? recoveryTracker.recordGatewayServiceProbe({
|
|
126
203
|
observedAtMs: props.observedAtMs,
|
|
204
|
+
recoveryBudgetClass,
|
|
127
205
|
result: props.serviceProbeResult,
|
|
128
206
|
zoneId: props.zoneId,
|
|
129
207
|
})
|
|
130
208
|
: recoveryTracker.recordGatewayControlLinkObservation({
|
|
131
209
|
observedAtMs: props.observedAtMs,
|
|
210
|
+
recoveryBudgetClass,
|
|
132
211
|
result: props.serviceProbeResult === 'ok'
|
|
133
212
|
? classifyGatewayControlLinkObservation({
|
|
134
213
|
nowMs: props.observedAtMs,
|
|
@@ -143,6 +222,7 @@ export function createGatewayServiceHealthMonitor(options) {
|
|
|
143
222
|
consecutiveFailures: decision.consecutiveFailures,
|
|
144
223
|
observedAtMs: props.observedAtMs,
|
|
145
224
|
reason: props.reason,
|
|
225
|
+
action: recoveryActionForBudgetClass(recoveryBudgetClass),
|
|
146
226
|
zoneId: decision.zoneId,
|
|
147
227
|
});
|
|
148
228
|
return;
|
|
@@ -156,17 +236,104 @@ export function createGatewayServiceHealthMonitor(options) {
|
|
|
156
236
|
}
|
|
157
237
|
recoveryTracker.markRecoveryStarted({
|
|
158
238
|
observedAtMs: props.observedAtMs,
|
|
239
|
+
recoveryBudgetClass,
|
|
159
240
|
zoneId: decision.zoneId,
|
|
160
241
|
});
|
|
161
|
-
const recoveryResult = await
|
|
242
|
+
const recoveryResult = await runRecovery({
|
|
162
243
|
consecutiveFailures: decision.consecutiveFailures,
|
|
163
244
|
reason: decision.reason,
|
|
164
245
|
zoneId: decision.zoneId,
|
|
246
|
+
}, recoveryActionForBudgetClass(recoveryBudgetClass));
|
|
247
|
+
const observedAtMs = options.now();
|
|
248
|
+
recoveryTracker.markRecoveryFinished({
|
|
249
|
+
observedAtMs,
|
|
250
|
+
recoveryBudgetClass,
|
|
251
|
+
result: budgetResultForRecoveryResult(recoveryResult),
|
|
252
|
+
zoneId: decision.zoneId,
|
|
165
253
|
});
|
|
254
|
+
recordGatewayRecoveryEvent({
|
|
255
|
+
consecutiveFailures: decision.consecutiveFailures,
|
|
256
|
+
observedAtMs,
|
|
257
|
+
reason: decision.reason,
|
|
258
|
+
result: recoveryResult,
|
|
259
|
+
zoneId: decision.zoneId,
|
|
260
|
+
});
|
|
261
|
+
};
|
|
262
|
+
const maybeRecoverAgentChannelProvider = async (props) => {
|
|
263
|
+
const latestEvents = latestChannelProviderHealthEvents(props.zoneId);
|
|
264
|
+
if (latestEvents.length === 0) {
|
|
265
|
+
return;
|
|
266
|
+
}
|
|
267
|
+
let selectedDecision;
|
|
268
|
+
for (const latestEvent of latestEvents) {
|
|
269
|
+
const eventKey = channelProviderEventKey(latestEvent);
|
|
270
|
+
if (recoveredChannelProviderEventKeys.has(eventKey)) {
|
|
271
|
+
continue;
|
|
272
|
+
}
|
|
273
|
+
const recoveryObservation = deriveChannelProviderRecoveryObservation({
|
|
274
|
+
allowRestartWhenUnrecoverable: channelProviderPolicy.restartGatewayOnUnrecoverable,
|
|
275
|
+
event: latestEvent,
|
|
276
|
+
nowMs: props.observedAtMs,
|
|
277
|
+
restartGatewayOnRecoverable: channelProviderPolicy.restartGatewayOnRecoverable,
|
|
278
|
+
staleAfterMs: options.staleAfterMs,
|
|
279
|
+
transitioningTimeoutMs: channelProviderPolicy.transitioningTimeoutMs,
|
|
280
|
+
});
|
|
281
|
+
if (recoveryObservation.kind === 'observe-only') {
|
|
282
|
+
continue;
|
|
283
|
+
}
|
|
284
|
+
const recoveryBudgetClass = classifyRecoveryBudgetClass({
|
|
285
|
+
consecutiveFailures: 0,
|
|
286
|
+
reason: recoveryObservation.reason ?? 'agent-channel-provider-unhealthy',
|
|
287
|
+
zoneId: props.zoneId,
|
|
288
|
+
});
|
|
289
|
+
const decision = recoveryTracker.recordAgentChannelProviderObservation({
|
|
290
|
+
...recoveryObservation.observation,
|
|
291
|
+
recoveryBudgetClass,
|
|
292
|
+
});
|
|
293
|
+
if (decision.kind === 'restart' || decision.kind === 'suspended') {
|
|
294
|
+
selectedDecision = {
|
|
295
|
+
decision,
|
|
296
|
+
eventKey,
|
|
297
|
+
reason: recoveryObservation.reason ?? 'agent-channel-provider-unhealthy',
|
|
298
|
+
recoveryBudgetClass,
|
|
299
|
+
};
|
|
300
|
+
break;
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
if (!selectedDecision) {
|
|
304
|
+
return;
|
|
305
|
+
}
|
|
306
|
+
const { decision, reason } = selectedDecision;
|
|
307
|
+
if (decision.kind === 'suspended') {
|
|
308
|
+
recordGatewayRecoverySuspendedEvent({
|
|
309
|
+
consecutiveFailedRecoveries: decision.consecutiveFailedRecoveries,
|
|
310
|
+
consecutiveFailures: decision.consecutiveFailures,
|
|
311
|
+
observedAtMs: props.observedAtMs,
|
|
312
|
+
reason,
|
|
313
|
+
action: recoveryActionForBudgetClass(selectedDecision.recoveryBudgetClass),
|
|
314
|
+
zoneId: decision.zoneId,
|
|
315
|
+
});
|
|
316
|
+
return;
|
|
317
|
+
}
|
|
318
|
+
if (stopped) {
|
|
319
|
+
writeGatewayServiceHealthMonitorLog(`recovery requested for zone '${decision.zoneId}' but monitor is stopped`);
|
|
320
|
+
return;
|
|
321
|
+
}
|
|
322
|
+
recoveryTracker.markRecoveryStarted({
|
|
323
|
+
observedAtMs: props.observedAtMs,
|
|
324
|
+
recoveryBudgetClass: selectedDecision.recoveryBudgetClass,
|
|
325
|
+
zoneId: decision.zoneId,
|
|
326
|
+
});
|
|
327
|
+
const recoveryResult = await runRecovery({
|
|
328
|
+
consecutiveFailures: decision.consecutiveFailures,
|
|
329
|
+
reason: decision.reason,
|
|
330
|
+
zoneId: decision.zoneId,
|
|
331
|
+
}, recoveryActionForBudgetClass(selectedDecision.recoveryBudgetClass));
|
|
166
332
|
const observedAtMs = options.now();
|
|
167
333
|
recoveryTracker.markRecoveryFinished({
|
|
168
334
|
observedAtMs,
|
|
169
|
-
|
|
335
|
+
recoveryBudgetClass: selectedDecision.recoveryBudgetClass,
|
|
336
|
+
result: budgetResultForRecoveryResult(recoveryResult),
|
|
170
337
|
zoneId: decision.zoneId,
|
|
171
338
|
});
|
|
172
339
|
recordGatewayRecoveryEvent({
|
|
@@ -176,6 +343,9 @@ export function createGatewayServiceHealthMonitor(options) {
|
|
|
176
343
|
result: recoveryResult,
|
|
177
344
|
zoneId: decision.zoneId,
|
|
178
345
|
});
|
|
346
|
+
if (recoveryResult.result === 'ok') {
|
|
347
|
+
recoveredChannelProviderEventKeys.add(selectedDecision.eventKey);
|
|
348
|
+
}
|
|
179
349
|
};
|
|
180
350
|
const tick = async () => {
|
|
181
351
|
if (runningTick) {
|
|
@@ -209,6 +379,10 @@ export function createGatewayServiceHealthMonitor(options) {
|
|
|
209
379
|
serviceProbeResult,
|
|
210
380
|
zoneId: result.zoneId,
|
|
211
381
|
});
|
|
382
|
+
await maybeRecoverAgentChannelProvider({
|
|
383
|
+
observedAtMs,
|
|
384
|
+
zoneId: result.zoneId,
|
|
385
|
+
});
|
|
212
386
|
}
|
|
213
387
|
}
|
|
214
388
|
catch (error) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gateway-service-health-monitor.js","sourceRoot":"","sources":["../../../src/controller/health/gateway-service-health-monitor.ts"],"names":[],"mappings":"AAEA,OAAO,EACN,8BAA8B,GAI9B,MAAM,iCAAiC,CAAC;AAgEzC,SAAS,mCAAmC,CAAC,OAAe;IAC3D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,oCAAoC,OAAO,IAAI,CAAC,CAAC;AACvE,CAAC;AAED,MAAM,iCAAiC,GAAG;IACzC,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,CAAC;CACE,CAAC;AAEX,MAAM,UAAU,iCAAiC,CAChD,OAAiD;IAEjD,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,WAAW,CAAC;IAC/D,MAAM,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,IAAI,aAAa,CAAC;IACrE,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,UAAU,CAAC;IAC5D,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,YAAY,CAAC;IAClE,MAAM,cAAc,GAAG,OAAO,CAAC,yBAAyB,CAAC;IACzD,MAAM,eAAe,GAAG,8BAA8B,CAAC,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC,CAAC;IACnF,IAAI,KAAiC,CAAC;IACtC,IAAI,WAAsC,CAAC;IAC3C,IAAI,OAAO,GAAG,KAAK,CAAC;IAEpB,MAAM,qCAAqC,GAAG,CAAC,KAG9C,EAAsC,EAAE;QACxC,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB;aAC/C,uBAAuB,CAAC,KAAK,CAAC,MAAM,CAAC;aACrC,IAAI,CAAC,CAAC,KAAK,EAA2E,EAAE;YACxF,OAAO,KAAK,CAAC,IAAI,KAAK,sBAAsB,CAAC;QAC9C,CAAC,CAAC,CAAC;QACJ,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACvB,OAAO,YAAY,CAAC;QACrB,CAAC;QACD,IAAI,KAAK,CAAC,KAAK,GAAG,gBAAgB,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;YACxE,OAAO,OAAO,CAAC;QAChB,CAAC;QACD,OAAO,gBAAgB,CAAC,MAAM,CAAC;IAChC,CAAC,CAAC;IAEF,MAAM,uBAAuB,GAAG,KAAK,EACpC,OAAiC,EACE,EAAE;QACrC,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC;YAC/B,OAAO;gBACN,SAAS,EAAE,CAAC;gBACZ,SAAS,EAAE,gCAAgC;gBAC3C,MAAM,EAAE,QAAQ;aAChB,CAAC;QACH,CAAC;QAED,IAAI,OAAmC,CAAC;QACxC,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QAClC,IAAI,CAAC;YACJ,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC;gBACzB,OAAO,CAAC,gBAAgB,CAAC,OAAO,CAAC;gBACjC,IAAI,OAAO,CAA0B,CAAC,OAAO,EAAE,EAAE;oBAChD,OAAO,GAAG,cAAc,CAAC,GAAG,EAAE;wBAC7B,OAAO,CAAC;4BACP,SAAS,EAAE,OAAO,CAAC,GAAG,EAAE,GAAG,WAAW;4BACtC,SAAS,EAAE,kBAAkB;4BAC7B,MAAM,EAAE,QAAQ;yBAChB,CAAC,CAAC;oBACJ,CAAC,EAAE,cAAc,CAAC,gBAAgB,CAAC,CAAC;oBACpC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;gBACnB,CAAC,CAAC;aACF,CAAC,CAAC;QACJ,CAAC;gBAAS,CAAC;YACV,IAAI,OAAO,EAAE,CAAC;gBACb,gBAAgB,CAAC,OAAO,CAAC,CAAC;YAC3B,CAAC;QACF,CAAC;IACF,CAAC,CAAC;IAEF,MAAM,0BAA0B,GAAG,CAAC,KAMnC,EAAQ,EAAE;QACV,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;YAClC,MAAM,KAAK,GAAG;gBACb,MAAM,EAAE,oBAAoB;gBAC5B,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;gBAC9C,UAAU,EAAE,cAAc,CAAC,UAAU;gBACrC,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;gBACjC,IAAI,EAAE,kBAAkB;gBACxB,wBAAwB,EAAE,KAAK,CAAC,MAAM,CAAC,wBAAwB;gBAC/D,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;gBACrC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU;gBACnC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO;gBAC7B,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;gBACrC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU;gBACnC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO;gBAC7B,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,MAAM,EAAE,IAAI;gBACZ,MAAM,EAAE,KAAK,CAAC,MAAM;aACS,CAAC;YAC/B,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACvC,OAAO;QACR,CAAC;QAED,MAAM,KAAK,GAAG;YACb,MAAM,EAAE,oBAAoB;YAC5B,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;YAC9C,UAAU,EAAE,cAAc,CAAC,UAAU;YACrC,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;YACjC,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;YACjC,IAAI,EAAE,kBAAkB;YACxB,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,wBAAwB,KAAK,SAAS;gBACtD,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,wBAAwB,EAAE,KAAK,CAAC,MAAM,CAAC,wBAAwB,EAAE,CAAC;YACvE,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;YACrC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU;YACnC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO;YAC7B,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,MAAM,EAAE,QAAQ;YAChB,MAAM,EAAE,KAAK,CAAC,MAAM;SACS,CAAC;QAC/B,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACxC,CAAC,CAAC;IAEF,MAAM,mCAAmC,GAAG,CAAC,KAM5C,EAAQ,EAAE;QACV,MAAM,KAAK,GAAG;YACb,MAAM,EAAE,oBAAoB;YAC5B,2BAA2B,EAAE,KAAK,CAAC,2BAA2B;YAC9D,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;YAC9C,UAAU,EAAE,cAAc,CAAC,UAAU;YACrC,SAAS,EAAE,uBAAuB;YAClC,qBAAqB,EAAE,cAAc,CAAC,qBAAqB;YAC3D,IAAI,EAAE,4BAA4B;YAClC,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,MAAM,EAAE,QAAQ;YAChB,MAAM,EAAE,KAAK,CAAC,MAAM;SACS,CAAC;QAC/B,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACxC,CAAC,CAAC;IAEF,MAAM,qBAAqB,GAAG,KAAK,EAAE,KAKpC,EAAiB,EAAE;QACnB,MAAM,QAAQ,GACb,KAAK,CAAC,MAAM,KAAK,2BAA2B;YAC3C,CAAC,CAAC,eAAe,CAAC,yBAAyB,CAAC;gBAC1C,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,MAAM,EAAE,KAAK,CAAC,kBAAkB;gBAChC,MAAM,EAAE,KAAK,CAAC,MAAM;aACpB,CAAC;YACH,CAAC,CAAC,eAAe,CAAC,mCAAmC,CAAC;gBACpD,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,MAAM,EACL,KAAK,CAAC,kBAAkB,KAAK,IAAI;oBAChC,CAAC,CAAC,qCAAqC,CAAC;wBACtC,KAAK,EAAE,KAAK,CAAC,YAAY;wBACzB,MAAM,EAAE,KAAK,CAAC,MAAM;qBACpB,CAAC;oBACH,CAAC,CAAC,YAAY;gBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;aACpB,CAAC,CAAC;QAEN,IAAI,QAAQ,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YACnC,mCAAmC,CAAC;gBACnC,2BAA2B,EAAE,QAAQ,CAAC,2BAA2B;gBACjE,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;gBACjD,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,MAAM,EAAE,QAAQ,CAAC,MAAM;aACvB,CAAC,CAAC;YACH,OAAO;QACR,CAAC;QACD,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YACjC,OAAO;QACR,CAAC;QACD,IAAI,OAAO,EAAE,CAAC;YACb,mCAAmC,CAClC,gCAAgC,QAAQ,CAAC,MAAM,0BAA0B,CACzE,CAAC;YACF,OAAO;QACR,CAAC;QAED,eAAe,CAAC,mBAAmB,CAAC;YACnC,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,CAAC,CAAC;QACH,MAAM,cAAc,GAAG,MAAM,uBAAuB,CAAC;YACpD,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;YACjD,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,CAAC,CAAC;QACH,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QACnC,eAAe,CAAC,oBAAoB,CAAC;YACpC,YAAY;YACZ,MAAM,EAAE,cAAc,CAAC,MAAM;YAC7B,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,CAAC,CAAC;QACH,0BAA0B,CAAC;YAC1B,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;YACjD,YAAY;YACZ,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,cAAc;YACtB,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,CAAC,CAAC;IACJ,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,KAAK,IAAmB,EAAE;QACtC,IAAI,WAAW,EAAE,CAAC;YACjB,OAAO,MAAM,WAAW,CAAC;QAC1B,CAAC;QACD,WAAW,GAAG,CAAC,KAAK,IAAI,EAAE;YACzB,MAAM,OAAO,CAAC,GAAG,CAChB,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;gBACpC,IAAI,CAAC;oBACJ,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;oBACrD,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;oBACnC,MAAM,kBAAkB,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC;oBACvD,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC;wBAC/B,IAAI,EAAE,wBAAwB;wBAC9B,YAAY;wBACZ,IAAI,EAAE,MAAM,CAAC,IAAI;wBACjB,IAAI,EAAE,MAAM,CAAC,IAAI;wBACjB,MAAM,EAAE,kBAAkB;wBAC1B,GAAG,CAAC,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC;wBAC7E,MAAM,EAAE,MAAM,CAAC,MAAM;qBACrB,CAAC,CAAC;oBACH,MAAM,qBAAqB,CAAC;wBAC3B,YAAY;wBACZ,MAAM,EAAE,2BAA2B;wBACnC,kBAAkB;wBAClB,MAAM,EAAE,MAAM,CAAC,MAAM;qBACrB,CAAC,CAAC;oBACH,IAAI,kBAAkB,KAAK,IAAI,EAAE,CAAC;wBACjC,MAAM,qBAAqB,CAAC;4BAC3B,YAAY;4BACZ,MAAM,EAAE,gCAAgC;4BACxC,kBAAkB;4BAClB,MAAM,EAAE,MAAM,CAAC,MAAM;yBACrB,CAAC,CAAC;oBACJ,CAAC;gBACF,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,mCAAmC,CAClC,0BAA0B,MAAM,MAAM,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC9F,CAAC;oBACF,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;oBACnC,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC;wBAC/B,IAAI,EAAE,wBAAwB;wBAC9B,YAAY;wBACZ,IAAI,EAAE,iCAAiC,CAAC,IAAI;wBAC5C,IAAI,EAAE,iCAAiC,CAAC,IAAI;wBAC5C,MAAM,EAAE,QAAQ;wBAChB,MAAM;qBACN,CAAC,CAAC;oBACH,MAAM,qBAAqB,CAAC;wBAC3B,YAAY;wBACZ,MAAM,EAAE,2BAA2B;wBACnC,kBAAkB,EAAE,QAAQ;wBAC5B,MAAM;qBACN,CAAC,CAAC;gBACJ,CAAC;YACF,CAAC,CAAC,CACF,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;YACjB,WAAW,GAAG,SAAS,CAAC;QACzB,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,WAAW,CAAC;IAC1B,CAAC,CAAC;IAEF,OAAO;QACN,KAAK,EAAE,GAAG,EAAE;YACX,IAAI,KAAK,EAAE,CAAC;gBACX,OAAO;YACR,CAAC;YACD,OAAO,GAAG,KAAK,CAAC;YAChB,KAAK,GAAG,eAAe,CAAC,KAAK,IAAI,EAAE;gBAClC,IAAI,CAAC;oBACJ,MAAM,IAAI,EAAE,CAAC;gBACd,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,mCAAmC,CAClC,0BAA0B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAClF,CAAC;gBACH,CAAC;YACF,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;YACvB,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;QACjB,CAAC;QACD,IAAI,EAAE,KAAK,IAAI,EAAE;YAChB,OAAO,GAAG,IAAI,CAAC;YACf,IAAI,KAAK,EAAE,CAAC;gBACX,iBAAiB,CAAC,KAAK,CAAC,CAAC;gBACzB,KAAK,GAAG,SAAS,CAAC;YACnB,CAAC;YACD,IAAI,WAAW,EAAE,CAAC;gBACjB,MAAM,WAAW,CAAC;YACnB,CAAC;QACF,CAAC;QACD,IAAI;KACJ,CAAC;AACH,CAAC"}
|
|
1
|
+
{"version":3,"file":"gateway-service-health-monitor.js","sourceRoot":"","sources":["../../../src/controller/health/gateway-service-health-monitor.ts"],"names":[],"mappings":"AAMA,OAAO,EACN,wCAAwC,GAExC,MAAM,4CAA4C,CAAC;AACpD,OAAO,EACN,8BAA8B,EAC9B,6CAA6C,GAM7C,MAAM,iCAAiC,CAAC;AA2GzC,SAAS,mCAAmC,CAAC,OAAe;IAC3D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,oCAAoC,OAAO,IAAI,CAAC,CAAC;AACvE,CAAC;AAED,MAAM,iCAAiC,GAAG;IACzC,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,CAAC;CACE,CAAC;AAEX,SAAS,uBAAuB,CAAC,KAAsC;IACtE,OAAO,GAAG,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,iBAAiB,KAAK,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC;AACrF,CAAC;AAED,SAAS,4BAA4B,CACpC,mBAAiD;IAEjD,OAAO,mBAAmB,KAAK,uBAAuB;QACrD,CAAC,CAAC,uBAAuB;QACzB,CAAC,CAAC,oBAAoB,CAAC;AACzB,CAAC;AAED,SAAS,6BAA6B,CACrC,MAA+B;IAE/B,OAAO,MAAM,CAAC,MAAM,KAAK,cAAc,IAAI,MAAM,CAAC,MAAM,KAAK,mBAAmB;QAC/E,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;AAClB,CAAC;AAED,SAAS,sCAAsC,CAC9C,KAAa;IAEb,OAAO,KAAK,KAAK,gCAAgC,IAAI,KAAK,KAAK,kBAAkB,CAAC;AACnF,CAAC;AAED,MAAM,UAAU,iCAAiC,CAChD,OAAiD;IAEjD,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,WAAW,CAAC;IAC/D,MAAM,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,IAAI,aAAa,CAAC;IACrE,MAAM,cAAc,GAAG,OAAO,CAAC,yBAAyB,CAAC;IACzD,MAAM,qBAAqB,GAC1B,cAAc,CAAC,qBAAqB,IAAI,6CAA6C,CAAC;IACvF,MAAM,eAAe,GAAG,8BAA8B,CAAC,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC,CAAC;IACnF,IAAI,KAAiC,CAAC;IACtC,IAAI,WAAsC,CAAC;IAC3C,MAAM,iCAAiC,GAAG,IAAI,GAAG,EAAU,CAAC;IAC5D,IAAI,OAAO,GAAG,KAAK,CAAC;IAEpB,MAAM,2BAA2B,GAAG,CACnC,OAA2C,EACZ,EAAE;QACjC,IAAI,CAAC;YACJ,OAAO,OAAO,CAAC,2BAA2B,EAAE,CAAC,OAAO,CAAC,IAAI,oBAAoB,CAAC;QAC/E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,mCAAmC,CAClC,mDAAmD,OAAO,CAAC,MAAM,MAChE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CACtD,EAAE,CACF,CAAC;YACF,OAAO,oBAAoB,CAAC;QAC7B,CAAC;IACF,CAAC,CAAC;IAEF,MAAM,qCAAqC,GAAG,CAAC,KAG9C,EAAsC,EAAE;QACxC,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB;aAC/C,uBAAuB,CAAC,KAAK,CAAC,MAAM,CAAC;aACrC,IAAI,CAAC,CAAC,KAAK,EAA2E,EAAE;YACxF,OAAO,KAAK,CAAC,IAAI,KAAK,sBAAsB,CAAC;QAC9C,CAAC,CAAC,CAAC;QACJ,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACvB,OAAO,YAAY,CAAC;QACrB,CAAC;QACD,IAAI,KAAK,CAAC,KAAK,GAAG,gBAAgB,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;YACxE,OAAO,OAAO,CAAC;QAChB,CAAC;QACD,OAAO,gBAAgB,CAAC,MAAM,CAAC;IAChC,CAAC,CAAC;IAEF,MAAM,iCAAiC,GAAG,CACzC,MAAc,EAC+B,EAAE,CAC/C,OAAO,CAAC,gBAAgB;SACtB,uBAAuB,CAAC,MAAM,CAAC;SAC/B,MAAM,CACN,CAAC,KAAK,EAA4C,EAAE,CACnD,KAAK,CAAC,IAAI,KAAK,+BAA+B,CAC/C,CAAC;IAEJ,MAAM,WAAW,GAAG,KAAK,EACxB,OAAiC,EACjC,cAGC,EACkC,EAAE;QACrC,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC;YAC/B,OAAO,cAAc,KAAK,oBAAoB;gBAC7C,CAAC,CAAC;oBACA,MAAM,EAAE,oBAAoB;oBAC5B,SAAS,EAAE,CAAC;oBACZ,SAAS,EAAE,gCAAgC;oBAC3C,MAAM,EAAE,QAAQ;iBAChB;gBACF,CAAC,CAAC;oBACA,MAAM,EAAE,uBAAuB;oBAC/B,SAAS,EAAE,CAAC;oBACZ,SAAS,EAAE,gCAAgC;oBAC3C,MAAM,EAAE,QAAQ;iBAChB,CAAC;QACL,CAAC;QACD,OAAO,MAAM,OAAO,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAChD,CAAC,CAAC;IAEF,MAAM,0BAA0B,GAAG,CAAC,KAMnC,EAAQ,EAAE;QACV,MAAM,cAAc,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,IAAI,oBAAoB,CAAC;QACnE,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;YAClC,MAAM,KAAK,GACV,KAAK,CAAC,MAAM,CAAC,MAAM,KAAK,uBAAuB;gBAC9C,CAAC,CAAE;oBACD,MAAM,EAAE,oBAAoB;oBAC5B,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;oBAC9C,UAAU,EAAE,cAAc,CAAC,UAAU;oBACrC,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;oBACjC,IAAI,EAAE,kBAAkB;oBACxB,wBAAwB,EAAE,KAAK,CAAC,MAAM,CAAC,wBAAwB;oBAC/D,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;oBACrC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU;oBACnC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO;oBAC7B,YAAY,EAAE,KAAK,CAAC,YAAY;oBAChC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;oBACrC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU;oBACnC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO;oBAC7B,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,KAAK,SAAS;wBACzC,CAAC,CAAC,EAAE;wBACJ,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;oBAC7C,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,MAAM,EAAE,IAAI;oBACZ,MAAM,EAAE,KAAK,CAAC,MAAM;iBACU;gBAChC,CAAC,CAAE;oBACD,MAAM,EAAE,uBAAuB;oBAC/B,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;oBAC9C,UAAU,EAAE,cAAc,CAAC,UAAU;oBACrC,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;oBACjC,IAAI,EAAE,kBAAkB;oBACxB,wBAAwB,EAAE,KAAK,CAAC,MAAM,CAAC,wBAAwB;oBAC/D,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;oBACrC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU;oBACnC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO;oBAC7B,YAAY,EAAE,KAAK,CAAC,YAAY;oBAChC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,KAAK,SAAS;wBACzC,CAAC,CAAC,EAAE;wBACJ,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;oBAC7C,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,MAAM,EAAE,IAAI;oBACZ,MAAM,EAAE,KAAK,CAAC,MAAM;iBACU,CAAC;YACnC,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACvC,OAAO;QACR,CAAC;QAED,MAAM,KAAK,GAAG;YACb,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;YAC9C,UAAU,EAAE,cAAc,CAAC,UAAU;YACrC,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;YACjC,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;YACjC,IAAI,EAAE,kBAAkB;YACxB,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,wBAAwB,KAAK,SAAS;gBACtD,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,wBAAwB,EAAE,KAAK,CAAC,MAAM,CAAC,wBAAwB,EAAE,CAAC;YACvE,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAC5F,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,MAAM,EAAE,QAAQ;YAChB,MAAM,EAAE,KAAK,CAAC,MAAM;SACX,CAAC;QACX,IAAI,cAAc,KAAK,oBAAoB,EAAE,CAAC;YAC7C,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;gBACxC,IAAI,CAAC,sCAAsC,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;oBACrE,MAAM,qBAAqB,GAAG;wBAC7B,GAAG,KAAK;wBACR,MAAM,EAAE,mBAAmB;wBAC3B,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;qBACJ,CAAC;oBAC/B,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;oBACvD,OAAO;gBACR,CAAC;gBACD,MAAM,YAAY,GAAG;oBACpB,GAAG,KAAK;oBACR,MAAM,EAAE,oBAAoB;oBAC5B,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;oBACjC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;oBACrC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU;oBACnC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO;iBACA,CAAC;gBAC/B,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;gBAC9C,OAAO;YACR,CAAC;YACD,MAAM,YAAY,GAAG;gBACpB,GAAG,KAAK;gBACR,MAAM,EAAE,oBAAoB;gBAC5B,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;gBACrC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU;gBACnC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO;aACA,CAAC;YAC/B,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YAC9C,OAAO;QACR,CAAC;QACD,MAAM,eAAe,GAAG;YACvB,GAAG,KAAK;YACR,MAAM,EAAE,cAAc;SACO,CAAC;QAC/B,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAClD,CAAC,CAAC;IAEF,MAAM,mCAAmC,GAAG,CAAC,KAO5C,EAAQ,EAAE;QACV,MAAM,KAAK,GAAG;YACb,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,2BAA2B,EAAE,KAAK,CAAC,2BAA2B;YAC9D,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;YAC9C,UAAU,EAAE,cAAc,CAAC,UAAU;YACrC,SAAS,EAAE,uBAAuB;YAClC,qBAAqB,EAAE,cAAc,CAAC,qBAAqB;YAC3D,IAAI,EAAE,4BAA4B;YAClC,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,MAAM,EAAE,QAAQ;YAChB,MAAM,EAAE,KAAK,CAAC,MAAM;SACS,CAAC;QAC/B,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACxC,CAAC,CAAC;IAEF,MAAM,qBAAqB,GAAG,KAAK,EAAE,KAKpC,EAAiB,EAAE;QACnB,MAAM,mBAAmB,GAAG,2BAA2B,CAAC;YACvD,mBAAmB,EAAE,CAAC;YACtB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,MAAM,EAAE,KAAK,CAAC,MAAM;SACpB,CAAC,CAAC;QACH,MAAM,QAAQ,GACb,KAAK,CAAC,MAAM,KAAK,2BAA2B;YAC3C,CAAC,CAAC,eAAe,CAAC,yBAAyB,CAAC;gBAC1C,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,mBAAmB;gBACnB,MAAM,EAAE,KAAK,CAAC,kBAAkB;gBAChC,MAAM,EAAE,KAAK,CAAC,MAAM;aACpB,CAAC;YACH,CAAC,CAAC,eAAe,CAAC,mCAAmC,CAAC;gBACpD,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,mBAAmB;gBACnB,MAAM,EACL,KAAK,CAAC,kBAAkB,KAAK,IAAI;oBAChC,CAAC,CAAC,qCAAqC,CAAC;wBACtC,KAAK,EAAE,KAAK,CAAC,YAAY;wBACzB,MAAM,EAAE,KAAK,CAAC,MAAM;qBACpB,CAAC;oBACH,CAAC,CAAC,YAAY;gBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;aACpB,CAAC,CAAC;QAEN,IAAI,QAAQ,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YACnC,mCAAmC,CAAC;gBACnC,2BAA2B,EAAE,QAAQ,CAAC,2BAA2B;gBACjE,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;gBACjD,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,MAAM,EAAE,4BAA4B,CAAC,mBAAmB,CAAC;gBACzD,MAAM,EAAE,QAAQ,CAAC,MAAM;aACvB,CAAC,CAAC;YACH,OAAO;QACR,CAAC;QACD,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YACjC,OAAO;QACR,CAAC;QACD,IAAI,OAAO,EAAE,CAAC;YACb,mCAAmC,CAClC,gCAAgC,QAAQ,CAAC,MAAM,0BAA0B,CACzE,CAAC;YACF,OAAO;QACR,CAAC;QAED,eAAe,CAAC,mBAAmB,CAAC;YACnC,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,mBAAmB;YACnB,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,CAAC,CAAC;QACH,MAAM,cAAc,GAAG,MAAM,WAAW,CACvC;YACC,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;YACjD,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,EACD,4BAA4B,CAAC,mBAAmB,CAAC,CACjD,CAAC;QACF,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QACnC,eAAe,CAAC,oBAAoB,CAAC;YACpC,YAAY;YACZ,mBAAmB;YACnB,MAAM,EAAE,6BAA6B,CAAC,cAAc,CAAC;YACrD,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,CAAC,CAAC;QACH,0BAA0B,CAAC;YAC1B,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;YACjD,YAAY;YACZ,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,cAAc;YACtB,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,CAAC,CAAC;IACJ,CAAC,CAAC;IAEF,MAAM,gCAAgC,GAAG,KAAK,EAAE,KAG/C,EAAiB,EAAE;QACnB,MAAM,YAAY,GAAG,iCAAiC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACrE,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO;QACR,CAAC;QACD,IAAI,gBAUQ,CAAC;QACb,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;YACxC,MAAM,QAAQ,GAAG,uBAAuB,CAAC,WAAW,CAAC,CAAC;YACtD,IAAI,iCAAiC,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACrD,SAAS;YACV,CAAC;YACD,MAAM,mBAAmB,GAAG,wCAAwC,CAAC;gBACpE,6BAA6B,EAAE,qBAAqB,CAAC,6BAA6B;gBAClF,KAAK,EAAE,WAAW;gBAClB,KAAK,EAAE,KAAK,CAAC,YAAY;gBACzB,2BAA2B,EAAE,qBAAqB,CAAC,2BAA2B;gBAC9E,YAAY,EAAE,OAAO,CAAC,YAAY;gBAClC,sBAAsB,EAAE,qBAAqB,CAAC,sBAAsB;aACpE,CAAC,CAAC;YACH,IAAI,mBAAmB,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;gBACjD,SAAS;YACV,CAAC;YACD,MAAM,mBAAmB,GAAG,2BAA2B,CAAC;gBACvD,mBAAmB,EAAE,CAAC;gBACtB,MAAM,EAAE,mBAAmB,CAAC,MAAM,IAAI,kCAAkC;gBACxE,MAAM,EAAE,KAAK,CAAC,MAAM;aACpB,CAAC,CAAC;YACH,MAAM,QAAQ,GAAG,eAAe,CAAC,qCAAqC,CAAC;gBACtE,GAAG,mBAAmB,CAAC,WAAW;gBAClC,mBAAmB;aACnB,CAAC,CAAC;YACH,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS,IAAI,QAAQ,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBAClE,gBAAgB,GAAG;oBAClB,QAAQ;oBACR,QAAQ;oBACR,MAAM,EAAE,mBAAmB,CAAC,MAAM,IAAI,kCAAkC;oBACxE,mBAAmB;iBACnB,CAAC;gBACF,MAAM;YACP,CAAC;QACF,CAAC;QACD,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACvB,OAAO;QACR,CAAC;QACD,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,gBAAgB,CAAC;QAC9C,IAAI,QAAQ,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YACnC,mCAAmC,CAAC;gBACnC,2BAA2B,EAAE,QAAQ,CAAC,2BAA2B;gBACjE,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;gBACjD,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,MAAM;gBACN,MAAM,EAAE,4BAA4B,CAAC,gBAAgB,CAAC,mBAAmB,CAAC;gBAC1E,MAAM,EAAE,QAAQ,CAAC,MAAM;aACvB,CAAC,CAAC;YACH,OAAO;QACR,CAAC;QACD,IAAI,OAAO,EAAE,CAAC;YACb,mCAAmC,CAClC,gCAAgC,QAAQ,CAAC,MAAM,0BAA0B,CACzE,CAAC;YACF,OAAO;QACR,CAAC;QAED,eAAe,CAAC,mBAAmB,CAAC;YACnC,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,mBAAmB,EAAE,gBAAgB,CAAC,mBAAmB;YACzD,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,CAAC,CAAC;QACH,MAAM,cAAc,GAAG,MAAM,WAAW,CACvC;YACC,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;YACjD,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,EACD,4BAA4B,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,CAClE,CAAC;QACF,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QACnC,eAAe,CAAC,oBAAoB,CAAC;YACpC,YAAY;YACZ,mBAAmB,EAAE,gBAAgB,CAAC,mBAAmB;YACzD,MAAM,EAAE,6BAA6B,CAAC,cAAc,CAAC;YACrD,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,CAAC,CAAC;QACH,0BAA0B,CAAC;YAC1B,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;YACjD,YAAY;YACZ,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,cAAc;YACtB,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,CAAC,CAAC;QACH,IAAI,cAAc,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;YACpC,iCAAiC,CAAC,GAAG,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAClE,CAAC;IACF,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,KAAK,IAAmB,EAAE;QACtC,IAAI,WAAW,EAAE,CAAC;YACjB,OAAO,MAAM,WAAW,CAAC;QAC1B,CAAC;QACD,WAAW,GAAG,CAAC,KAAK,IAAI,EAAE;YACzB,MAAM,OAAO,CAAC,GAAG,CAChB,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;gBACpC,IAAI,CAAC;oBACJ,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;oBACrD,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;oBACnC,MAAM,kBAAkB,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC;oBACvD,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC;wBAC/B,IAAI,EAAE,wBAAwB;wBAC9B,YAAY;wBACZ,IAAI,EAAE,MAAM,CAAC,IAAI;wBACjB,IAAI,EAAE,MAAM,CAAC,IAAI;wBACjB,MAAM,EAAE,kBAAkB;wBAC1B,GAAG,CAAC,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC;wBAC7E,MAAM,EAAE,MAAM,CAAC,MAAM;qBACrB,CAAC,CAAC;oBACH,MAAM,qBAAqB,CAAC;wBAC3B,YAAY;wBACZ,MAAM,EAAE,2BAA2B;wBACnC,kBAAkB;wBAClB,MAAM,EAAE,MAAM,CAAC,MAAM;qBACrB,CAAC,CAAC;oBACH,IAAI,kBAAkB,KAAK,IAAI,EAAE,CAAC;wBACjC,MAAM,qBAAqB,CAAC;4BAC3B,YAAY;4BACZ,MAAM,EAAE,gCAAgC;4BACxC,kBAAkB;4BAClB,MAAM,EAAE,MAAM,CAAC,MAAM;yBACrB,CAAC,CAAC;wBACH,MAAM,gCAAgC,CAAC;4BACtC,YAAY;4BACZ,MAAM,EAAE,MAAM,CAAC,MAAM;yBACrB,CAAC,CAAC;oBACJ,CAAC;gBACF,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,mCAAmC,CAClC,0BAA0B,MAAM,MAAM,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC9F,CAAC;oBACF,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;oBACnC,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC;wBAC/B,IAAI,EAAE,wBAAwB;wBAC9B,YAAY;wBACZ,IAAI,EAAE,iCAAiC,CAAC,IAAI;wBAC5C,IAAI,EAAE,iCAAiC,CAAC,IAAI;wBAC5C,MAAM,EAAE,QAAQ;wBAChB,MAAM;qBACN,CAAC,CAAC;oBACH,MAAM,qBAAqB,CAAC;wBAC3B,YAAY;wBACZ,MAAM,EAAE,2BAA2B;wBACnC,kBAAkB,EAAE,QAAQ;wBAC5B,MAAM;qBACN,CAAC,CAAC;gBACJ,CAAC;YACF,CAAC,CAAC,CACF,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;YACjB,WAAW,GAAG,SAAS,CAAC;QACzB,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,WAAW,CAAC;IAC1B,CAAC,CAAC;IAEF,OAAO;QACN,KAAK,EAAE,GAAG,EAAE;YACX,IAAI,KAAK,EAAE,CAAC;gBACX,OAAO;YACR,CAAC;YACD,OAAO,GAAG,KAAK,CAAC;YAChB,KAAK,GAAG,eAAe,CAAC,KAAK,IAAI,EAAE;gBAClC,IAAI,CAAC;oBACJ,MAAM,IAAI,EAAE,CAAC;gBACd,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,mCAAmC,CAClC,0BAA0B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAClF,CAAC;gBACH,CAAC;YACF,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;YACvB,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;QACjB,CAAC;QACD,IAAI,EAAE,KAAK,IAAI,EAAE;YAChB,OAAO,GAAG,IAAI,CAAC;YACf,IAAI,KAAK,EAAE,CAAC;gBACX,iBAAiB,CAAC,KAAK,CAAC,CAAC;gBACzB,KAAK,GAAG,SAAS,CAAC;YACnB,CAAC;YACD,IAAI,WAAW,EAAE,CAAC;gBACjB,MAAM,WAAW,CAAC;YACnB,CAAC;QACF,CAAC;QACD,IAAI;KACJ,CAAC;AACH,CAAC"}
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import type { GatewayRecoveryHealthReason } from '@agent-vm/gateway-interface';
|
|
2
2
|
export type GatewayVmRecoveryObservationResult = 'failed' | 'ok' | 'stale' | 'timeout' | 'unobserved';
|
|
3
3
|
export type GatewayVmRecoveryReason = GatewayRecoveryHealthReason;
|
|
4
|
+
export type GatewayVmRecoveryBudgetClass = 'gateway-vm-cold-start' | 'gateway-vm-restart';
|
|
4
5
|
export interface GatewayVmAutoRecoveryPolicy {
|
|
6
|
+
readonly channelProviderHealth?: GatewayVmChannelProviderRecoveryPolicy | undefined;
|
|
5
7
|
readonly consecutiveFailureThreshold: number;
|
|
6
8
|
readonly cooldownMs: number;
|
|
7
9
|
readonly enabled: boolean;
|
|
@@ -9,16 +11,33 @@ export interface GatewayVmAutoRecoveryPolicy {
|
|
|
9
11
|
readonly maxConsecutiveFailedRecoveries: number;
|
|
10
12
|
readonly restartTimeoutMs: number;
|
|
11
13
|
}
|
|
14
|
+
export interface GatewayVmChannelProviderRecoveryPolicy {
|
|
15
|
+
readonly consecutiveFailureThreshold: number;
|
|
16
|
+
readonly enabled: boolean;
|
|
17
|
+
readonly restartGatewayOnRecoverable: boolean;
|
|
18
|
+
readonly restartGatewayOnUnrecoverable: boolean;
|
|
19
|
+
readonly transitioningTimeoutMs: number;
|
|
20
|
+
}
|
|
21
|
+
export declare const defaultGatewayVmChannelProviderRecoveryPolicy: {
|
|
22
|
+
consecutiveFailureThreshold: number;
|
|
23
|
+
enabled: true;
|
|
24
|
+
restartGatewayOnRecoverable: true;
|
|
25
|
+
restartGatewayOnUnrecoverable: false;
|
|
26
|
+
transitioningTimeoutMs: number;
|
|
27
|
+
};
|
|
12
28
|
export interface CreateGatewayVmRecoveryTrackerOptions {
|
|
13
29
|
readonly policy: GatewayVmAutoRecoveryPolicy;
|
|
14
30
|
}
|
|
15
31
|
export interface GatewayVmRecoveryObservation {
|
|
32
|
+
readonly channelProviderId?: string | undefined;
|
|
16
33
|
readonly observedAtMs: number;
|
|
34
|
+
readonly recoveryBudgetClass?: GatewayVmRecoveryBudgetClass | undefined;
|
|
17
35
|
readonly result: GatewayVmRecoveryObservationResult;
|
|
18
36
|
readonly zoneId: string;
|
|
19
37
|
}
|
|
20
38
|
export interface GatewayVmRecoveryLifecycleEvent {
|
|
21
39
|
readonly observedAtMs: number;
|
|
40
|
+
readonly recoveryBudgetClass?: GatewayVmRecoveryBudgetClass | undefined;
|
|
22
41
|
readonly result?: 'failed' | 'ok' | undefined;
|
|
23
42
|
readonly zoneId: string;
|
|
24
43
|
}
|
|
@@ -41,6 +60,7 @@ export type GatewayVmRecoveryDecision = {
|
|
|
41
60
|
export interface GatewayVmRecoveryTracker {
|
|
42
61
|
readonly markRecoveryFinished: (event: GatewayVmRecoveryLifecycleEvent) => void;
|
|
43
62
|
readonly markRecoveryStarted: (event: GatewayVmRecoveryLifecycleEvent) => void;
|
|
63
|
+
readonly recordAgentChannelProviderObservation: (observation: GatewayVmRecoveryObservation) => GatewayVmRecoveryDecision;
|
|
44
64
|
readonly recordGatewayControlLinkObservation: (observation: GatewayVmRecoveryObservation) => GatewayVmRecoveryDecision;
|
|
45
65
|
readonly recordGatewayServiceProbe: (observation: GatewayVmRecoveryObservation) => GatewayVmRecoveryDecision;
|
|
46
66
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gateway-vm-recovery-policy.d.ts","sourceRoot":"","sources":["../../../src/controller/health/gateway-vm-recovery-policy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,6BAA6B,CAAC;AAE/E,MAAM,MAAM,kCAAkC,GAC3C,QAAQ,GACR,IAAI,GACJ,OAAO,GACP,SAAS,GACT,YAAY,CAAC;AAEhB,MAAM,MAAM,uBAAuB,GAAG,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"gateway-vm-recovery-policy.d.ts","sourceRoot":"","sources":["../../../src/controller/health/gateway-vm-recovery-policy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,6BAA6B,CAAC;AAE/E,MAAM,MAAM,kCAAkC,GAC3C,QAAQ,GACR,IAAI,GACJ,OAAO,GACP,SAAS,GACT,YAAY,CAAC;AAEhB,MAAM,MAAM,uBAAuB,GAAG,2BAA2B,CAAC;AAClE,MAAM,MAAM,4BAA4B,GAAG,uBAAuB,GAAG,oBAAoB,CAAC;AAE1F,MAAM,WAAW,2BAA2B;IAC3C,QAAQ,CAAC,qBAAqB,CAAC,EAAE,sCAAsC,GAAG,SAAS,CAAC;IACpF,QAAQ,CAAC,2BAA2B,EAAE,MAAM,CAAC;IAC7C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC,QAAQ,CAAC,8BAA8B,EAAE,MAAM,CAAC;IAChD,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;CAClC;AAED,MAAM,WAAW,sCAAsC;IACtD,QAAQ,CAAC,2BAA2B,EAAE,MAAM,CAAC;IAC7C,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,2BAA2B,EAAE,OAAO,CAAC;IAC9C,QAAQ,CAAC,6BAA6B,EAAE,OAAO,CAAC;IAChD,QAAQ,CAAC,sBAAsB,EAAE,MAAM,CAAC;CACxC;AAED,eAAO,MAAM,6CAA6C;;;;;;CAMR,CAAC;AAEnD,MAAM,WAAW,qCAAqC;IACrD,QAAQ,CAAC,MAAM,EAAE,2BAA2B,CAAC;CAC7C;AAED,MAAM,WAAW,4BAA4B;IAC5C,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,mBAAmB,CAAC,EAAE,4BAA4B,GAAG,SAAS,CAAC;IACxE,QAAQ,CAAC,MAAM,EAAE,kCAAkC,CAAC;IACpD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,+BAA+B;IAC/C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,mBAAmB,CAAC,EAAE,4BAA4B,GAAG,SAAS,CAAC;IACxE,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,GAAG,IAAI,GAAG,SAAS,CAAC;IAC9C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,yBAAyB,GAClC;IACA,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,CAAC,EAAE,UAAU,GAAG,UAAU,GAAG,WAAW,GAAG,YAAY,GAAG,SAAS,CAAC;CAClF,GACD;IACA,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,uBAAuB,CAAC;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACvB,GACD;IACA,QAAQ,CAAC,2BAA2B,EAAE,MAAM,CAAC;IAC7C,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,uBAAuB,CAAC;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACvB,CAAC;AAEL,MAAM,WAAW,wBAAwB;IACxC,QAAQ,CAAC,oBAAoB,EAAE,CAAC,KAAK,EAAE,+BAA+B,KAAK,IAAI,CAAC;IAChF,QAAQ,CAAC,mBAAmB,EAAE,CAAC,KAAK,EAAE,+BAA+B,KAAK,IAAI,CAAC;IAC/E,QAAQ,CAAC,qCAAqC,EAAE,CAC/C,WAAW,EAAE,4BAA4B,KACrC,yBAAyB,CAAC;IAC/B,QAAQ,CAAC,mCAAmC,EAAE,CAC7C,WAAW,EAAE,4BAA4B,KACrC,yBAAyB,CAAC;IAC/B,QAAQ,CAAC,yBAAyB,EAAE,CACnC,WAAW,EAAE,4BAA4B,KACrC,yBAAyB,CAAC;CAC/B;AAgID,wBAAgB,8BAA8B,CAC7C,OAAO,EAAE,qCAAqC,GAC5C,wBAAwB,CAuH1B"}
|