@agent-vm/agent-vm 0.0.92 → 0.0.93

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/dist/build/managed-image-dockerfile.d.ts +2 -1
  2. package/dist/build/managed-image-dockerfile.d.ts.map +1 -1
  3. package/dist/build/managed-image-dockerfile.js +51 -27
  4. package/dist/build/managed-image-dockerfile.js.map +1 -1
  5. package/dist/cli/commands/controller-definition.d.ts +42 -42
  6. package/dist/cli/commands/create-app.d.ts +42 -42
  7. package/dist/cli/manual-templates.d.ts.map +1 -1
  8. package/dist/cli/manual-templates.js +11 -2
  9. package/dist/cli/manual-templates.js.map +1 -1
  10. package/dist/config/system-config.d.ts +7 -0
  11. package/dist/config/system-config.d.ts.map +1 -1
  12. package/dist/config/system-config.js +35 -0
  13. package/dist/config/system-config.js.map +1 -1
  14. package/dist/controller/controller-runtime-operations.d.ts +1 -0
  15. package/dist/controller/controller-runtime-operations.d.ts.map +1 -1
  16. package/dist/controller/controller-runtime-operations.js +2 -0
  17. package/dist/controller/controller-runtime-operations.js.map +1 -1
  18. package/dist/controller/controller-runtime-types.d.ts +3 -0
  19. package/dist/controller/controller-runtime-types.d.ts.map +1 -1
  20. package/dist/controller/controller-runtime.d.ts +1 -1
  21. package/dist/controller/controller-runtime.d.ts.map +1 -1
  22. package/dist/controller/controller-runtime.js +207 -116
  23. package/dist/controller/controller-runtime.js.map +1 -1
  24. package/dist/controller/health/channel-provider-recovery-observation.d.ts +23 -0
  25. package/dist/controller/health/channel-provider-recovery-observation.d.ts.map +1 -0
  26. package/dist/controller/health/channel-provider-recovery-observation.js +69 -0
  27. package/dist/controller/health/channel-provider-recovery-observation.js.map +1 -0
  28. package/dist/controller/health/durable-health-event-log.d.ts +24 -0
  29. package/dist/controller/health/durable-health-event-log.d.ts.map +1 -0
  30. package/dist/controller/health/durable-health-event-log.js +89 -0
  31. package/dist/controller/health/durable-health-event-log.js.map +1 -0
  32. package/dist/controller/health/gateway-recovery-actions.d.ts +27 -0
  33. package/dist/controller/health/gateway-recovery-actions.d.ts.map +1 -0
  34. package/dist/controller/health/gateway-recovery-actions.js +71 -0
  35. package/dist/controller/health/gateway-recovery-actions.js.map +1 -0
  36. package/dist/controller/health/gateway-service-health-monitor.d.ts +41 -3
  37. package/dist/controller/health/gateway-service-health-monitor.d.ts.map +1 -1
  38. package/dist/controller/health/gateway-service-health-monitor.js +231 -57
  39. package/dist/controller/health/gateway-service-health-monitor.js.map +1 -1
  40. package/dist/controller/health/gateway-vm-recovery-policy.d.ts +20 -0
  41. package/dist/controller/health/gateway-vm-recovery-policy.d.ts.map +1 -1
  42. package/dist/controller/health/gateway-vm-recovery-policy.js +85 -21
  43. package/dist/controller/health/gateway-vm-recovery-policy.js.map +1 -1
  44. package/dist/controller/health/gateway-vm-recovery-runner.d.ts +39 -0
  45. package/dist/controller/health/gateway-vm-recovery-runner.d.ts.map +1 -0
  46. package/dist/controller/health/gateway-vm-recovery-runner.js +251 -0
  47. package/dist/controller/health/gateway-vm-recovery-runner.js.map +1 -0
  48. package/dist/controller/health/health-event-store.d.ts +4 -0
  49. package/dist/controller/health/health-event-store.d.ts.map +1 -1
  50. package/dist/controller/health/health-event-store.js +19 -0
  51. package/dist/controller/health/health-event-store.js.map +1 -1
  52. package/dist/controller/http/controller-health-event-routes.d.ts +6 -0
  53. package/dist/controller/http/controller-health-event-routes.d.ts.map +1 -1
  54. package/dist/controller/http/controller-health-event-routes.js +49 -0
  55. package/dist/controller/http/controller-health-event-routes.js.map +1 -1
  56. package/dist/controller/http/controller-http-routes.d.ts.map +1 -1
  57. package/dist/controller/http/controller-http-routes.js +6 -0
  58. package/dist/controller/http/controller-http-routes.js.map +1 -1
  59. package/dist/controller/leases/lease-manager.d.ts.map +1 -1
  60. package/dist/controller/leases/lease-manager.js +37 -16
  61. package/dist/controller/leases/lease-manager.js.map +1 -1
  62. package/dist/controller/leases/tool-vm-lease-lifecycle.d.ts +44 -0
  63. package/dist/controller/leases/tool-vm-lease-lifecycle.d.ts.map +1 -0
  64. package/dist/controller/leases/tool-vm-lease-lifecycle.js +28 -0
  65. package/dist/controller/leases/tool-vm-lease-lifecycle.js.map +1 -0
  66. package/dist/controller/zone-runtimes/gateway-lifecycle-operation-record.d.ts +37 -0
  67. package/dist/controller/zone-runtimes/gateway-lifecycle-operation-record.d.ts.map +1 -0
  68. package/dist/controller/zone-runtimes/gateway-lifecycle-operation-record.js +133 -0
  69. package/dist/controller/zone-runtimes/gateway-lifecycle-operation-record.js.map +1 -0
  70. package/dist/controller/zone-runtimes/gateway-zone-state-machine.d.ts +101 -0
  71. package/dist/controller/zone-runtimes/gateway-zone-state-machine.d.ts.map +1 -0
  72. package/dist/controller/zone-runtimes/gateway-zone-state-machine.js +143 -0
  73. package/dist/controller/zone-runtimes/gateway-zone-state-machine.js.map +1 -0
  74. package/dist/controller/zone-runtimes/openclaw-zone-runtime.d.ts +8 -1
  75. package/dist/controller/zone-runtimes/openclaw-zone-runtime.d.ts.map +1 -1
  76. package/dist/controller/zone-runtimes/openclaw-zone-runtime.js +621 -65
  77. package/dist/controller/zone-runtimes/openclaw-zone-runtime.js.map +1 -1
  78. package/dist/controller/zone-runtimes/zone-runtime-errors.d.ts +7 -1
  79. package/dist/controller/zone-runtimes/zone-runtime-errors.d.ts.map +1 -1
  80. package/dist/controller/zone-runtimes/zone-runtime-errors.js +5 -1
  81. package/dist/controller/zone-runtimes/zone-runtime-errors.js.map +1 -1
  82. package/dist/controller/zone-runtimes/zone-runtime-registry.d.ts +2 -0
  83. package/dist/controller/zone-runtimes/zone-runtime-registry.d.ts.map +1 -1
  84. package/dist/controller/zone-runtimes/zone-runtime-registry.js +23 -0
  85. package/dist/controller/zone-runtimes/zone-runtime-registry.js.map +1 -1
  86. package/dist/controller/zone-runtimes/zone-runtime-types.d.ts +7 -0
  87. package/dist/controller/zone-runtimes/zone-runtime-types.d.ts.map +1 -1
  88. package/dist/gateway/gateway-ownership-evidence.d.ts +35 -0
  89. package/dist/gateway/gateway-ownership-evidence.d.ts.map +1 -0
  90. package/dist/gateway/gateway-ownership-evidence.js +10 -0
  91. package/dist/gateway/gateway-ownership-evidence.js.map +1 -0
  92. package/dist/gateway/gateway-recovery.d.ts +16 -0
  93. package/dist/gateway/gateway-recovery.d.ts.map +1 -1
  94. package/dist/gateway/gateway-recovery.js +105 -9
  95. package/dist/gateway/gateway-recovery.js.map +1 -1
  96. package/dist/gateway/gateway-zone-orchestrator.d.ts.map +1 -1
  97. package/dist/gateway/gateway-zone-orchestrator.js +50 -39
  98. package/dist/gateway/gateway-zone-orchestrator.js.map +1 -1
  99. package/dist/integration-tests/{smoke-harness.d.ts → e2e-harness.d.ts} +45 -37
  100. package/dist/integration-tests/e2e-harness.d.ts.map +1 -0
  101. package/dist/integration-tests/{smoke-harness.js → e2e-harness.js} +112 -94
  102. package/dist/integration-tests/e2e-harness.js.map +1 -0
  103. package/dist/integration-tests/e2e-workspace-build-global-setup.d.ts +16 -0
  104. package/dist/integration-tests/e2e-workspace-build-global-setup.d.ts.map +1 -0
  105. package/dist/integration-tests/e2e-workspace-build-global-setup.js +27 -0
  106. package/dist/integration-tests/e2e-workspace-build-global-setup.js.map +1 -0
  107. package/dist/integration-tests/live-agent-model-roundtrip-deployment.d.ts +11 -0
  108. package/dist/integration-tests/live-agent-model-roundtrip-deployment.d.ts.map +1 -0
  109. package/dist/integration-tests/live-agent-model-roundtrip-deployment.js +48 -0
  110. package/dist/integration-tests/live-agent-model-roundtrip-deployment.js.map +1 -0
  111. package/dist/integration-tests/live-agent-model-roundtrip-gates.d.ts +11 -0
  112. package/dist/integration-tests/live-agent-model-roundtrip-gates.d.ts.map +1 -0
  113. package/dist/integration-tests/live-agent-model-roundtrip-gates.js +21 -0
  114. package/dist/integration-tests/live-agent-model-roundtrip-gates.js.map +1 -0
  115. package/dist/integration-tests/live-vm-e2e-gates.d.ts +2 -0
  116. package/dist/integration-tests/live-vm-e2e-gates.d.ts.map +1 -0
  117. package/dist/integration-tests/live-vm-e2e-gates.js +4 -0
  118. package/dist/integration-tests/live-vm-e2e-gates.js.map +1 -0
  119. package/dist/operations/controller-status.d.ts +5 -0
  120. package/dist/operations/controller-status.d.ts.map +1 -1
  121. package/dist/operations/controller-status.js +42 -0
  122. package/dist/operations/controller-status.js.map +1 -1
  123. package/package.json +11 -11
  124. package/dist/integration-tests/live-integration-gates.d.ts +0 -2
  125. package/dist/integration-tests/live-integration-gates.d.ts.map +0 -1
  126. package/dist/integration-tests/live-integration-gates.js +0 -4
  127. package/dist/integration-tests/live-integration-gates.js.map +0 -1
  128. package/dist/integration-tests/smoke-harness.d.ts.map +0 -1
  129. package/dist/integration-tests/smoke-harness.js.map +0 -1
@@ -1,4 +1,5 @@
1
- import { createGatewayVmRecoveryTracker, } from './gateway-vm-recovery-policy.js';
1
+ import { deriveChannelProviderRecoveryObservation, } from './channel-provider-recovery-observation.js';
2
+ import { createGatewayVmRecoveryTracker, defaultGatewayVmChannelProviderRecoveryPolicy, } from './gateway-vm-recovery-policy.js';
2
3
  function writeGatewayServiceHealthMonitorLog(message) {
3
4
  process.stderr.write(`[gateway-service-health-monitor] ${message}\n`);
4
5
  }
@@ -6,16 +7,41 @@ const unknownGatewayServiceHealthTarget = {
6
7
  path: '(unknown)',
7
8
  port: 0,
8
9
  };
10
+ function channelProviderEventKey(event) {
11
+ return `${event.zoneId}\0${event.channelProviderId}\0${String(event.observedAtMs)}`;
12
+ }
13
+ function recoveryActionForBudgetClass(recoveryBudgetClass) {
14
+ return recoveryBudgetClass === 'gateway-vm-cold-start'
15
+ ? 'gateway-vm-cold-start'
16
+ : 'gateway-vm-restart';
17
+ }
18
+ function budgetResultForRecoveryResult(result) {
19
+ return result.action === 'observe-only' || result.action === 'operator-required'
20
+ ? undefined
21
+ : result.result;
22
+ }
23
+ function isGatewayRecoveryTimeoutErrorCodeValue(value) {
24
+ return value === 'recovery-callback-unconfigured' || value === 'recovery-timeout';
25
+ }
9
26
  export function createGatewayServiceHealthMonitor(options) {
10
27
  const setIntervalImpl = options.setIntervalImpl ?? setInterval;
11
28
  const clearIntervalImpl = options.clearIntervalImpl ?? clearInterval;
12
- const setTimeoutImpl = options.setTimeoutImpl ?? setTimeout;
13
- const clearTimeoutImpl = options.clearTimeoutImpl ?? clearTimeout;
14
29
  const recoveryPolicy = options.gatewayServiceAutoRestart;
30
+ const channelProviderPolicy = recoveryPolicy.channelProviderHealth ?? defaultGatewayVmChannelProviderRecoveryPolicy;
15
31
  const recoveryTracker = createGatewayVmRecoveryTracker({ policy: recoveryPolicy });
16
32
  let timer;
17
33
  let runningTick;
34
+ const recoveredChannelProviderEventKeys = new Set();
18
35
  let stopped = false;
36
+ const classifyRecoveryBudgetClass = (request) => {
37
+ try {
38
+ return options.classifyRecoveryBudgetClass?.(request) ?? 'gateway-vm-restart';
39
+ }
40
+ catch (error) {
41
+ writeGatewayServiceHealthMonitorLog(`recovery budget classification failed for zone '${request.zoneId}': ${error instanceof Error ? error.message : String(error)}`);
42
+ return 'gateway-vm-restart';
43
+ }
44
+ };
19
45
  const classifyGatewayControlLinkObservation = (props) => {
20
46
  const controlLinkEvent = options.healthEventStore
21
47
  .listLatestEventsForZone(props.zoneId)
@@ -30,62 +56,74 @@ export function createGatewayServiceHealthMonitor(options) {
30
56
  }
31
57
  return controlLinkEvent.result;
32
58
  };
33
- const runRecoveryWithDeadline = async (request) => {
59
+ const latestChannelProviderHealthEvents = (zoneId) => options.healthEventStore
60
+ .listLatestEventsForZone(zoneId)
61
+ .filter((event) => event.kind === 'agent-channel-provider-health');
62
+ const runRecovery = async (request, intendedAction) => {
34
63
  if (!options.recoverGatewayVm) {
35
- return {
36
- elapsedMs: 0,
37
- errorCode: 'recovery-callback-unconfigured',
38
- result: 'failed',
39
- };
40
- }
41
- let timeout;
42
- const startedAtMs = options.now();
43
- try {
44
- return await Promise.race([
45
- options.recoverGatewayVm(request),
46
- new Promise((resolve) => {
47
- timeout = setTimeoutImpl(() => {
48
- resolve({
49
- elapsedMs: options.now() - startedAtMs,
50
- errorCode: 'recovery-timeout',
51
- result: 'failed',
52
- });
53
- }, recoveryPolicy.restartTimeoutMs);
54
- timeout.unref?.();
55
- }),
56
- ]);
57
- }
58
- finally {
59
- if (timeout) {
60
- clearTimeoutImpl(timeout);
61
- }
64
+ return intendedAction === 'gateway-vm-restart'
65
+ ? {
66
+ action: 'gateway-vm-restart',
67
+ elapsedMs: 0,
68
+ errorCode: 'recovery-callback-unconfigured',
69
+ result: 'failed',
70
+ }
71
+ : {
72
+ action: 'gateway-vm-cold-start',
73
+ elapsedMs: 0,
74
+ errorCode: 'recovery-callback-unconfigured',
75
+ result: 'failed',
76
+ };
62
77
  }
78
+ return await options.recoverGatewayVm(request);
63
79
  };
64
80
  const recordGatewayRecoveryEvent = (props) => {
81
+ const recoveryAction = props.result.action ?? 'gateway-vm-restart';
65
82
  if (props.result.result === 'ok') {
66
- const event = {
67
- action: 'gateway-vm-restart',
68
- consecutiveFailures: props.consecutiveFailures,
69
- cooldownMs: recoveryPolicy.cooldownMs,
70
- elapsedMs: props.result.elapsedMs,
71
- kind: 'gateway-recovery',
72
- leaseReleaseFailureCount: props.result.leaseReleaseFailureCount,
73
- newBootedAt: props.result.newBootedAt,
74
- newHostPid: props.result.newHostPid,
75
- newVmId: props.result.newVmId,
76
- observedAtMs: props.observedAtMs,
77
- oldBootedAt: props.result.oldBootedAt,
78
- oldHostPid: props.result.oldHostPid,
79
- oldVmId: props.result.oldVmId,
80
- reason: props.reason,
81
- result: 'ok',
82
- zoneId: props.zoneId,
83
- };
83
+ const event = props.result.action !== 'gateway-vm-cold-start'
84
+ ? {
85
+ action: 'gateway-vm-restart',
86
+ consecutiveFailures: props.consecutiveFailures,
87
+ cooldownMs: recoveryPolicy.cooldownMs,
88
+ elapsedMs: props.result.elapsedMs,
89
+ kind: 'gateway-recovery',
90
+ leaseReleaseFailureCount: props.result.leaseReleaseFailureCount,
91
+ newBootedAt: props.result.newBootedAt,
92
+ newHostPid: props.result.newHostPid,
93
+ newVmId: props.result.newVmId,
94
+ observedAtMs: props.observedAtMs,
95
+ oldBootedAt: props.result.oldBootedAt,
96
+ oldHostPid: props.result.oldHostPid,
97
+ oldVmId: props.result.oldVmId,
98
+ ...(props.result.operationId === undefined
99
+ ? {}
100
+ : { operationId: props.result.operationId }),
101
+ reason: props.reason,
102
+ result: 'ok',
103
+ zoneId: props.zoneId,
104
+ }
105
+ : {
106
+ action: 'gateway-vm-cold-start',
107
+ consecutiveFailures: props.consecutiveFailures,
108
+ cooldownMs: recoveryPolicy.cooldownMs,
109
+ elapsedMs: props.result.elapsedMs,
110
+ kind: 'gateway-recovery',
111
+ leaseReleaseFailureCount: props.result.leaseReleaseFailureCount,
112
+ newBootedAt: props.result.newBootedAt,
113
+ newHostPid: props.result.newHostPid,
114
+ newVmId: props.result.newVmId,
115
+ observedAtMs: props.observedAtMs,
116
+ ...(props.result.operationId === undefined
117
+ ? {}
118
+ : { operationId: props.result.operationId }),
119
+ reason: props.reason,
120
+ result: 'ok',
121
+ zoneId: props.zoneId,
122
+ };
84
123
  options.healthEventStore.record(event);
85
124
  return;
86
125
  }
87
126
  const event = {
88
- action: 'gateway-vm-restart',
89
127
  consecutiveFailures: props.consecutiveFailures,
90
128
  cooldownMs: recoveryPolicy.cooldownMs,
91
129
  elapsedMs: props.result.elapsedMs,
@@ -95,18 +133,52 @@ export function createGatewayServiceHealthMonitor(options) {
95
133
  ? {}
96
134
  : { leaseReleaseFailureCount: props.result.leaseReleaseFailureCount }),
97
135
  observedAtMs: props.observedAtMs,
98
- oldBootedAt: props.result.oldBootedAt,
99
- oldHostPid: props.result.oldHostPid,
100
- oldVmId: props.result.oldVmId,
136
+ ...(props.result.operationId === undefined ? {} : { operationId: props.result.operationId }),
101
137
  reason: props.reason,
102
138
  result: 'failed',
103
139
  zoneId: props.zoneId,
104
140
  };
105
- options.healthEventStore.record(event);
141
+ if (recoveryAction === 'gateway-vm-restart') {
142
+ if (props.result.oldVmId === undefined) {
143
+ if (!isGatewayRecoveryTimeoutErrorCodeValue(props.result.errorCode)) {
144
+ const malformedRestartEvent = {
145
+ ...event,
146
+ action: 'operator-required',
147
+ errorCode: props.result.errorCode,
148
+ };
149
+ options.healthEventStore.record(malformedRestartEvent);
150
+ return;
151
+ }
152
+ const timeoutEvent = {
153
+ ...event,
154
+ action: 'gateway-vm-restart',
155
+ errorCode: props.result.errorCode,
156
+ oldBootedAt: props.result.oldBootedAt,
157
+ oldHostPid: props.result.oldHostPid,
158
+ oldVmId: props.result.oldVmId,
159
+ };
160
+ options.healthEventStore.record(timeoutEvent);
161
+ return;
162
+ }
163
+ const restartEvent = {
164
+ ...event,
165
+ action: 'gateway-vm-restart',
166
+ oldBootedAt: props.result.oldBootedAt,
167
+ oldHostPid: props.result.oldHostPid,
168
+ oldVmId: props.result.oldVmId,
169
+ };
170
+ options.healthEventStore.record(restartEvent);
171
+ return;
172
+ }
173
+ const nonRestartEvent = {
174
+ ...event,
175
+ action: recoveryAction,
176
+ };
177
+ options.healthEventStore.record(nonRestartEvent);
106
178
  };
107
179
  const recordGatewayRecoverySuspendedEvent = (props) => {
108
180
  const event = {
109
- action: 'gateway-vm-restart',
181
+ action: props.action,
110
182
  consecutiveFailedRecoveries: props.consecutiveFailedRecoveries,
111
183
  consecutiveFailures: props.consecutiveFailures,
112
184
  cooldownMs: recoveryPolicy.cooldownMs,
@@ -121,14 +193,21 @@ export function createGatewayServiceHealthMonitor(options) {
121
193
  options.healthEventStore.record(event);
122
194
  };
123
195
  const maybeRecoverGatewayVm = async (props) => {
196
+ const recoveryBudgetClass = classifyRecoveryBudgetClass({
197
+ consecutiveFailures: 0,
198
+ reason: props.reason,
199
+ zoneId: props.zoneId,
200
+ });
124
201
  const decision = props.reason === 'gateway-service-unhealthy'
125
202
  ? recoveryTracker.recordGatewayServiceProbe({
126
203
  observedAtMs: props.observedAtMs,
204
+ recoveryBudgetClass,
127
205
  result: props.serviceProbeResult,
128
206
  zoneId: props.zoneId,
129
207
  })
130
208
  : recoveryTracker.recordGatewayControlLinkObservation({
131
209
  observedAtMs: props.observedAtMs,
210
+ recoveryBudgetClass,
132
211
  result: props.serviceProbeResult === 'ok'
133
212
  ? classifyGatewayControlLinkObservation({
134
213
  nowMs: props.observedAtMs,
@@ -143,6 +222,7 @@ export function createGatewayServiceHealthMonitor(options) {
143
222
  consecutiveFailures: decision.consecutiveFailures,
144
223
  observedAtMs: props.observedAtMs,
145
224
  reason: props.reason,
225
+ action: recoveryActionForBudgetClass(recoveryBudgetClass),
146
226
  zoneId: decision.zoneId,
147
227
  });
148
228
  return;
@@ -156,17 +236,104 @@ export function createGatewayServiceHealthMonitor(options) {
156
236
  }
157
237
  recoveryTracker.markRecoveryStarted({
158
238
  observedAtMs: props.observedAtMs,
239
+ recoveryBudgetClass,
159
240
  zoneId: decision.zoneId,
160
241
  });
161
- const recoveryResult = await runRecoveryWithDeadline({
242
+ const recoveryResult = await runRecovery({
162
243
  consecutiveFailures: decision.consecutiveFailures,
163
244
  reason: decision.reason,
164
245
  zoneId: decision.zoneId,
246
+ }, recoveryActionForBudgetClass(recoveryBudgetClass));
247
+ const observedAtMs = options.now();
248
+ recoveryTracker.markRecoveryFinished({
249
+ observedAtMs,
250
+ recoveryBudgetClass,
251
+ result: budgetResultForRecoveryResult(recoveryResult),
252
+ zoneId: decision.zoneId,
165
253
  });
254
+ recordGatewayRecoveryEvent({
255
+ consecutiveFailures: decision.consecutiveFailures,
256
+ observedAtMs,
257
+ reason: decision.reason,
258
+ result: recoveryResult,
259
+ zoneId: decision.zoneId,
260
+ });
261
+ };
262
+ const maybeRecoverAgentChannelProvider = async (props) => {
263
+ const latestEvents = latestChannelProviderHealthEvents(props.zoneId);
264
+ if (latestEvents.length === 0) {
265
+ return;
266
+ }
267
+ let selectedDecision;
268
+ for (const latestEvent of latestEvents) {
269
+ const eventKey = channelProviderEventKey(latestEvent);
270
+ if (recoveredChannelProviderEventKeys.has(eventKey)) {
271
+ continue;
272
+ }
273
+ const recoveryObservation = deriveChannelProviderRecoveryObservation({
274
+ allowRestartWhenUnrecoverable: channelProviderPolicy.restartGatewayOnUnrecoverable,
275
+ event: latestEvent,
276
+ nowMs: props.observedAtMs,
277
+ restartGatewayOnRecoverable: channelProviderPolicy.restartGatewayOnRecoverable,
278
+ staleAfterMs: options.staleAfterMs,
279
+ transitioningTimeoutMs: channelProviderPolicy.transitioningTimeoutMs,
280
+ });
281
+ if (recoveryObservation.kind === 'observe-only') {
282
+ continue;
283
+ }
284
+ const recoveryBudgetClass = classifyRecoveryBudgetClass({
285
+ consecutiveFailures: 0,
286
+ reason: recoveryObservation.reason ?? 'agent-channel-provider-unhealthy',
287
+ zoneId: props.zoneId,
288
+ });
289
+ const decision = recoveryTracker.recordAgentChannelProviderObservation({
290
+ ...recoveryObservation.observation,
291
+ recoveryBudgetClass,
292
+ });
293
+ if (decision.kind === 'restart' || decision.kind === 'suspended') {
294
+ selectedDecision = {
295
+ decision,
296
+ eventKey,
297
+ reason: recoveryObservation.reason ?? 'agent-channel-provider-unhealthy',
298
+ recoveryBudgetClass,
299
+ };
300
+ break;
301
+ }
302
+ }
303
+ if (!selectedDecision) {
304
+ return;
305
+ }
306
+ const { decision, reason } = selectedDecision;
307
+ if (decision.kind === 'suspended') {
308
+ recordGatewayRecoverySuspendedEvent({
309
+ consecutiveFailedRecoveries: decision.consecutiveFailedRecoveries,
310
+ consecutiveFailures: decision.consecutiveFailures,
311
+ observedAtMs: props.observedAtMs,
312
+ reason,
313
+ action: recoveryActionForBudgetClass(selectedDecision.recoveryBudgetClass),
314
+ zoneId: decision.zoneId,
315
+ });
316
+ return;
317
+ }
318
+ if (stopped) {
319
+ writeGatewayServiceHealthMonitorLog(`recovery requested for zone '${decision.zoneId}' but monitor is stopped`);
320
+ return;
321
+ }
322
+ recoveryTracker.markRecoveryStarted({
323
+ observedAtMs: props.observedAtMs,
324
+ recoveryBudgetClass: selectedDecision.recoveryBudgetClass,
325
+ zoneId: decision.zoneId,
326
+ });
327
+ const recoveryResult = await runRecovery({
328
+ consecutiveFailures: decision.consecutiveFailures,
329
+ reason: decision.reason,
330
+ zoneId: decision.zoneId,
331
+ }, recoveryActionForBudgetClass(selectedDecision.recoveryBudgetClass));
166
332
  const observedAtMs = options.now();
167
333
  recoveryTracker.markRecoveryFinished({
168
334
  observedAtMs,
169
- result: recoveryResult.result,
335
+ recoveryBudgetClass: selectedDecision.recoveryBudgetClass,
336
+ result: budgetResultForRecoveryResult(recoveryResult),
170
337
  zoneId: decision.zoneId,
171
338
  });
172
339
  recordGatewayRecoveryEvent({
@@ -176,6 +343,9 @@ export function createGatewayServiceHealthMonitor(options) {
176
343
  result: recoveryResult,
177
344
  zoneId: decision.zoneId,
178
345
  });
346
+ if (recoveryResult.result === 'ok') {
347
+ recoveredChannelProviderEventKeys.add(selectedDecision.eventKey);
348
+ }
179
349
  };
180
350
  const tick = async () => {
181
351
  if (runningTick) {
@@ -209,6 +379,10 @@ export function createGatewayServiceHealthMonitor(options) {
209
379
  serviceProbeResult,
210
380
  zoneId: result.zoneId,
211
381
  });
382
+ await maybeRecoverAgentChannelProvider({
383
+ observedAtMs,
384
+ zoneId: result.zoneId,
385
+ });
212
386
  }
213
387
  }
214
388
  catch (error) {
@@ -1 +1 @@
1
- {"version":3,"file":"gateway-service-health-monitor.js","sourceRoot":"","sources":["../../../src/controller/health/gateway-service-health-monitor.ts"],"names":[],"mappings":"AAEA,OAAO,EACN,8BAA8B,GAI9B,MAAM,iCAAiC,CAAC;AAgEzC,SAAS,mCAAmC,CAAC,OAAe;IAC3D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,oCAAoC,OAAO,IAAI,CAAC,CAAC;AACvE,CAAC;AAED,MAAM,iCAAiC,GAAG;IACzC,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,CAAC;CACE,CAAC;AAEX,MAAM,UAAU,iCAAiC,CAChD,OAAiD;IAEjD,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,WAAW,CAAC;IAC/D,MAAM,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,IAAI,aAAa,CAAC;IACrE,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,UAAU,CAAC;IAC5D,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,YAAY,CAAC;IAClE,MAAM,cAAc,GAAG,OAAO,CAAC,yBAAyB,CAAC;IACzD,MAAM,eAAe,GAAG,8BAA8B,CAAC,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC,CAAC;IACnF,IAAI,KAAiC,CAAC;IACtC,IAAI,WAAsC,CAAC;IAC3C,IAAI,OAAO,GAAG,KAAK,CAAC;IAEpB,MAAM,qCAAqC,GAAG,CAAC,KAG9C,EAAsC,EAAE;QACxC,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB;aAC/C,uBAAuB,CAAC,KAAK,CAAC,MAAM,CAAC;aACrC,IAAI,CAAC,CAAC,KAAK,EAA2E,EAAE;YACxF,OAAO,KAAK,CAAC,IAAI,KAAK,sBAAsB,CAAC;QAC9C,CAAC,CAAC,CAAC;QACJ,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACvB,OAAO,YAAY,CAAC;QACrB,CAAC;QACD,IAAI,KAAK,CAAC,KAAK,GAAG,gBAAgB,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;YACxE,OAAO,OAAO,CAAC;QAChB,CAAC;QACD,OAAO,gBAAgB,CAAC,MAAM,CAAC;IAChC,CAAC,CAAC;IAEF,MAAM,uBAAuB,GAAG,KAAK,EACpC,OAAiC,EACE,EAAE;QACrC,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC;YAC/B,OAAO;gBACN,SAAS,EAAE,CAAC;gBACZ,SAAS,EAAE,gCAAgC;gBAC3C,MAAM,EAAE,QAAQ;aAChB,CAAC;QACH,CAAC;QAED,IAAI,OAAmC,CAAC;QACxC,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QAClC,IAAI,CAAC;YACJ,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC;gBACzB,OAAO,CAAC,gBAAgB,CAAC,OAAO,CAAC;gBACjC,IAAI,OAAO,CAA0B,CAAC,OAAO,EAAE,EAAE;oBAChD,OAAO,GAAG,cAAc,CAAC,GAAG,EAAE;wBAC7B,OAAO,CAAC;4BACP,SAAS,EAAE,OAAO,CAAC,GAAG,EAAE,GAAG,WAAW;4BACtC,SAAS,EAAE,kBAAkB;4BAC7B,MAAM,EAAE,QAAQ;yBAChB,CAAC,CAAC;oBACJ,CAAC,EAAE,cAAc,CAAC,gBAAgB,CAAC,CAAC;oBACpC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;gBACnB,CAAC,CAAC;aACF,CAAC,CAAC;QACJ,CAAC;gBAAS,CAAC;YACV,IAAI,OAAO,EAAE,CAAC;gBACb,gBAAgB,CAAC,OAAO,CAAC,CAAC;YAC3B,CAAC;QACF,CAAC;IACF,CAAC,CAAC;IAEF,MAAM,0BAA0B,GAAG,CAAC,KAMnC,EAAQ,EAAE;QACV,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;YAClC,MAAM,KAAK,GAAG;gBACb,MAAM,EAAE,oBAAoB;gBAC5B,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;gBAC9C,UAAU,EAAE,cAAc,CAAC,UAAU;gBACrC,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;gBACjC,IAAI,EAAE,kBAAkB;gBACxB,wBAAwB,EAAE,KAAK,CAAC,MAAM,CAAC,wBAAwB;gBAC/D,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;gBACrC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU;gBACnC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO;gBAC7B,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;gBACrC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU;gBACnC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO;gBAC7B,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,MAAM,EAAE,IAAI;gBACZ,MAAM,EAAE,KAAK,CAAC,MAAM;aACS,CAAC;YAC/B,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACvC,OAAO;QACR,CAAC;QAED,MAAM,KAAK,GAAG;YACb,MAAM,EAAE,oBAAoB;YAC5B,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;YAC9C,UAAU,EAAE,cAAc,CAAC,UAAU;YACrC,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;YACjC,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;YACjC,IAAI,EAAE,kBAAkB;YACxB,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,wBAAwB,KAAK,SAAS;gBACtD,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,wBAAwB,EAAE,KAAK,CAAC,MAAM,CAAC,wBAAwB,EAAE,CAAC;YACvE,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;YACrC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU;YACnC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO;YAC7B,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,MAAM,EAAE,QAAQ;YAChB,MAAM,EAAE,KAAK,CAAC,MAAM;SACS,CAAC;QAC/B,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACxC,CAAC,CAAC;IAEF,MAAM,mCAAmC,GAAG,CAAC,KAM5C,EAAQ,EAAE;QACV,MAAM,KAAK,GAAG;YACb,MAAM,EAAE,oBAAoB;YAC5B,2BAA2B,EAAE,KAAK,CAAC,2BAA2B;YAC9D,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;YAC9C,UAAU,EAAE,cAAc,CAAC,UAAU;YACrC,SAAS,EAAE,uBAAuB;YAClC,qBAAqB,EAAE,cAAc,CAAC,qBAAqB;YAC3D,IAAI,EAAE,4BAA4B;YAClC,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,MAAM,EAAE,QAAQ;YAChB,MAAM,EAAE,KAAK,CAAC,MAAM;SACS,CAAC;QAC/B,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACxC,CAAC,CAAC;IAEF,MAAM,qBAAqB,GAAG,KAAK,EAAE,KAKpC,EAAiB,EAAE;QACnB,MAAM,QAAQ,GACb,KAAK,CAAC,MAAM,KAAK,2BAA2B;YAC3C,CAAC,CAAC,eAAe,CAAC,yBAAyB,CAAC;gBAC1C,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,MAAM,EAAE,KAAK,CAAC,kBAAkB;gBAChC,MAAM,EAAE,KAAK,CAAC,MAAM;aACpB,CAAC;YACH,CAAC,CAAC,eAAe,CAAC,mCAAmC,CAAC;gBACpD,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,MAAM,EACL,KAAK,CAAC,kBAAkB,KAAK,IAAI;oBAChC,CAAC,CAAC,qCAAqC,CAAC;wBACtC,KAAK,EAAE,KAAK,CAAC,YAAY;wBACzB,MAAM,EAAE,KAAK,CAAC,MAAM;qBACpB,CAAC;oBACH,CAAC,CAAC,YAAY;gBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;aACpB,CAAC,CAAC;QAEN,IAAI,QAAQ,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YACnC,mCAAmC,CAAC;gBACnC,2BAA2B,EAAE,QAAQ,CAAC,2BAA2B;gBACjE,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;gBACjD,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,MAAM,EAAE,QAAQ,CAAC,MAAM;aACvB,CAAC,CAAC;YACH,OAAO;QACR,CAAC;QACD,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YACjC,OAAO;QACR,CAAC;QACD,IAAI,OAAO,EAAE,CAAC;YACb,mCAAmC,CAClC,gCAAgC,QAAQ,CAAC,MAAM,0BAA0B,CACzE,CAAC;YACF,OAAO;QACR,CAAC;QAED,eAAe,CAAC,mBAAmB,CAAC;YACnC,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,CAAC,CAAC;QACH,MAAM,cAAc,GAAG,MAAM,uBAAuB,CAAC;YACpD,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;YACjD,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,CAAC,CAAC;QACH,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QACnC,eAAe,CAAC,oBAAoB,CAAC;YACpC,YAAY;YACZ,MAAM,EAAE,cAAc,CAAC,MAAM;YAC7B,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,CAAC,CAAC;QACH,0BAA0B,CAAC;YAC1B,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;YACjD,YAAY;YACZ,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,cAAc;YACtB,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,CAAC,CAAC;IACJ,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,KAAK,IAAmB,EAAE;QACtC,IAAI,WAAW,EAAE,CAAC;YACjB,OAAO,MAAM,WAAW,CAAC;QAC1B,CAAC;QACD,WAAW,GAAG,CAAC,KAAK,IAAI,EAAE;YACzB,MAAM,OAAO,CAAC,GAAG,CAChB,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;gBACpC,IAAI,CAAC;oBACJ,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;oBACrD,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;oBACnC,MAAM,kBAAkB,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC;oBACvD,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC;wBAC/B,IAAI,EAAE,wBAAwB;wBAC9B,YAAY;wBACZ,IAAI,EAAE,MAAM,CAAC,IAAI;wBACjB,IAAI,EAAE,MAAM,CAAC,IAAI;wBACjB,MAAM,EAAE,kBAAkB;wBAC1B,GAAG,CAAC,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC;wBAC7E,MAAM,EAAE,MAAM,CAAC,MAAM;qBACrB,CAAC,CAAC;oBACH,MAAM,qBAAqB,CAAC;wBAC3B,YAAY;wBACZ,MAAM,EAAE,2BAA2B;wBACnC,kBAAkB;wBAClB,MAAM,EAAE,MAAM,CAAC,MAAM;qBACrB,CAAC,CAAC;oBACH,IAAI,kBAAkB,KAAK,IAAI,EAAE,CAAC;wBACjC,MAAM,qBAAqB,CAAC;4BAC3B,YAAY;4BACZ,MAAM,EAAE,gCAAgC;4BACxC,kBAAkB;4BAClB,MAAM,EAAE,MAAM,CAAC,MAAM;yBACrB,CAAC,CAAC;oBACJ,CAAC;gBACF,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,mCAAmC,CAClC,0BAA0B,MAAM,MAAM,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC9F,CAAC;oBACF,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;oBACnC,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC;wBAC/B,IAAI,EAAE,wBAAwB;wBAC9B,YAAY;wBACZ,IAAI,EAAE,iCAAiC,CAAC,IAAI;wBAC5C,IAAI,EAAE,iCAAiC,CAAC,IAAI;wBAC5C,MAAM,EAAE,QAAQ;wBAChB,MAAM;qBACN,CAAC,CAAC;oBACH,MAAM,qBAAqB,CAAC;wBAC3B,YAAY;wBACZ,MAAM,EAAE,2BAA2B;wBACnC,kBAAkB,EAAE,QAAQ;wBAC5B,MAAM;qBACN,CAAC,CAAC;gBACJ,CAAC;YACF,CAAC,CAAC,CACF,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;YACjB,WAAW,GAAG,SAAS,CAAC;QACzB,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,WAAW,CAAC;IAC1B,CAAC,CAAC;IAEF,OAAO;QACN,KAAK,EAAE,GAAG,EAAE;YACX,IAAI,KAAK,EAAE,CAAC;gBACX,OAAO;YACR,CAAC;YACD,OAAO,GAAG,KAAK,CAAC;YAChB,KAAK,GAAG,eAAe,CAAC,KAAK,IAAI,EAAE;gBAClC,IAAI,CAAC;oBACJ,MAAM,IAAI,EAAE,CAAC;gBACd,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,mCAAmC,CAClC,0BAA0B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAClF,CAAC;gBACH,CAAC;YACF,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;YACvB,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;QACjB,CAAC;QACD,IAAI,EAAE,KAAK,IAAI,EAAE;YAChB,OAAO,GAAG,IAAI,CAAC;YACf,IAAI,KAAK,EAAE,CAAC;gBACX,iBAAiB,CAAC,KAAK,CAAC,CAAC;gBACzB,KAAK,GAAG,SAAS,CAAC;YACnB,CAAC;YACD,IAAI,WAAW,EAAE,CAAC;gBACjB,MAAM,WAAW,CAAC;YACnB,CAAC;QACF,CAAC;QACD,IAAI;KACJ,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"gateway-service-health-monitor.js","sourceRoot":"","sources":["../../../src/controller/health/gateway-service-health-monitor.ts"],"names":[],"mappings":"AAMA,OAAO,EACN,wCAAwC,GAExC,MAAM,4CAA4C,CAAC;AACpD,OAAO,EACN,8BAA8B,EAC9B,6CAA6C,GAM7C,MAAM,iCAAiC,CAAC;AA2GzC,SAAS,mCAAmC,CAAC,OAAe;IAC3D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,oCAAoC,OAAO,IAAI,CAAC,CAAC;AACvE,CAAC;AAED,MAAM,iCAAiC,GAAG;IACzC,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,CAAC;CACE,CAAC;AAEX,SAAS,uBAAuB,CAAC,KAAsC;IACtE,OAAO,GAAG,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,iBAAiB,KAAK,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC;AACrF,CAAC;AAED,SAAS,4BAA4B,CACpC,mBAAiD;IAEjD,OAAO,mBAAmB,KAAK,uBAAuB;QACrD,CAAC,CAAC,uBAAuB;QACzB,CAAC,CAAC,oBAAoB,CAAC;AACzB,CAAC;AAED,SAAS,6BAA6B,CACrC,MAA+B;IAE/B,OAAO,MAAM,CAAC,MAAM,KAAK,cAAc,IAAI,MAAM,CAAC,MAAM,KAAK,mBAAmB;QAC/E,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;AAClB,CAAC;AAED,SAAS,sCAAsC,CAC9C,KAAa;IAEb,OAAO,KAAK,KAAK,gCAAgC,IAAI,KAAK,KAAK,kBAAkB,CAAC;AACnF,CAAC;AAED,MAAM,UAAU,iCAAiC,CAChD,OAAiD;IAEjD,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,WAAW,CAAC;IAC/D,MAAM,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,IAAI,aAAa,CAAC;IACrE,MAAM,cAAc,GAAG,OAAO,CAAC,yBAAyB,CAAC;IACzD,MAAM,qBAAqB,GAC1B,cAAc,CAAC,qBAAqB,IAAI,6CAA6C,CAAC;IACvF,MAAM,eAAe,GAAG,8BAA8B,CAAC,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC,CAAC;IACnF,IAAI,KAAiC,CAAC;IACtC,IAAI,WAAsC,CAAC;IAC3C,MAAM,iCAAiC,GAAG,IAAI,GAAG,EAAU,CAAC;IAC5D,IAAI,OAAO,GAAG,KAAK,CAAC;IAEpB,MAAM,2BAA2B,GAAG,CACnC,OAA2C,EACZ,EAAE;QACjC,IAAI,CAAC;YACJ,OAAO,OAAO,CAAC,2BAA2B,EAAE,CAAC,OAAO,CAAC,IAAI,oBAAoB,CAAC;QAC/E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,mCAAmC,CAClC,mDAAmD,OAAO,CAAC,MAAM,MAChE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CACtD,EAAE,CACF,CAAC;YACF,OAAO,oBAAoB,CAAC;QAC7B,CAAC;IACF,CAAC,CAAC;IAEF,MAAM,qCAAqC,GAAG,CAAC,KAG9C,EAAsC,EAAE;QACxC,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB;aAC/C,uBAAuB,CAAC,KAAK,CAAC,MAAM,CAAC;aACrC,IAAI,CAAC,CAAC,KAAK,EAA2E,EAAE;YACxF,OAAO,KAAK,CAAC,IAAI,KAAK,sBAAsB,CAAC;QAC9C,CAAC,CAAC,CAAC;QACJ,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACvB,OAAO,YAAY,CAAC;QACrB,CAAC;QACD,IAAI,KAAK,CAAC,KAAK,GAAG,gBAAgB,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;YACxE,OAAO,OAAO,CAAC;QAChB,CAAC;QACD,OAAO,gBAAgB,CAAC,MAAM,CAAC;IAChC,CAAC,CAAC;IAEF,MAAM,iCAAiC,GAAG,CACzC,MAAc,EAC+B,EAAE,CAC/C,OAAO,CAAC,gBAAgB;SACtB,uBAAuB,CAAC,MAAM,CAAC;SAC/B,MAAM,CACN,CAAC,KAAK,EAA4C,EAAE,CACnD,KAAK,CAAC,IAAI,KAAK,+BAA+B,CAC/C,CAAC;IAEJ,MAAM,WAAW,GAAG,KAAK,EACxB,OAAiC,EACjC,cAGC,EACkC,EAAE;QACrC,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC;YAC/B,OAAO,cAAc,KAAK,oBAAoB;gBAC7C,CAAC,CAAC;oBACA,MAAM,EAAE,oBAAoB;oBAC5B,SAAS,EAAE,CAAC;oBACZ,SAAS,EAAE,gCAAgC;oBAC3C,MAAM,EAAE,QAAQ;iBAChB;gBACF,CAAC,CAAC;oBACA,MAAM,EAAE,uBAAuB;oBAC/B,SAAS,EAAE,CAAC;oBACZ,SAAS,EAAE,gCAAgC;oBAC3C,MAAM,EAAE,QAAQ;iBAChB,CAAC;QACL,CAAC;QACD,OAAO,MAAM,OAAO,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAChD,CAAC,CAAC;IAEF,MAAM,0BAA0B,GAAG,CAAC,KAMnC,EAAQ,EAAE;QACV,MAAM,cAAc,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,IAAI,oBAAoB,CAAC;QACnE,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;YAClC,MAAM,KAAK,GACV,KAAK,CAAC,MAAM,CAAC,MAAM,KAAK,uBAAuB;gBAC9C,CAAC,CAAE;oBACD,MAAM,EAAE,oBAAoB;oBAC5B,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;oBAC9C,UAAU,EAAE,cAAc,CAAC,UAAU;oBACrC,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;oBACjC,IAAI,EAAE,kBAAkB;oBACxB,wBAAwB,EAAE,KAAK,CAAC,MAAM,CAAC,wBAAwB;oBAC/D,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;oBACrC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU;oBACnC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO;oBAC7B,YAAY,EAAE,KAAK,CAAC,YAAY;oBAChC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;oBACrC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU;oBACnC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO;oBAC7B,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,KAAK,SAAS;wBACzC,CAAC,CAAC,EAAE;wBACJ,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;oBAC7C,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,MAAM,EAAE,IAAI;oBACZ,MAAM,EAAE,KAAK,CAAC,MAAM;iBACU;gBAChC,CAAC,CAAE;oBACD,MAAM,EAAE,uBAAuB;oBAC/B,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;oBAC9C,UAAU,EAAE,cAAc,CAAC,UAAU;oBACrC,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;oBACjC,IAAI,EAAE,kBAAkB;oBACxB,wBAAwB,EAAE,KAAK,CAAC,MAAM,CAAC,wBAAwB;oBAC/D,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;oBACrC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU;oBACnC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO;oBAC7B,YAAY,EAAE,KAAK,CAAC,YAAY;oBAChC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,KAAK,SAAS;wBACzC,CAAC,CAAC,EAAE;wBACJ,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;oBAC7C,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,MAAM,EAAE,IAAI;oBACZ,MAAM,EAAE,KAAK,CAAC,MAAM;iBACU,CAAC;YACnC,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACvC,OAAO;QACR,CAAC;QAED,MAAM,KAAK,GAAG;YACb,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;YAC9C,UAAU,EAAE,cAAc,CAAC,UAAU;YACrC,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;YACjC,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;YACjC,IAAI,EAAE,kBAAkB;YACxB,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,wBAAwB,KAAK,SAAS;gBACtD,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,wBAAwB,EAAE,KAAK,CAAC,MAAM,CAAC,wBAAwB,EAAE,CAAC;YACvE,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAC5F,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,MAAM,EAAE,QAAQ;YAChB,MAAM,EAAE,KAAK,CAAC,MAAM;SACX,CAAC;QACX,IAAI,cAAc,KAAK,oBAAoB,EAAE,CAAC;YAC7C,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;gBACxC,IAAI,CAAC,sCAAsC,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;oBACrE,MAAM,qBAAqB,GAAG;wBAC7B,GAAG,KAAK;wBACR,MAAM,EAAE,mBAAmB;wBAC3B,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;qBACJ,CAAC;oBAC/B,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;oBACvD,OAAO;gBACR,CAAC;gBACD,MAAM,YAAY,GAAG;oBACpB,GAAG,KAAK;oBACR,MAAM,EAAE,oBAAoB;oBAC5B,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;oBACjC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;oBACrC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU;oBACnC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO;iBACA,CAAC;gBAC/B,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;gBAC9C,OAAO;YACR,CAAC;YACD,MAAM,YAAY,GAAG;gBACpB,GAAG,KAAK;gBACR,MAAM,EAAE,oBAAoB;gBAC5B,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;gBACrC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU;gBACnC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO;aACA,CAAC;YAC/B,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YAC9C,OAAO;QACR,CAAC;QACD,MAAM,eAAe,GAAG;YACvB,GAAG,KAAK;YACR,MAAM,EAAE,cAAc;SACO,CAAC;QAC/B,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAClD,CAAC,CAAC;IAEF,MAAM,mCAAmC,GAAG,CAAC,KAO5C,EAAQ,EAAE;QACV,MAAM,KAAK,GAAG;YACb,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,2BAA2B,EAAE,KAAK,CAAC,2BAA2B;YAC9D,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;YAC9C,UAAU,EAAE,cAAc,CAAC,UAAU;YACrC,SAAS,EAAE,uBAAuB;YAClC,qBAAqB,EAAE,cAAc,CAAC,qBAAqB;YAC3D,IAAI,EAAE,4BAA4B;YAClC,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,MAAM,EAAE,QAAQ;YAChB,MAAM,EAAE,KAAK,CAAC,MAAM;SACS,CAAC;QAC/B,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACxC,CAAC,CAAC;IAEF,MAAM,qBAAqB,GAAG,KAAK,EAAE,KAKpC,EAAiB,EAAE;QACnB,MAAM,mBAAmB,GAAG,2BAA2B,CAAC;YACvD,mBAAmB,EAAE,CAAC;YACtB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,MAAM,EAAE,KAAK,CAAC,MAAM;SACpB,CAAC,CAAC;QACH,MAAM,QAAQ,GACb,KAAK,CAAC,MAAM,KAAK,2BAA2B;YAC3C,CAAC,CAAC,eAAe,CAAC,yBAAyB,CAAC;gBAC1C,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,mBAAmB;gBACnB,MAAM,EAAE,KAAK,CAAC,kBAAkB;gBAChC,MAAM,EAAE,KAAK,CAAC,MAAM;aACpB,CAAC;YACH,CAAC,CAAC,eAAe,CAAC,mCAAmC,CAAC;gBACpD,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,mBAAmB;gBACnB,MAAM,EACL,KAAK,CAAC,kBAAkB,KAAK,IAAI;oBAChC,CAAC,CAAC,qCAAqC,CAAC;wBACtC,KAAK,EAAE,KAAK,CAAC,YAAY;wBACzB,MAAM,EAAE,KAAK,CAAC,MAAM;qBACpB,CAAC;oBACH,CAAC,CAAC,YAAY;gBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;aACpB,CAAC,CAAC;QAEN,IAAI,QAAQ,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YACnC,mCAAmC,CAAC;gBACnC,2BAA2B,EAAE,QAAQ,CAAC,2BAA2B;gBACjE,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;gBACjD,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,MAAM,EAAE,4BAA4B,CAAC,mBAAmB,CAAC;gBACzD,MAAM,EAAE,QAAQ,CAAC,MAAM;aACvB,CAAC,CAAC;YACH,OAAO;QACR,CAAC;QACD,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YACjC,OAAO;QACR,CAAC;QACD,IAAI,OAAO,EAAE,CAAC;YACb,mCAAmC,CAClC,gCAAgC,QAAQ,CAAC,MAAM,0BAA0B,CACzE,CAAC;YACF,OAAO;QACR,CAAC;QAED,eAAe,CAAC,mBAAmB,CAAC;YACnC,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,mBAAmB;YACnB,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,CAAC,CAAC;QACH,MAAM,cAAc,GAAG,MAAM,WAAW,CACvC;YACC,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;YACjD,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,EACD,4BAA4B,CAAC,mBAAmB,CAAC,CACjD,CAAC;QACF,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QACnC,eAAe,CAAC,oBAAoB,CAAC;YACpC,YAAY;YACZ,mBAAmB;YACnB,MAAM,EAAE,6BAA6B,CAAC,cAAc,CAAC;YACrD,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,CAAC,CAAC;QACH,0BAA0B,CAAC;YAC1B,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;YACjD,YAAY;YACZ,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,cAAc;YACtB,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,CAAC,CAAC;IACJ,CAAC,CAAC;IAEF,MAAM,gCAAgC,GAAG,KAAK,EAAE,KAG/C,EAAiB,EAAE;QACnB,MAAM,YAAY,GAAG,iCAAiC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACrE,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO;QACR,CAAC;QACD,IAAI,gBAUQ,CAAC;QACb,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;YACxC,MAAM,QAAQ,GAAG,uBAAuB,CAAC,WAAW,CAAC,CAAC;YACtD,IAAI,iCAAiC,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACrD,SAAS;YACV,CAAC;YACD,MAAM,mBAAmB,GAAG,wCAAwC,CAAC;gBACpE,6BAA6B,EAAE,qBAAqB,CAAC,6BAA6B;gBAClF,KAAK,EAAE,WAAW;gBAClB,KAAK,EAAE,KAAK,CAAC,YAAY;gBACzB,2BAA2B,EAAE,qBAAqB,CAAC,2BAA2B;gBAC9E,YAAY,EAAE,OAAO,CAAC,YAAY;gBAClC,sBAAsB,EAAE,qBAAqB,CAAC,sBAAsB;aACpE,CAAC,CAAC;YACH,IAAI,mBAAmB,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;gBACjD,SAAS;YACV,CAAC;YACD,MAAM,mBAAmB,GAAG,2BAA2B,CAAC;gBACvD,mBAAmB,EAAE,CAAC;gBACtB,MAAM,EAAE,mBAAmB,CAAC,MAAM,IAAI,kCAAkC;gBACxE,MAAM,EAAE,KAAK,CAAC,MAAM;aACpB,CAAC,CAAC;YACH,MAAM,QAAQ,GAAG,eAAe,CAAC,qCAAqC,CAAC;gBACtE,GAAG,mBAAmB,CAAC,WAAW;gBAClC,mBAAmB;aACnB,CAAC,CAAC;YACH,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS,IAAI,QAAQ,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBAClE,gBAAgB,GAAG;oBAClB,QAAQ;oBACR,QAAQ;oBACR,MAAM,EAAE,mBAAmB,CAAC,MAAM,IAAI,kCAAkC;oBACxE,mBAAmB;iBACnB,CAAC;gBACF,MAAM;YACP,CAAC;QACF,CAAC;QACD,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACvB,OAAO;QACR,CAAC;QACD,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,gBAAgB,CAAC;QAC9C,IAAI,QAAQ,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YACnC,mCAAmC,CAAC;gBACnC,2BAA2B,EAAE,QAAQ,CAAC,2BAA2B;gBACjE,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;gBACjD,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,MAAM;gBACN,MAAM,EAAE,4BAA4B,CAAC,gBAAgB,CAAC,mBAAmB,CAAC;gBAC1E,MAAM,EAAE,QAAQ,CAAC,MAAM;aACvB,CAAC,CAAC;YACH,OAAO;QACR,CAAC;QACD,IAAI,OAAO,EAAE,CAAC;YACb,mCAAmC,CAClC,gCAAgC,QAAQ,CAAC,MAAM,0BAA0B,CACzE,CAAC;YACF,OAAO;QACR,CAAC;QAED,eAAe,CAAC,mBAAmB,CAAC;YACnC,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,mBAAmB,EAAE,gBAAgB,CAAC,mBAAmB;YACzD,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,CAAC,CAAC;QACH,MAAM,cAAc,GAAG,MAAM,WAAW,CACvC;YACC,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;YACjD,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,EACD,4BAA4B,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,CAClE,CAAC;QACF,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QACnC,eAAe,CAAC,oBAAoB,CAAC;YACpC,YAAY;YACZ,mBAAmB,EAAE,gBAAgB,CAAC,mBAAmB;YACzD,MAAM,EAAE,6BAA6B,CAAC,cAAc,CAAC;YACrD,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,CAAC,CAAC;QACH,0BAA0B,CAAC;YAC1B,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;YACjD,YAAY;YACZ,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,cAAc;YACtB,MAAM,EAAE,QAAQ,CAAC,MAAM;SACvB,CAAC,CAAC;QACH,IAAI,cAAc,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;YACpC,iCAAiC,CAAC,GAAG,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAClE,CAAC;IACF,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,KAAK,IAAmB,EAAE;QACtC,IAAI,WAAW,EAAE,CAAC;YACjB,OAAO,MAAM,WAAW,CAAC;QAC1B,CAAC;QACD,WAAW,GAAG,CAAC,KAAK,IAAI,EAAE;YACzB,MAAM,OAAO,CAAC,GAAG,CAChB,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;gBACpC,IAAI,CAAC;oBACJ,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;oBACrD,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;oBACnC,MAAM,kBAAkB,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC;oBACvD,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC;wBAC/B,IAAI,EAAE,wBAAwB;wBAC9B,YAAY;wBACZ,IAAI,EAAE,MAAM,CAAC,IAAI;wBACjB,IAAI,EAAE,MAAM,CAAC,IAAI;wBACjB,MAAM,EAAE,kBAAkB;wBAC1B,GAAG,CAAC,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC;wBAC7E,MAAM,EAAE,MAAM,CAAC,MAAM;qBACrB,CAAC,CAAC;oBACH,MAAM,qBAAqB,CAAC;wBAC3B,YAAY;wBACZ,MAAM,EAAE,2BAA2B;wBACnC,kBAAkB;wBAClB,MAAM,EAAE,MAAM,CAAC,MAAM;qBACrB,CAAC,CAAC;oBACH,IAAI,kBAAkB,KAAK,IAAI,EAAE,CAAC;wBACjC,MAAM,qBAAqB,CAAC;4BAC3B,YAAY;4BACZ,MAAM,EAAE,gCAAgC;4BACxC,kBAAkB;4BAClB,MAAM,EAAE,MAAM,CAAC,MAAM;yBACrB,CAAC,CAAC;wBACH,MAAM,gCAAgC,CAAC;4BACtC,YAAY;4BACZ,MAAM,EAAE,MAAM,CAAC,MAAM;yBACrB,CAAC,CAAC;oBACJ,CAAC;gBACF,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,mCAAmC,CAClC,0BAA0B,MAAM,MAAM,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC9F,CAAC;oBACF,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;oBACnC,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC;wBAC/B,IAAI,EAAE,wBAAwB;wBAC9B,YAAY;wBACZ,IAAI,EAAE,iCAAiC,CAAC,IAAI;wBAC5C,IAAI,EAAE,iCAAiC,CAAC,IAAI;wBAC5C,MAAM,EAAE,QAAQ;wBAChB,MAAM;qBACN,CAAC,CAAC;oBACH,MAAM,qBAAqB,CAAC;wBAC3B,YAAY;wBACZ,MAAM,EAAE,2BAA2B;wBACnC,kBAAkB,EAAE,QAAQ;wBAC5B,MAAM;qBACN,CAAC,CAAC;gBACJ,CAAC;YACF,CAAC,CAAC,CACF,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;YACjB,WAAW,GAAG,SAAS,CAAC;QACzB,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,WAAW,CAAC;IAC1B,CAAC,CAAC;IAEF,OAAO;QACN,KAAK,EAAE,GAAG,EAAE;YACX,IAAI,KAAK,EAAE,CAAC;gBACX,OAAO;YACR,CAAC;YACD,OAAO,GAAG,KAAK,CAAC;YAChB,KAAK,GAAG,eAAe,CAAC,KAAK,IAAI,EAAE;gBAClC,IAAI,CAAC;oBACJ,MAAM,IAAI,EAAE,CAAC;gBACd,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,mCAAmC,CAClC,0BAA0B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAClF,CAAC;gBACH,CAAC;YACF,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;YACvB,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;QACjB,CAAC;QACD,IAAI,EAAE,KAAK,IAAI,EAAE;YAChB,OAAO,GAAG,IAAI,CAAC;YACf,IAAI,KAAK,EAAE,CAAC;gBACX,iBAAiB,CAAC,KAAK,CAAC,CAAC;gBACzB,KAAK,GAAG,SAAS,CAAC;YACnB,CAAC;YACD,IAAI,WAAW,EAAE,CAAC;gBACjB,MAAM,WAAW,CAAC;YACnB,CAAC;QACF,CAAC;QACD,IAAI;KACJ,CAAC;AACH,CAAC"}
@@ -1,7 +1,9 @@
1
1
  import type { GatewayRecoveryHealthReason } from '@agent-vm/gateway-interface';
2
2
  export type GatewayVmRecoveryObservationResult = 'failed' | 'ok' | 'stale' | 'timeout' | 'unobserved';
3
3
  export type GatewayVmRecoveryReason = GatewayRecoveryHealthReason;
4
+ export type GatewayVmRecoveryBudgetClass = 'gateway-vm-cold-start' | 'gateway-vm-restart';
4
5
  export interface GatewayVmAutoRecoveryPolicy {
6
+ readonly channelProviderHealth?: GatewayVmChannelProviderRecoveryPolicy | undefined;
5
7
  readonly consecutiveFailureThreshold: number;
6
8
  readonly cooldownMs: number;
7
9
  readonly enabled: boolean;
@@ -9,16 +11,33 @@ export interface GatewayVmAutoRecoveryPolicy {
9
11
  readonly maxConsecutiveFailedRecoveries: number;
10
12
  readonly restartTimeoutMs: number;
11
13
  }
14
+ export interface GatewayVmChannelProviderRecoveryPolicy {
15
+ readonly consecutiveFailureThreshold: number;
16
+ readonly enabled: boolean;
17
+ readonly restartGatewayOnRecoverable: boolean;
18
+ readonly restartGatewayOnUnrecoverable: boolean;
19
+ readonly transitioningTimeoutMs: number;
20
+ }
21
+ export declare const defaultGatewayVmChannelProviderRecoveryPolicy: {
22
+ consecutiveFailureThreshold: number;
23
+ enabled: true;
24
+ restartGatewayOnRecoverable: true;
25
+ restartGatewayOnUnrecoverable: false;
26
+ transitioningTimeoutMs: number;
27
+ };
12
28
  export interface CreateGatewayVmRecoveryTrackerOptions {
13
29
  readonly policy: GatewayVmAutoRecoveryPolicy;
14
30
  }
15
31
  export interface GatewayVmRecoveryObservation {
32
+ readonly channelProviderId?: string | undefined;
16
33
  readonly observedAtMs: number;
34
+ readonly recoveryBudgetClass?: GatewayVmRecoveryBudgetClass | undefined;
17
35
  readonly result: GatewayVmRecoveryObservationResult;
18
36
  readonly zoneId: string;
19
37
  }
20
38
  export interface GatewayVmRecoveryLifecycleEvent {
21
39
  readonly observedAtMs: number;
40
+ readonly recoveryBudgetClass?: GatewayVmRecoveryBudgetClass | undefined;
22
41
  readonly result?: 'failed' | 'ok' | undefined;
23
42
  readonly zoneId: string;
24
43
  }
@@ -41,6 +60,7 @@ export type GatewayVmRecoveryDecision = {
41
60
  export interface GatewayVmRecoveryTracker {
42
61
  readonly markRecoveryFinished: (event: GatewayVmRecoveryLifecycleEvent) => void;
43
62
  readonly markRecoveryStarted: (event: GatewayVmRecoveryLifecycleEvent) => void;
63
+ readonly recordAgentChannelProviderObservation: (observation: GatewayVmRecoveryObservation) => GatewayVmRecoveryDecision;
44
64
  readonly recordGatewayControlLinkObservation: (observation: GatewayVmRecoveryObservation) => GatewayVmRecoveryDecision;
45
65
  readonly recordGatewayServiceProbe: (observation: GatewayVmRecoveryObservation) => GatewayVmRecoveryDecision;
46
66
  }
@@ -1 +1 @@
1
- {"version":3,"file":"gateway-vm-recovery-policy.d.ts","sourceRoot":"","sources":["../../../src/controller/health/gateway-vm-recovery-policy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,6BAA6B,CAAC;AAE/E,MAAM,MAAM,kCAAkC,GAC3C,QAAQ,GACR,IAAI,GACJ,OAAO,GACP,SAAS,GACT,YAAY,CAAC;AAEhB,MAAM,MAAM,uBAAuB,GAAG,2BAA2B,CAAC;AAElE,MAAM,WAAW,2BAA2B;IAC3C,QAAQ,CAAC,2BAA2B,EAAE,MAAM,CAAC;IAC7C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC,QAAQ,CAAC,8BAA8B,EAAE,MAAM,CAAC;IAChD,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;CAClC;AAED,MAAM,WAAW,qCAAqC;IACrD,QAAQ,CAAC,MAAM,EAAE,2BAA2B,CAAC;CAC7C;AAED,MAAM,WAAW,4BAA4B;IAC5C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,kCAAkC,CAAC;IACpD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,+BAA+B;IAC/C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,GAAG,IAAI,GAAG,SAAS,CAAC;IAC9C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,yBAAyB,GAClC;IACA,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,CAAC,EAAE,UAAU,GAAG,UAAU,GAAG,WAAW,GAAG,YAAY,GAAG,SAAS,CAAC;CAClF,GACD;IACA,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,uBAAuB,CAAC;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACvB,GACD;IACA,QAAQ,CAAC,2BAA2B,EAAE,MAAM,CAAC;IAC7C,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,uBAAuB,CAAC;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACvB,CAAC;AAEL,MAAM,WAAW,wBAAwB;IACxC,QAAQ,CAAC,oBAAoB,EAAE,CAAC,KAAK,EAAE,+BAA+B,KAAK,IAAI,CAAC;IAChF,QAAQ,CAAC,mBAAmB,EAAE,CAAC,KAAK,EAAE,+BAA+B,KAAK,IAAI,CAAC;IAC/E,QAAQ,CAAC,mCAAmC,EAAE,CAC7C,WAAW,EAAE,4BAA4B,KACrC,yBAAyB,CAAC;IAC/B,QAAQ,CAAC,yBAAyB,EAAE,CACnC,WAAW,EAAE,4BAA4B,KACrC,yBAAyB,CAAC;CAC/B;AAqGD,wBAAgB,8BAA8B,CAC7C,OAAO,EAAE,qCAAqC,GAC5C,wBAAwB,CA4E1B"}
1
+ {"version":3,"file":"gateway-vm-recovery-policy.d.ts","sourceRoot":"","sources":["../../../src/controller/health/gateway-vm-recovery-policy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,6BAA6B,CAAC;AAE/E,MAAM,MAAM,kCAAkC,GAC3C,QAAQ,GACR,IAAI,GACJ,OAAO,GACP,SAAS,GACT,YAAY,CAAC;AAEhB,MAAM,MAAM,uBAAuB,GAAG,2BAA2B,CAAC;AAClE,MAAM,MAAM,4BAA4B,GAAG,uBAAuB,GAAG,oBAAoB,CAAC;AAE1F,MAAM,WAAW,2BAA2B;IAC3C,QAAQ,CAAC,qBAAqB,CAAC,EAAE,sCAAsC,GAAG,SAAS,CAAC;IACpF,QAAQ,CAAC,2BAA2B,EAAE,MAAM,CAAC;IAC7C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC,QAAQ,CAAC,8BAA8B,EAAE,MAAM,CAAC;IAChD,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;CAClC;AAED,MAAM,WAAW,sCAAsC;IACtD,QAAQ,CAAC,2BAA2B,EAAE,MAAM,CAAC;IAC7C,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,2BAA2B,EAAE,OAAO,CAAC;IAC9C,QAAQ,CAAC,6BAA6B,EAAE,OAAO,CAAC;IAChD,QAAQ,CAAC,sBAAsB,EAAE,MAAM,CAAC;CACxC;AAED,eAAO,MAAM,6CAA6C;;;;;;CAMR,CAAC;AAEnD,MAAM,WAAW,qCAAqC;IACrD,QAAQ,CAAC,MAAM,EAAE,2BAA2B,CAAC;CAC7C;AAED,MAAM,WAAW,4BAA4B;IAC5C,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,mBAAmB,CAAC,EAAE,4BAA4B,GAAG,SAAS,CAAC;IACxE,QAAQ,CAAC,MAAM,EAAE,kCAAkC,CAAC;IACpD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,+BAA+B;IAC/C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,mBAAmB,CAAC,EAAE,4BAA4B,GAAG,SAAS,CAAC;IACxE,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,GAAG,IAAI,GAAG,SAAS,CAAC;IAC9C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,yBAAyB,GAClC;IACA,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,CAAC,EAAE,UAAU,GAAG,UAAU,GAAG,WAAW,GAAG,YAAY,GAAG,SAAS,CAAC;CAClF,GACD;IACA,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,uBAAuB,CAAC;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACvB,GACD;IACA,QAAQ,CAAC,2BAA2B,EAAE,MAAM,CAAC;IAC7C,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,uBAAuB,CAAC;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACvB,CAAC;AAEL,MAAM,WAAW,wBAAwB;IACxC,QAAQ,CAAC,oBAAoB,EAAE,CAAC,KAAK,EAAE,+BAA+B,KAAK,IAAI,CAAC;IAChF,QAAQ,CAAC,mBAAmB,EAAE,CAAC,KAAK,EAAE,+BAA+B,KAAK,IAAI,CAAC;IAC/E,QAAQ,CAAC,qCAAqC,EAAE,CAC/C,WAAW,EAAE,4BAA4B,KACrC,yBAAyB,CAAC;IAC/B,QAAQ,CAAC,mCAAmC,EAAE,CAC7C,WAAW,EAAE,4BAA4B,KACrC,yBAAyB,CAAC;IAC/B,QAAQ,CAAC,yBAAyB,EAAE,CACnC,WAAW,EAAE,4BAA4B,KACrC,yBAAyB,CAAC;CAC/B;AAgID,wBAAgB,8BAA8B,CAC7C,OAAO,EAAE,qCAAqC,GAC5C,wBAAwB,CAuH1B"}