@agent-vm/gateway-interface 0.0.81 → 0.0.84
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +421 -17
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +643 -10
- package/dist/index.js.map +1 -1
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -25,6 +25,428 @@ function gatewayVmAllowedHosts(egressHosts) {
|
|
|
25
25
|
return Array.from(new Set([controllerVmHost, ...egressHostsForAudience(egressHosts, "gateway")]));
|
|
26
26
|
}
|
|
27
27
|
//#endregion
|
|
28
|
+
//#region src/health/controller-request-policy.ts
|
|
29
|
+
const gatewayInternalControllerRequestOperations = [
|
|
30
|
+
"controller-health",
|
|
31
|
+
"health-event-publish",
|
|
32
|
+
"openclaw-runtime-status",
|
|
33
|
+
"zone-git-push",
|
|
34
|
+
"lease-create",
|
|
35
|
+
"lease-get",
|
|
36
|
+
"lease-peek",
|
|
37
|
+
"lease-list",
|
|
38
|
+
"lease-renew",
|
|
39
|
+
"lease-release",
|
|
40
|
+
"lease-use-start",
|
|
41
|
+
"lease-heartbeat",
|
|
42
|
+
"lease-use-end"
|
|
43
|
+
];
|
|
44
|
+
const workerInternalControllerRequestOperations = ["worker-push-branches", "worker-pull-default"];
|
|
45
|
+
const dedicatedControllerRequestHealthEventOperationSet = new Set(["lease-heartbeat", "lease-renew"]);
|
|
46
|
+
function isGenericControllerRequestEventOperation(operation) {
|
|
47
|
+
return !dedicatedControllerRequestHealthEventOperationSet.has(operation);
|
|
48
|
+
}
|
|
49
|
+
const genericControllerRequestEventOperations = [...gatewayInternalControllerRequestOperations, ...workerInternalControllerRequestOperations].filter(isGenericControllerRequestEventOperation);
|
|
50
|
+
const externalControllerRoutes = [
|
|
51
|
+
"GET /controller-status",
|
|
52
|
+
"GET /zones/:zoneId/status",
|
|
53
|
+
"GET /zones/:zoneId/health",
|
|
54
|
+
"GET /zones/:zoneId/zone-git/status",
|
|
55
|
+
"GET /zones/:zoneId/logs",
|
|
56
|
+
"POST /zones/:zoneId/credentials/refresh",
|
|
57
|
+
"POST /zones/:zoneId/destroy",
|
|
58
|
+
"POST /zones/:zoneId/upgrade",
|
|
59
|
+
"GET /zones/:zoneId/tasks/:taskId",
|
|
60
|
+
"POST /zones/:zoneId/worker-tasks",
|
|
61
|
+
"POST /zones/:zoneId/tasks/:taskId/close",
|
|
62
|
+
"POST /zones/:zoneId/enable-ssh",
|
|
63
|
+
"POST /zones/:zoneId/execute-command",
|
|
64
|
+
"POST /stop-controller"
|
|
65
|
+
];
|
|
66
|
+
var ControllerRequestPolicyTransportError = class extends Error {
|
|
67
|
+
code;
|
|
68
|
+
operation;
|
|
69
|
+
constructor(options) {
|
|
70
|
+
const causeMessage = options.cause instanceof Error ? options.cause.message : String(options.cause);
|
|
71
|
+
super(`${options.operation} ${options.code}: ${causeMessage}`, { cause: options.cause });
|
|
72
|
+
this.code = options.code;
|
|
73
|
+
this.operation = options.operation;
|
|
74
|
+
}
|
|
75
|
+
};
|
|
76
|
+
function sleep(ms, signal) {
|
|
77
|
+
return new Promise((resolve, reject) => {
|
|
78
|
+
if (signal?.aborted === true) {
|
|
79
|
+
reject(signal.reason);
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
const timeout = setTimeout(() => {
|
|
83
|
+
signal?.removeEventListener("abort", onAbort);
|
|
84
|
+
resolve();
|
|
85
|
+
}, ms);
|
|
86
|
+
const onAbort = () => {
|
|
87
|
+
clearTimeout(timeout);
|
|
88
|
+
reject(signal?.reason);
|
|
89
|
+
};
|
|
90
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
function shouldRetryResponse(response, policy) {
|
|
94
|
+
return policy.retryEnabled && policy.retryStatuses.includes(response.status);
|
|
95
|
+
}
|
|
96
|
+
async function drainControllerResponseBody(response) {
|
|
97
|
+
if (response.bodyUsed) return;
|
|
98
|
+
await response.text().catch(() => void 0);
|
|
99
|
+
}
|
|
100
|
+
async function fetchWithTimeout(options) {
|
|
101
|
+
const abortController = new AbortController();
|
|
102
|
+
let callerAborted = options.init?.signal?.aborted ?? false;
|
|
103
|
+
let timedOut = false;
|
|
104
|
+
const abortFromCaller = () => {
|
|
105
|
+
callerAborted = true;
|
|
106
|
+
abortController.abort(options.init?.signal?.reason);
|
|
107
|
+
};
|
|
108
|
+
const timeout = setTimeout(() => {
|
|
109
|
+
timedOut = true;
|
|
110
|
+
abortController.abort(/* @__PURE__ */ new Error(`${options.operation} timed out after ${String(options.timeoutMs)}ms`));
|
|
111
|
+
}, options.timeoutMs);
|
|
112
|
+
if (callerAborted) abortController.abort(options.init?.signal?.reason);
|
|
113
|
+
else options.init?.signal?.addEventListener("abort", abortFromCaller, { once: true });
|
|
114
|
+
try {
|
|
115
|
+
return await options.fetchImpl(options.input, {
|
|
116
|
+
...options.init,
|
|
117
|
+
signal: abortController.signal
|
|
118
|
+
});
|
|
119
|
+
} catch (error) {
|
|
120
|
+
if (callerAborted) throw error;
|
|
121
|
+
throw new ControllerRequestPolicyTransportError({
|
|
122
|
+
cause: error,
|
|
123
|
+
code: timedOut ? "controller-request-timeout" : "controller-request-failed",
|
|
124
|
+
operation: options.operation
|
|
125
|
+
});
|
|
126
|
+
} finally {
|
|
127
|
+
clearTimeout(timeout);
|
|
128
|
+
options.init?.signal?.removeEventListener("abort", abortFromCaller);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
async function fetchControllerWithPolicy(options) {
|
|
132
|
+
const policy = options.policy ?? controllerRequestPolicies[options.operation];
|
|
133
|
+
const fetchImpl = options.fetchImpl ?? fetch;
|
|
134
|
+
let lastTransportError;
|
|
135
|
+
for (let attempt = 1; attempt <= policy.maxAttempts; attempt += 1) try {
|
|
136
|
+
const response = await fetchWithTimeout({
|
|
137
|
+
fetchImpl,
|
|
138
|
+
init: options.init,
|
|
139
|
+
input: options.input,
|
|
140
|
+
operation: options.operation,
|
|
141
|
+
timeoutMs: policy.timeoutMs
|
|
142
|
+
});
|
|
143
|
+
if (attempt < policy.maxAttempts && shouldRetryResponse(response, policy)) {
|
|
144
|
+
await drainControllerResponseBody(response);
|
|
145
|
+
if (policy.retryBaseDelayMs > 0) await sleep(policy.retryBaseDelayMs, options.init?.signal ?? void 0);
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
return response;
|
|
149
|
+
} catch (error) {
|
|
150
|
+
if (!(error instanceof ControllerRequestPolicyTransportError)) throw error;
|
|
151
|
+
lastTransportError = error;
|
|
152
|
+
if (!(policy.retryEnabled && attempt < policy.maxAttempts)) throw error;
|
|
153
|
+
if (policy.retryBaseDelayMs > 0) await sleep(policy.retryBaseDelayMs, options.init?.signal ?? void 0);
|
|
154
|
+
}
|
|
155
|
+
throw lastTransportError ?? /* @__PURE__ */ new Error(`${options.operation} failed without a response`);
|
|
156
|
+
}
|
|
157
|
+
const controllerRequestPolicies = {
|
|
158
|
+
"controller-health": {
|
|
159
|
+
idempotency: "read",
|
|
160
|
+
maxAttempts: 1,
|
|
161
|
+
retryBaseDelayMs: 0,
|
|
162
|
+
retryEnabled: false,
|
|
163
|
+
retryStatuses: [],
|
|
164
|
+
timeoutMs: 3e3
|
|
165
|
+
},
|
|
166
|
+
"health-event-publish": {
|
|
167
|
+
idempotency: "safe-mutation",
|
|
168
|
+
maxAttempts: 1,
|
|
169
|
+
retryBaseDelayMs: 0,
|
|
170
|
+
retryEnabled: false,
|
|
171
|
+
retryStatuses: [],
|
|
172
|
+
timeoutMs: 3e3
|
|
173
|
+
},
|
|
174
|
+
"openclaw-runtime-status": {
|
|
175
|
+
idempotency: "safe-mutation",
|
|
176
|
+
maxAttempts: 30,
|
|
177
|
+
retryBaseDelayMs: 1e3,
|
|
178
|
+
retryEnabled: true,
|
|
179
|
+
retryStatuses: [
|
|
180
|
+
429,
|
|
181
|
+
503,
|
|
182
|
+
504
|
|
183
|
+
],
|
|
184
|
+
timeoutMs: 3e3
|
|
185
|
+
},
|
|
186
|
+
"zone-git-push": {
|
|
187
|
+
idempotency: "unsafe-mutation",
|
|
188
|
+
maxAttempts: 1,
|
|
189
|
+
retryBaseDelayMs: 0,
|
|
190
|
+
retryEnabled: false,
|
|
191
|
+
retryStatuses: [],
|
|
192
|
+
timeoutMs: 12e4
|
|
193
|
+
},
|
|
194
|
+
"lease-create": {
|
|
195
|
+
idempotency: "unsafe-mutation",
|
|
196
|
+
maxAttempts: 1,
|
|
197
|
+
retryBaseDelayMs: 0,
|
|
198
|
+
retryEnabled: false,
|
|
199
|
+
retryStatuses: [],
|
|
200
|
+
timeoutMs: 18e4
|
|
201
|
+
},
|
|
202
|
+
"lease-get": {
|
|
203
|
+
idempotency: "read",
|
|
204
|
+
maxAttempts: 2,
|
|
205
|
+
retryBaseDelayMs: 250,
|
|
206
|
+
retryEnabled: true,
|
|
207
|
+
retryStatuses: [503, 504],
|
|
208
|
+
timeoutMs: 5e3
|
|
209
|
+
},
|
|
210
|
+
"lease-peek": {
|
|
211
|
+
idempotency: "read",
|
|
212
|
+
maxAttempts: 2,
|
|
213
|
+
retryBaseDelayMs: 250,
|
|
214
|
+
retryEnabled: true,
|
|
215
|
+
retryStatuses: [503, 504],
|
|
216
|
+
timeoutMs: 5e3
|
|
217
|
+
},
|
|
218
|
+
"lease-list": {
|
|
219
|
+
idempotency: "read",
|
|
220
|
+
maxAttempts: 2,
|
|
221
|
+
retryBaseDelayMs: 250,
|
|
222
|
+
retryEnabled: true,
|
|
223
|
+
retryStatuses: [503, 504],
|
|
224
|
+
timeoutMs: 5e3
|
|
225
|
+
},
|
|
226
|
+
"lease-renew": {
|
|
227
|
+
idempotency: "safe-mutation",
|
|
228
|
+
maxAttempts: 3,
|
|
229
|
+
retryBaseDelayMs: 250,
|
|
230
|
+
retryEnabled: true,
|
|
231
|
+
retryStatuses: [
|
|
232
|
+
429,
|
|
233
|
+
503,
|
|
234
|
+
504
|
|
235
|
+
],
|
|
236
|
+
timeoutMs: 1e4
|
|
237
|
+
},
|
|
238
|
+
"lease-release": {
|
|
239
|
+
idempotency: "safe-mutation",
|
|
240
|
+
maxAttempts: 2,
|
|
241
|
+
retryBaseDelayMs: 250,
|
|
242
|
+
retryEnabled: true,
|
|
243
|
+
retryStatuses: [503, 504],
|
|
244
|
+
timeoutMs: 5e3
|
|
245
|
+
},
|
|
246
|
+
"lease-use-start": {
|
|
247
|
+
idempotency: "safe-mutation",
|
|
248
|
+
maxAttempts: 2,
|
|
249
|
+
retryBaseDelayMs: 250,
|
|
250
|
+
retryEnabled: true,
|
|
251
|
+
retryStatuses: [
|
|
252
|
+
429,
|
|
253
|
+
503,
|
|
254
|
+
504
|
|
255
|
+
],
|
|
256
|
+
timeoutMs: 1e4
|
|
257
|
+
},
|
|
258
|
+
"lease-heartbeat": {
|
|
259
|
+
idempotency: "safe-mutation",
|
|
260
|
+
maxAttempts: 2,
|
|
261
|
+
retryBaseDelayMs: 250,
|
|
262
|
+
retryEnabled: true,
|
|
263
|
+
retryStatuses: [
|
|
264
|
+
429,
|
|
265
|
+
503,
|
|
266
|
+
504
|
|
267
|
+
],
|
|
268
|
+
timeoutMs: 5e3
|
|
269
|
+
},
|
|
270
|
+
"lease-use-end": {
|
|
271
|
+
idempotency: "safe-mutation",
|
|
272
|
+
maxAttempts: 2,
|
|
273
|
+
retryBaseDelayMs: 250,
|
|
274
|
+
retryEnabled: true,
|
|
275
|
+
retryStatuses: [503, 504],
|
|
276
|
+
timeoutMs: 5e3
|
|
277
|
+
},
|
|
278
|
+
"worker-push-branches": {
|
|
279
|
+
idempotency: "unsafe-mutation",
|
|
280
|
+
maxAttempts: 1,
|
|
281
|
+
retryBaseDelayMs: 0,
|
|
282
|
+
retryEnabled: false,
|
|
283
|
+
retryStatuses: [],
|
|
284
|
+
timeoutMs: 12e4
|
|
285
|
+
},
|
|
286
|
+
"worker-pull-default": {
|
|
287
|
+
idempotency: "unsafe-mutation",
|
|
288
|
+
maxAttempts: 1,
|
|
289
|
+
retryBaseDelayMs: 0,
|
|
290
|
+
retryEnabled: false,
|
|
291
|
+
retryStatuses: [],
|
|
292
|
+
timeoutMs: 12e4
|
|
293
|
+
}
|
|
294
|
+
};
|
|
295
|
+
//#endregion
|
|
296
|
+
//#region src/health/agent-vm-health.ts
|
|
297
|
+
const agentVmHealthEventKinds = [
|
|
298
|
+
"gateway-service-health",
|
|
299
|
+
"gateway-control-link",
|
|
300
|
+
"controller-request",
|
|
301
|
+
"lease-renew",
|
|
302
|
+
"lease-heartbeat",
|
|
303
|
+
"tool-vm-ssh",
|
|
304
|
+
"gateway-plugin-health"
|
|
305
|
+
];
|
|
306
|
+
const agentVmHealthResultKinds = [
|
|
307
|
+
"ok",
|
|
308
|
+
"failed",
|
|
309
|
+
"timeout",
|
|
310
|
+
"stale"
|
|
311
|
+
];
|
|
312
|
+
const gatewayControlLinkHealthPins = {
|
|
313
|
+
controllerHost: "controller.vm.host",
|
|
314
|
+
controllerPort: 18800,
|
|
315
|
+
operation: "controller-health",
|
|
316
|
+
path: "/health"
|
|
317
|
+
};
|
|
318
|
+
const zoneHealthStateKinds = [
|
|
319
|
+
"unknown",
|
|
320
|
+
"ok",
|
|
321
|
+
"stale",
|
|
322
|
+
"failed"
|
|
323
|
+
];
|
|
324
|
+
const zoneHealthIssueKinds = [
|
|
325
|
+
"gateway-service-unhealthy",
|
|
326
|
+
"gateway-control-link-unhealthy",
|
|
327
|
+
"controller-request-failing",
|
|
328
|
+
"lease-heartbeat-failing",
|
|
329
|
+
"lease-renew-failing",
|
|
330
|
+
"tool-vm-ssh-failing",
|
|
331
|
+
"gateway-plugin-unhealthy",
|
|
332
|
+
"health-event-stale"
|
|
333
|
+
];
|
|
334
|
+
function isRecord(value) {
|
|
335
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
336
|
+
}
|
|
337
|
+
function isOneOf(values, value) {
|
|
338
|
+
return typeof value === "string" && values.includes(value);
|
|
339
|
+
}
|
|
340
|
+
function isNonNegativeFiniteNumber(value) {
|
|
341
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0;
|
|
342
|
+
}
|
|
343
|
+
function hasBaseEventFields(record) {
|
|
344
|
+
return isNonNegativeFiniteNumber(record.observedAtMs) && isOneOf(agentVmHealthResultKinds, record.result) && typeof record.zoneId === "string" && record.zoneId.length > 0;
|
|
345
|
+
}
|
|
346
|
+
function optionalString(value) {
|
|
347
|
+
return value === void 0 || typeof value === "string";
|
|
348
|
+
}
|
|
349
|
+
function optionalStatusCode(value) {
|
|
350
|
+
return value === void 0 || Number.isInteger(value);
|
|
351
|
+
}
|
|
352
|
+
function isAgentVmHealthEvent(value) {
|
|
353
|
+
if (!isRecord(value) || !hasBaseEventFields(value)) return false;
|
|
354
|
+
switch (value.kind) {
|
|
355
|
+
case "gateway-service-health": return typeof value.path === "string" && value.path.length > 0 && Number.isInteger(value.port) && optionalStatusCode(value.statusCode);
|
|
356
|
+
case "gateway-control-link": return value.controllerHost === gatewayControlLinkHealthPins.controllerHost && value.controllerPort === gatewayControlLinkHealthPins.controllerPort && isNonNegativeFiniteNumber(value.elapsedMs) && value.operation === gatewayControlLinkHealthPins.operation && value.path === gatewayControlLinkHealthPins.path;
|
|
357
|
+
case "controller-request": return Number.isInteger(value.attempt) && isNonNegativeFiniteNumber(value.elapsedMs) && optionalString(value.errorCode) && Number.isInteger(value.maxAttempts) && isOneOf(genericControllerRequestEventOperations, value.operation) && optionalStatusCode(value.statusCode);
|
|
358
|
+
case "lease-renew": return typeof value.agentId === "string" && isNonNegativeFiniteNumber(value.elapsedMs) && optionalString(value.errorCode) && typeof value.leaseId === "string";
|
|
359
|
+
case "lease-heartbeat": return typeof value.agentId === "string" && isNonNegativeFiniteNumber(value.elapsedMs) && optionalString(value.errorCode) && typeof value.leaseId === "string" && typeof value.useId === "string";
|
|
360
|
+
case "tool-vm-ssh": return typeof value.agentId === "string" && isNonNegativeFiniteNumber(value.elapsedMs) && optionalString(value.errorCode) && typeof value.leaseId === "string" && isOneOf([
|
|
361
|
+
"command",
|
|
362
|
+
"file-bridge",
|
|
363
|
+
"finalize",
|
|
364
|
+
"probe"
|
|
365
|
+
], value.operation);
|
|
366
|
+
case "gateway-plugin-health": return isOneOf(gatewayTypeValues, value.gatewayService) && isOneOf([
|
|
367
|
+
"starting",
|
|
368
|
+
"ready",
|
|
369
|
+
"stopping",
|
|
370
|
+
"failed"
|
|
371
|
+
], value.state);
|
|
372
|
+
default: return false;
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
function healthEventBucketKey(event) {
|
|
376
|
+
switch (event.kind) {
|
|
377
|
+
case "gateway-control-link": return `${event.zoneId}:${event.kind}`;
|
|
378
|
+
case "gateway-service-health": return `${event.zoneId}:${event.kind}:${event.port}:${event.path}`;
|
|
379
|
+
case "controller-request": return `${event.zoneId}:${event.kind}:${event.operation}`;
|
|
380
|
+
case "lease-heartbeat": return `${event.zoneId}:${event.kind}:${event.leaseId}:${event.useId}`;
|
|
381
|
+
case "lease-renew": return `${event.zoneId}:${event.kind}:${event.leaseId}`;
|
|
382
|
+
case "tool-vm-ssh": return `${event.zoneId}:${event.kind}:${event.leaseId}:${event.operation}`;
|
|
383
|
+
case "gateway-plugin-health": return `${event.zoneId}:${event.kind}:${event.gatewayService}`;
|
|
384
|
+
}
|
|
385
|
+
return assertNeverHealthEvent(event);
|
|
386
|
+
}
|
|
387
|
+
function failedIssueKindForEvent(event) {
|
|
388
|
+
switch (event.kind) {
|
|
389
|
+
case "gateway-service-health": return "gateway-service-unhealthy";
|
|
390
|
+
case "gateway-control-link": return "gateway-control-link-unhealthy";
|
|
391
|
+
case "controller-request": return "controller-request-failing";
|
|
392
|
+
case "lease-heartbeat": return "lease-heartbeat-failing";
|
|
393
|
+
case "lease-renew": return "lease-renew-failing";
|
|
394
|
+
case "tool-vm-ssh": return "tool-vm-ssh-failing";
|
|
395
|
+
case "gateway-plugin-health": return "gateway-plugin-unhealthy";
|
|
396
|
+
}
|
|
397
|
+
return assertNeverHealthEvent(event);
|
|
398
|
+
}
|
|
399
|
+
function assertNeverHealthEvent(event) {
|
|
400
|
+
throw new Error(`Unhandled health event kind: ${JSON.stringify(event)}`);
|
|
401
|
+
}
|
|
402
|
+
function issueForEvent(event, options) {
|
|
403
|
+
if (options.nowMs - event.observedAtMs > options.staleAfterMs) return {
|
|
404
|
+
kind: "health-event-stale",
|
|
405
|
+
latestEvent: event,
|
|
406
|
+
message: `${event.kind} health event is stale`,
|
|
407
|
+
sinceMs: event.observedAtMs
|
|
408
|
+
};
|
|
409
|
+
if (event.result === "failed" || event.result === "timeout" || event.result === "stale") return {
|
|
410
|
+
kind: failedIssueKindForEvent(event),
|
|
411
|
+
latestEvent: event,
|
|
412
|
+
message: `${event.kind} health event reported ${event.result}`,
|
|
413
|
+
sinceMs: event.observedAtMs
|
|
414
|
+
};
|
|
415
|
+
}
|
|
416
|
+
function deriveZoneHealthSnapshot(events, options) {
|
|
417
|
+
const latestByKey = /* @__PURE__ */ new Map();
|
|
418
|
+
for (const event of events) {
|
|
419
|
+
if (event.zoneId !== options.zoneId) continue;
|
|
420
|
+
const key = healthEventBucketKey(event);
|
|
421
|
+
const previous = latestByKey.get(key);
|
|
422
|
+
if (!previous || previous.observedAtMs <= event.observedAtMs) latestByKey.set(key, event);
|
|
423
|
+
}
|
|
424
|
+
const latestEvents = [...latestByKey.values()].toSorted((first, second) => second.observedAtMs - first.observedAtMs);
|
|
425
|
+
if (latestEvents.length === 0) return {
|
|
426
|
+
kind: "unknown",
|
|
427
|
+
reason: "no-events",
|
|
428
|
+
zoneId: options.zoneId
|
|
429
|
+
};
|
|
430
|
+
const issues = latestEvents.map((event) => issueForEvent(event, options)).filter((issue) => issue !== void 0);
|
|
431
|
+
if (issues.length === 0) return {
|
|
432
|
+
kind: "ok",
|
|
433
|
+
latestEvents,
|
|
434
|
+
zoneId: options.zoneId
|
|
435
|
+
};
|
|
436
|
+
if (issues.some((issue) => issue.kind === "health-event-stale")) return {
|
|
437
|
+
issues,
|
|
438
|
+
kind: "stale",
|
|
439
|
+
latestEvents,
|
|
440
|
+
zoneId: options.zoneId
|
|
441
|
+
};
|
|
442
|
+
return {
|
|
443
|
+
issues,
|
|
444
|
+
kind: "failed",
|
|
445
|
+
latestEvents,
|
|
446
|
+
zoneId: options.zoneId
|
|
447
|
+
};
|
|
448
|
+
}
|
|
449
|
+
//#endregion
|
|
28
450
|
//#region src/force-ipv4-egress.ts
|
|
29
451
|
/**
|
|
30
452
|
* Canonical NODE_OPTIONS value for forcing IPv4-preference egress
|
|
@@ -62,6 +484,7 @@ function gatewayVmAllowedHosts(egressHosts) {
|
|
|
62
484
|
* revert recommendation by the Node core team).
|
|
63
485
|
*/
|
|
64
486
|
const FORCE_IPV4_EGRESS_NODE_OPTIONS = "--dns-result-order=ipv4first --no-network-family-autoselection";
|
|
487
|
+
const FORCE_IPV4_EGRESS_NODE_OPTION_FLAGS = FORCE_IPV4_EGRESS_NODE_OPTIONS.split(/\s+/u);
|
|
65
488
|
/**
|
|
66
489
|
* Compose the forced IPv4-preference flags with a user-provided
|
|
67
490
|
* NODE_OPTIONS value (if any).
|
|
@@ -72,8 +495,9 @@ const FORCE_IPV4_EGRESS_NODE_OPTIONS = "--dns-result-order=ipv4first --no-networ
|
|
|
72
495
|
* a zone secret happens to provide its own NODE_OPTIONS.
|
|
73
496
|
*
|
|
74
497
|
* Forced flags come FIRST so they are unambiguously applied.
|
|
75
|
-
* User-provided flags are appended verbatim
|
|
76
|
-
*
|
|
498
|
+
* User-provided flags are appended verbatim except for duplicate
|
|
499
|
+
* forced IPv4-preference flags. Node treats NODE_OPTIONS as a
|
|
500
|
+
* whitespace-separated list and all flags apply.
|
|
77
501
|
*
|
|
78
502
|
* Returns just the forced flags if the user value is undefined,
|
|
79
503
|
* empty, or whitespace-only.
|
|
@@ -93,7 +517,9 @@ const FORCE_IPV4_EGRESS_NODE_OPTIONS = "--dns-result-order=ipv4first --no-networ
|
|
|
93
517
|
function composeNodeOptions(userValue) {
|
|
94
518
|
const trimmed = userValue?.trim() ?? "";
|
|
95
519
|
if (trimmed === "") return FORCE_IPV4_EGRESS_NODE_OPTIONS;
|
|
96
|
-
|
|
520
|
+
const userFlags = trimmed.split(/\s+/u).filter((flag) => !FORCE_IPV4_EGRESS_NODE_OPTION_FLAGS.includes(flag));
|
|
521
|
+
if (userFlags.length === 0) return FORCE_IPV4_EGRESS_NODE_OPTIONS;
|
|
522
|
+
return `${FORCE_IPV4_EGRESS_NODE_OPTIONS} ${userFlags.join(" ")}`;
|
|
97
523
|
}
|
|
98
524
|
//#endregion
|
|
99
525
|
//#region src/split-resolved-gateway-secrets.ts
|
|
@@ -153,6 +579,12 @@ function mergeRuntimeGatewaySecrets(baseSecrets, options = {}) {
|
|
|
153
579
|
//#endregion
|
|
154
580
|
//#region src/tool-vm-active-use.ts
|
|
155
581
|
const defaultMaxHeartbeatDurationMs = 720 * 60 * 1e3;
|
|
582
|
+
function jitterDelayMs(params) {
|
|
583
|
+
if (params.jitterRatio <= 0) return params.delayMs;
|
|
584
|
+
const spreadMs = params.delayMs * params.jitterRatio;
|
|
585
|
+
const jitteredMs = params.delayMs - spreadMs + params.random() * spreadMs * 2;
|
|
586
|
+
return Math.max(1, Math.round(jitteredMs));
|
|
587
|
+
}
|
|
156
588
|
function createToolVmActiveUseId() {
|
|
157
589
|
return v7();
|
|
158
590
|
}
|
|
@@ -170,8 +602,12 @@ async function createToolVmActiveUseHandle(options) {
|
|
|
170
602
|
const now = options.nowImpl ?? Date.now;
|
|
171
603
|
const startedAt = now();
|
|
172
604
|
const maxHeartbeatDurationMs = options.maxHeartbeatDurationMs ?? defaultMaxHeartbeatDurationMs;
|
|
605
|
+
const heartbeatJitterRatio = options.heartbeatJitterRatio ?? .1;
|
|
606
|
+
const random = options.randomImpl ?? Math.random;
|
|
607
|
+
const operationAbortController = new AbortController();
|
|
173
608
|
let ended = false;
|
|
174
609
|
let heartbeatTimer;
|
|
610
|
+
let latestReport;
|
|
175
611
|
const clearHeartbeatTimer = () => {
|
|
176
612
|
if (heartbeatTimer) {
|
|
177
613
|
clearTimeoutImpl(heartbeatTimer);
|
|
@@ -183,13 +619,27 @@ async function createToolVmActiveUseHandle(options) {
|
|
|
183
619
|
clearHeartbeatTimer();
|
|
184
620
|
heartbeatTimer = setTimeoutImpl(() => {
|
|
185
621
|
if (now() - startedAt >= maxHeartbeatDurationMs) return;
|
|
186
|
-
|
|
622
|
+
const heartbeatRequest = latestReport === void 0 ? {} : { report: latestReport };
|
|
623
|
+
options.heartbeatActiveUse(startedUse.useId, heartbeatRequest).then((heartbeat) => {
|
|
187
624
|
if (!ended) scheduleHeartbeat(heartbeat.heartbeatAfterMs);
|
|
188
625
|
}).catch((error) => {
|
|
189
626
|
options.logHeartbeatFailure?.(error);
|
|
627
|
+
if (options.isHeartbeatErrorRefreshable?.(error) === true && options.onRefreshableHeartbeatFailure) {
|
|
628
|
+
operationAbortController.abort(error);
|
|
629
|
+
ended = true;
|
|
630
|
+
clearHeartbeatTimer();
|
|
631
|
+
options.onRefreshableHeartbeatFailure(error).catch((staleError) => {
|
|
632
|
+
options.logHeartbeatFailure?.(staleError);
|
|
633
|
+
});
|
|
634
|
+
return;
|
|
635
|
+
}
|
|
190
636
|
if (!ended) scheduleHeartbeat(startedUse.heartbeatAfterMs);
|
|
191
637
|
});
|
|
192
|
-
},
|
|
638
|
+
}, jitterDelayMs({
|
|
639
|
+
delayMs,
|
|
640
|
+
jitterRatio: heartbeatJitterRatio,
|
|
641
|
+
random
|
|
642
|
+
}));
|
|
193
643
|
};
|
|
194
644
|
scheduleHeartbeat(startedUse.heartbeatAfterMs);
|
|
195
645
|
const end = async (outcome = "completed") => {
|
|
@@ -197,7 +647,10 @@ async function createToolVmActiveUseHandle(options) {
|
|
|
197
647
|
ended = true;
|
|
198
648
|
clearHeartbeatTimer();
|
|
199
649
|
try {
|
|
200
|
-
await options.endActiveUse(startedUse.useId, {
|
|
650
|
+
await options.endActiveUse(startedUse.useId, {
|
|
651
|
+
outcome,
|
|
652
|
+
...latestReport === void 0 ? {} : { report: latestReport }
|
|
653
|
+
});
|
|
201
654
|
} catch (error) {
|
|
202
655
|
if (options.isEndErrorTolerable?.(error) === true) {
|
|
203
656
|
options.logEndFailure?.(error);
|
|
@@ -207,11 +660,190 @@ async function createToolVmActiveUseHandle(options) {
|
|
|
207
660
|
}
|
|
208
661
|
};
|
|
209
662
|
return {
|
|
663
|
+
signal: operationAbortController.signal,
|
|
210
664
|
useId: startedUse.useId,
|
|
211
665
|
dispose: end,
|
|
212
|
-
end
|
|
666
|
+
end,
|
|
667
|
+
report: (report) => {
|
|
668
|
+
if (ended) return;
|
|
669
|
+
latestReport = report;
|
|
670
|
+
}
|
|
671
|
+
};
|
|
672
|
+
}
|
|
673
|
+
//#endregion
|
|
674
|
+
//#region src/runtime-paths/runtime-path-mapping.ts
|
|
675
|
+
const TOOL_VM_WORKSPACE_GUEST_ROOT = "/workspace";
|
|
676
|
+
const TOOL_VM_SCRATCH_GUEST_ROOT = "/work";
|
|
677
|
+
const OPENCLAW_STATE_VM_ROOT = "/home/openclaw/.openclaw/state";
|
|
678
|
+
const OPENCLAW_STATE_SANDBOXES_VM_ROOT = `${OPENCLAW_STATE_VM_ROOT}/sandboxes`;
|
|
679
|
+
const guidanceNamespaceOrder = [
|
|
680
|
+
"tool-vm-guest",
|
|
681
|
+
"openclaw-gateway",
|
|
682
|
+
"controller-host"
|
|
683
|
+
];
|
|
684
|
+
function pathContainsParentTraversal(inputPath) {
|
|
685
|
+
return inputPath.split(/\/+/u).includes("..");
|
|
686
|
+
}
|
|
687
|
+
function normalizeAbsolutePath(inputPath) {
|
|
688
|
+
return `/${inputPath.split("/").filter((segment) => segment !== "" && segment !== ".").join("/")}`;
|
|
689
|
+
}
|
|
690
|
+
function normalizeRoot(rootPath) {
|
|
691
|
+
const normalizedRoot = normalizeAbsolutePath(rootPath);
|
|
692
|
+
return normalizedRoot === "/" ? normalizedRoot : normalizedRoot.replace(/\/+$/u, "");
|
|
693
|
+
}
|
|
694
|
+
function pathMatchesRoot(candidatePath, rootPath) {
|
|
695
|
+
return candidatePath === rootPath || candidatePath.startsWith(`${rootPath}/`);
|
|
696
|
+
}
|
|
697
|
+
function relativePathForRoot(candidatePath, rootPath) {
|
|
698
|
+
return candidatePath === rootPath ? "" : candidatePath.slice(rootPath.length + 1);
|
|
699
|
+
}
|
|
700
|
+
function joinRootAndRelative(rootPath, relativePath) {
|
|
701
|
+
return relativePath === "" ? rootPath : `${rootPath}/${relativePath}`;
|
|
702
|
+
}
|
|
703
|
+
function runtimeRootIsInvalid(rootPath) {
|
|
704
|
+
return rootPath.trim() === "" || !rootPath.startsWith("/") || pathContainsParentTraversal(rootPath);
|
|
705
|
+
}
|
|
706
|
+
function namespaceShouldShowInGuidance(root, namespace) {
|
|
707
|
+
if (root.showInGuidance?.[namespace] !== void 0) return root.showInGuidance[namespace];
|
|
708
|
+
return namespace !== "controller-host";
|
|
709
|
+
}
|
|
710
|
+
function allowedPathFormsForMapping(mapping, purpose) {
|
|
711
|
+
return mapping.roots.flatMap((root) => {
|
|
712
|
+
if (!root.capabilities[purpose]) return [];
|
|
713
|
+
const suffix = root.rootPathAllowed ? "[/subpath]" : "/<child>";
|
|
714
|
+
return guidanceNamespaceOrder.flatMap((namespace) => {
|
|
715
|
+
const rootPath = root.locations[namespace];
|
|
716
|
+
if (rootPath === void 0 || !namespaceShouldShowInGuidance(root, namespace)) return [];
|
|
717
|
+
return [`${normalizeRoot(rootPath)}${suffix}`];
|
|
718
|
+
});
|
|
719
|
+
});
|
|
720
|
+
}
|
|
721
|
+
function retryGuidanceForMapping(mapping, purpose) {
|
|
722
|
+
return `Use one of the allowed path forms for ${mapping.id} ${purpose}: ${allowedPathFormsForMapping(mapping, purpose).join(", ")}.`;
|
|
723
|
+
}
|
|
724
|
+
function errorResult(params) {
|
|
725
|
+
return {
|
|
726
|
+
error: {
|
|
727
|
+
allowedPathForms: allowedPathFormsForMapping(params.mapping, params.purpose),
|
|
728
|
+
code: params.code,
|
|
729
|
+
inputPath: params.inputPath,
|
|
730
|
+
mappingId: params.mapping.id,
|
|
731
|
+
message: params.message,
|
|
732
|
+
purpose: params.purpose,
|
|
733
|
+
retryGuidance: retryGuidanceForMapping(params.mapping, params.purpose)
|
|
734
|
+
},
|
|
735
|
+
ok: false
|
|
213
736
|
};
|
|
214
737
|
}
|
|
738
|
+
function findBestRootMatch(params) {
|
|
739
|
+
return params.mapping.roots.flatMap((root) => guidanceNamespaceOrder.flatMap((inputNamespace) => {
|
|
740
|
+
const rootPath = root.locations[inputNamespace];
|
|
741
|
+
if (rootPath === void 0) return [];
|
|
742
|
+
if (params.sourceNamespace !== void 0 && inputNamespace !== params.sourceNamespace) return [];
|
|
743
|
+
const normalizedRoot = normalizeRoot(rootPath);
|
|
744
|
+
return pathMatchesRoot(params.inputPath, normalizedRoot) ? [{
|
|
745
|
+
inputNamespace,
|
|
746
|
+
matchedRoot: normalizedRoot,
|
|
747
|
+
root
|
|
748
|
+
}] : [];
|
|
749
|
+
})).toSorted((left, right) => right.matchedRoot.length - left.matchedRoot.length)[0];
|
|
750
|
+
}
|
|
751
|
+
function findInvalidRoot(mapping) {
|
|
752
|
+
for (const root of mapping.roots) for (const namespace of guidanceNamespaceOrder) {
|
|
753
|
+
const rootPath = root.locations[namespace];
|
|
754
|
+
if (rootPath !== void 0 && runtimeRootIsInvalid(rootPath)) return {
|
|
755
|
+
rootId: root.id,
|
|
756
|
+
rootPath
|
|
757
|
+
};
|
|
758
|
+
}
|
|
759
|
+
}
|
|
760
|
+
function translateRuntimePath(input) {
|
|
761
|
+
if (!input.inputPath.startsWith("/")) return errorResult({
|
|
762
|
+
code: "path-not-absolute",
|
|
763
|
+
inputPath: input.inputPath,
|
|
764
|
+
mapping: input.mapping,
|
|
765
|
+
message: `Path '${input.inputPath}' must be absolute.`,
|
|
766
|
+
purpose: input.purpose
|
|
767
|
+
});
|
|
768
|
+
if (pathContainsParentTraversal(input.inputPath)) return errorResult({
|
|
769
|
+
code: "path-parent-traversal",
|
|
770
|
+
inputPath: input.inputPath,
|
|
771
|
+
mapping: input.mapping,
|
|
772
|
+
message: `Path '${input.inputPath}' must not contain parent traversal.`,
|
|
773
|
+
purpose: input.purpose
|
|
774
|
+
});
|
|
775
|
+
const invalidRoot = findInvalidRoot(input.mapping);
|
|
776
|
+
if (invalidRoot !== void 0) return errorResult({
|
|
777
|
+
code: "invalid-runtime-root",
|
|
778
|
+
inputPath: input.inputPath,
|
|
779
|
+
mapping: input.mapping,
|
|
780
|
+
message: `Runtime path root '${invalidRoot.rootId}' has invalid path '${invalidRoot.rootPath}'.`,
|
|
781
|
+
purpose: input.purpose
|
|
782
|
+
});
|
|
783
|
+
const normalizedInputPath = normalizeAbsolutePath(input.inputPath);
|
|
784
|
+
const match = findBestRootMatch({
|
|
785
|
+
inputPath: normalizedInputPath,
|
|
786
|
+
mapping: input.mapping,
|
|
787
|
+
...input.sourceNamespace === void 0 ? {} : { sourceNamespace: input.sourceNamespace }
|
|
788
|
+
});
|
|
789
|
+
if (match === void 0) return errorResult({
|
|
790
|
+
code: "unknown-runtime-path",
|
|
791
|
+
inputPath: normalizedInputPath,
|
|
792
|
+
mapping: input.mapping,
|
|
793
|
+
message: `Path '${normalizedInputPath}' is not part of runtime path mapping '${input.mapping.id}'.`,
|
|
794
|
+
purpose: input.purpose
|
|
795
|
+
});
|
|
796
|
+
const relativePath = relativePathForRoot(normalizedInputPath, match.matchedRoot);
|
|
797
|
+
if (relativePath === "" && !match.root.rootPathAllowed) return errorResult({
|
|
798
|
+
code: "root-path-not-allowed",
|
|
799
|
+
inputPath: normalizedInputPath,
|
|
800
|
+
mapping: input.mapping,
|
|
801
|
+
message: `Path '${normalizedInputPath}' matched ${match.root.guidanceLabel}, but the root itself is not allowed for ${input.purpose}.`,
|
|
802
|
+
purpose: input.purpose
|
|
803
|
+
});
|
|
804
|
+
if (!match.root.capabilities[input.purpose]) return errorResult({
|
|
805
|
+
code: "purpose-not-allowed",
|
|
806
|
+
inputPath: normalizedInputPath,
|
|
807
|
+
mapping: input.mapping,
|
|
808
|
+
message: `Path '${normalizedInputPath}' matched ${match.root.guidanceLabel} but cannot be used for ${input.purpose}.`,
|
|
809
|
+
purpose: input.purpose
|
|
810
|
+
});
|
|
811
|
+
const targetRoot = match.root.locations[input.targetNamespace];
|
|
812
|
+
if (targetRoot === void 0) return errorResult({
|
|
813
|
+
code: "target-namespace-not-available",
|
|
814
|
+
inputPath: normalizedInputPath,
|
|
815
|
+
mapping: input.mapping,
|
|
816
|
+
message: `Path '${normalizedInputPath}' matched ${match.root.guidanceLabel}, but '${input.targetNamespace}' is not available for that root.`,
|
|
817
|
+
purpose: input.purpose
|
|
818
|
+
});
|
|
819
|
+
const normalizedTargetRoot = normalizeRoot(targetRoot);
|
|
820
|
+
return {
|
|
821
|
+
ok: true,
|
|
822
|
+
value: {
|
|
823
|
+
backing: match.root.backing,
|
|
824
|
+
capabilities: match.root.capabilities,
|
|
825
|
+
inputNamespace: match.inputNamespace,
|
|
826
|
+
inputPath: normalizedInputPath,
|
|
827
|
+
mappingId: input.mapping.id,
|
|
828
|
+
outputNamespace: input.targetNamespace,
|
|
829
|
+
outputPath: joinRootAndRelative(normalizedTargetRoot, relativePath),
|
|
830
|
+
relativePath,
|
|
831
|
+
rootId: match.root.id
|
|
832
|
+
}
|
|
833
|
+
};
|
|
834
|
+
}
|
|
835
|
+
//#endregion
|
|
836
|
+
//#region src/tool-vm-lease-id.ts
|
|
837
|
+
function createToolVmLeaseId() {
|
|
838
|
+
return parseToolVmLeaseId(v7());
|
|
839
|
+
}
|
|
840
|
+
function isToolVmLeaseId(value) {
|
|
841
|
+
return typeof value === "string" && validate(value) && version(value) === 7;
|
|
842
|
+
}
|
|
843
|
+
function parseToolVmLeaseId(value) {
|
|
844
|
+
if (isToolVmLeaseId(value)) return value;
|
|
845
|
+
throw new TypeError("Tool VM lease id must be an opaque UUIDv7 string.");
|
|
846
|
+
}
|
|
215
847
|
//#endregion
|
|
216
848
|
//#region src/vm-capability-lease.ts
|
|
217
849
|
const VM_SSH_PUBLIC_ENDPOINT_KEYS = new Set([
|
|
@@ -244,15 +876,16 @@ function isVmSshPublicEndpoint(value) {
|
|
|
244
876
|
function objectValue(value) {
|
|
245
877
|
return typeof value === "object" && value !== null ? value : void 0;
|
|
246
878
|
}
|
|
879
|
+
const deprecatedScopeKeyPropertyName = ["scope", "Key"].join("");
|
|
247
880
|
function isToolVmSshLease(value) {
|
|
248
881
|
const record = objectValue(value);
|
|
249
|
-
return isVmCapabilityLease(record, "ssh-sandbox") && isVmSshEndpoint(Reflect.get(record, "ssh")) && typeof Reflect.get(record, "agentId") === "string" && typeof Reflect.get(record, "
|
|
882
|
+
return isVmCapabilityLease(record, "ssh-sandbox") && isToolVmLeaseId(Reflect.get(record, "leaseId")) && isVmSshEndpoint(Reflect.get(record, "ssh")) && typeof Reflect.get(record, "agentId") === "string" && typeof Reflect.get(record, "idleTtlMs") === "number" && typeof Reflect.get(record, "tcpSlot") === "number" && typeof Reflect.get(record, "workdir") === "string" && !Reflect.has(record, deprecatedScopeKeyPropertyName);
|
|
250
883
|
}
|
|
251
884
|
function isToolVmLeasePeek(value) {
|
|
252
885
|
const record = objectValue(value);
|
|
253
|
-
return isVmCapabilityLease(record, "ssh-sandbox") && typeof Reflect.get(record, "agentId") === "string" && typeof Reflect.get(record, "createdAt") === "number" && typeof Reflect.get(record, "
|
|
886
|
+
return isVmCapabilityLease(record, "ssh-sandbox") && isToolVmLeaseId(Reflect.get(record, "leaseId")) && typeof Reflect.get(record, "agentId") === "string" && typeof Reflect.get(record, "createdAt") === "number" && typeof Reflect.get(record, "idleTtlMs") === "number" && typeof Reflect.get(record, "lastUsedAt") === "number" && typeof Reflect.get(record, "profileId") === "string" && isVmSshPublicEndpoint(Reflect.get(record, "ssh")) && typeof Reflect.get(record, "tcpSlot") === "number" && typeof Reflect.get(record, "workdir") === "string" && typeof Reflect.get(record, "zoneId") === "string" && !Reflect.has(record, deprecatedScopeKeyPropertyName);
|
|
254
887
|
}
|
|
255
888
|
//#endregion
|
|
256
|
-
export { FORCE_IPV4_EGRESS_NODE_OPTIONS, buildGatewaySessionLabel, buildToolSessionLabel, composeNodeOptions, controllerVmHost, createToolVmActiveUseHandle, createToolVmActiveUseId, egressHostsForAudience, gatewayTypeValues, gatewayVmAllowedHosts, isToolVmActiveUseId, isToolVmLeasePeek, isToolVmSshLease, isVmCapabilityLease, isVmSshEndpoint, isVmSshPublicEndpoint, mergeRuntimeGatewaySecrets, splitResolvedGatewaySecrets, splitResolvedSecretsByInjection, targetsAudience, vmAudienceValues };
|
|
889
|
+
export { ControllerRequestPolicyTransportError, FORCE_IPV4_EGRESS_NODE_OPTIONS, OPENCLAW_STATE_SANDBOXES_VM_ROOT, OPENCLAW_STATE_VM_ROOT, TOOL_VM_SCRATCH_GUEST_ROOT, TOOL_VM_WORKSPACE_GUEST_ROOT, agentVmHealthEventKinds, agentVmHealthResultKinds, buildGatewaySessionLabel, buildToolSessionLabel, composeNodeOptions, controllerRequestPolicies, controllerVmHost, createToolVmActiveUseHandle, createToolVmActiveUseId, createToolVmLeaseId, deriveZoneHealthSnapshot, drainControllerResponseBody, egressHostsForAudience, externalControllerRoutes, fetchControllerWithPolicy, gatewayControlLinkHealthPins, gatewayInternalControllerRequestOperations, gatewayTypeValues, gatewayVmAllowedHosts, genericControllerRequestEventOperations, healthEventBucketKey, isAgentVmHealthEvent, isToolVmActiveUseId, isToolVmLeaseId, isToolVmLeasePeek, isToolVmSshLease, isVmCapabilityLease, isVmSshEndpoint, isVmSshPublicEndpoint, mergeRuntimeGatewaySecrets, parseToolVmLeaseId, splitResolvedGatewaySecrets, splitResolvedSecretsByInjection, targetsAudience, translateRuntimePath, vmAudienceValues, workerInternalControllerRequestOperations, zoneHealthIssueKinds, zoneHealthStateKinds };
|
|
257
890
|
|
|
258
891
|
//# sourceMappingURL=index.js.map
|