@algosuite/vo-mcp 0.2.0-beta.55 → 0.2.0-beta.57
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/autostart-cli.js +6 -3
- package/dist/autostart-cli.js.map +2 -2
- package/dist/install-cli.js +6 -3
- package/dist/install-cli.js.map +2 -2
- package/dist/runner-cli.js +649 -127
- package/dist/runner-cli.js.map +4 -4
- package/dist/runner-supervisor.js +871 -88
- package/dist/runner-supervisor.js.map +4 -4
- package/package.json +1 -1
|
@@ -31,7 +31,7 @@ var init_control_plane_auth_stub = __esm({
|
|
|
31
31
|
|
|
32
32
|
// src/runner-supervisor.mjs
|
|
33
33
|
import { spawn } from "node:child_process";
|
|
34
|
-
import { randomUUID as
|
|
34
|
+
import { randomUUID as randomUUID4 } from "node:crypto";
|
|
35
35
|
import { createRequire } from "node:module";
|
|
36
36
|
import { fileURLToPath as fileURLToPath3 } from "node:url";
|
|
37
37
|
import { dirname as dirname4, join as join5 } from "node:path";
|
|
@@ -202,8 +202,9 @@ async function mergeVerifiedPrRequest(req, prNumber, automationContext, onUnauth
|
|
|
202
202
|
actionReceiptId: typeof json.action_receipt_id === "string" ? json.action_receipt_id : null
|
|
203
203
|
};
|
|
204
204
|
}
|
|
205
|
-
|
|
206
|
-
|
|
205
|
+
const captureStoreRetry = json?.action_status === "not_attempted" && (json?.error === "capture_preflight_unavailable" || json?.error === "decision_outcome_intent_failed");
|
|
206
|
+
if (res.status === 503 && (json?.error === "verify_unavailable" || json?.error === "merge_unavailable" || captureStoreRetry)) {
|
|
207
|
+
return { status: "retry", reason: json.reason || json.message || json.error || "verification unavailable" };
|
|
207
208
|
}
|
|
208
209
|
return {
|
|
209
210
|
status: "blocked",
|
|
@@ -287,6 +288,58 @@ function makeClaimGateNotice({ log = () => {
|
|
|
287
288
|
};
|
|
288
289
|
}
|
|
289
290
|
|
|
291
|
+
// ../../scripts/virtual-office/code-runner/control-plane-knowledge-context.mjs
|
|
292
|
+
var MIN_KNOWLEDGE_CONTEXT_TIMEOUT_MS = 15e3;
|
|
293
|
+
var RETRY_DELAYS_MS = [2e3, 6e3];
|
|
294
|
+
var MAX_ATTEMPTS = RETRY_DELAYS_MS.length + 1;
|
|
295
|
+
function isRetryableStatus(status) {
|
|
296
|
+
return status >= 500 && status <= 599;
|
|
297
|
+
}
|
|
298
|
+
function defaultSleep(ms) {
|
|
299
|
+
return new Promise((resolve5) => {
|
|
300
|
+
setTimeout(resolve5, ms);
|
|
301
|
+
});
|
|
302
|
+
}
|
|
303
|
+
async function getTaskKnowledgeContextRequest(req, taskId, { query } = {}, {
|
|
304
|
+
taskRequestTimeoutMs,
|
|
305
|
+
invalidateToken = () => {
|
|
306
|
+
},
|
|
307
|
+
sleep: sleep2 = defaultSleep,
|
|
308
|
+
log = () => {
|
|
309
|
+
}
|
|
310
|
+
} = {}) {
|
|
311
|
+
const body = {};
|
|
312
|
+
if (typeof query === "string" && query.trim()) body.query = query;
|
|
313
|
+
const path2 = `/api/v1/code-task/${encodeURIComponent(taskId)}/knowledge-context`;
|
|
314
|
+
const timeoutMs = Math.max(Number(taskRequestTimeoutMs) || 0, MIN_KNOWLEDGE_CONTEXT_TIMEOUT_MS);
|
|
315
|
+
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt += 1) {
|
|
316
|
+
let res;
|
|
317
|
+
let cause;
|
|
318
|
+
try {
|
|
319
|
+
res = await req("POST", path2, body, { timeoutMs });
|
|
320
|
+
} catch (err) {
|
|
321
|
+
cause = err;
|
|
322
|
+
}
|
|
323
|
+
if (!cause) {
|
|
324
|
+
if (res.status === 401) {
|
|
325
|
+
invalidateToken();
|
|
326
|
+
throw new Error("knowledge-context unauthorized (401)");
|
|
327
|
+
}
|
|
328
|
+
if (res.status === 404) return null;
|
|
329
|
+
if (res.ok) return res.json();
|
|
330
|
+
if (!isRetryableStatus(res.status)) {
|
|
331
|
+
throw new Error(`knowledge-context failed: HTTP ${res.status}`);
|
|
332
|
+
}
|
|
333
|
+
cause = new Error(`knowledge-context failed: HTTP ${res.status}`);
|
|
334
|
+
}
|
|
335
|
+
if (attempt === MAX_ATTEMPTS) throw cause;
|
|
336
|
+
const delayMs = RETRY_DELAYS_MS[attempt - 1];
|
|
337
|
+
log(`knowledge-context attempt ${attempt}/${MAX_ATTEMPTS} failed (${cause.message}); retrying in ${delayMs}ms`);
|
|
338
|
+
await sleep2(delayMs);
|
|
339
|
+
}
|
|
340
|
+
throw new Error("knowledge-context retry loop exited unexpectedly");
|
|
341
|
+
}
|
|
342
|
+
|
|
290
343
|
// ../../scripts/virtual-office/code-runner/control-plane-client.mjs
|
|
291
344
|
var cachedFirebaseToken = null;
|
|
292
345
|
var ClaimAuthorityChangedError = class extends Error {
|
|
@@ -323,7 +376,8 @@ function createControlPlaneClient({
|
|
|
323
376
|
6e4
|
|
324
377
|
),
|
|
325
378
|
runnerId: runnerId2,
|
|
326
|
-
runnerInstanceId
|
|
379
|
+
runnerInstanceId,
|
|
380
|
+
sleep: sleep2
|
|
327
381
|
} = {}) {
|
|
328
382
|
const resolvedBaseUrl = baseUrl ?? env.VO_CONTROL_PLANE_URL ?? "";
|
|
329
383
|
if (!resolvedBaseUrl) throw new Error("VO_CONTROL_PLANE_URL is required for the code-runner daemon");
|
|
@@ -504,17 +558,16 @@ function createControlPlaneClient({
|
|
|
504
558
|
if (!res.ok) throw new Error(`attachment download failed: HTTP ${res.status}`);
|
|
505
559
|
return Buffer.from(await res.arrayBuffer());
|
|
506
560
|
},
|
|
561
|
+
// Raised per-attempt timeout (>=15s) + bounded retry — see control-plane-knowledge-context.mjs.
|
|
507
562
|
async getTaskKnowledgeContext(taskId, { query } = {}) {
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
if (!res.ok) throw new Error(`knowledge-context failed: HTTP ${res.status}`);
|
|
517
|
-
return res.json();
|
|
563
|
+
return getTaskKnowledgeContextRequest(req, taskId, { query }, {
|
|
564
|
+
taskRequestTimeoutMs,
|
|
565
|
+
invalidateToken: () => {
|
|
566
|
+
cachedFirebaseToken = null;
|
|
567
|
+
},
|
|
568
|
+
sleep: sleep2,
|
|
569
|
+
log: (m) => console.warn(`[code-runner ${(/* @__PURE__ */ new Date()).toISOString()}] ${m}`)
|
|
570
|
+
});
|
|
518
571
|
},
|
|
519
572
|
/** Weekly Claude token usage report — see control-plane-weekly-tokens.mjs. */
|
|
520
573
|
async postWeeklyTokens(report) {
|
|
@@ -2037,6 +2090,7 @@ async function finishPendingActivation({
|
|
|
2037
2090
|
packageVersion: packageVersion2,
|
|
2038
2091
|
supervisorIdentity,
|
|
2039
2092
|
waitForLocalRunner: waitForLocalRunner2,
|
|
2093
|
+
isReadinessDeferred = () => false,
|
|
2040
2094
|
localStatus: localStatus2,
|
|
2041
2095
|
waitForCloudRunner = waitForAuthoritativeRunnerHeartbeat,
|
|
2042
2096
|
cloudTimeoutMs,
|
|
@@ -2071,7 +2125,10 @@ async function finishPendingActivation({
|
|
|
2071
2125
|
try {
|
|
2072
2126
|
attestation = attestCurrentSupervisor({ runtimeRoot, selfPath: selfPath2, version: packageVersion2 });
|
|
2073
2127
|
if (!attestation.ok) throw new Error(attestation.detail);
|
|
2074
|
-
if (!await waitForLocalRunner2(child))
|
|
2128
|
+
if (!await waitForLocalRunner2(child)) {
|
|
2129
|
+
if (isReadinessDeferred(child)) return false;
|
|
2130
|
+
throw new Error("activated runner did not become locally ready");
|
|
2131
|
+
}
|
|
2075
2132
|
} catch (error) {
|
|
2076
2133
|
let detail = `activation attestation failed: ${error instanceof Error ? error.message : String(error)}`.slice(0, 400);
|
|
2077
2134
|
let rolledBack = null;
|
|
@@ -2265,8 +2322,97 @@ function resolveSupervisorChildEntry({
|
|
|
2265
2322
|
import { hostname as systemHostname } from "node:os";
|
|
2266
2323
|
|
|
2267
2324
|
// src/runner-readiness.mjs
|
|
2268
|
-
|
|
2269
|
-
|
|
2325
|
+
import { createHash as createHash5, randomUUID as randomUUID3 } from "node:crypto";
|
|
2326
|
+
var RUNNER_GITHUB_READINESS_TIMEOUT_MS = 5e4;
|
|
2327
|
+
var RUNNER_IDENTITY_READINESS_TIMEOUT_MS = 1e4;
|
|
2328
|
+
var RUNNER_READINESS_MAX_REPOSITORIES = 4;
|
|
2329
|
+
var RUNNER_READINESS_TRANSIENT_MAX_RETRY_AFTER_MS = 3e5;
|
|
2330
|
+
var RUNNER_READINESS_DEFERRAL_MESSAGE_TYPE = "vo-runner-readiness-deferred-v1";
|
|
2331
|
+
var RUNNER_READINESS_DEFERRAL_ACK_TYPE = "vo-runner-readiness-deferred-ack-v1";
|
|
2332
|
+
var RUNNER_REPOSITORY = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u;
|
|
2333
|
+
var UUID_RE2 = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
|
|
2334
|
+
function runnerRepositoryScopeFromEnv(env = {}) {
|
|
2335
|
+
return String(env.VO_CODE_RUNNER_REPOS || "").split(/[\s,]+/u).map((repo) => repo.trim()).filter(Boolean);
|
|
2336
|
+
}
|
|
2337
|
+
function normalizeRunnerRepositoryScope(repositories) {
|
|
2338
|
+
if (!Array.isArray(repositories) || repositories.length < 1) {
|
|
2339
|
+
throw new Error("VO_CODE_RUNNER_REPOS must name at least one owner/name repository");
|
|
2340
|
+
}
|
|
2341
|
+
if (repositories.length > RUNNER_READINESS_MAX_REPOSITORIES) {
|
|
2342
|
+
throw new Error(`VO_CODE_RUNNER_REPOS supports at most ${RUNNER_READINESS_MAX_REPOSITORIES} repositories`);
|
|
2343
|
+
}
|
|
2344
|
+
const normalized = repositories.map((repo) => {
|
|
2345
|
+
const [owner, name] = typeof repo === "string" ? repo.split("/") : [];
|
|
2346
|
+
if (typeof repo !== "string" || repo.length > 140 || !RUNNER_REPOSITORY.test(repo) || owner === "." || owner === ".." || name === "." || name === "..") {
|
|
2347
|
+
throw new Error("VO_CODE_RUNNER_REPOS entries must be canonical owner/name repositories");
|
|
2348
|
+
}
|
|
2349
|
+
return repo.toLowerCase();
|
|
2350
|
+
});
|
|
2351
|
+
if (new Set(normalized).size !== normalized.length) {
|
|
2352
|
+
throw new Error("VO_CODE_RUNNER_REPOS entries must be unique");
|
|
2353
|
+
}
|
|
2354
|
+
const owners = new Set(normalized.map((repo) => repo.split("/")[0]));
|
|
2355
|
+
if (owners.size !== 1) {
|
|
2356
|
+
throw new Error("VO_CODE_RUNNER_REPOS entries must share one owner");
|
|
2357
|
+
}
|
|
2358
|
+
return Object.freeze(normalized.sort());
|
|
2359
|
+
}
|
|
2360
|
+
function runnerRepositoryScopeDigest(repositories) {
|
|
2361
|
+
const scope = normalizeRunnerRepositoryScope(repositories);
|
|
2362
|
+
return createHash5("sha256").update(JSON.stringify({
|
|
2363
|
+
version: 1,
|
|
2364
|
+
repositories: scope
|
|
2365
|
+
}), "utf8").digest("hex");
|
|
2366
|
+
}
|
|
2367
|
+
function runnerReadinessRetryDelayMs(readiness) {
|
|
2368
|
+
if (readiness?.paired !== true || readiness?.githubReady !== false) return null;
|
|
2369
|
+
if (Number.isFinite(readiness.retryAfterMs) && readiness.retryAfterMs > 0) {
|
|
2370
|
+
return Math.ceil(readiness.retryAfterMs);
|
|
2371
|
+
}
|
|
2372
|
+
return null;
|
|
2373
|
+
}
|
|
2374
|
+
function parseRunnerReadinessDeferralRequest(message) {
|
|
2375
|
+
if (message?.type !== RUNNER_READINESS_DEFERRAL_MESSAGE_TYPE || !UUID_RE2.test(String(message.nonce || "")) || !Number.isFinite(message.retryAfterMs) || message.retryAfterMs <= 0) return null;
|
|
2376
|
+
return Object.freeze({
|
|
2377
|
+
nonce: message.nonce,
|
|
2378
|
+
retryAfterMs: Math.ceil(message.retryAfterMs),
|
|
2379
|
+
error: typeof message.error === "string" ? message.error : null
|
|
2380
|
+
});
|
|
2381
|
+
}
|
|
2382
|
+
function failed({
|
|
2383
|
+
paired = false,
|
|
2384
|
+
operatorId = null,
|
|
2385
|
+
tenantId = null,
|
|
2386
|
+
githubReady = null,
|
|
2387
|
+
retryAfterMs = null,
|
|
2388
|
+
error,
|
|
2389
|
+
message
|
|
2390
|
+
}) {
|
|
2391
|
+
return {
|
|
2392
|
+
ok: false,
|
|
2393
|
+
paired,
|
|
2394
|
+
operatorId,
|
|
2395
|
+
tenantId,
|
|
2396
|
+
githubReady,
|
|
2397
|
+
...Number.isFinite(retryAfterMs) && retryAfterMs > 0 ? { retryAfterMs } : {},
|
|
2398
|
+
error,
|
|
2399
|
+
message
|
|
2400
|
+
};
|
|
2401
|
+
}
|
|
2402
|
+
function responseRetryAfterMs(response, body) {
|
|
2403
|
+
const transientReadinessFailure = response.status === 503 && body?.error === "github_installation_readiness_retryable";
|
|
2404
|
+
if (response.status !== 429 && !transientReadinessFailure) return null;
|
|
2405
|
+
const bound = (retryAfterMs) => transientReadinessFailure ? Math.min(RUNNER_READINESS_TRANSIENT_MAX_RETRY_AFTER_MS, retryAfterMs) : retryAfterMs;
|
|
2406
|
+
const value = response.headers?.get?.("retry-after")?.trim() ?? "";
|
|
2407
|
+
const seconds = Number(value);
|
|
2408
|
+
if (value && Number.isFinite(seconds) && seconds >= 0) {
|
|
2409
|
+
return bound(Math.max(1e3, Math.ceil(seconds * 1e3)));
|
|
2410
|
+
}
|
|
2411
|
+
const at = value ? Date.parse(value) : Number.NaN;
|
|
2412
|
+
if (Number.isFinite(at)) {
|
|
2413
|
+
return bound(Math.max(1e3, Math.ceil(at - Date.now())));
|
|
2414
|
+
}
|
|
2415
|
+
return bound(6e4);
|
|
2270
2416
|
}
|
|
2271
2417
|
async function responseBody(response) {
|
|
2272
2418
|
try {
|
|
@@ -2279,11 +2425,21 @@ async function responseBody(response) {
|
|
|
2279
2425
|
function serverMessage(body, fallback) {
|
|
2280
2426
|
return typeof body.message === "string" && body.message.trim() ? body.message.trim() : fallback;
|
|
2281
2427
|
}
|
|
2282
|
-
async function
|
|
2428
|
+
async function fetchJsonWithTimeout(fetchImpl, url, init, timeoutMs) {
|
|
2283
2429
|
const controller = new AbortController();
|
|
2284
|
-
|
|
2430
|
+
let timer;
|
|
2431
|
+
const timeout = new Promise((_, reject) => {
|
|
2432
|
+
timer = setTimeout(() => {
|
|
2433
|
+
controller.abort();
|
|
2434
|
+
reject(new Error(`request aborted after ${timeoutMs}ms`));
|
|
2435
|
+
}, timeoutMs);
|
|
2436
|
+
});
|
|
2285
2437
|
try {
|
|
2286
|
-
|
|
2438
|
+
const requestAndBody = (async () => {
|
|
2439
|
+
const response = await fetchImpl(url, { ...init, signal: controller.signal });
|
|
2440
|
+
return { response, body: await responseBody(response) };
|
|
2441
|
+
})();
|
|
2442
|
+
return await Promise.race([requestAndBody, timeout]);
|
|
2287
2443
|
} finally {
|
|
2288
2444
|
clearTimeout(timer);
|
|
2289
2445
|
}
|
|
@@ -2293,18 +2449,24 @@ async function probeRunnerReadiness({
|
|
|
2293
2449
|
token,
|
|
2294
2450
|
fetchImpl = fetch,
|
|
2295
2451
|
requireGithub = false,
|
|
2296
|
-
|
|
2452
|
+
repositories = [],
|
|
2453
|
+
timeoutMs = RUNNER_IDENTITY_READINESS_TIMEOUT_MS,
|
|
2454
|
+
// The server performs at most five sequential App-JWT GETs (installation plus
|
|
2455
|
+
// every one of at most four configured repositories), each bounded at 8s.
|
|
2456
|
+
// Keep a transport margin while preventing a stuck proof from hanging startup.
|
|
2457
|
+
githubTimeoutMs = RUNNER_GITHUB_READINESS_TIMEOUT_MS
|
|
2297
2458
|
}) {
|
|
2298
2459
|
const base = controlPlaneUrl.replace(/\/+$/u, "");
|
|
2299
2460
|
const headers = { authorization: `Bearer ${token}` };
|
|
2300
2461
|
let identityResponse;
|
|
2462
|
+
let identity;
|
|
2301
2463
|
try {
|
|
2302
|
-
identityResponse = await
|
|
2464
|
+
({ response: identityResponse, body: identity } = await fetchJsonWithTimeout(
|
|
2303
2465
|
fetchImpl,
|
|
2304
2466
|
`${base}/api/v1/auth/me`,
|
|
2305
2467
|
{ headers },
|
|
2306
2468
|
timeoutMs
|
|
2307
|
-
);
|
|
2469
|
+
));
|
|
2308
2470
|
} catch (error) {
|
|
2309
2471
|
const detail = error instanceof Error ? error.message : String(error);
|
|
2310
2472
|
return failed({
|
|
@@ -2312,7 +2474,6 @@ async function probeRunnerReadiness({
|
|
|
2312
2474
|
message: `AlgoHQ could not be reached: ${detail}`
|
|
2313
2475
|
});
|
|
2314
2476
|
}
|
|
2315
|
-
const identity = await responseBody(identityResponse);
|
|
2316
2477
|
if (!identityResponse.ok) {
|
|
2317
2478
|
return failed({
|
|
2318
2479
|
error: "credential_rejected",
|
|
@@ -2338,18 +2499,33 @@ async function probeRunnerReadiness({
|
|
|
2338
2499
|
message: "Paired to AlgoHQ."
|
|
2339
2500
|
};
|
|
2340
2501
|
}
|
|
2502
|
+
let repositoryScope = null;
|
|
2503
|
+
try {
|
|
2504
|
+
if (!Array.isArray(repositories)) {
|
|
2505
|
+
throw new Error("VO_CODE_RUNNER_REPOS must be a repository array");
|
|
2506
|
+
}
|
|
2507
|
+
if (repositories.length > 0) repositoryScope = normalizeRunnerRepositoryScope(repositories);
|
|
2508
|
+
} catch (error) {
|
|
2509
|
+
return failed({
|
|
2510
|
+
paired: true,
|
|
2511
|
+
operatorId,
|
|
2512
|
+
tenantId,
|
|
2513
|
+
githubReady: false,
|
|
2514
|
+
error: "github_repository_scope_invalid",
|
|
2515
|
+
message: error instanceof Error ? error.message : String(error)
|
|
2516
|
+
});
|
|
2517
|
+
}
|
|
2341
2518
|
let githubResponse;
|
|
2519
|
+
let github;
|
|
2342
2520
|
try {
|
|
2343
|
-
|
|
2521
|
+
const readinessUrl = new URL(`${base}/api/v1/github/installation-readiness`);
|
|
2522
|
+
for (const repo of repositoryScope ?? []) readinessUrl.searchParams.append("repo", repo);
|
|
2523
|
+
({ response: githubResponse, body: github } = await fetchJsonWithTimeout(
|
|
2344
2524
|
fetchImpl,
|
|
2345
|
-
|
|
2346
|
-
{
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
body: "{}"
|
|
2350
|
-
},
|
|
2351
|
-
timeoutMs
|
|
2352
|
-
);
|
|
2525
|
+
readinessUrl.toString(),
|
|
2526
|
+
{ headers },
|
|
2527
|
+
githubTimeoutMs
|
|
2528
|
+
));
|
|
2353
2529
|
} catch (error) {
|
|
2354
2530
|
const detail = error instanceof Error ? error.message : String(error);
|
|
2355
2531
|
return failed({
|
|
@@ -2361,14 +2537,25 @@ async function probeRunnerReadiness({
|
|
|
2361
2537
|
message: `GitHub publication readiness could not be checked: ${detail}`
|
|
2362
2538
|
});
|
|
2363
2539
|
}
|
|
2364
|
-
const
|
|
2365
|
-
|
|
2540
|
+
const repositorySelection = github.repository_selection;
|
|
2541
|
+
const repositoriesVerified = github.repositories_verified;
|
|
2542
|
+
const returnedScope = github.repository_scope;
|
|
2543
|
+
let normalizedReturnedScope = null;
|
|
2544
|
+
try {
|
|
2545
|
+
normalizedReturnedScope = normalizeRunnerRepositoryScope(returnedScope);
|
|
2546
|
+
} catch {
|
|
2547
|
+
}
|
|
2548
|
+
const effectiveScope = repositoryScope ?? normalizedReturnedScope;
|
|
2549
|
+
const sourceVerified = repositoryScope === null ? github.repository_scope_source === "persisted_installation_singleton" && normalizedReturnedScope?.length === 1 : github.repository_scope_source === "runner_config";
|
|
2550
|
+
const repositoryScopeVerified = effectiveScope !== null && normalizedReturnedScope !== null && Array.isArray(returnedScope) && returnedScope.length === normalizedReturnedScope.length && returnedScope.every((repo, index) => repo === normalizedReturnedScope[index]) && normalizedReturnedScope.length === effectiveScope.length && normalizedReturnedScope.every((repo, index) => repo === effectiveScope[index]) && github.repository_scope_sha256 === runnerRepositoryScopeDigest(effectiveScope) && repositoriesVerified === effectiveScope.length && sourceVerified && (repositorySelection === "all" || repositorySelection === "selected");
|
|
2551
|
+
if (!githubResponse.ok || github.configured !== true || github.verified !== true || github.publication_ready !== true || !repositoryScopeVerified) {
|
|
2366
2552
|
const error = typeof github.error === "string" && github.error ? github.error : "github_not_ready";
|
|
2367
2553
|
return failed({
|
|
2368
2554
|
paired: true,
|
|
2369
2555
|
operatorId,
|
|
2370
2556
|
tenantId,
|
|
2371
2557
|
githubReady: false,
|
|
2558
|
+
retryAfterMs: responseRetryAfterMs(githubResponse, github),
|
|
2372
2559
|
error,
|
|
2373
2560
|
message: serverMessage(github, `GitHub publication preflight failed (HTTP ${githubResponse.status}).`)
|
|
2374
2561
|
});
|
|
@@ -2379,8 +2566,10 @@ async function probeRunnerReadiness({
|
|
|
2379
2566
|
operatorId,
|
|
2380
2567
|
tenantId,
|
|
2381
2568
|
githubReady: true,
|
|
2569
|
+
repositoryScope: effectiveScope,
|
|
2570
|
+
repositoryScopeSource: github.repository_scope_source,
|
|
2382
2571
|
error: null,
|
|
2383
|
-
message:
|
|
2572
|
+
message: `Paired with a verified Algosuite GitHub App installation (${repositoriesVerified} repos).`
|
|
2384
2573
|
};
|
|
2385
2574
|
}
|
|
2386
2575
|
function pairedOperatorScope(readiness) {
|
|
@@ -2419,25 +2608,51 @@ async function prepareSupervisorAuth({
|
|
|
2419
2608
|
const token = explicitAdminToken || storedCredential?.vo_credential;
|
|
2420
2609
|
if (!token) throw new Error("runner is not paired; run `vo-mcp pair` once on this host");
|
|
2421
2610
|
let operatorId = String(baseEnv.VO_CODE_RUNNER_OPERATOR_IDS || "").split(/[\s,]+/u).filter(Boolean)[0] || void 0;
|
|
2611
|
+
let repositoryScope = runnerRepositoryScopeFromEnv(baseEnv);
|
|
2612
|
+
let startupRetryAfterMs = null;
|
|
2613
|
+
let githubReady = null;
|
|
2614
|
+
let readinessError = null;
|
|
2422
2615
|
if (!explicitAdminToken) {
|
|
2423
|
-
const readiness = await probeReadiness({
|
|
2424
|
-
|
|
2425
|
-
|
|
2616
|
+
const readiness = await probeReadiness({
|
|
2617
|
+
controlPlaneUrl,
|
|
2618
|
+
token,
|
|
2619
|
+
requireGithub: true,
|
|
2620
|
+
repositories: repositoryScope
|
|
2621
|
+
});
|
|
2622
|
+
const pairedScope = pairedOperatorScope(readiness) || (readiness?.paired === true && typeof readiness.operatorId === "string" ? readiness.operatorId.trim() : "");
|
|
2623
|
+
operatorId = pairedScope || void 0;
|
|
2426
2624
|
if (!operatorId) throw new Error("runner readiness failed: paired operator scope is missing");
|
|
2625
|
+
if (readiness.ok && Array.isArray(readiness.repositoryScope)) {
|
|
2626
|
+
repositoryScope = [...readiness.repositoryScope];
|
|
2627
|
+
githubReady = true;
|
|
2628
|
+
} else {
|
|
2629
|
+
githubReady = false;
|
|
2630
|
+
readinessError = typeof readiness?.error === "string" ? readiness.error : "github_not_ready";
|
|
2631
|
+
startupRetryAfterMs = runnerReadinessRetryDelayMs(readiness);
|
|
2632
|
+
}
|
|
2427
2633
|
}
|
|
2634
|
+
const effectiveBaseEnv = repositoryScope.length > 0 ? { ...baseEnv, VO_CODE_RUNNER_REPOS: repositoryScope.join(",") } : baseEnv;
|
|
2428
2635
|
const childEnv = buildSupervisorChildEnv({
|
|
2429
|
-
baseEnv,
|
|
2636
|
+
baseEnv: effectiveBaseEnv,
|
|
2430
2637
|
controlPlaneUrl,
|
|
2431
2638
|
explicitAdminToken,
|
|
2432
2639
|
pairedOperatorId: explicitAdminToken ? null : operatorId
|
|
2433
2640
|
});
|
|
2434
2641
|
const clientEnv = {
|
|
2435
|
-
...
|
|
2642
|
+
...effectiveBaseEnv,
|
|
2436
2643
|
VO_CONTROL_PLANE_ADMIN_TOKEN: token,
|
|
2437
2644
|
VO_CONTROL_PLANE_URL: controlPlaneUrl,
|
|
2438
2645
|
...operatorId ? { VO_CODE_RUNNER_OPERATOR_IDS: operatorId } : {}
|
|
2439
2646
|
};
|
|
2440
|
-
return {
|
|
2647
|
+
return {
|
|
2648
|
+
childEnv,
|
|
2649
|
+
clientEnv,
|
|
2650
|
+
operatorId,
|
|
2651
|
+
repositoryScope,
|
|
2652
|
+
githubReady,
|
|
2653
|
+
readinessError,
|
|
2654
|
+
startupRetryAfterMs
|
|
2655
|
+
};
|
|
2441
2656
|
}
|
|
2442
2657
|
|
|
2443
2658
|
// src/runner/supervisor-credential-reader.mjs
|
|
@@ -2477,12 +2692,309 @@ function readStoredCredentialIsolated({
|
|
|
2477
2692
|
}
|
|
2478
2693
|
}
|
|
2479
2694
|
|
|
2695
|
+
// src/runner/respawn-circuit.mjs
|
|
2696
|
+
var DEFAULT_RESPAWN_CIRCUIT = Object.freeze({
|
|
2697
|
+
baseDelayMs: 2e3,
|
|
2698
|
+
maxDelayMs: 3e4,
|
|
2699
|
+
healthyResetMs: 6e4,
|
|
2700
|
+
maxRapidExits: 5
|
|
2701
|
+
});
|
|
2702
|
+
function createRespawnCircuit(options = {}) {
|
|
2703
|
+
const config = { ...DEFAULT_RESPAWN_CIRCUIT, ...options };
|
|
2704
|
+
for (const [key, value] of Object.entries(config)) {
|
|
2705
|
+
if (!Number.isInteger(value) || value < 1) throw new Error(`${key} must be a positive integer`);
|
|
2706
|
+
}
|
|
2707
|
+
let rapidExits = 0;
|
|
2708
|
+
return Object.freeze({
|
|
2709
|
+
recordExit(startedAt, exitedAt) {
|
|
2710
|
+
if (!Number.isFinite(startedAt) || !Number.isFinite(exitedAt) || exitedAt < startedAt) {
|
|
2711
|
+
throw new Error("respawn circuit timestamps are invalid");
|
|
2712
|
+
}
|
|
2713
|
+
const uptimeMs = exitedAt - startedAt;
|
|
2714
|
+
rapidExits = uptimeMs >= config.healthyResetMs ? 0 : rapidExits + 1;
|
|
2715
|
+
const tripped = rapidExits >= config.maxRapidExits;
|
|
2716
|
+
const delayMs = Math.min(
|
|
2717
|
+
config.maxDelayMs,
|
|
2718
|
+
config.baseDelayMs * 2 ** Math.max(0, rapidExits - 1)
|
|
2719
|
+
);
|
|
2720
|
+
return Object.freeze({ rapidExits, uptimeMs, delayMs, tripped });
|
|
2721
|
+
},
|
|
2722
|
+
snapshot() {
|
|
2723
|
+
return Object.freeze({ rapidExits, tripped: rapidExits >= config.maxRapidExits });
|
|
2724
|
+
}
|
|
2725
|
+
});
|
|
2726
|
+
}
|
|
2727
|
+
|
|
2728
|
+
// src/runner/supervisor-child-health.mjs
|
|
2729
|
+
var terminatedChildren = /* @__PURE__ */ new WeakSet();
|
|
2730
|
+
function attachSupervisorChildTerminationCustody(target, onTerminated, onObservedError = () => {
|
|
2731
|
+
}) {
|
|
2732
|
+
let accounted = false;
|
|
2733
|
+
const account = (kind, error = null) => {
|
|
2734
|
+
terminatedChildren.add(target);
|
|
2735
|
+
if (accounted) return false;
|
|
2736
|
+
accounted = true;
|
|
2737
|
+
onTerminated(Object.freeze({ target, kind, error }));
|
|
2738
|
+
return true;
|
|
2739
|
+
};
|
|
2740
|
+
target.on("error", (error) => {
|
|
2741
|
+
onObservedError(error);
|
|
2742
|
+
if (target.pid === void 0 || target.pid === null) account("error", error);
|
|
2743
|
+
});
|
|
2744
|
+
target.on("exit", () => {
|
|
2745
|
+
account("exit");
|
|
2746
|
+
});
|
|
2747
|
+
return Object.freeze({ accounted: () => accounted });
|
|
2748
|
+
}
|
|
2749
|
+
function supervisorChildHasExited(target) {
|
|
2750
|
+
return terminatedChildren.has(target) || target.exitCode !== null || (target.signalCode ?? null) !== null;
|
|
2751
|
+
}
|
|
2752
|
+
function supervisorChildIsRunning(target) {
|
|
2753
|
+
return !supervisorChildHasExited(target);
|
|
2754
|
+
}
|
|
2755
|
+
function shouldRespawnSupervisorChild({ stopping, degraded, handling, child }) {
|
|
2756
|
+
return !stopping && !degraded && !handling && (!child || supervisorChildHasExited(child));
|
|
2757
|
+
}
|
|
2758
|
+
function shouldDeferSupervisorReadinessExit({
|
|
2759
|
+
target,
|
|
2760
|
+
currentChild,
|
|
2761
|
+
retryAfterMs
|
|
2762
|
+
}) {
|
|
2763
|
+
return target === currentChild && target?.exitCode === 75 && Number.isFinite(retryAfterMs) && retryAfterMs > 0;
|
|
2764
|
+
}
|
|
2765
|
+
function createSupervisorReadinessDeferralTracker({
|
|
2766
|
+
currentChild,
|
|
2767
|
+
releaseCurrentChild,
|
|
2768
|
+
onCaptured
|
|
2769
|
+
}) {
|
|
2770
|
+
const pending = /* @__PURE__ */ new WeakMap();
|
|
2771
|
+
const terminal = /* @__PURE__ */ new WeakSet();
|
|
2772
|
+
const recoveryGenerations = /* @__PURE__ */ new WeakMap();
|
|
2773
|
+
return Object.freeze({
|
|
2774
|
+
recordPending(target, request) {
|
|
2775
|
+
if (!target || target !== currentChild() || terminal.has(target) || pending.has(target) || typeof request?.nonce !== "string" || !request.nonce || !Number.isFinite(request.retryAfterMs) || request.retryAfterMs <= 0) return false;
|
|
2776
|
+
pending.set(target, Object.freeze({
|
|
2777
|
+
nonce: request.nonce,
|
|
2778
|
+
retryAfterMs: Math.ceil(request.retryAfterMs)
|
|
2779
|
+
}));
|
|
2780
|
+
return true;
|
|
2781
|
+
},
|
|
2782
|
+
recordRecoveryGeneration(target, generation) {
|
|
2783
|
+
if (!target || !Number.isInteger(generation)) return false;
|
|
2784
|
+
recoveryGenerations.set(target, generation);
|
|
2785
|
+
return true;
|
|
2786
|
+
},
|
|
2787
|
+
isDeferred(target) {
|
|
2788
|
+
const request = target ? pending.get(target) : null;
|
|
2789
|
+
return Boolean(target) && (terminal.has(target) || shouldDeferSupervisorReadinessExit({
|
|
2790
|
+
target,
|
|
2791
|
+
currentChild: currentChild(),
|
|
2792
|
+
retryAfterMs: request?.retryAfterMs
|
|
2793
|
+
}));
|
|
2794
|
+
},
|
|
2795
|
+
capture(target) {
|
|
2796
|
+
if (!target) return false;
|
|
2797
|
+
if (terminal.has(target)) return true;
|
|
2798
|
+
const request = pending.get(target);
|
|
2799
|
+
if (!shouldDeferSupervisorReadinessExit({
|
|
2800
|
+
target,
|
|
2801
|
+
currentChild: currentChild(),
|
|
2802
|
+
retryAfterMs: request?.retryAfterMs
|
|
2803
|
+
})) return false;
|
|
2804
|
+
const recoveryGeneration = recoveryGenerations.get(target);
|
|
2805
|
+
terminal.add(target);
|
|
2806
|
+
pending.delete(target);
|
|
2807
|
+
recoveryGenerations.delete(target);
|
|
2808
|
+
releaseCurrentChild(target);
|
|
2809
|
+
onCaptured(Object.freeze({
|
|
2810
|
+
target,
|
|
2811
|
+
retryAfterMs: request.retryAfterMs,
|
|
2812
|
+
recoveryGeneration: Number.isInteger(recoveryGeneration) ? recoveryGeneration : null
|
|
2813
|
+
}));
|
|
2814
|
+
return true;
|
|
2815
|
+
},
|
|
2816
|
+
forget(target) {
|
|
2817
|
+
if (!target) return;
|
|
2818
|
+
pending.delete(target);
|
|
2819
|
+
recoveryGenerations.delete(target);
|
|
2820
|
+
}
|
|
2821
|
+
});
|
|
2822
|
+
}
|
|
2823
|
+
function resolveSupervisorChildStartAuthority({
|
|
2824
|
+
degradationState,
|
|
2825
|
+
deferredRecoveryGeneration
|
|
2826
|
+
}) {
|
|
2827
|
+
const snapshot = degradationState.snapshot();
|
|
2828
|
+
const exactRecovery = Number.isInteger(deferredRecoveryGeneration) && deferredRecoveryGeneration === snapshot.generation;
|
|
2829
|
+
return Object.freeze({
|
|
2830
|
+
allowed: !snapshot.degraded || exactRecovery,
|
|
2831
|
+
recoveryGeneration: exactRecovery ? deferredRecoveryGeneration : null
|
|
2832
|
+
});
|
|
2833
|
+
}
|
|
2834
|
+
function createSupervisorStartupDeferral({
|
|
2835
|
+
retryAfterMs = null,
|
|
2836
|
+
nowMs = () => Date.now()
|
|
2837
|
+
} = {}) {
|
|
2838
|
+
let started = false;
|
|
2839
|
+
let notBefore = null;
|
|
2840
|
+
const defer = (delayMs) => {
|
|
2841
|
+
if (started || !Number.isFinite(delayMs) || delayMs <= 0) return false;
|
|
2842
|
+
const candidate = nowMs() + Math.ceil(delayMs);
|
|
2843
|
+
notBefore = notBefore === null ? candidate : Math.max(notBefore, candidate);
|
|
2844
|
+
return true;
|
|
2845
|
+
};
|
|
2846
|
+
defer(retryAfterMs);
|
|
2847
|
+
return Object.freeze({
|
|
2848
|
+
defer,
|
|
2849
|
+
reopen(delayMs) {
|
|
2850
|
+
if (!Number.isFinite(delayMs) || delayMs <= 0) return false;
|
|
2851
|
+
started = false;
|
|
2852
|
+
notBefore = null;
|
|
2853
|
+
return defer(delayMs);
|
|
2854
|
+
},
|
|
2855
|
+
isPending() {
|
|
2856
|
+
return !started && notBefore !== null && nowMs() < notBefore;
|
|
2857
|
+
},
|
|
2858
|
+
remainingMs() {
|
|
2859
|
+
return started || notBefore === null ? 0 : Math.max(0, notBefore - nowMs());
|
|
2860
|
+
},
|
|
2861
|
+
consumeIfReady() {
|
|
2862
|
+
if (started || notBefore !== null && nowMs() < notBefore) return false;
|
|
2863
|
+
started = true;
|
|
2864
|
+
notBefore = null;
|
|
2865
|
+
return true;
|
|
2866
|
+
},
|
|
2867
|
+
hasStarted() {
|
|
2868
|
+
return started;
|
|
2869
|
+
}
|
|
2870
|
+
});
|
|
2871
|
+
}
|
|
2872
|
+
function createSupervisorDegradationState() {
|
|
2873
|
+
let degraded = false;
|
|
2874
|
+
let generation = 0;
|
|
2875
|
+
return {
|
|
2876
|
+
isDegraded() {
|
|
2877
|
+
return degraded;
|
|
2878
|
+
},
|
|
2879
|
+
markDegraded() {
|
|
2880
|
+
degraded = true;
|
|
2881
|
+
generation += 1;
|
|
2882
|
+
},
|
|
2883
|
+
beginHealthProof({ target, currentChild, allowRecovery, onRecovery }) {
|
|
2884
|
+
const observedGeneration = generation;
|
|
2885
|
+
return () => {
|
|
2886
|
+
if (currentChild() !== target || generation !== observedGeneration) return false;
|
|
2887
|
+
if (degraded && !allowRecovery) return false;
|
|
2888
|
+
const recovered = degraded;
|
|
2889
|
+
degraded = false;
|
|
2890
|
+
if (recovered) onRecovery?.();
|
|
2891
|
+
return true;
|
|
2892
|
+
};
|
|
2893
|
+
},
|
|
2894
|
+
beginDegradationProof({ target, currentChild, onDegraded }) {
|
|
2895
|
+
const observedGeneration = generation;
|
|
2896
|
+
return (message) => {
|
|
2897
|
+
if (currentChild() !== target || generation !== observedGeneration) return false;
|
|
2898
|
+
onDegraded(message);
|
|
2899
|
+
return true;
|
|
2900
|
+
};
|
|
2901
|
+
},
|
|
2902
|
+
snapshot() {
|
|
2903
|
+
return { degraded, generation };
|
|
2904
|
+
}
|
|
2905
|
+
};
|
|
2906
|
+
}
|
|
2907
|
+
function beginSupervisorHealthyStateCommit({
|
|
2908
|
+
degradationState,
|
|
2909
|
+
target,
|
|
2910
|
+
currentChild,
|
|
2911
|
+
allowRecovery,
|
|
2912
|
+
clearStaleExitCode
|
|
2913
|
+
}) {
|
|
2914
|
+
return degradationState.beginHealthProof({
|
|
2915
|
+
target,
|
|
2916
|
+
currentChild,
|
|
2917
|
+
allowRecovery,
|
|
2918
|
+
onRecovery: clearStaleExitCode
|
|
2919
|
+
});
|
|
2920
|
+
}
|
|
2921
|
+
var RUNNER_RECOVERY_ACTIONS = /* @__PURE__ */ new Set(["update", "reinstall", "reconnect"]);
|
|
2922
|
+
function shouldRecoverSupervisorChildAfterAction({
|
|
2923
|
+
stoppedChild,
|
|
2924
|
+
actionKind,
|
|
2925
|
+
actionSucceeded
|
|
2926
|
+
}) {
|
|
2927
|
+
return Boolean(stoppedChild) || actionSucceeded === true && RUNNER_RECOVERY_ACTIONS.has(actionKind);
|
|
2928
|
+
}
|
|
2929
|
+
function createSupervisorControlRecoveryFence() {
|
|
2930
|
+
let targetStoppedForControl = null;
|
|
2931
|
+
const consume = () => {
|
|
2932
|
+
const target = targetStoppedForControl;
|
|
2933
|
+
targetStoppedForControl = null;
|
|
2934
|
+
return target;
|
|
2935
|
+
};
|
|
2936
|
+
return {
|
|
2937
|
+
markBeforeStop(target) {
|
|
2938
|
+
targetStoppedForControl = target && supervisorChildIsRunning(target) ? target : null;
|
|
2939
|
+
return targetStoppedForControl !== null;
|
|
2940
|
+
},
|
|
2941
|
+
consume,
|
|
2942
|
+
async recoverOnce(recover) {
|
|
2943
|
+
const target = consume();
|
|
2944
|
+
if (!target) return false;
|
|
2945
|
+
await recover(target);
|
|
2946
|
+
return true;
|
|
2947
|
+
}
|
|
2948
|
+
};
|
|
2949
|
+
}
|
|
2950
|
+
async function verifySupervisorChildHealth({
|
|
2951
|
+
target,
|
|
2952
|
+
degradeOnExit,
|
|
2953
|
+
context,
|
|
2954
|
+
waitBeforeProbe,
|
|
2955
|
+
waitForLocalRunner: waitForLocalRunner2,
|
|
2956
|
+
stopExpectedly,
|
|
2957
|
+
markHealthy,
|
|
2958
|
+
markDegraded,
|
|
2959
|
+
isReadinessDeferredExit = () => false,
|
|
2960
|
+
isExpectedStop = () => false
|
|
2961
|
+
}) {
|
|
2962
|
+
try {
|
|
2963
|
+
await waitBeforeProbe();
|
|
2964
|
+
const healthy = supervisorChildIsRunning(target) && await waitForLocalRunner2(target);
|
|
2965
|
+
const exited = supervisorChildHasExited(target);
|
|
2966
|
+
if (isExpectedStop(target)) return false;
|
|
2967
|
+
if (exited && isReadinessDeferredExit(target)) return false;
|
|
2968
|
+
if (healthy && !exited) {
|
|
2969
|
+
if (markHealthy() !== false) return true;
|
|
2970
|
+
if (supervisorChildIsRunning(target)) await stopExpectedly(target);
|
|
2971
|
+
return false;
|
|
2972
|
+
}
|
|
2973
|
+
if (!exited || degradeOnExit) {
|
|
2974
|
+
const committed = markDegraded(`${context} did not become healthy; entering degraded mode`) !== false;
|
|
2975
|
+
if (!committed) return false;
|
|
2976
|
+
if (supervisorChildIsRunning(target)) await stopExpectedly(target);
|
|
2977
|
+
}
|
|
2978
|
+
return false;
|
|
2979
|
+
} catch (error) {
|
|
2980
|
+
if (isExpectedStop(target)) return false;
|
|
2981
|
+
const committed = markDegraded(
|
|
2982
|
+
`${context} health proof failed; entering degraded mode: ${error instanceof Error ? error.message : String(error)}`
|
|
2983
|
+
) !== false;
|
|
2984
|
+
if (!committed) return false;
|
|
2985
|
+
if (supervisorChildIsRunning(target)) await stopExpectedly(target).catch(() => {
|
|
2986
|
+
});
|
|
2987
|
+
return false;
|
|
2988
|
+
}
|
|
2989
|
+
}
|
|
2990
|
+
|
|
2480
2991
|
// src/runner-supervisor.mjs
|
|
2481
2992
|
var DEFAULT_CONTROL_PLANE_URL = "https://vo-control-plane-bzjphrajaq-uc.a.run.app";
|
|
2482
2993
|
var POLL_MS = 5e3;
|
|
2483
2994
|
var CHILD_START_MS = 1500;
|
|
2995
|
+
var CHILD_READINESS_TIMEOUT_MS = RUNNER_IDENTITY_READINESS_TIMEOUT_MS + RUNNER_GITHUB_READINESS_TIMEOUT_MS + 1e4;
|
|
2484
2996
|
var SUPERVISOR_CAPABILITIES = ["bundled-runtime-slots-v1", "legacy-orphan-purge-v1"];
|
|
2485
|
-
var
|
|
2997
|
+
var UUID_RE3 = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
|
|
2486
2998
|
var selfPath = fileURLToPath3(import.meta.url);
|
|
2487
2999
|
var bundledChildEntry = join5(dirname4(selfPath), "runner-cli.js");
|
|
2488
3000
|
var supervisorRuntimeRoot = runtimeRootFromEnv(process.env);
|
|
@@ -2495,7 +3007,7 @@ function packageVersion() {
|
|
|
2495
3007
|
return "unknown";
|
|
2496
3008
|
}
|
|
2497
3009
|
}
|
|
2498
|
-
var requestedSupervisorInstanceId =
|
|
3010
|
+
var requestedSupervisorInstanceId = UUID_RE3.test(String(process.env.VO_RUNNER_SUPERVISOR_INSTANCE_ID || "")) ? process.env.VO_RUNNER_SUPERVISOR_INSTANCE_ID : randomUUID4();
|
|
2499
3011
|
var supervisorInstanceId = activationSupervisorInstanceId(
|
|
2500
3012
|
runtimeRootFromEnv(process.env),
|
|
2501
3013
|
requestedSupervisorInstanceId
|
|
@@ -2519,16 +3031,27 @@ function spawnChild(childEnv) {
|
|
|
2519
3031
|
}
|
|
2520
3032
|
return spawn(process.execPath, resolved.args, {
|
|
2521
3033
|
env: childEnv,
|
|
2522
|
-
stdio: "inherit",
|
|
3034
|
+
stdio: ["inherit", "inherit", "inherit", "ipc"],
|
|
2523
3035
|
windowsHide: true
|
|
2524
3036
|
});
|
|
2525
3037
|
}
|
|
2526
3038
|
function spawnPreviousChild(entry, childEnv) {
|
|
2527
|
-
|
|
3039
|
+
const previous = spawn(process.execPath, [entry, "runner"], {
|
|
2528
3040
|
env: childEnv,
|
|
2529
|
-
stdio: "inherit",
|
|
3041
|
+
stdio: ["inherit", "inherit", "inherit", "ipc"],
|
|
2530
3042
|
windowsHide: true
|
|
2531
3043
|
});
|
|
3044
|
+
attachSupervisorChildTerminationCustody(
|
|
3045
|
+
previous,
|
|
3046
|
+
() => {
|
|
3047
|
+
},
|
|
3048
|
+
(error) => {
|
|
3049
|
+
console.error(
|
|
3050
|
+
`[vo-runner supervisor] rollback child process error: ${error instanceof Error ? error.message : String(error)}`
|
|
3051
|
+
);
|
|
3052
|
+
}
|
|
3053
|
+
);
|
|
3054
|
+
return previous;
|
|
2532
3055
|
}
|
|
2533
3056
|
async function localStatus() {
|
|
2534
3057
|
try {
|
|
@@ -2542,32 +3065,54 @@ async function localStatus() {
|
|
|
2542
3065
|
}
|
|
2543
3066
|
}
|
|
2544
3067
|
async function waitForLocalRunner(child) {
|
|
2545
|
-
const deadline = Date.now() +
|
|
3068
|
+
const deadline = Date.now() + CHILD_READINESS_TIMEOUT_MS;
|
|
2546
3069
|
while (Date.now() < deadline) {
|
|
2547
|
-
if (child
|
|
3070
|
+
if (supervisorChildHasExited(child)) return false;
|
|
2548
3071
|
const status = await localStatus();
|
|
2549
3072
|
if (status?.running === true && Number(status.pid) === Number(child.pid)) return true;
|
|
2550
3073
|
await sleep(500);
|
|
2551
3074
|
}
|
|
2552
3075
|
return false;
|
|
2553
3076
|
}
|
|
3077
|
+
async function waitForChildExit(child, timeoutMs) {
|
|
3078
|
+
if (!child || supervisorChildHasExited(child)) return true;
|
|
3079
|
+
return new Promise((resolve5) => {
|
|
3080
|
+
let settled = false;
|
|
3081
|
+
const finish = (exited) => {
|
|
3082
|
+
if (settled) return;
|
|
3083
|
+
settled = true;
|
|
3084
|
+
clearTimeout(timer);
|
|
3085
|
+
child.off("exit", onExit);
|
|
3086
|
+
resolve5(exited);
|
|
3087
|
+
};
|
|
3088
|
+
const onExit = () => finish(true);
|
|
3089
|
+
const timer = setTimeout(() => finish(false), timeoutMs);
|
|
3090
|
+
child.once("exit", onExit);
|
|
3091
|
+
if (supervisorChildHasExited(child)) finish(true);
|
|
3092
|
+
});
|
|
3093
|
+
}
|
|
2554
3094
|
async function stopChild(child) {
|
|
2555
|
-
if (!child || child
|
|
3095
|
+
if (!child || supervisorChildHasExited(child)) return;
|
|
2556
3096
|
child.kill(process.platform === "win32" ? void 0 : "SIGTERM");
|
|
2557
|
-
await
|
|
2558
|
-
|
|
2559
|
-
|
|
2560
|
-
]);
|
|
2561
|
-
if (child.exitCode === null) child.kill("SIGKILL");
|
|
3097
|
+
if (await waitForChildExit(child, 15e3)) return;
|
|
3098
|
+
if (supervisorChildIsRunning(child)) child.kill("SIGKILL");
|
|
3099
|
+
await waitForChildExit(child, 5e3);
|
|
2562
3100
|
}
|
|
2563
3101
|
async function main() {
|
|
2564
3102
|
const stored = readStoredCredentialIsolated();
|
|
2565
3103
|
const controlPlaneUrl = process.env.VO_CONTROL_PLANE_URL || DEFAULT_CONTROL_PLANE_URL;
|
|
2566
|
-
const
|
|
3104
|
+
const pairedSupervisor = !String(process.env.VO_CONTROL_PLANE_ADMIN_TOKEN || "").trim();
|
|
3105
|
+
const supervisorAuthInput = {
|
|
2567
3106
|
baseEnv: process.env,
|
|
2568
3107
|
storedCredential: stored,
|
|
2569
3108
|
controlPlaneUrl
|
|
2570
|
-
}
|
|
3109
|
+
};
|
|
3110
|
+
const {
|
|
3111
|
+
childEnv,
|
|
3112
|
+
clientEnv,
|
|
3113
|
+
operatorId,
|
|
3114
|
+
startupRetryAfterMs
|
|
3115
|
+
} = await prepareSupervisorAuth(supervisorAuthInput);
|
|
2571
3116
|
Object.assign(childEnv, {
|
|
2572
3117
|
VO_RUNNER_SUPERVISOR_INSTANCE_ID: supervisorInstanceId,
|
|
2573
3118
|
VO_RUNNER_SUPERVISOR_VERSION: supervisorVersion,
|
|
@@ -2578,37 +3123,266 @@ async function main() {
|
|
|
2578
3123
|
let child = null;
|
|
2579
3124
|
let stopping = false;
|
|
2580
3125
|
let handling = false;
|
|
2581
|
-
|
|
3126
|
+
const expectedChildStops = /* @__PURE__ */ new WeakSet();
|
|
3127
|
+
const healthNeutralizedChildren = /* @__PURE__ */ new WeakSet();
|
|
3128
|
+
const degradationState = createSupervisorDegradationState();
|
|
3129
|
+
const markSupervisorDegraded = () => {
|
|
3130
|
+
degradationState.markDegraded();
|
|
3131
|
+
process.exitCode = 1;
|
|
3132
|
+
};
|
|
3133
|
+
const respawnCircuit = createRespawnCircuit();
|
|
3134
|
+
const controlRecoveryFence = createSupervisorControlRecoveryFence();
|
|
3135
|
+
const startupDeferral = createSupervisorStartupDeferral({
|
|
3136
|
+
retryAfterMs: startupRetryAfterMs
|
|
3137
|
+
});
|
|
3138
|
+
let startupNeedsReadinessRefresh = startupRetryAfterMs !== null;
|
|
3139
|
+
let deferredControlRecoveryGeneration = null;
|
|
3140
|
+
const stopChildExpectedly = async (target) => {
|
|
3141
|
+
if (!target) return;
|
|
3142
|
+
expectedChildStops.add(target);
|
|
3143
|
+
healthNeutralizedChildren.add(target);
|
|
3144
|
+
try {
|
|
3145
|
+
await stopChild(target);
|
|
3146
|
+
} finally {
|
|
3147
|
+
if (supervisorChildHasExited(target)) expectedChildStops.delete(target);
|
|
3148
|
+
}
|
|
3149
|
+
};
|
|
3150
|
+
const readinessDeferrals = createSupervisorReadinessDeferralTracker({
|
|
3151
|
+
currentChild: () => child,
|
|
3152
|
+
releaseCurrentChild: (target) => {
|
|
3153
|
+
if (child === target) child = null;
|
|
3154
|
+
},
|
|
3155
|
+
onCaptured: ({ retryAfterMs, recoveryGeneration }) => {
|
|
3156
|
+
startupNeedsReadinessRefresh = true;
|
|
3157
|
+
if (Number.isInteger(recoveryGeneration) && degradationState.snapshot().generation === recoveryGeneration) {
|
|
3158
|
+
deferredControlRecoveryGeneration = recoveryGeneration;
|
|
3159
|
+
}
|
|
3160
|
+
startupDeferral.reopen(retryAfterMs);
|
|
3161
|
+
console.warn(
|
|
3162
|
+
`[vo-runner supervisor] child deferred by GitHub readiness for ${retryAfterMs}ms; preserving control polling without consuming rapid-exit breaker budget`
|
|
3163
|
+
);
|
|
3164
|
+
}
|
|
3165
|
+
});
|
|
3166
|
+
const isChildReadinessDeferred = (target) => readinessDeferrals.isDeferred(target);
|
|
3167
|
+
const captureChildReadinessDeferral = (target) => readinessDeferrals.capture(target);
|
|
2582
3168
|
const respawn = () => {
|
|
2583
|
-
if (!
|
|
3169
|
+
if (!startupDeferral.hasStarted()) return;
|
|
3170
|
+
if (!shouldRespawnSupervisorChild({
|
|
3171
|
+
stopping,
|
|
3172
|
+
degraded: degradationState.isDegraded(),
|
|
3173
|
+
handling,
|
|
3174
|
+
child
|
|
3175
|
+
})) return;
|
|
3176
|
+
try {
|
|
3177
|
+
child = launchChild();
|
|
3178
|
+
void verifyChildHealth(child, false, "automatic relaunch");
|
|
3179
|
+
} catch (error) {
|
|
3180
|
+
markSupervisorDegraded();
|
|
3181
|
+
console.error(
|
|
3182
|
+
`[vo-runner supervisor] child relaunch failed; entering degraded mode: ${error instanceof Error ? error.message : String(error)}`
|
|
3183
|
+
);
|
|
3184
|
+
}
|
|
2584
3185
|
};
|
|
2585
3186
|
const launchChild = () => {
|
|
2586
3187
|
const next = spawnChild(childEnv);
|
|
2587
|
-
|
|
3188
|
+
const startedAt = Date.now();
|
|
3189
|
+
next.on("message", (message) => {
|
|
3190
|
+
const request = parseRunnerReadinessDeferralRequest(message);
|
|
3191
|
+
if (!request || !readinessDeferrals.recordPending(next, request)) return;
|
|
3192
|
+
try {
|
|
3193
|
+
next.send({
|
|
3194
|
+
type: RUNNER_READINESS_DEFERRAL_ACK_TYPE,
|
|
3195
|
+
nonce: request.nonce
|
|
3196
|
+
}, (error) => {
|
|
3197
|
+
if (error) console.warn(`[vo-runner supervisor] readiness deferral ACK failed: ${error.message}`);
|
|
3198
|
+
});
|
|
3199
|
+
} catch (error) {
|
|
3200
|
+
console.warn(
|
|
3201
|
+
`[vo-runner supervisor] readiness deferral ACK failed: ${error instanceof Error ? error.message : String(error)}`
|
|
3202
|
+
);
|
|
3203
|
+
}
|
|
3204
|
+
});
|
|
3205
|
+
attachSupervisorChildTerminationCustody(
|
|
3206
|
+
next,
|
|
3207
|
+
() => {
|
|
3208
|
+
if (stopping || expectedChildStops.delete(next)) {
|
|
3209
|
+
readinessDeferrals.forget(next);
|
|
3210
|
+
return;
|
|
3211
|
+
}
|
|
3212
|
+
if (captureChildReadinessDeferral(next)) return;
|
|
3213
|
+
readinessDeferrals.forget(next);
|
|
3214
|
+
const decision = respawnCircuit.recordExit(startedAt, Date.now());
|
|
3215
|
+
if (decision.tripped) {
|
|
3216
|
+
markSupervisorDegraded();
|
|
3217
|
+
console.error(
|
|
3218
|
+
`[vo-runner supervisor] child exited rapidly ${decision.rapidExits} times; entering degraded mode to stop startup/token churn while remote repair remains available`
|
|
3219
|
+
);
|
|
3220
|
+
return;
|
|
3221
|
+
}
|
|
3222
|
+
setTimeout(respawn, decision.delayMs);
|
|
3223
|
+
},
|
|
3224
|
+
(error) => {
|
|
3225
|
+
console.error(
|
|
3226
|
+
`[vo-runner supervisor] child process error: ${error instanceof Error ? error.message : String(error)}`
|
|
3227
|
+
);
|
|
3228
|
+
}
|
|
3229
|
+
);
|
|
2588
3230
|
return next;
|
|
2589
3231
|
};
|
|
2590
|
-
|
|
2591
|
-
|
|
2592
|
-
|
|
2593
|
-
|
|
2594
|
-
|
|
2595
|
-
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
|
|
2602
|
-
|
|
2603
|
-
|
|
2604
|
-
|
|
2605
|
-
|
|
2606
|
-
|
|
2607
|
-
|
|
2608
|
-
|
|
2609
|
-
|
|
2610
|
-
|
|
2611
|
-
|
|
3232
|
+
const verifyChildHealth = async (target, degradeOnExit, context) => {
|
|
3233
|
+
const commitSupervisorHealthyState = beginSupervisorHealthyStateCommit({
|
|
3234
|
+
degradationState,
|
|
3235
|
+
target,
|
|
3236
|
+
currentChild: () => child,
|
|
3237
|
+
allowRecovery: degradeOnExit,
|
|
3238
|
+
clearStaleExitCode: () => {
|
|
3239
|
+
process.exitCode = void 0;
|
|
3240
|
+
}
|
|
3241
|
+
});
|
|
3242
|
+
const commitSupervisorDegradedState = degradationState.beginDegradationProof({
|
|
3243
|
+
target,
|
|
3244
|
+
currentChild: () => child,
|
|
3245
|
+
onDegraded: (message) => {
|
|
3246
|
+
markSupervisorDegraded();
|
|
3247
|
+
console.error(`[vo-runner supervisor] ${message}`);
|
|
3248
|
+
}
|
|
3249
|
+
});
|
|
3250
|
+
return verifySupervisorChildHealth({
|
|
3251
|
+
target,
|
|
3252
|
+
degradeOnExit,
|
|
3253
|
+
context,
|
|
3254
|
+
waitBeforeProbe: () => sleep(CHILD_START_MS),
|
|
3255
|
+
waitForLocalRunner,
|
|
3256
|
+
stopExpectedly: stopChildExpectedly,
|
|
3257
|
+
isReadinessDeferredExit: isChildReadinessDeferred,
|
|
3258
|
+
isExpectedStop: (candidate) => healthNeutralizedChildren.has(candidate),
|
|
3259
|
+
markHealthy: commitSupervisorHealthyState,
|
|
3260
|
+
markDegraded: commitSupervisorDegradedState
|
|
3261
|
+
});
|
|
3262
|
+
};
|
|
3263
|
+
const deferChildAdmission = (retryAfterMs) => {
|
|
3264
|
+
startupNeedsReadinessRefresh = true;
|
|
3265
|
+
if (startupDeferral.hasStarted()) startupDeferral.reopen(retryAfterMs);
|
|
3266
|
+
else startupDeferral.defer(retryAfterMs);
|
|
3267
|
+
console.warn(
|
|
3268
|
+
`[vo-runner supervisor] GitHub readiness embargoed child startup for ${retryAfterMs}ms; control polling remains active`
|
|
3269
|
+
);
|
|
3270
|
+
};
|
|
3271
|
+
const refreshSupervisorChildAdmission = async () => {
|
|
3272
|
+
if (!pairedSupervisor) return true;
|
|
3273
|
+
const refreshed = await prepareSupervisorAuth({
|
|
3274
|
+
...supervisorAuthInput,
|
|
3275
|
+
baseEnv: childEnv
|
|
3276
|
+
});
|
|
3277
|
+
if (refreshed.operatorId !== operatorId) {
|
|
3278
|
+
throw new Error("runner readiness operator identity changed during supervisor admission");
|
|
3279
|
+
}
|
|
3280
|
+
if (refreshed.startupRetryAfterMs !== null) {
|
|
3281
|
+
deferChildAdmission(refreshed.startupRetryAfterMs);
|
|
3282
|
+
return false;
|
|
3283
|
+
}
|
|
3284
|
+
if (refreshed.repositoryScope.length > 0) {
|
|
3285
|
+
const repositoryScope = refreshed.repositoryScope.join(",");
|
|
3286
|
+
childEnv.VO_CODE_RUNNER_REPOS = repositoryScope;
|
|
3287
|
+
clientEnv.VO_CODE_RUNNER_REPOS = repositoryScope;
|
|
3288
|
+
}
|
|
3289
|
+
startupNeedsReadinessRefresh = false;
|
|
3290
|
+
return true;
|
|
3291
|
+
};
|
|
3292
|
+
const ensureHealthyChildAfterControl = async () => {
|
|
3293
|
+
if (!child || supervisorChildHasExited(child)) {
|
|
3294
|
+
const recoveryGeneration = degradationState.snapshot().generation;
|
|
3295
|
+
if (!startupDeferral.hasStarted() && startupDeferral.isPending()) {
|
|
3296
|
+
deferredControlRecoveryGeneration = recoveryGeneration;
|
|
3297
|
+
return false;
|
|
3298
|
+
}
|
|
3299
|
+
try {
|
|
3300
|
+
if (!await refreshSupervisorChildAdmission()) {
|
|
3301
|
+
deferredControlRecoveryGeneration = recoveryGeneration;
|
|
3302
|
+
return false;
|
|
3303
|
+
}
|
|
3304
|
+
if (!startupDeferral.hasStarted() && !startupDeferral.consumeIfReady()) return false;
|
|
3305
|
+
child = launchChild();
|
|
3306
|
+
readinessDeferrals.recordRecoveryGeneration(child, recoveryGeneration);
|
|
3307
|
+
} catch (error) {
|
|
3308
|
+
markSupervisorDegraded();
|
|
3309
|
+
console.error(
|
|
3310
|
+
`[vo-runner supervisor] control recovery relaunch failed; entering degraded mode: ${error instanceof Error ? error.message : String(error)}`
|
|
3311
|
+
);
|
|
3312
|
+
return false;
|
|
3313
|
+
}
|
|
3314
|
+
}
|
|
3315
|
+
return verifyChildHealth(child, true, "control recovery relaunch");
|
|
3316
|
+
};
|
|
3317
|
+
const startInitialChild = async (recoveryGeneration = null) => {
|
|
3318
|
+
const activationChild = launchChild();
|
|
3319
|
+
child = activationChild;
|
|
3320
|
+
if (Number.isInteger(recoveryGeneration)) {
|
|
3321
|
+
readinessDeferrals.recordRecoveryGeneration(activationChild, recoveryGeneration);
|
|
3322
|
+
}
|
|
3323
|
+
void verifyChildHealth(
|
|
3324
|
+
activationChild,
|
|
3325
|
+
Number.isInteger(recoveryGeneration),
|
|
3326
|
+
Number.isInteger(recoveryGeneration) ? "deferred control recovery" : "initial child"
|
|
3327
|
+
);
|
|
3328
|
+
if (!await finishPendingActivation({
|
|
3329
|
+
client,
|
|
3330
|
+
child: activationChild,
|
|
3331
|
+
runtimeRoot,
|
|
3332
|
+
operatorId,
|
|
3333
|
+
runnerId,
|
|
3334
|
+
selfPath,
|
|
3335
|
+
packageVersion: supervisorVersion,
|
|
3336
|
+
supervisorIdentity: supervisorControlIdentity,
|
|
3337
|
+
waitForLocalRunner,
|
|
3338
|
+
isReadinessDeferred: isChildReadinessDeferred,
|
|
3339
|
+
localStatus,
|
|
3340
|
+
stopChild: stopChildExpectedly,
|
|
3341
|
+
launchPreviousChild: (entry) => spawnPreviousChild(entry, childEnv),
|
|
3342
|
+
log: (message) => console.error(`[vo-runner supervisor] ${message}`)
|
|
3343
|
+
})) {
|
|
3344
|
+
if (captureChildReadinessDeferral(activationChild)) {
|
|
3345
|
+
console.warn("[vo-runner supervisor] activation attestation deferred with GitHub readiness; pending slot remains durable");
|
|
3346
|
+
return false;
|
|
3347
|
+
}
|
|
3348
|
+
markSupervisorDegraded();
|
|
3349
|
+
await stopChildExpectedly(activationChild);
|
|
3350
|
+
if (child === activationChild) child = null;
|
|
3351
|
+
console.error("[vo-runner supervisor] activation FAILED \u2014 entering degraded mode: not serving tasks, still polling for remote control actions");
|
|
3352
|
+
return false;
|
|
3353
|
+
}
|
|
3354
|
+
return true;
|
|
3355
|
+
};
|
|
3356
|
+
const attemptInitialChildStart = async () => {
|
|
3357
|
+
if (startupDeferral.hasStarted() || startupDeferral.isPending()) return false;
|
|
3358
|
+
const beforeRefresh = resolveSupervisorChildStartAuthority({
|
|
3359
|
+
degradationState,
|
|
3360
|
+
deferredRecoveryGeneration: deferredControlRecoveryGeneration
|
|
3361
|
+
});
|
|
3362
|
+
if (!beforeRefresh.allowed) return false;
|
|
3363
|
+
if (startupNeedsReadinessRefresh) {
|
|
3364
|
+
try {
|
|
3365
|
+
if (!await refreshSupervisorChildAdmission()) return false;
|
|
3366
|
+
} catch (error) {
|
|
3367
|
+
markSupervisorDegraded();
|
|
3368
|
+
deferredControlRecoveryGeneration = null;
|
|
3369
|
+
startupDeferral.consumeIfReady();
|
|
3370
|
+
console.error(
|
|
3371
|
+
`[vo-runner supervisor] deferred readiness refresh failed; entering degraded mode: ${error instanceof Error ? error.message : String(error)}`
|
|
3372
|
+
);
|
|
3373
|
+
return false;
|
|
3374
|
+
}
|
|
3375
|
+
}
|
|
3376
|
+
const afterRefresh = resolveSupervisorChildStartAuthority({
|
|
3377
|
+
degradationState,
|
|
3378
|
+
deferredRecoveryGeneration: deferredControlRecoveryGeneration
|
|
3379
|
+
});
|
|
3380
|
+
if (!afterRefresh.allowed) return false;
|
|
3381
|
+
if (!startupDeferral.consumeIfReady()) return false;
|
|
3382
|
+
deferredControlRecoveryGeneration = null;
|
|
3383
|
+
return startInitialChild(afterRefresh.recoveryGeneration);
|
|
3384
|
+
};
|
|
3385
|
+
await attemptInitialChildStart();
|
|
2612
3386
|
const shutdown = async () => {
|
|
2613
3387
|
if (stopping) return;
|
|
2614
3388
|
stopping = true;
|
|
@@ -2622,6 +3396,7 @@ async function main() {
|
|
|
2622
3396
|
});
|
|
2623
3397
|
while (!stopping) {
|
|
2624
3398
|
try {
|
|
3399
|
+
await attemptInitialChildStart();
|
|
2625
3400
|
const action = await client.pollRunnerControl({
|
|
2626
3401
|
runnerId,
|
|
2627
3402
|
...operatorId ? { operatorId } : {},
|
|
@@ -2631,6 +3406,7 @@ async function main() {
|
|
|
2631
3406
|
await sleep(POLL_MS);
|
|
2632
3407
|
continue;
|
|
2633
3408
|
}
|
|
3409
|
+
deferredControlRecoveryGeneration = null;
|
|
2634
3410
|
handling = true;
|
|
2635
3411
|
const beforeStop = await localStatus();
|
|
2636
3412
|
if (beforeStop && Number(beforeStop.activeTasks || 0) > 0) {
|
|
@@ -2642,9 +3418,11 @@ async function main() {
|
|
|
2642
3418
|
detail: `deferred safely: ${beforeStop.activeTasks} active task(s); retry when the runner is idle`
|
|
2643
3419
|
});
|
|
2644
3420
|
handling = false;
|
|
3421
|
+
respawn();
|
|
2645
3422
|
continue;
|
|
2646
3423
|
}
|
|
2647
|
-
|
|
3424
|
+
controlRecoveryFence.markBeforeStop(child);
|
|
3425
|
+
await stopChildExpectedly(child);
|
|
2648
3426
|
const bundledAction = action.kind === "update" || action.kind === "reinstall";
|
|
2649
3427
|
const result = bundledAction ? stageAndActivateBundledUpdate({
|
|
2650
3428
|
runtimeRoot,
|
|
@@ -2670,20 +3448,25 @@ async function main() {
|
|
|
2670
3448
|
console.warn(`[vo-runner supervisor] activated ${result.active.version}; exiting for new-process attestation`);
|
|
2671
3449
|
return;
|
|
2672
3450
|
}
|
|
2673
|
-
|
|
2674
|
-
|
|
2675
|
-
|
|
2676
|
-
|
|
3451
|
+
const recoveryAuthorized = shouldRecoverSupervisorChildAfterAction({
|
|
3452
|
+
stoppedChild: controlRecoveryFence.consume(),
|
|
3453
|
+
actionKind: action.kind,
|
|
3454
|
+
actionSucceeded: result.ok
|
|
3455
|
+
});
|
|
3456
|
+
const relaunchedHealthy = recoveryAuthorized ? await ensureHealthyChildAfterControl() : false;
|
|
3457
|
+
if (!relaunchedHealthy) result.ok = false;
|
|
2677
3458
|
await client.completeRunnerControl(action.actionId, {
|
|
2678
3459
|
runnerId,
|
|
2679
3460
|
...operatorId ? { operatorId } : {},
|
|
2680
3461
|
...supervisorControlIdentity,
|
|
2681
3462
|
status: result.ok ? "succeeded" : "failed",
|
|
2682
|
-
detail: result.ok ? result.detail ? `${result.detail}; runner ${packageVersion()} reconnected`.slice(0, 1e3) : `runner ${packageVersion()} reconnected` : `maintenance exited ${result.status}${result.detail ? `: ${result.detail}` : ""}`
|
|
3463
|
+
detail: result.ok ? result.detail ? `${result.detail}; runner ${packageVersion()} reconnected`.slice(0, 1e3) : `runner ${packageVersion()} reconnected` : `maintenance exited ${result.status}${result.detail ? `: ${result.detail}` : ""}${relaunchedHealthy ? "" : "; runner relaunch did not become healthy"}`
|
|
2683
3464
|
});
|
|
2684
3465
|
handling = false;
|
|
3466
|
+
respawn();
|
|
2685
3467
|
} catch (error) {
|
|
2686
3468
|
console.error(`[vo-runner supervisor] ${error instanceof Error ? error.message : String(error)}`);
|
|
3469
|
+
await controlRecoveryFence.recoverOnce(() => ensureHealthyChildAfterControl());
|
|
2687
3470
|
handling = false;
|
|
2688
3471
|
await sleep(POLL_MS);
|
|
2689
3472
|
}
|