@algosuite/vo-mcp 0.2.0-beta.56 → 0.2.0-beta.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/autostart-cli.js +6 -3
- package/dist/autostart-cli.js.map +2 -2
- package/dist/install-cli.js +6 -3
- package/dist/install-cli.js.map +2 -2
- package/dist/runner-cli.js +344 -63
- package/dist/runner-cli.js.map +4 -4
- package/dist/runner-supervisor.js +970 -88
- package/dist/runner-supervisor.js.map +4 -4
- package/package.json +1 -1
|
@@ -31,7 +31,7 @@ var init_control_plane_auth_stub = __esm({
|
|
|
31
31
|
|
|
32
32
|
// src/runner-supervisor.mjs
|
|
33
33
|
import { spawn } from "node:child_process";
|
|
34
|
-
import { randomUUID as
|
|
34
|
+
import { randomUUID as randomUUID4 } from "node:crypto";
|
|
35
35
|
import { createRequire } from "node:module";
|
|
36
36
|
import { fileURLToPath as fileURLToPath3 } from "node:url";
|
|
37
37
|
import { dirname as dirname4, join as join5 } from "node:path";
|
|
@@ -2090,6 +2090,7 @@ async function finishPendingActivation({
|
|
|
2090
2090
|
packageVersion: packageVersion2,
|
|
2091
2091
|
supervisorIdentity,
|
|
2092
2092
|
waitForLocalRunner: waitForLocalRunner2,
|
|
2093
|
+
isReadinessDeferred = () => false,
|
|
2093
2094
|
localStatus: localStatus2,
|
|
2094
2095
|
waitForCloudRunner = waitForAuthoritativeRunnerHeartbeat,
|
|
2095
2096
|
cloudTimeoutMs,
|
|
@@ -2124,7 +2125,10 @@ async function finishPendingActivation({
|
|
|
2124
2125
|
try {
|
|
2125
2126
|
attestation = attestCurrentSupervisor({ runtimeRoot, selfPath: selfPath2, version: packageVersion2 });
|
|
2126
2127
|
if (!attestation.ok) throw new Error(attestation.detail);
|
|
2127
|
-
if (!await waitForLocalRunner2(child))
|
|
2128
|
+
if (!await waitForLocalRunner2(child)) {
|
|
2129
|
+
if (isReadinessDeferred(child)) return false;
|
|
2130
|
+
throw new Error("activated runner did not become locally ready");
|
|
2131
|
+
}
|
|
2128
2132
|
} catch (error) {
|
|
2129
2133
|
let detail = `activation attestation failed: ${error instanceof Error ? error.message : String(error)}`.slice(0, 400);
|
|
2130
2134
|
let rolledBack = null;
|
|
@@ -2314,12 +2318,252 @@ function resolveSupervisorChildEntry({
|
|
|
2314
2318
|
};
|
|
2315
2319
|
}
|
|
2316
2320
|
|
|
2321
|
+
// src/runner/update-drain-gate.mjs
|
|
2322
|
+
var DEFAULT_DRAIN_CAP_MS = 45 * 60 * 1e3;
|
|
2323
|
+
var DEFAULT_DRAIN_CHECK_MS = 3e4;
|
|
2324
|
+
var MAX_DRAIN_CAP_MS = 6 * 60 * 60 * 1e3;
|
|
2325
|
+
var MIN_DRAIN_CHECK_MS = 1e3;
|
|
2326
|
+
var MAX_DRAIN_CHECK_MS = 5 * 60 * 1e3;
|
|
2327
|
+
var UPDATE_RESTART_ERROR_MESSAGE = "runner update restart after drain timeout";
|
|
2328
|
+
var UPDATE_RESTART_RESULT = "runner_update_restart";
|
|
2329
|
+
var sleepMs = (ms) => new Promise((resolve5) => setTimeout(resolve5, ms));
|
|
2330
|
+
function positiveNumber(raw) {
|
|
2331
|
+
if (raw === void 0 || raw === null || String(raw).trim() === "") return null;
|
|
2332
|
+
const value = Number(raw);
|
|
2333
|
+
return Number.isFinite(value) && value >= 0 ? value : null;
|
|
2334
|
+
}
|
|
2335
|
+
function resolveDrainCapMs(env = {}) {
|
|
2336
|
+
const ms = positiveNumber(env.VO_RUNNER_UPDATE_DRAIN_CAP_MS);
|
|
2337
|
+
if (ms !== null) return Math.min(ms, MAX_DRAIN_CAP_MS);
|
|
2338
|
+
const minutes = positiveNumber(env.VO_RUNNER_UPDATE_DRAIN_CAP_MIN);
|
|
2339
|
+
if (minutes !== null) return Math.min(minutes * 6e4, MAX_DRAIN_CAP_MS);
|
|
2340
|
+
return DEFAULT_DRAIN_CAP_MS;
|
|
2341
|
+
}
|
|
2342
|
+
function resolveDrainCheckMs(env = {}) {
|
|
2343
|
+
const ms = positiveNumber(env.VO_RUNNER_UPDATE_DRAIN_CHECK_MS);
|
|
2344
|
+
if (ms === null) return DEFAULT_DRAIN_CHECK_MS;
|
|
2345
|
+
return Math.min(Math.max(ms, MIN_DRAIN_CHECK_MS), MAX_DRAIN_CHECK_MS);
|
|
2346
|
+
}
|
|
2347
|
+
function readActiveTasks(status, { childRunning = true } = {}) {
|
|
2348
|
+
if (!childRunning) {
|
|
2349
|
+
return { count: 0, ids: [], known: true, runnerId: null, runnerInstanceId: null };
|
|
2350
|
+
}
|
|
2351
|
+
if (!status || status.ok === false) {
|
|
2352
|
+
return { count: 0, ids: [], known: false, runnerId: null, runnerInstanceId: null };
|
|
2353
|
+
}
|
|
2354
|
+
const ids = Array.isArray(status.activeTaskIds) ? status.activeTaskIds.map((id) => String(id)).filter(Boolean) : [];
|
|
2355
|
+
const reported = Number(status.activeTasks);
|
|
2356
|
+
const count = Number.isFinite(reported) && reported >= 0 ? reported : ids.length;
|
|
2357
|
+
return {
|
|
2358
|
+
count: Math.max(count, ids.length),
|
|
2359
|
+
ids,
|
|
2360
|
+
known: true,
|
|
2361
|
+
runnerId: status.runnerId ? String(status.runnerId) : null,
|
|
2362
|
+
runnerInstanceId: status.runnerInstanceId ? String(status.runnerInstanceId) : null
|
|
2363
|
+
};
|
|
2364
|
+
}
|
|
2365
|
+
function decideDrainStep({ active, elapsedMs, capMs }) {
|
|
2366
|
+
const busy = active.known ? active.count : "unknown";
|
|
2367
|
+
if (active.known && active.count === 0) return { action: "proceed", busy };
|
|
2368
|
+
if (elapsedMs >= capMs) return { action: "cap", busy };
|
|
2369
|
+
return { action: "wait", busy };
|
|
2370
|
+
}
|
|
2371
|
+
function describeBusy(active) {
|
|
2372
|
+
if (!active.known) return "status unreadable from a running child (assumed busy)";
|
|
2373
|
+
const ids = active.ids.length > 0 ? ` [${active.ids.join(", ")}]` : " [ids unavailable]";
|
|
2374
|
+
return `${active.count} active task(s)${ids}`;
|
|
2375
|
+
}
|
|
2376
|
+
async function markCappedTasks({ active, failInFlightTask, log }) {
|
|
2377
|
+
if (active.ids.length === 0) {
|
|
2378
|
+
if (active.count > 0 || !active.known) {
|
|
2379
|
+
log(`update drain cap reached with ${describeBusy(active)} \u2014 cannot annotate tasks the runner did not name`);
|
|
2380
|
+
}
|
|
2381
|
+
return [];
|
|
2382
|
+
}
|
|
2383
|
+
const marked = [];
|
|
2384
|
+
for (const taskId of active.ids) {
|
|
2385
|
+
try {
|
|
2386
|
+
await failInFlightTask(taskId, {
|
|
2387
|
+
message: UPDATE_RESTART_ERROR_MESSAGE,
|
|
2388
|
+
result: UPDATE_RESTART_RESULT,
|
|
2389
|
+
runnerId: active.runnerId,
|
|
2390
|
+
runnerInstanceId: active.runnerInstanceId
|
|
2391
|
+
});
|
|
2392
|
+
marked.push(taskId);
|
|
2393
|
+
} catch (error) {
|
|
2394
|
+
log(`could not mark task ${taskId} before update restart: ${error instanceof Error ? error.message : String(error)}`);
|
|
2395
|
+
}
|
|
2396
|
+
}
|
|
2397
|
+
return marked;
|
|
2398
|
+
}
|
|
2399
|
+
async function awaitRunnerUpdateDrain({
|
|
2400
|
+
readStatus,
|
|
2401
|
+
isChildRunning = () => true,
|
|
2402
|
+
failInFlightTask = async () => {
|
|
2403
|
+
},
|
|
2404
|
+
drainable = false,
|
|
2405
|
+
env = {},
|
|
2406
|
+
now = Date.now,
|
|
2407
|
+
sleep: sleep2 = sleepMs,
|
|
2408
|
+
log = () => {
|
|
2409
|
+
}
|
|
2410
|
+
} = {}) {
|
|
2411
|
+
const capMs = resolveDrainCapMs(env);
|
|
2412
|
+
const checkMs = resolveDrainCheckMs(env);
|
|
2413
|
+
const startedAt = now();
|
|
2414
|
+
let checks = 0;
|
|
2415
|
+
let lastReported = null;
|
|
2416
|
+
const snapshot = async () => {
|
|
2417
|
+
checks += 1;
|
|
2418
|
+
const childRunning = Boolean(isChildRunning());
|
|
2419
|
+
let status;
|
|
2420
|
+
try {
|
|
2421
|
+
status = await readStatus();
|
|
2422
|
+
} catch {
|
|
2423
|
+
status = null;
|
|
2424
|
+
}
|
|
2425
|
+
return readActiveTasks(status, { childRunning });
|
|
2426
|
+
};
|
|
2427
|
+
if (!drainable) {
|
|
2428
|
+
const active = await snapshot();
|
|
2429
|
+
if (active.known && active.count > 0) {
|
|
2430
|
+
return {
|
|
2431
|
+
proceed: false,
|
|
2432
|
+
detail: `deferred safely: ${active.count} active task(s); retry when the runner is idle`,
|
|
2433
|
+
capped: false,
|
|
2434
|
+
waitedMs: 0,
|
|
2435
|
+
checks,
|
|
2436
|
+
markedTaskIds: []
|
|
2437
|
+
};
|
|
2438
|
+
}
|
|
2439
|
+
return { proceed: true, detail: "runner idle", capped: false, waitedMs: 0, checks, markedTaskIds: [] };
|
|
2440
|
+
}
|
|
2441
|
+
for (; ; ) {
|
|
2442
|
+
const active = await snapshot();
|
|
2443
|
+
const elapsedMs = now() - startedAt;
|
|
2444
|
+
const step = decideDrainStep({ active, elapsedMs, capMs });
|
|
2445
|
+
if (step.action === "proceed") {
|
|
2446
|
+
if (lastReported !== null) {
|
|
2447
|
+
log(`update drain complete after ${Math.round(elapsedMs / 1e3)}s \u2014 runner idle, applying staged update restart`);
|
|
2448
|
+
}
|
|
2449
|
+
return { proceed: true, detail: "runner idle", capped: false, waitedMs: elapsedMs, checks, markedTaskIds: [] };
|
|
2450
|
+
}
|
|
2451
|
+
if (step.action === "cap") {
|
|
2452
|
+
log(`update drain cap reached after ${Math.round(elapsedMs / 1e3)}s (cap ${Math.round(capMs / 1e3)}s) \u2014 ${describeBusy(active)}; marking task(s) before restart`);
|
|
2453
|
+
const markedTaskIds = await markCappedTasks({ active, failInFlightTask, log });
|
|
2454
|
+
return {
|
|
2455
|
+
proceed: true,
|
|
2456
|
+
capped: true,
|
|
2457
|
+
detail: `update restart applied after drain cap; marked ${markedTaskIds.length} task(s) as "${UPDATE_RESTART_ERROR_MESSAGE}"`,
|
|
2458
|
+
waitedMs: elapsedMs,
|
|
2459
|
+
checks,
|
|
2460
|
+
markedTaskIds
|
|
2461
|
+
};
|
|
2462
|
+
}
|
|
2463
|
+
const fingerprint = `${step.busy}:${active.ids.join(",")}`;
|
|
2464
|
+
if (fingerprint !== lastReported) {
|
|
2465
|
+
lastReported = fingerprint;
|
|
2466
|
+
log(`deferring staged update restart \u2014 ${describeBusy(active)}; re-checking every ${Math.round(checkMs / 1e3)}s until idle (cap ${Math.round(capMs / 1e3)}s)`);
|
|
2467
|
+
}
|
|
2468
|
+
await sleep2(Math.max(1, Math.min(checkMs, capMs - elapsedMs)));
|
|
2469
|
+
}
|
|
2470
|
+
}
|
|
2471
|
+
|
|
2317
2472
|
// src/runner/supervisor-child-env.mjs
|
|
2318
2473
|
import { hostname as systemHostname } from "node:os";
|
|
2319
2474
|
|
|
2320
2475
|
// src/runner-readiness.mjs
|
|
2321
|
-
|
|
2322
|
-
|
|
2476
|
+
import { createHash as createHash5, randomUUID as randomUUID3 } from "node:crypto";
|
|
2477
|
+
var RUNNER_GITHUB_READINESS_TIMEOUT_MS = 5e4;
|
|
2478
|
+
var RUNNER_IDENTITY_READINESS_TIMEOUT_MS = 1e4;
|
|
2479
|
+
var RUNNER_READINESS_MAX_REPOSITORIES = 4;
|
|
2480
|
+
var RUNNER_READINESS_TRANSIENT_MAX_RETRY_AFTER_MS = 3e5;
|
|
2481
|
+
var RUNNER_READINESS_DEFERRAL_MESSAGE_TYPE = "vo-runner-readiness-deferred-v1";
|
|
2482
|
+
var RUNNER_READINESS_DEFERRAL_ACK_TYPE = "vo-runner-readiness-deferred-ack-v1";
|
|
2483
|
+
var RUNNER_REPOSITORY = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u;
|
|
2484
|
+
var UUID_RE2 = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
|
|
2485
|
+
function runnerRepositoryScopeFromEnv(env = {}) {
|
|
2486
|
+
return String(env.VO_CODE_RUNNER_REPOS || "").split(/[\s,]+/u).map((repo) => repo.trim()).filter(Boolean);
|
|
2487
|
+
}
|
|
2488
|
+
function normalizeRunnerRepositoryScope(repositories) {
|
|
2489
|
+
if (!Array.isArray(repositories) || repositories.length < 1) {
|
|
2490
|
+
throw new Error("VO_CODE_RUNNER_REPOS must name at least one owner/name repository");
|
|
2491
|
+
}
|
|
2492
|
+
if (repositories.length > RUNNER_READINESS_MAX_REPOSITORIES) {
|
|
2493
|
+
throw new Error(`VO_CODE_RUNNER_REPOS supports at most ${RUNNER_READINESS_MAX_REPOSITORIES} repositories`);
|
|
2494
|
+
}
|
|
2495
|
+
const normalized = repositories.map((repo) => {
|
|
2496
|
+
const [owner, name] = typeof repo === "string" ? repo.split("/") : [];
|
|
2497
|
+
if (typeof repo !== "string" || repo.length > 140 || !RUNNER_REPOSITORY.test(repo) || owner === "." || owner === ".." || name === "." || name === "..") {
|
|
2498
|
+
throw new Error("VO_CODE_RUNNER_REPOS entries must be canonical owner/name repositories");
|
|
2499
|
+
}
|
|
2500
|
+
return repo.toLowerCase();
|
|
2501
|
+
});
|
|
2502
|
+
if (new Set(normalized).size !== normalized.length) {
|
|
2503
|
+
throw new Error("VO_CODE_RUNNER_REPOS entries must be unique");
|
|
2504
|
+
}
|
|
2505
|
+
const owners = new Set(normalized.map((repo) => repo.split("/")[0]));
|
|
2506
|
+
if (owners.size !== 1) {
|
|
2507
|
+
throw new Error("VO_CODE_RUNNER_REPOS entries must share one owner");
|
|
2508
|
+
}
|
|
2509
|
+
return Object.freeze(normalized.sort());
|
|
2510
|
+
}
|
|
2511
|
+
function runnerRepositoryScopeDigest(repositories) {
|
|
2512
|
+
const scope = normalizeRunnerRepositoryScope(repositories);
|
|
2513
|
+
return createHash5("sha256").update(JSON.stringify({
|
|
2514
|
+
version: 1,
|
|
2515
|
+
repositories: scope
|
|
2516
|
+
}), "utf8").digest("hex");
|
|
2517
|
+
}
|
|
2518
|
+
function runnerReadinessRetryDelayMs(readiness) {
|
|
2519
|
+
if (readiness?.paired !== true || readiness?.githubReady !== false) return null;
|
|
2520
|
+
if (Number.isFinite(readiness.retryAfterMs) && readiness.retryAfterMs > 0) {
|
|
2521
|
+
return Math.ceil(readiness.retryAfterMs);
|
|
2522
|
+
}
|
|
2523
|
+
return null;
|
|
2524
|
+
}
|
|
2525
|
+
function parseRunnerReadinessDeferralRequest(message) {
|
|
2526
|
+
if (message?.type !== RUNNER_READINESS_DEFERRAL_MESSAGE_TYPE || !UUID_RE2.test(String(message.nonce || "")) || !Number.isFinite(message.retryAfterMs) || message.retryAfterMs <= 0) return null;
|
|
2527
|
+
return Object.freeze({
|
|
2528
|
+
nonce: message.nonce,
|
|
2529
|
+
retryAfterMs: Math.ceil(message.retryAfterMs),
|
|
2530
|
+
error: typeof message.error === "string" ? message.error : null
|
|
2531
|
+
});
|
|
2532
|
+
}
|
|
2533
|
+
function failed({
|
|
2534
|
+
paired = false,
|
|
2535
|
+
operatorId = null,
|
|
2536
|
+
tenantId = null,
|
|
2537
|
+
githubReady = null,
|
|
2538
|
+
retryAfterMs = null,
|
|
2539
|
+
error,
|
|
2540
|
+
message
|
|
2541
|
+
}) {
|
|
2542
|
+
return {
|
|
2543
|
+
ok: false,
|
|
2544
|
+
paired,
|
|
2545
|
+
operatorId,
|
|
2546
|
+
tenantId,
|
|
2547
|
+
githubReady,
|
|
2548
|
+
...Number.isFinite(retryAfterMs) && retryAfterMs > 0 ? { retryAfterMs } : {},
|
|
2549
|
+
error,
|
|
2550
|
+
message
|
|
2551
|
+
};
|
|
2552
|
+
}
|
|
2553
|
+
function responseRetryAfterMs(response, body) {
|
|
2554
|
+
const transientReadinessFailure = response.status === 503 && body?.error === "github_installation_readiness_retryable";
|
|
2555
|
+
if (response.status !== 429 && !transientReadinessFailure) return null;
|
|
2556
|
+
const bound = (retryAfterMs) => transientReadinessFailure ? Math.min(RUNNER_READINESS_TRANSIENT_MAX_RETRY_AFTER_MS, retryAfterMs) : retryAfterMs;
|
|
2557
|
+
const value = response.headers?.get?.("retry-after")?.trim() ?? "";
|
|
2558
|
+
const seconds = Number(value);
|
|
2559
|
+
if (value && Number.isFinite(seconds) && seconds >= 0) {
|
|
2560
|
+
return bound(Math.max(1e3, Math.ceil(seconds * 1e3)));
|
|
2561
|
+
}
|
|
2562
|
+
const at = value ? Date.parse(value) : Number.NaN;
|
|
2563
|
+
if (Number.isFinite(at)) {
|
|
2564
|
+
return bound(Math.max(1e3, Math.ceil(at - Date.now())));
|
|
2565
|
+
}
|
|
2566
|
+
return bound(6e4);
|
|
2323
2567
|
}
|
|
2324
2568
|
async function responseBody(response) {
|
|
2325
2569
|
try {
|
|
@@ -2332,11 +2576,21 @@ async function responseBody(response) {
|
|
|
2332
2576
|
function serverMessage(body, fallback) {
|
|
2333
2577
|
return typeof body.message === "string" && body.message.trim() ? body.message.trim() : fallback;
|
|
2334
2578
|
}
|
|
2335
|
-
async function
|
|
2579
|
+
async function fetchJsonWithTimeout(fetchImpl, url, init, timeoutMs) {
|
|
2336
2580
|
const controller = new AbortController();
|
|
2337
|
-
|
|
2581
|
+
let timer;
|
|
2582
|
+
const timeout = new Promise((_, reject) => {
|
|
2583
|
+
timer = setTimeout(() => {
|
|
2584
|
+
controller.abort();
|
|
2585
|
+
reject(new Error(`request aborted after ${timeoutMs}ms`));
|
|
2586
|
+
}, timeoutMs);
|
|
2587
|
+
});
|
|
2338
2588
|
try {
|
|
2339
|
-
|
|
2589
|
+
const requestAndBody = (async () => {
|
|
2590
|
+
const response = await fetchImpl(url, { ...init, signal: controller.signal });
|
|
2591
|
+
return { response, body: await responseBody(response) };
|
|
2592
|
+
})();
|
|
2593
|
+
return await Promise.race([requestAndBody, timeout]);
|
|
2340
2594
|
} finally {
|
|
2341
2595
|
clearTimeout(timer);
|
|
2342
2596
|
}
|
|
@@ -2346,18 +2600,24 @@ async function probeRunnerReadiness({
|
|
|
2346
2600
|
token,
|
|
2347
2601
|
fetchImpl = fetch,
|
|
2348
2602
|
requireGithub = false,
|
|
2349
|
-
|
|
2603
|
+
repositories = [],
|
|
2604
|
+
timeoutMs = RUNNER_IDENTITY_READINESS_TIMEOUT_MS,
|
|
2605
|
+
// The server performs at most five sequential App-JWT GETs (installation plus
|
|
2606
|
+
// every one of at most four configured repositories), each bounded at 8s.
|
|
2607
|
+
// Keep a transport margin while preventing a stuck proof from hanging startup.
|
|
2608
|
+
githubTimeoutMs = RUNNER_GITHUB_READINESS_TIMEOUT_MS
|
|
2350
2609
|
}) {
|
|
2351
2610
|
const base = controlPlaneUrl.replace(/\/+$/u, "");
|
|
2352
2611
|
const headers = { authorization: `Bearer ${token}` };
|
|
2353
2612
|
let identityResponse;
|
|
2613
|
+
let identity;
|
|
2354
2614
|
try {
|
|
2355
|
-
identityResponse = await
|
|
2615
|
+
({ response: identityResponse, body: identity } = await fetchJsonWithTimeout(
|
|
2356
2616
|
fetchImpl,
|
|
2357
2617
|
`${base}/api/v1/auth/me`,
|
|
2358
2618
|
{ headers },
|
|
2359
2619
|
timeoutMs
|
|
2360
|
-
);
|
|
2620
|
+
));
|
|
2361
2621
|
} catch (error) {
|
|
2362
2622
|
const detail = error instanceof Error ? error.message : String(error);
|
|
2363
2623
|
return failed({
|
|
@@ -2365,7 +2625,6 @@ async function probeRunnerReadiness({
|
|
|
2365
2625
|
message: `AlgoHQ could not be reached: ${detail}`
|
|
2366
2626
|
});
|
|
2367
2627
|
}
|
|
2368
|
-
const identity = await responseBody(identityResponse);
|
|
2369
2628
|
if (!identityResponse.ok) {
|
|
2370
2629
|
return failed({
|
|
2371
2630
|
error: "credential_rejected",
|
|
@@ -2391,18 +2650,33 @@ async function probeRunnerReadiness({
|
|
|
2391
2650
|
message: "Paired to AlgoHQ."
|
|
2392
2651
|
};
|
|
2393
2652
|
}
|
|
2653
|
+
let repositoryScope = null;
|
|
2654
|
+
try {
|
|
2655
|
+
if (!Array.isArray(repositories)) {
|
|
2656
|
+
throw new Error("VO_CODE_RUNNER_REPOS must be a repository array");
|
|
2657
|
+
}
|
|
2658
|
+
if (repositories.length > 0) repositoryScope = normalizeRunnerRepositoryScope(repositories);
|
|
2659
|
+
} catch (error) {
|
|
2660
|
+
return failed({
|
|
2661
|
+
paired: true,
|
|
2662
|
+
operatorId,
|
|
2663
|
+
tenantId,
|
|
2664
|
+
githubReady: false,
|
|
2665
|
+
error: "github_repository_scope_invalid",
|
|
2666
|
+
message: error instanceof Error ? error.message : String(error)
|
|
2667
|
+
});
|
|
2668
|
+
}
|
|
2394
2669
|
let githubResponse;
|
|
2670
|
+
let github;
|
|
2395
2671
|
try {
|
|
2396
|
-
|
|
2672
|
+
const readinessUrl = new URL(`${base}/api/v1/github/installation-readiness`);
|
|
2673
|
+
for (const repo of repositoryScope ?? []) readinessUrl.searchParams.append("repo", repo);
|
|
2674
|
+
({ response: githubResponse, body: github } = await fetchJsonWithTimeout(
|
|
2397
2675
|
fetchImpl,
|
|
2398
|
-
|
|
2399
|
-
{
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
body: "{}"
|
|
2403
|
-
},
|
|
2404
|
-
timeoutMs
|
|
2405
|
-
);
|
|
2676
|
+
readinessUrl.toString(),
|
|
2677
|
+
{ headers },
|
|
2678
|
+
githubTimeoutMs
|
|
2679
|
+
));
|
|
2406
2680
|
} catch (error) {
|
|
2407
2681
|
const detail = error instanceof Error ? error.message : String(error);
|
|
2408
2682
|
return failed({
|
|
@@ -2414,14 +2688,25 @@ async function probeRunnerReadiness({
|
|
|
2414
2688
|
message: `GitHub publication readiness could not be checked: ${detail}`
|
|
2415
2689
|
});
|
|
2416
2690
|
}
|
|
2417
|
-
const
|
|
2418
|
-
|
|
2691
|
+
const repositorySelection = github.repository_selection;
|
|
2692
|
+
const repositoriesVerified = github.repositories_verified;
|
|
2693
|
+
const returnedScope = github.repository_scope;
|
|
2694
|
+
let normalizedReturnedScope = null;
|
|
2695
|
+
try {
|
|
2696
|
+
normalizedReturnedScope = normalizeRunnerRepositoryScope(returnedScope);
|
|
2697
|
+
} catch {
|
|
2698
|
+
}
|
|
2699
|
+
const effectiveScope = repositoryScope ?? normalizedReturnedScope;
|
|
2700
|
+
const sourceVerified = repositoryScope === null ? github.repository_scope_source === "persisted_installation_singleton" && normalizedReturnedScope?.length === 1 : github.repository_scope_source === "runner_config";
|
|
2701
|
+
const repositoryScopeVerified = effectiveScope !== null && normalizedReturnedScope !== null && Array.isArray(returnedScope) && returnedScope.length === normalizedReturnedScope.length && returnedScope.every((repo, index) => repo === normalizedReturnedScope[index]) && normalizedReturnedScope.length === effectiveScope.length && normalizedReturnedScope.every((repo, index) => repo === effectiveScope[index]) && github.repository_scope_sha256 === runnerRepositoryScopeDigest(effectiveScope) && repositoriesVerified === effectiveScope.length && sourceVerified && (repositorySelection === "all" || repositorySelection === "selected");
|
|
2702
|
+
if (!githubResponse.ok || github.configured !== true || github.verified !== true || github.publication_ready !== true || !repositoryScopeVerified) {
|
|
2419
2703
|
const error = typeof github.error === "string" && github.error ? github.error : "github_not_ready";
|
|
2420
2704
|
return failed({
|
|
2421
2705
|
paired: true,
|
|
2422
2706
|
operatorId,
|
|
2423
2707
|
tenantId,
|
|
2424
2708
|
githubReady: false,
|
|
2709
|
+
retryAfterMs: responseRetryAfterMs(githubResponse, github),
|
|
2425
2710
|
error,
|
|
2426
2711
|
message: serverMessage(github, `GitHub publication preflight failed (HTTP ${githubResponse.status}).`)
|
|
2427
2712
|
});
|
|
@@ -2432,8 +2717,10 @@ async function probeRunnerReadiness({
|
|
|
2432
2717
|
operatorId,
|
|
2433
2718
|
tenantId,
|
|
2434
2719
|
githubReady: true,
|
|
2720
|
+
repositoryScope: effectiveScope,
|
|
2721
|
+
repositoryScopeSource: github.repository_scope_source,
|
|
2435
2722
|
error: null,
|
|
2436
|
-
message:
|
|
2723
|
+
message: `Paired with a verified Algosuite GitHub App installation (${repositoriesVerified} repos).`
|
|
2437
2724
|
};
|
|
2438
2725
|
}
|
|
2439
2726
|
function pairedOperatorScope(readiness) {
|
|
@@ -2472,25 +2759,51 @@ async function prepareSupervisorAuth({
|
|
|
2472
2759
|
const token = explicitAdminToken || storedCredential?.vo_credential;
|
|
2473
2760
|
if (!token) throw new Error("runner is not paired; run `vo-mcp pair` once on this host");
|
|
2474
2761
|
let operatorId = String(baseEnv.VO_CODE_RUNNER_OPERATOR_IDS || "").split(/[\s,]+/u).filter(Boolean)[0] || void 0;
|
|
2762
|
+
let repositoryScope = runnerRepositoryScopeFromEnv(baseEnv);
|
|
2763
|
+
let startupRetryAfterMs = null;
|
|
2764
|
+
let githubReady = null;
|
|
2765
|
+
let readinessError = null;
|
|
2475
2766
|
if (!explicitAdminToken) {
|
|
2476
|
-
const readiness = await probeReadiness({
|
|
2477
|
-
|
|
2478
|
-
|
|
2767
|
+
const readiness = await probeReadiness({
|
|
2768
|
+
controlPlaneUrl,
|
|
2769
|
+
token,
|
|
2770
|
+
requireGithub: true,
|
|
2771
|
+
repositories: repositoryScope
|
|
2772
|
+
});
|
|
2773
|
+
const pairedScope = pairedOperatorScope(readiness) || (readiness?.paired === true && typeof readiness.operatorId === "string" ? readiness.operatorId.trim() : "");
|
|
2774
|
+
operatorId = pairedScope || void 0;
|
|
2479
2775
|
if (!operatorId) throw new Error("runner readiness failed: paired operator scope is missing");
|
|
2776
|
+
if (readiness.ok && Array.isArray(readiness.repositoryScope)) {
|
|
2777
|
+
repositoryScope = [...readiness.repositoryScope];
|
|
2778
|
+
githubReady = true;
|
|
2779
|
+
} else {
|
|
2780
|
+
githubReady = false;
|
|
2781
|
+
readinessError = typeof readiness?.error === "string" ? readiness.error : "github_not_ready";
|
|
2782
|
+
startupRetryAfterMs = runnerReadinessRetryDelayMs(readiness);
|
|
2783
|
+
}
|
|
2480
2784
|
}
|
|
2785
|
+
const effectiveBaseEnv = repositoryScope.length > 0 ? { ...baseEnv, VO_CODE_RUNNER_REPOS: repositoryScope.join(",") } : baseEnv;
|
|
2481
2786
|
const childEnv = buildSupervisorChildEnv({
|
|
2482
|
-
baseEnv,
|
|
2787
|
+
baseEnv: effectiveBaseEnv,
|
|
2483
2788
|
controlPlaneUrl,
|
|
2484
2789
|
explicitAdminToken,
|
|
2485
2790
|
pairedOperatorId: explicitAdminToken ? null : operatorId
|
|
2486
2791
|
});
|
|
2487
2792
|
const clientEnv = {
|
|
2488
|
-
...
|
|
2793
|
+
...effectiveBaseEnv,
|
|
2489
2794
|
VO_CONTROL_PLANE_ADMIN_TOKEN: token,
|
|
2490
2795
|
VO_CONTROL_PLANE_URL: controlPlaneUrl,
|
|
2491
2796
|
...operatorId ? { VO_CODE_RUNNER_OPERATOR_IDS: operatorId } : {}
|
|
2492
2797
|
};
|
|
2493
|
-
return {
|
|
2798
|
+
return {
|
|
2799
|
+
childEnv,
|
|
2800
|
+
clientEnv,
|
|
2801
|
+
operatorId,
|
|
2802
|
+
repositoryScope,
|
|
2803
|
+
githubReady,
|
|
2804
|
+
readinessError,
|
|
2805
|
+
startupRetryAfterMs
|
|
2806
|
+
};
|
|
2494
2807
|
}
|
|
2495
2808
|
|
|
2496
2809
|
// src/runner/supervisor-credential-reader.mjs
|
|
@@ -2530,12 +2843,309 @@ function readStoredCredentialIsolated({
|
|
|
2530
2843
|
}
|
|
2531
2844
|
}
|
|
2532
2845
|
|
|
2846
|
+
// src/runner/respawn-circuit.mjs
|
|
2847
|
+
var DEFAULT_RESPAWN_CIRCUIT = Object.freeze({
|
|
2848
|
+
baseDelayMs: 2e3,
|
|
2849
|
+
maxDelayMs: 3e4,
|
|
2850
|
+
healthyResetMs: 6e4,
|
|
2851
|
+
maxRapidExits: 5
|
|
2852
|
+
});
|
|
2853
|
+
function createRespawnCircuit(options = {}) {
|
|
2854
|
+
const config = { ...DEFAULT_RESPAWN_CIRCUIT, ...options };
|
|
2855
|
+
for (const [key, value] of Object.entries(config)) {
|
|
2856
|
+
if (!Number.isInteger(value) || value < 1) throw new Error(`${key} must be a positive integer`);
|
|
2857
|
+
}
|
|
2858
|
+
let rapidExits = 0;
|
|
2859
|
+
return Object.freeze({
|
|
2860
|
+
recordExit(startedAt, exitedAt) {
|
|
2861
|
+
if (!Number.isFinite(startedAt) || !Number.isFinite(exitedAt) || exitedAt < startedAt) {
|
|
2862
|
+
throw new Error("respawn circuit timestamps are invalid");
|
|
2863
|
+
}
|
|
2864
|
+
const uptimeMs = exitedAt - startedAt;
|
|
2865
|
+
rapidExits = uptimeMs >= config.healthyResetMs ? 0 : rapidExits + 1;
|
|
2866
|
+
const tripped = rapidExits >= config.maxRapidExits;
|
|
2867
|
+
const delayMs = Math.min(
|
|
2868
|
+
config.maxDelayMs,
|
|
2869
|
+
config.baseDelayMs * 2 ** Math.max(0, rapidExits - 1)
|
|
2870
|
+
);
|
|
2871
|
+
return Object.freeze({ rapidExits, uptimeMs, delayMs, tripped });
|
|
2872
|
+
},
|
|
2873
|
+
snapshot() {
|
|
2874
|
+
return Object.freeze({ rapidExits, tripped: rapidExits >= config.maxRapidExits });
|
|
2875
|
+
}
|
|
2876
|
+
});
|
|
2877
|
+
}
|
|
2878
|
+
|
|
2879
|
+
// src/runner/supervisor-child-health.mjs
|
|
2880
|
+
var terminatedChildren = /* @__PURE__ */ new WeakSet();
|
|
2881
|
+
function attachSupervisorChildTerminationCustody(target, onTerminated, onObservedError = () => {
|
|
2882
|
+
}) {
|
|
2883
|
+
let accounted = false;
|
|
2884
|
+
const account = (kind, error = null) => {
|
|
2885
|
+
terminatedChildren.add(target);
|
|
2886
|
+
if (accounted) return false;
|
|
2887
|
+
accounted = true;
|
|
2888
|
+
onTerminated(Object.freeze({ target, kind, error }));
|
|
2889
|
+
return true;
|
|
2890
|
+
};
|
|
2891
|
+
target.on("error", (error) => {
|
|
2892
|
+
onObservedError(error);
|
|
2893
|
+
if (target.pid === void 0 || target.pid === null) account("error", error);
|
|
2894
|
+
});
|
|
2895
|
+
target.on("exit", () => {
|
|
2896
|
+
account("exit");
|
|
2897
|
+
});
|
|
2898
|
+
return Object.freeze({ accounted: () => accounted });
|
|
2899
|
+
}
|
|
2900
|
+
function supervisorChildHasExited(target) {
|
|
2901
|
+
return terminatedChildren.has(target) || target.exitCode !== null || (target.signalCode ?? null) !== null;
|
|
2902
|
+
}
|
|
2903
|
+
function supervisorChildIsRunning(target) {
|
|
2904
|
+
return !supervisorChildHasExited(target);
|
|
2905
|
+
}
|
|
2906
|
+
function shouldRespawnSupervisorChild({ stopping, degraded, handling, child }) {
|
|
2907
|
+
return !stopping && !degraded && !handling && (!child || supervisorChildHasExited(child));
|
|
2908
|
+
}
|
|
2909
|
+
function shouldDeferSupervisorReadinessExit({
|
|
2910
|
+
target,
|
|
2911
|
+
currentChild,
|
|
2912
|
+
retryAfterMs
|
|
2913
|
+
}) {
|
|
2914
|
+
return target === currentChild && target?.exitCode === 75 && Number.isFinite(retryAfterMs) && retryAfterMs > 0;
|
|
2915
|
+
}
|
|
2916
|
+
function createSupervisorReadinessDeferralTracker({
|
|
2917
|
+
currentChild,
|
|
2918
|
+
releaseCurrentChild,
|
|
2919
|
+
onCaptured
|
|
2920
|
+
}) {
|
|
2921
|
+
const pending = /* @__PURE__ */ new WeakMap();
|
|
2922
|
+
const terminal = /* @__PURE__ */ new WeakSet();
|
|
2923
|
+
const recoveryGenerations = /* @__PURE__ */ new WeakMap();
|
|
2924
|
+
return Object.freeze({
|
|
2925
|
+
recordPending(target, request) {
|
|
2926
|
+
if (!target || target !== currentChild() || terminal.has(target) || pending.has(target) || typeof request?.nonce !== "string" || !request.nonce || !Number.isFinite(request.retryAfterMs) || request.retryAfterMs <= 0) return false;
|
|
2927
|
+
pending.set(target, Object.freeze({
|
|
2928
|
+
nonce: request.nonce,
|
|
2929
|
+
retryAfterMs: Math.ceil(request.retryAfterMs)
|
|
2930
|
+
}));
|
|
2931
|
+
return true;
|
|
2932
|
+
},
|
|
2933
|
+
recordRecoveryGeneration(target, generation) {
|
|
2934
|
+
if (!target || !Number.isInteger(generation)) return false;
|
|
2935
|
+
recoveryGenerations.set(target, generation);
|
|
2936
|
+
return true;
|
|
2937
|
+
},
|
|
2938
|
+
isDeferred(target) {
|
|
2939
|
+
const request = target ? pending.get(target) : null;
|
|
2940
|
+
return Boolean(target) && (terminal.has(target) || shouldDeferSupervisorReadinessExit({
|
|
2941
|
+
target,
|
|
2942
|
+
currentChild: currentChild(),
|
|
2943
|
+
retryAfterMs: request?.retryAfterMs
|
|
2944
|
+
}));
|
|
2945
|
+
},
|
|
2946
|
+
capture(target) {
|
|
2947
|
+
if (!target) return false;
|
|
2948
|
+
if (terminal.has(target)) return true;
|
|
2949
|
+
const request = pending.get(target);
|
|
2950
|
+
if (!shouldDeferSupervisorReadinessExit({
|
|
2951
|
+
target,
|
|
2952
|
+
currentChild: currentChild(),
|
|
2953
|
+
retryAfterMs: request?.retryAfterMs
|
|
2954
|
+
})) return false;
|
|
2955
|
+
const recoveryGeneration = recoveryGenerations.get(target);
|
|
2956
|
+
terminal.add(target);
|
|
2957
|
+
pending.delete(target);
|
|
2958
|
+
recoveryGenerations.delete(target);
|
|
2959
|
+
releaseCurrentChild(target);
|
|
2960
|
+
onCaptured(Object.freeze({
|
|
2961
|
+
target,
|
|
2962
|
+
retryAfterMs: request.retryAfterMs,
|
|
2963
|
+
recoveryGeneration: Number.isInteger(recoveryGeneration) ? recoveryGeneration : null
|
|
2964
|
+
}));
|
|
2965
|
+
return true;
|
|
2966
|
+
},
|
|
2967
|
+
forget(target) {
|
|
2968
|
+
if (!target) return;
|
|
2969
|
+
pending.delete(target);
|
|
2970
|
+
recoveryGenerations.delete(target);
|
|
2971
|
+
}
|
|
2972
|
+
});
|
|
2973
|
+
}
|
|
2974
|
+
function resolveSupervisorChildStartAuthority({
|
|
2975
|
+
degradationState,
|
|
2976
|
+
deferredRecoveryGeneration
|
|
2977
|
+
}) {
|
|
2978
|
+
const snapshot = degradationState.snapshot();
|
|
2979
|
+
const exactRecovery = Number.isInteger(deferredRecoveryGeneration) && deferredRecoveryGeneration === snapshot.generation;
|
|
2980
|
+
return Object.freeze({
|
|
2981
|
+
allowed: !snapshot.degraded || exactRecovery,
|
|
2982
|
+
recoveryGeneration: exactRecovery ? deferredRecoveryGeneration : null
|
|
2983
|
+
});
|
|
2984
|
+
}
|
|
2985
|
+
function createSupervisorStartupDeferral({
|
|
2986
|
+
retryAfterMs = null,
|
|
2987
|
+
nowMs = () => Date.now()
|
|
2988
|
+
} = {}) {
|
|
2989
|
+
let started = false;
|
|
2990
|
+
let notBefore = null;
|
|
2991
|
+
const defer = (delayMs) => {
|
|
2992
|
+
if (started || !Number.isFinite(delayMs) || delayMs <= 0) return false;
|
|
2993
|
+
const candidate = nowMs() + Math.ceil(delayMs);
|
|
2994
|
+
notBefore = notBefore === null ? candidate : Math.max(notBefore, candidate);
|
|
2995
|
+
return true;
|
|
2996
|
+
};
|
|
2997
|
+
defer(retryAfterMs);
|
|
2998
|
+
return Object.freeze({
|
|
2999
|
+
defer,
|
|
3000
|
+
reopen(delayMs) {
|
|
3001
|
+
if (!Number.isFinite(delayMs) || delayMs <= 0) return false;
|
|
3002
|
+
started = false;
|
|
3003
|
+
notBefore = null;
|
|
3004
|
+
return defer(delayMs);
|
|
3005
|
+
},
|
|
3006
|
+
isPending() {
|
|
3007
|
+
return !started && notBefore !== null && nowMs() < notBefore;
|
|
3008
|
+
},
|
|
3009
|
+
remainingMs() {
|
|
3010
|
+
return started || notBefore === null ? 0 : Math.max(0, notBefore - nowMs());
|
|
3011
|
+
},
|
|
3012
|
+
consumeIfReady() {
|
|
3013
|
+
if (started || notBefore !== null && nowMs() < notBefore) return false;
|
|
3014
|
+
started = true;
|
|
3015
|
+
notBefore = null;
|
|
3016
|
+
return true;
|
|
3017
|
+
},
|
|
3018
|
+
hasStarted() {
|
|
3019
|
+
return started;
|
|
3020
|
+
}
|
|
3021
|
+
});
|
|
3022
|
+
}
|
|
3023
|
+
function createSupervisorDegradationState() {
|
|
3024
|
+
let degraded = false;
|
|
3025
|
+
let generation = 0;
|
|
3026
|
+
return {
|
|
3027
|
+
isDegraded() {
|
|
3028
|
+
return degraded;
|
|
3029
|
+
},
|
|
3030
|
+
markDegraded() {
|
|
3031
|
+
degraded = true;
|
|
3032
|
+
generation += 1;
|
|
3033
|
+
},
|
|
3034
|
+
beginHealthProof({ target, currentChild, allowRecovery, onRecovery }) {
|
|
3035
|
+
const observedGeneration = generation;
|
|
3036
|
+
return () => {
|
|
3037
|
+
if (currentChild() !== target || generation !== observedGeneration) return false;
|
|
3038
|
+
if (degraded && !allowRecovery) return false;
|
|
3039
|
+
const recovered = degraded;
|
|
3040
|
+
degraded = false;
|
|
3041
|
+
if (recovered) onRecovery?.();
|
|
3042
|
+
return true;
|
|
3043
|
+
};
|
|
3044
|
+
},
|
|
3045
|
+
beginDegradationProof({ target, currentChild, onDegraded }) {
|
|
3046
|
+
const observedGeneration = generation;
|
|
3047
|
+
return (message) => {
|
|
3048
|
+
if (currentChild() !== target || generation !== observedGeneration) return false;
|
|
3049
|
+
onDegraded(message);
|
|
3050
|
+
return true;
|
|
3051
|
+
};
|
|
3052
|
+
},
|
|
3053
|
+
snapshot() {
|
|
3054
|
+
return { degraded, generation };
|
|
3055
|
+
}
|
|
3056
|
+
};
|
|
3057
|
+
}
|
|
3058
|
+
function beginSupervisorHealthyStateCommit({
|
|
3059
|
+
degradationState,
|
|
3060
|
+
target,
|
|
3061
|
+
currentChild,
|
|
3062
|
+
allowRecovery,
|
|
3063
|
+
clearStaleExitCode
|
|
3064
|
+
}) {
|
|
3065
|
+
return degradationState.beginHealthProof({
|
|
3066
|
+
target,
|
|
3067
|
+
currentChild,
|
|
3068
|
+
allowRecovery,
|
|
3069
|
+
onRecovery: clearStaleExitCode
|
|
3070
|
+
});
|
|
3071
|
+
}
|
|
3072
|
+
var RUNNER_RECOVERY_ACTIONS = /* @__PURE__ */ new Set(["update", "reinstall", "reconnect"]);
|
|
3073
|
+
function shouldRecoverSupervisorChildAfterAction({
|
|
3074
|
+
stoppedChild,
|
|
3075
|
+
actionKind,
|
|
3076
|
+
actionSucceeded
|
|
3077
|
+
}) {
|
|
3078
|
+
return Boolean(stoppedChild) || actionSucceeded === true && RUNNER_RECOVERY_ACTIONS.has(actionKind);
|
|
3079
|
+
}
|
|
3080
|
+
function createSupervisorControlRecoveryFence() {
|
|
3081
|
+
let targetStoppedForControl = null;
|
|
3082
|
+
const consume = () => {
|
|
3083
|
+
const target = targetStoppedForControl;
|
|
3084
|
+
targetStoppedForControl = null;
|
|
3085
|
+
return target;
|
|
3086
|
+
};
|
|
3087
|
+
return {
|
|
3088
|
+
markBeforeStop(target) {
|
|
3089
|
+
targetStoppedForControl = target && supervisorChildIsRunning(target) ? target : null;
|
|
3090
|
+
return targetStoppedForControl !== null;
|
|
3091
|
+
},
|
|
3092
|
+
consume,
|
|
3093
|
+
async recoverOnce(recover) {
|
|
3094
|
+
const target = consume();
|
|
3095
|
+
if (!target) return false;
|
|
3096
|
+
await recover(target);
|
|
3097
|
+
return true;
|
|
3098
|
+
}
|
|
3099
|
+
};
|
|
3100
|
+
}
|
|
3101
|
+
async function verifySupervisorChildHealth({
|
|
3102
|
+
target,
|
|
3103
|
+
degradeOnExit,
|
|
3104
|
+
context,
|
|
3105
|
+
waitBeforeProbe,
|
|
3106
|
+
waitForLocalRunner: waitForLocalRunner2,
|
|
3107
|
+
stopExpectedly,
|
|
3108
|
+
markHealthy,
|
|
3109
|
+
markDegraded,
|
|
3110
|
+
isReadinessDeferredExit = () => false,
|
|
3111
|
+
isExpectedStop = () => false
|
|
3112
|
+
}) {
|
|
3113
|
+
try {
|
|
3114
|
+
await waitBeforeProbe();
|
|
3115
|
+
const healthy = supervisorChildIsRunning(target) && await waitForLocalRunner2(target);
|
|
3116
|
+
const exited = supervisorChildHasExited(target);
|
|
3117
|
+
if (isExpectedStop(target)) return false;
|
|
3118
|
+
if (exited && isReadinessDeferredExit(target)) return false;
|
|
3119
|
+
if (healthy && !exited) {
|
|
3120
|
+
if (markHealthy() !== false) return true;
|
|
3121
|
+
if (supervisorChildIsRunning(target)) await stopExpectedly(target);
|
|
3122
|
+
return false;
|
|
3123
|
+
}
|
|
3124
|
+
if (!exited || degradeOnExit) {
|
|
3125
|
+
const committed = markDegraded(`${context} did not become healthy; entering degraded mode`) !== false;
|
|
3126
|
+
if (!committed) return false;
|
|
3127
|
+
if (supervisorChildIsRunning(target)) await stopExpectedly(target);
|
|
3128
|
+
}
|
|
3129
|
+
return false;
|
|
3130
|
+
} catch (error) {
|
|
3131
|
+
if (isExpectedStop(target)) return false;
|
|
3132
|
+
const committed = markDegraded(
|
|
3133
|
+
`${context} health proof failed; entering degraded mode: ${error instanceof Error ? error.message : String(error)}`
|
|
3134
|
+
) !== false;
|
|
3135
|
+
if (!committed) return false;
|
|
3136
|
+
if (supervisorChildIsRunning(target)) await stopExpectedly(target).catch(() => {
|
|
3137
|
+
});
|
|
3138
|
+
return false;
|
|
3139
|
+
}
|
|
3140
|
+
}
|
|
3141
|
+
|
|
2533
3142
|
// src/runner-supervisor.mjs
|
|
2534
3143
|
var DEFAULT_CONTROL_PLANE_URL = "https://vo-control-plane-bzjphrajaq-uc.a.run.app";
|
|
2535
3144
|
var POLL_MS = 5e3;
|
|
2536
3145
|
var CHILD_START_MS = 1500;
|
|
3146
|
+
var CHILD_READINESS_TIMEOUT_MS = RUNNER_IDENTITY_READINESS_TIMEOUT_MS + RUNNER_GITHUB_READINESS_TIMEOUT_MS + 1e4;
|
|
2537
3147
|
var SUPERVISOR_CAPABILITIES = ["bundled-runtime-slots-v1", "legacy-orphan-purge-v1"];
|
|
2538
|
-
var
|
|
3148
|
+
var UUID_RE3 = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
|
|
2539
3149
|
var selfPath = fileURLToPath3(import.meta.url);
|
|
2540
3150
|
var bundledChildEntry = join5(dirname4(selfPath), "runner-cli.js");
|
|
2541
3151
|
var supervisorRuntimeRoot = runtimeRootFromEnv(process.env);
|
|
@@ -2548,7 +3158,7 @@ function packageVersion() {
|
|
|
2548
3158
|
return "unknown";
|
|
2549
3159
|
}
|
|
2550
3160
|
}
|
|
2551
|
-
var requestedSupervisorInstanceId =
|
|
3161
|
+
var requestedSupervisorInstanceId = UUID_RE3.test(String(process.env.VO_RUNNER_SUPERVISOR_INSTANCE_ID || "")) ? process.env.VO_RUNNER_SUPERVISOR_INSTANCE_ID : randomUUID4();
|
|
2552
3162
|
var supervisorInstanceId = activationSupervisorInstanceId(
|
|
2553
3163
|
runtimeRootFromEnv(process.env),
|
|
2554
3164
|
requestedSupervisorInstanceId
|
|
@@ -2572,16 +3182,27 @@ function spawnChild(childEnv) {
|
|
|
2572
3182
|
}
|
|
2573
3183
|
return spawn(process.execPath, resolved.args, {
|
|
2574
3184
|
env: childEnv,
|
|
2575
|
-
stdio: "inherit",
|
|
3185
|
+
stdio: ["inherit", "inherit", "inherit", "ipc"],
|
|
2576
3186
|
windowsHide: true
|
|
2577
3187
|
});
|
|
2578
3188
|
}
|
|
2579
3189
|
function spawnPreviousChild(entry, childEnv) {
|
|
2580
|
-
|
|
3190
|
+
const previous = spawn(process.execPath, [entry, "runner"], {
|
|
2581
3191
|
env: childEnv,
|
|
2582
|
-
stdio: "inherit",
|
|
3192
|
+
stdio: ["inherit", "inherit", "inherit", "ipc"],
|
|
2583
3193
|
windowsHide: true
|
|
2584
3194
|
});
|
|
3195
|
+
attachSupervisorChildTerminationCustody(
|
|
3196
|
+
previous,
|
|
3197
|
+
() => {
|
|
3198
|
+
},
|
|
3199
|
+
(error) => {
|
|
3200
|
+
console.error(
|
|
3201
|
+
`[vo-runner supervisor] rollback child process error: ${error instanceof Error ? error.message : String(error)}`
|
|
3202
|
+
);
|
|
3203
|
+
}
|
|
3204
|
+
);
|
|
3205
|
+
return previous;
|
|
2585
3206
|
}
|
|
2586
3207
|
async function localStatus() {
|
|
2587
3208
|
try {
|
|
@@ -2595,32 +3216,54 @@ async function localStatus() {
|
|
|
2595
3216
|
}
|
|
2596
3217
|
}
|
|
2597
3218
|
async function waitForLocalRunner(child) {
|
|
2598
|
-
const deadline = Date.now() +
|
|
3219
|
+
const deadline = Date.now() + CHILD_READINESS_TIMEOUT_MS;
|
|
2599
3220
|
while (Date.now() < deadline) {
|
|
2600
|
-
if (child
|
|
3221
|
+
if (supervisorChildHasExited(child)) return false;
|
|
2601
3222
|
const status = await localStatus();
|
|
2602
3223
|
if (status?.running === true && Number(status.pid) === Number(child.pid)) return true;
|
|
2603
3224
|
await sleep(500);
|
|
2604
3225
|
}
|
|
2605
3226
|
return false;
|
|
2606
3227
|
}
|
|
3228
|
+
async function waitForChildExit(child, timeoutMs) {
|
|
3229
|
+
if (!child || supervisorChildHasExited(child)) return true;
|
|
3230
|
+
return new Promise((resolve5) => {
|
|
3231
|
+
let settled = false;
|
|
3232
|
+
const finish = (exited) => {
|
|
3233
|
+
if (settled) return;
|
|
3234
|
+
settled = true;
|
|
3235
|
+
clearTimeout(timer);
|
|
3236
|
+
child.off("exit", onExit);
|
|
3237
|
+
resolve5(exited);
|
|
3238
|
+
};
|
|
3239
|
+
const onExit = () => finish(true);
|
|
3240
|
+
const timer = setTimeout(() => finish(false), timeoutMs);
|
|
3241
|
+
child.once("exit", onExit);
|
|
3242
|
+
if (supervisorChildHasExited(child)) finish(true);
|
|
3243
|
+
});
|
|
3244
|
+
}
|
|
2607
3245
|
async function stopChild(child) {
|
|
2608
|
-
if (!child || child
|
|
3246
|
+
if (!child || supervisorChildHasExited(child)) return;
|
|
2609
3247
|
child.kill(process.platform === "win32" ? void 0 : "SIGTERM");
|
|
2610
|
-
await
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
]);
|
|
2614
|
-
if (child.exitCode === null) child.kill("SIGKILL");
|
|
3248
|
+
if (await waitForChildExit(child, 15e3)) return;
|
|
3249
|
+
if (supervisorChildIsRunning(child)) child.kill("SIGKILL");
|
|
3250
|
+
await waitForChildExit(child, 5e3);
|
|
2615
3251
|
}
|
|
2616
3252
|
async function main() {
|
|
2617
3253
|
const stored = readStoredCredentialIsolated();
|
|
2618
3254
|
const controlPlaneUrl = process.env.VO_CONTROL_PLANE_URL || DEFAULT_CONTROL_PLANE_URL;
|
|
2619
|
-
const
|
|
3255
|
+
const pairedSupervisor = !String(process.env.VO_CONTROL_PLANE_ADMIN_TOKEN || "").trim();
|
|
3256
|
+
const supervisorAuthInput = {
|
|
2620
3257
|
baseEnv: process.env,
|
|
2621
3258
|
storedCredential: stored,
|
|
2622
3259
|
controlPlaneUrl
|
|
2623
|
-
}
|
|
3260
|
+
};
|
|
3261
|
+
const {
|
|
3262
|
+
childEnv,
|
|
3263
|
+
clientEnv,
|
|
3264
|
+
operatorId,
|
|
3265
|
+
startupRetryAfterMs
|
|
3266
|
+
} = await prepareSupervisorAuth(supervisorAuthInput);
|
|
2624
3267
|
Object.assign(childEnv, {
|
|
2625
3268
|
VO_RUNNER_SUPERVISOR_INSTANCE_ID: supervisorInstanceId,
|
|
2626
3269
|
VO_RUNNER_SUPERVISOR_VERSION: supervisorVersion,
|
|
@@ -2631,37 +3274,266 @@ async function main() {
|
|
|
2631
3274
|
let child = null;
|
|
2632
3275
|
let stopping = false;
|
|
2633
3276
|
let handling = false;
|
|
2634
|
-
|
|
3277
|
+
const expectedChildStops = /* @__PURE__ */ new WeakSet();
|
|
3278
|
+
const healthNeutralizedChildren = /* @__PURE__ */ new WeakSet();
|
|
3279
|
+
const degradationState = createSupervisorDegradationState();
|
|
3280
|
+
const markSupervisorDegraded = () => {
|
|
3281
|
+
degradationState.markDegraded();
|
|
3282
|
+
process.exitCode = 1;
|
|
3283
|
+
};
|
|
3284
|
+
const respawnCircuit = createRespawnCircuit();
|
|
3285
|
+
const controlRecoveryFence = createSupervisorControlRecoveryFence();
|
|
3286
|
+
const startupDeferral = createSupervisorStartupDeferral({
|
|
3287
|
+
retryAfterMs: startupRetryAfterMs
|
|
3288
|
+
});
|
|
3289
|
+
let startupNeedsReadinessRefresh = startupRetryAfterMs !== null;
|
|
3290
|
+
let deferredControlRecoveryGeneration = null;
|
|
3291
|
+
const stopChildExpectedly = async (target) => {
|
|
3292
|
+
if (!target) return;
|
|
3293
|
+
expectedChildStops.add(target);
|
|
3294
|
+
healthNeutralizedChildren.add(target);
|
|
3295
|
+
try {
|
|
3296
|
+
await stopChild(target);
|
|
3297
|
+
} finally {
|
|
3298
|
+
if (supervisorChildHasExited(target)) expectedChildStops.delete(target);
|
|
3299
|
+
}
|
|
3300
|
+
};
|
|
3301
|
+
const readinessDeferrals = createSupervisorReadinessDeferralTracker({
|
|
3302
|
+
currentChild: () => child,
|
|
3303
|
+
releaseCurrentChild: (target) => {
|
|
3304
|
+
if (child === target) child = null;
|
|
3305
|
+
},
|
|
3306
|
+
onCaptured: ({ retryAfterMs, recoveryGeneration }) => {
|
|
3307
|
+
startupNeedsReadinessRefresh = true;
|
|
3308
|
+
if (Number.isInteger(recoveryGeneration) && degradationState.snapshot().generation === recoveryGeneration) {
|
|
3309
|
+
deferredControlRecoveryGeneration = recoveryGeneration;
|
|
3310
|
+
}
|
|
3311
|
+
startupDeferral.reopen(retryAfterMs);
|
|
3312
|
+
console.warn(
|
|
3313
|
+
`[vo-runner supervisor] child deferred by GitHub readiness for ${retryAfterMs}ms; preserving control polling without consuming rapid-exit breaker budget`
|
|
3314
|
+
);
|
|
3315
|
+
}
|
|
3316
|
+
});
|
|
3317
|
+
const isChildReadinessDeferred = (target) => readinessDeferrals.isDeferred(target);
|
|
3318
|
+
const captureChildReadinessDeferral = (target) => readinessDeferrals.capture(target);
|
|
2635
3319
|
const respawn = () => {
|
|
2636
|
-
if (!
|
|
3320
|
+
if (!startupDeferral.hasStarted()) return;
|
|
3321
|
+
if (!shouldRespawnSupervisorChild({
|
|
3322
|
+
stopping,
|
|
3323
|
+
degraded: degradationState.isDegraded(),
|
|
3324
|
+
handling,
|
|
3325
|
+
child
|
|
3326
|
+
})) return;
|
|
3327
|
+
try {
|
|
3328
|
+
child = launchChild();
|
|
3329
|
+
void verifyChildHealth(child, false, "automatic relaunch");
|
|
3330
|
+
} catch (error) {
|
|
3331
|
+
markSupervisorDegraded();
|
|
3332
|
+
console.error(
|
|
3333
|
+
`[vo-runner supervisor] child relaunch failed; entering degraded mode: ${error instanceof Error ? error.message : String(error)}`
|
|
3334
|
+
);
|
|
3335
|
+
}
|
|
2637
3336
|
};
|
|
2638
3337
|
const launchChild = () => {
|
|
2639
3338
|
const next = spawnChild(childEnv);
|
|
2640
|
-
|
|
3339
|
+
const startedAt = Date.now();
|
|
3340
|
+
next.on("message", (message) => {
|
|
3341
|
+
const request = parseRunnerReadinessDeferralRequest(message);
|
|
3342
|
+
if (!request || !readinessDeferrals.recordPending(next, request)) return;
|
|
3343
|
+
try {
|
|
3344
|
+
next.send({
|
|
3345
|
+
type: RUNNER_READINESS_DEFERRAL_ACK_TYPE,
|
|
3346
|
+
nonce: request.nonce
|
|
3347
|
+
}, (error) => {
|
|
3348
|
+
if (error) console.warn(`[vo-runner supervisor] readiness deferral ACK failed: ${error.message}`);
|
|
3349
|
+
});
|
|
3350
|
+
} catch (error) {
|
|
3351
|
+
console.warn(
|
|
3352
|
+
`[vo-runner supervisor] readiness deferral ACK failed: ${error instanceof Error ? error.message : String(error)}`
|
|
3353
|
+
);
|
|
3354
|
+
}
|
|
3355
|
+
});
|
|
3356
|
+
attachSupervisorChildTerminationCustody(
|
|
3357
|
+
next,
|
|
3358
|
+
() => {
|
|
3359
|
+
if (stopping || expectedChildStops.delete(next)) {
|
|
3360
|
+
readinessDeferrals.forget(next);
|
|
3361
|
+
return;
|
|
3362
|
+
}
|
|
3363
|
+
if (captureChildReadinessDeferral(next)) return;
|
|
3364
|
+
readinessDeferrals.forget(next);
|
|
3365
|
+
const decision = respawnCircuit.recordExit(startedAt, Date.now());
|
|
3366
|
+
if (decision.tripped) {
|
|
3367
|
+
markSupervisorDegraded();
|
|
3368
|
+
console.error(
|
|
3369
|
+
`[vo-runner supervisor] child exited rapidly ${decision.rapidExits} times; entering degraded mode to stop startup/token churn while remote repair remains available`
|
|
3370
|
+
);
|
|
3371
|
+
return;
|
|
3372
|
+
}
|
|
3373
|
+
setTimeout(respawn, decision.delayMs);
|
|
3374
|
+
},
|
|
3375
|
+
(error) => {
|
|
3376
|
+
console.error(
|
|
3377
|
+
`[vo-runner supervisor] child process error: ${error instanceof Error ? error.message : String(error)}`
|
|
3378
|
+
);
|
|
3379
|
+
}
|
|
3380
|
+
);
|
|
2641
3381
|
return next;
|
|
2642
3382
|
};
|
|
2643
|
-
|
|
2644
|
-
|
|
2645
|
-
|
|
2646
|
-
|
|
2647
|
-
|
|
2648
|
-
|
|
2649
|
-
|
|
2650
|
-
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
|
|
3383
|
+
const verifyChildHealth = async (target, degradeOnExit, context) => {
|
|
3384
|
+
const commitSupervisorHealthyState = beginSupervisorHealthyStateCommit({
|
|
3385
|
+
degradationState,
|
|
3386
|
+
target,
|
|
3387
|
+
currentChild: () => child,
|
|
3388
|
+
allowRecovery: degradeOnExit,
|
|
3389
|
+
clearStaleExitCode: () => {
|
|
3390
|
+
process.exitCode = void 0;
|
|
3391
|
+
}
|
|
3392
|
+
});
|
|
3393
|
+
const commitSupervisorDegradedState = degradationState.beginDegradationProof({
|
|
3394
|
+
target,
|
|
3395
|
+
currentChild: () => child,
|
|
3396
|
+
onDegraded: (message) => {
|
|
3397
|
+
markSupervisorDegraded();
|
|
3398
|
+
console.error(`[vo-runner supervisor] ${message}`);
|
|
3399
|
+
}
|
|
3400
|
+
});
|
|
3401
|
+
return verifySupervisorChildHealth({
|
|
3402
|
+
target,
|
|
3403
|
+
degradeOnExit,
|
|
3404
|
+
context,
|
|
3405
|
+
waitBeforeProbe: () => sleep(CHILD_START_MS),
|
|
3406
|
+
waitForLocalRunner,
|
|
3407
|
+
stopExpectedly: stopChildExpectedly,
|
|
3408
|
+
isReadinessDeferredExit: isChildReadinessDeferred,
|
|
3409
|
+
isExpectedStop: (candidate) => healthNeutralizedChildren.has(candidate),
|
|
3410
|
+
markHealthy: commitSupervisorHealthyState,
|
|
3411
|
+
markDegraded: commitSupervisorDegradedState
|
|
3412
|
+
});
|
|
3413
|
+
};
|
|
3414
|
+
const deferChildAdmission = (retryAfterMs) => {
|
|
3415
|
+
startupNeedsReadinessRefresh = true;
|
|
3416
|
+
if (startupDeferral.hasStarted()) startupDeferral.reopen(retryAfterMs);
|
|
3417
|
+
else startupDeferral.defer(retryAfterMs);
|
|
3418
|
+
console.warn(
|
|
3419
|
+
`[vo-runner supervisor] GitHub readiness embargoed child startup for ${retryAfterMs}ms; control polling remains active`
|
|
3420
|
+
);
|
|
3421
|
+
};
|
|
3422
|
+
const refreshSupervisorChildAdmission = async () => {
|
|
3423
|
+
if (!pairedSupervisor) return true;
|
|
3424
|
+
const refreshed = await prepareSupervisorAuth({
|
|
3425
|
+
...supervisorAuthInput,
|
|
3426
|
+
baseEnv: childEnv
|
|
3427
|
+
});
|
|
3428
|
+
if (refreshed.operatorId !== operatorId) {
|
|
3429
|
+
throw new Error("runner readiness operator identity changed during supervisor admission");
|
|
3430
|
+
}
|
|
3431
|
+
if (refreshed.startupRetryAfterMs !== null) {
|
|
3432
|
+
deferChildAdmission(refreshed.startupRetryAfterMs);
|
|
3433
|
+
return false;
|
|
3434
|
+
}
|
|
3435
|
+
if (refreshed.repositoryScope.length > 0) {
|
|
3436
|
+
const repositoryScope = refreshed.repositoryScope.join(",");
|
|
3437
|
+
childEnv.VO_CODE_RUNNER_REPOS = repositoryScope;
|
|
3438
|
+
clientEnv.VO_CODE_RUNNER_REPOS = repositoryScope;
|
|
3439
|
+
}
|
|
3440
|
+
startupNeedsReadinessRefresh = false;
|
|
3441
|
+
return true;
|
|
3442
|
+
};
|
|
3443
|
+
const ensureHealthyChildAfterControl = async () => {
|
|
3444
|
+
if (!child || supervisorChildHasExited(child)) {
|
|
3445
|
+
const recoveryGeneration = degradationState.snapshot().generation;
|
|
3446
|
+
if (!startupDeferral.hasStarted() && startupDeferral.isPending()) {
|
|
3447
|
+
deferredControlRecoveryGeneration = recoveryGeneration;
|
|
3448
|
+
return false;
|
|
3449
|
+
}
|
|
3450
|
+
try {
|
|
3451
|
+
if (!await refreshSupervisorChildAdmission()) {
|
|
3452
|
+
deferredControlRecoveryGeneration = recoveryGeneration;
|
|
3453
|
+
return false;
|
|
3454
|
+
}
|
|
3455
|
+
if (!startupDeferral.hasStarted() && !startupDeferral.consumeIfReady()) return false;
|
|
3456
|
+
child = launchChild();
|
|
3457
|
+
readinessDeferrals.recordRecoveryGeneration(child, recoveryGeneration);
|
|
3458
|
+
} catch (error) {
|
|
3459
|
+
markSupervisorDegraded();
|
|
3460
|
+
console.error(
|
|
3461
|
+
`[vo-runner supervisor] control recovery relaunch failed; entering degraded mode: ${error instanceof Error ? error.message : String(error)}`
|
|
3462
|
+
);
|
|
3463
|
+
return false;
|
|
3464
|
+
}
|
|
3465
|
+
}
|
|
3466
|
+
return verifyChildHealth(child, true, "control recovery relaunch");
|
|
3467
|
+
};
|
|
3468
|
+
const startInitialChild = async (recoveryGeneration = null) => {
|
|
3469
|
+
const activationChild = launchChild();
|
|
3470
|
+
child = activationChild;
|
|
3471
|
+
if (Number.isInteger(recoveryGeneration)) {
|
|
3472
|
+
readinessDeferrals.recordRecoveryGeneration(activationChild, recoveryGeneration);
|
|
3473
|
+
}
|
|
3474
|
+
void verifyChildHealth(
|
|
3475
|
+
activationChild,
|
|
3476
|
+
Number.isInteger(recoveryGeneration),
|
|
3477
|
+
Number.isInteger(recoveryGeneration) ? "deferred control recovery" : "initial child"
|
|
3478
|
+
);
|
|
3479
|
+
if (!await finishPendingActivation({
|
|
3480
|
+
client,
|
|
3481
|
+
child: activationChild,
|
|
3482
|
+
runtimeRoot,
|
|
3483
|
+
operatorId,
|
|
3484
|
+
runnerId,
|
|
3485
|
+
selfPath,
|
|
3486
|
+
packageVersion: supervisorVersion,
|
|
3487
|
+
supervisorIdentity: supervisorControlIdentity,
|
|
3488
|
+
waitForLocalRunner,
|
|
3489
|
+
isReadinessDeferred: isChildReadinessDeferred,
|
|
3490
|
+
localStatus,
|
|
3491
|
+
stopChild: stopChildExpectedly,
|
|
3492
|
+
launchPreviousChild: (entry) => spawnPreviousChild(entry, childEnv),
|
|
3493
|
+
log: (message) => console.error(`[vo-runner supervisor] ${message}`)
|
|
3494
|
+
})) {
|
|
3495
|
+
if (captureChildReadinessDeferral(activationChild)) {
|
|
3496
|
+
console.warn("[vo-runner supervisor] activation attestation deferred with GitHub readiness; pending slot remains durable");
|
|
3497
|
+
return false;
|
|
3498
|
+
}
|
|
3499
|
+
markSupervisorDegraded();
|
|
3500
|
+
await stopChildExpectedly(activationChild);
|
|
3501
|
+
if (child === activationChild) child = null;
|
|
3502
|
+
console.error("[vo-runner supervisor] activation FAILED \u2014 entering degraded mode: not serving tasks, still polling for remote control actions");
|
|
3503
|
+
return false;
|
|
3504
|
+
}
|
|
3505
|
+
return true;
|
|
3506
|
+
};
|
|
3507
|
+
const attemptInitialChildStart = async () => {
|
|
3508
|
+
if (startupDeferral.hasStarted() || startupDeferral.isPending()) return false;
|
|
3509
|
+
const beforeRefresh = resolveSupervisorChildStartAuthority({
|
|
3510
|
+
degradationState,
|
|
3511
|
+
deferredRecoveryGeneration: deferredControlRecoveryGeneration
|
|
3512
|
+
});
|
|
3513
|
+
if (!beforeRefresh.allowed) return false;
|
|
3514
|
+
if (startupNeedsReadinessRefresh) {
|
|
3515
|
+
try {
|
|
3516
|
+
if (!await refreshSupervisorChildAdmission()) return false;
|
|
3517
|
+
} catch (error) {
|
|
3518
|
+
markSupervisorDegraded();
|
|
3519
|
+
deferredControlRecoveryGeneration = null;
|
|
3520
|
+
startupDeferral.consumeIfReady();
|
|
3521
|
+
console.error(
|
|
3522
|
+
`[vo-runner supervisor] deferred readiness refresh failed; entering degraded mode: ${error instanceof Error ? error.message : String(error)}`
|
|
3523
|
+
);
|
|
3524
|
+
return false;
|
|
3525
|
+
}
|
|
3526
|
+
}
|
|
3527
|
+
const afterRefresh = resolveSupervisorChildStartAuthority({
|
|
3528
|
+
degradationState,
|
|
3529
|
+
deferredRecoveryGeneration: deferredControlRecoveryGeneration
|
|
3530
|
+
});
|
|
3531
|
+
if (!afterRefresh.allowed) return false;
|
|
3532
|
+
if (!startupDeferral.consumeIfReady()) return false;
|
|
3533
|
+
deferredControlRecoveryGeneration = null;
|
|
3534
|
+
return startInitialChild(afterRefresh.recoveryGeneration);
|
|
3535
|
+
};
|
|
3536
|
+
await attemptInitialChildStart();
|
|
2665
3537
|
const shutdown = async () => {
|
|
2666
3538
|
if (stopping) return;
|
|
2667
3539
|
stopping = true;
|
|
@@ -2675,6 +3547,7 @@ async function main() {
|
|
|
2675
3547
|
});
|
|
2676
3548
|
while (!stopping) {
|
|
2677
3549
|
try {
|
|
3550
|
+
await attemptInitialChildStart();
|
|
2678
3551
|
const action = await client.pollRunnerControl({
|
|
2679
3552
|
runnerId,
|
|
2680
3553
|
...operatorId ? { operatorId } : {},
|
|
@@ -2684,21 +3557,26 @@ async function main() {
|
|
|
2684
3557
|
await sleep(POLL_MS);
|
|
2685
3558
|
continue;
|
|
2686
3559
|
}
|
|
3560
|
+
deferredControlRecoveryGeneration = null;
|
|
2687
3561
|
handling = true;
|
|
2688
|
-
const
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
|
|
2696
|
-
})
|
|
3562
|
+
const controlIdentity = { runnerId, ...operatorId ? { operatorId } : {}, ...supervisorControlIdentity };
|
|
3563
|
+
const bundledAction = action.kind === "update" || action.kind === "reinstall";
|
|
3564
|
+
const drain = await awaitRunnerUpdateDrain({
|
|
3565
|
+
readStatus: localStatus,
|
|
3566
|
+
drainable: bundledAction,
|
|
3567
|
+
env: process.env,
|
|
3568
|
+
isChildRunning: () => supervisorChildIsRunning(child),
|
|
3569
|
+
failInFlightTask: (taskId, patch) => client.postProgress(taskId, { status: "failed", message: patch.message, result: patch.result, ...patch.runnerId ? { runner_id: patch.runnerId } : {}, ...patch.runnerInstanceId ? { runner_instance_id: patch.runnerInstanceId } : {} }),
|
|
3570
|
+
log: (message) => console.warn(`[vo-runner supervisor] ${message}`)
|
|
3571
|
+
});
|
|
3572
|
+
if (!drain.proceed) {
|
|
3573
|
+
await client.completeRunnerControl(action.actionId, { ...controlIdentity, status: "failed", detail: drain.detail });
|
|
2697
3574
|
handling = false;
|
|
3575
|
+
respawn();
|
|
2698
3576
|
continue;
|
|
2699
3577
|
}
|
|
2700
|
-
|
|
2701
|
-
|
|
3578
|
+
controlRecoveryFence.markBeforeStop(child);
|
|
3579
|
+
await stopChildExpectedly(child);
|
|
2702
3580
|
const result = bundledAction ? stageAndActivateBundledUpdate({
|
|
2703
3581
|
runtimeRoot,
|
|
2704
3582
|
packageSpec: `@algosuite/vo-mcp@${action.desired_package_version}`,
|
|
@@ -2720,23 +3598,27 @@ async function main() {
|
|
|
2720
3598
|
log: (message) => console.warn(`[vo-runner supervisor] ${message}`)
|
|
2721
3599
|
});
|
|
2722
3600
|
if (result.ok && result.handoff) {
|
|
3601
|
+
if (drain.capped) console.warn(`[vo-runner supervisor] ${drain.detail}`);
|
|
2723
3602
|
console.warn(`[vo-runner supervisor] activated ${result.active.version}; exiting for new-process attestation`);
|
|
2724
3603
|
return;
|
|
2725
3604
|
}
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
3605
|
+
const recoveryAuthorized = shouldRecoverSupervisorChildAfterAction({
|
|
3606
|
+
stoppedChild: controlRecoveryFence.consume(),
|
|
3607
|
+
actionKind: action.kind,
|
|
3608
|
+
actionSucceeded: result.ok
|
|
3609
|
+
});
|
|
3610
|
+
const relaunchedHealthy = recoveryAuthorized ? await ensureHealthyChildAfterControl() : false;
|
|
3611
|
+
if (!relaunchedHealthy) result.ok = false;
|
|
2730
3612
|
await client.completeRunnerControl(action.actionId, {
|
|
2731
|
-
|
|
2732
|
-
...operatorId ? { operatorId } : {},
|
|
2733
|
-
...supervisorControlIdentity,
|
|
3613
|
+
...controlIdentity,
|
|
2734
3614
|
status: result.ok ? "succeeded" : "failed",
|
|
2735
|
-
detail: result.ok ? result.detail ? `${result.detail}; runner ${packageVersion()} reconnected`.slice(0, 1e3) : `runner ${packageVersion()} reconnected` : `maintenance exited ${result.status}${result.detail ? `: ${result.detail}` : ""}`
|
|
3615
|
+
detail: result.ok ? result.detail ? `${result.detail}; runner ${packageVersion()} reconnected`.slice(0, 1e3) : `runner ${packageVersion()} reconnected` : `maintenance exited ${result.status}${result.detail ? `: ${result.detail}` : ""}${relaunchedHealthy ? "" : "; runner relaunch did not become healthy"}`
|
|
2736
3616
|
});
|
|
2737
3617
|
handling = false;
|
|
3618
|
+
respawn();
|
|
2738
3619
|
} catch (error) {
|
|
2739
3620
|
console.error(`[vo-runner supervisor] ${error instanceof Error ? error.message : String(error)}`);
|
|
3621
|
+
await controlRecoveryFence.recoverOnce(() => ensureHealthyChildAfterControl());
|
|
2740
3622
|
handling = false;
|
|
2741
3623
|
await sleep(POLL_MS);
|
|
2742
3624
|
}
|