@algosuite/vo-mcp 0.2.0-beta.56 → 0.2.0-beta.57
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/autostart-cli.js +6 -3
- package/dist/autostart-cli.js.map +2 -2
- package/dist/install-cli.js +6 -3
- package/dist/install-cli.js.map +2 -2
- package/dist/runner-cli.js +286 -55
- package/dist/runner-cli.js.map +3 -3
- package/dist/runner-supervisor.js +805 -75
- package/dist/runner-supervisor.js.map +4 -4
- package/package.json +1 -1
|
@@ -31,7 +31,7 @@ var init_control_plane_auth_stub = __esm({
|
|
|
31
31
|
|
|
32
32
|
// src/runner-supervisor.mjs
|
|
33
33
|
import { spawn } from "node:child_process";
|
|
34
|
-
import { randomUUID as
|
|
34
|
+
import { randomUUID as randomUUID4 } from "node:crypto";
|
|
35
35
|
import { createRequire } from "node:module";
|
|
36
36
|
import { fileURLToPath as fileURLToPath3 } from "node:url";
|
|
37
37
|
import { dirname as dirname4, join as join5 } from "node:path";
|
|
@@ -2090,6 +2090,7 @@ async function finishPendingActivation({
|
|
|
2090
2090
|
packageVersion: packageVersion2,
|
|
2091
2091
|
supervisorIdentity,
|
|
2092
2092
|
waitForLocalRunner: waitForLocalRunner2,
|
|
2093
|
+
isReadinessDeferred = () => false,
|
|
2093
2094
|
localStatus: localStatus2,
|
|
2094
2095
|
waitForCloudRunner = waitForAuthoritativeRunnerHeartbeat,
|
|
2095
2096
|
cloudTimeoutMs,
|
|
@@ -2124,7 +2125,10 @@ async function finishPendingActivation({
|
|
|
2124
2125
|
try {
|
|
2125
2126
|
attestation = attestCurrentSupervisor({ runtimeRoot, selfPath: selfPath2, version: packageVersion2 });
|
|
2126
2127
|
if (!attestation.ok) throw new Error(attestation.detail);
|
|
2127
|
-
if (!await waitForLocalRunner2(child))
|
|
2128
|
+
if (!await waitForLocalRunner2(child)) {
|
|
2129
|
+
if (isReadinessDeferred(child)) return false;
|
|
2130
|
+
throw new Error("activated runner did not become locally ready");
|
|
2131
|
+
}
|
|
2128
2132
|
} catch (error) {
|
|
2129
2133
|
let detail = `activation attestation failed: ${error instanceof Error ? error.message : String(error)}`.slice(0, 400);
|
|
2130
2134
|
let rolledBack = null;
|
|
@@ -2318,8 +2322,97 @@ function resolveSupervisorChildEntry({
|
|
|
2318
2322
|
import { hostname as systemHostname } from "node:os";
|
|
2319
2323
|
|
|
2320
2324
|
// src/runner-readiness.mjs
|
|
2321
|
-
|
|
2322
|
-
|
|
2325
|
+
import { createHash as createHash5, randomUUID as randomUUID3 } from "node:crypto";
|
|
2326
|
+
var RUNNER_GITHUB_READINESS_TIMEOUT_MS = 5e4;
|
|
2327
|
+
var RUNNER_IDENTITY_READINESS_TIMEOUT_MS = 1e4;
|
|
2328
|
+
var RUNNER_READINESS_MAX_REPOSITORIES = 4;
|
|
2329
|
+
var RUNNER_READINESS_TRANSIENT_MAX_RETRY_AFTER_MS = 3e5;
|
|
2330
|
+
var RUNNER_READINESS_DEFERRAL_MESSAGE_TYPE = "vo-runner-readiness-deferred-v1";
|
|
2331
|
+
var RUNNER_READINESS_DEFERRAL_ACK_TYPE = "vo-runner-readiness-deferred-ack-v1";
|
|
2332
|
+
var RUNNER_REPOSITORY = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u;
|
|
2333
|
+
var UUID_RE2 = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
|
|
2334
|
+
function runnerRepositoryScopeFromEnv(env = {}) {
|
|
2335
|
+
return String(env.VO_CODE_RUNNER_REPOS || "").split(/[\s,]+/u).map((repo) => repo.trim()).filter(Boolean);
|
|
2336
|
+
}
|
|
2337
|
+
function normalizeRunnerRepositoryScope(repositories) {
|
|
2338
|
+
if (!Array.isArray(repositories) || repositories.length < 1) {
|
|
2339
|
+
throw new Error("VO_CODE_RUNNER_REPOS must name at least one owner/name repository");
|
|
2340
|
+
}
|
|
2341
|
+
if (repositories.length > RUNNER_READINESS_MAX_REPOSITORIES) {
|
|
2342
|
+
throw new Error(`VO_CODE_RUNNER_REPOS supports at most ${RUNNER_READINESS_MAX_REPOSITORIES} repositories`);
|
|
2343
|
+
}
|
|
2344
|
+
const normalized = repositories.map((repo) => {
|
|
2345
|
+
const [owner, name] = typeof repo === "string" ? repo.split("/") : [];
|
|
2346
|
+
if (typeof repo !== "string" || repo.length > 140 || !RUNNER_REPOSITORY.test(repo) || owner === "." || owner === ".." || name === "." || name === "..") {
|
|
2347
|
+
throw new Error("VO_CODE_RUNNER_REPOS entries must be canonical owner/name repositories");
|
|
2348
|
+
}
|
|
2349
|
+
return repo.toLowerCase();
|
|
2350
|
+
});
|
|
2351
|
+
if (new Set(normalized).size !== normalized.length) {
|
|
2352
|
+
throw new Error("VO_CODE_RUNNER_REPOS entries must be unique");
|
|
2353
|
+
}
|
|
2354
|
+
const owners = new Set(normalized.map((repo) => repo.split("/")[0]));
|
|
2355
|
+
if (owners.size !== 1) {
|
|
2356
|
+
throw new Error("VO_CODE_RUNNER_REPOS entries must share one owner");
|
|
2357
|
+
}
|
|
2358
|
+
return Object.freeze(normalized.sort());
|
|
2359
|
+
}
|
|
2360
|
+
function runnerRepositoryScopeDigest(repositories) {
|
|
2361
|
+
const scope = normalizeRunnerRepositoryScope(repositories);
|
|
2362
|
+
return createHash5("sha256").update(JSON.stringify({
|
|
2363
|
+
version: 1,
|
|
2364
|
+
repositories: scope
|
|
2365
|
+
}), "utf8").digest("hex");
|
|
2366
|
+
}
|
|
2367
|
+
function runnerReadinessRetryDelayMs(readiness) {
|
|
2368
|
+
if (readiness?.paired !== true || readiness?.githubReady !== false) return null;
|
|
2369
|
+
if (Number.isFinite(readiness.retryAfterMs) && readiness.retryAfterMs > 0) {
|
|
2370
|
+
return Math.ceil(readiness.retryAfterMs);
|
|
2371
|
+
}
|
|
2372
|
+
return null;
|
|
2373
|
+
}
|
|
2374
|
+
function parseRunnerReadinessDeferralRequest(message) {
|
|
2375
|
+
if (message?.type !== RUNNER_READINESS_DEFERRAL_MESSAGE_TYPE || !UUID_RE2.test(String(message.nonce || "")) || !Number.isFinite(message.retryAfterMs) || message.retryAfterMs <= 0) return null;
|
|
2376
|
+
return Object.freeze({
|
|
2377
|
+
nonce: message.nonce,
|
|
2378
|
+
retryAfterMs: Math.ceil(message.retryAfterMs),
|
|
2379
|
+
error: typeof message.error === "string" ? message.error : null
|
|
2380
|
+
});
|
|
2381
|
+
}
|
|
2382
|
+
function failed({
|
|
2383
|
+
paired = false,
|
|
2384
|
+
operatorId = null,
|
|
2385
|
+
tenantId = null,
|
|
2386
|
+
githubReady = null,
|
|
2387
|
+
retryAfterMs = null,
|
|
2388
|
+
error,
|
|
2389
|
+
message
|
|
2390
|
+
}) {
|
|
2391
|
+
return {
|
|
2392
|
+
ok: false,
|
|
2393
|
+
paired,
|
|
2394
|
+
operatorId,
|
|
2395
|
+
tenantId,
|
|
2396
|
+
githubReady,
|
|
2397
|
+
...Number.isFinite(retryAfterMs) && retryAfterMs > 0 ? { retryAfterMs } : {},
|
|
2398
|
+
error,
|
|
2399
|
+
message
|
|
2400
|
+
};
|
|
2401
|
+
}
|
|
2402
|
+
function responseRetryAfterMs(response, body) {
|
|
2403
|
+
const transientReadinessFailure = response.status === 503 && body?.error === "github_installation_readiness_retryable";
|
|
2404
|
+
if (response.status !== 429 && !transientReadinessFailure) return null;
|
|
2405
|
+
const bound = (retryAfterMs) => transientReadinessFailure ? Math.min(RUNNER_READINESS_TRANSIENT_MAX_RETRY_AFTER_MS, retryAfterMs) : retryAfterMs;
|
|
2406
|
+
const value = response.headers?.get?.("retry-after")?.trim() ?? "";
|
|
2407
|
+
const seconds = Number(value);
|
|
2408
|
+
if (value && Number.isFinite(seconds) && seconds >= 0) {
|
|
2409
|
+
return bound(Math.max(1e3, Math.ceil(seconds * 1e3)));
|
|
2410
|
+
}
|
|
2411
|
+
const at = value ? Date.parse(value) : Number.NaN;
|
|
2412
|
+
if (Number.isFinite(at)) {
|
|
2413
|
+
return bound(Math.max(1e3, Math.ceil(at - Date.now())));
|
|
2414
|
+
}
|
|
2415
|
+
return bound(6e4);
|
|
2323
2416
|
}
|
|
2324
2417
|
async function responseBody(response) {
|
|
2325
2418
|
try {
|
|
@@ -2332,11 +2425,21 @@ async function responseBody(response) {
|
|
|
2332
2425
|
function serverMessage(body, fallback) {
|
|
2333
2426
|
return typeof body.message === "string" && body.message.trim() ? body.message.trim() : fallback;
|
|
2334
2427
|
}
|
|
2335
|
-
async function
|
|
2428
|
+
async function fetchJsonWithTimeout(fetchImpl, url, init, timeoutMs) {
|
|
2336
2429
|
const controller = new AbortController();
|
|
2337
|
-
|
|
2430
|
+
let timer;
|
|
2431
|
+
const timeout = new Promise((_, reject) => {
|
|
2432
|
+
timer = setTimeout(() => {
|
|
2433
|
+
controller.abort();
|
|
2434
|
+
reject(new Error(`request aborted after ${timeoutMs}ms`));
|
|
2435
|
+
}, timeoutMs);
|
|
2436
|
+
});
|
|
2338
2437
|
try {
|
|
2339
|
-
|
|
2438
|
+
const requestAndBody = (async () => {
|
|
2439
|
+
const response = await fetchImpl(url, { ...init, signal: controller.signal });
|
|
2440
|
+
return { response, body: await responseBody(response) };
|
|
2441
|
+
})();
|
|
2442
|
+
return await Promise.race([requestAndBody, timeout]);
|
|
2340
2443
|
} finally {
|
|
2341
2444
|
clearTimeout(timer);
|
|
2342
2445
|
}
|
|
@@ -2346,18 +2449,24 @@ async function probeRunnerReadiness({
|
|
|
2346
2449
|
token,
|
|
2347
2450
|
fetchImpl = fetch,
|
|
2348
2451
|
requireGithub = false,
|
|
2349
|
-
|
|
2452
|
+
repositories = [],
|
|
2453
|
+
timeoutMs = RUNNER_IDENTITY_READINESS_TIMEOUT_MS,
|
|
2454
|
+
// The server performs at most five sequential App-JWT GETs (installation plus
|
|
2455
|
+
// every one of at most four configured repositories), each bounded at 8s.
|
|
2456
|
+
// Keep a transport margin while preventing a stuck proof from hanging startup.
|
|
2457
|
+
githubTimeoutMs = RUNNER_GITHUB_READINESS_TIMEOUT_MS
|
|
2350
2458
|
}) {
|
|
2351
2459
|
const base = controlPlaneUrl.replace(/\/+$/u, "");
|
|
2352
2460
|
const headers = { authorization: `Bearer ${token}` };
|
|
2353
2461
|
let identityResponse;
|
|
2462
|
+
let identity;
|
|
2354
2463
|
try {
|
|
2355
|
-
identityResponse = await
|
|
2464
|
+
({ response: identityResponse, body: identity } = await fetchJsonWithTimeout(
|
|
2356
2465
|
fetchImpl,
|
|
2357
2466
|
`${base}/api/v1/auth/me`,
|
|
2358
2467
|
{ headers },
|
|
2359
2468
|
timeoutMs
|
|
2360
|
-
);
|
|
2469
|
+
));
|
|
2361
2470
|
} catch (error) {
|
|
2362
2471
|
const detail = error instanceof Error ? error.message : String(error);
|
|
2363
2472
|
return failed({
|
|
@@ -2365,7 +2474,6 @@ async function probeRunnerReadiness({
|
|
|
2365
2474
|
message: `AlgoHQ could not be reached: ${detail}`
|
|
2366
2475
|
});
|
|
2367
2476
|
}
|
|
2368
|
-
const identity = await responseBody(identityResponse);
|
|
2369
2477
|
if (!identityResponse.ok) {
|
|
2370
2478
|
return failed({
|
|
2371
2479
|
error: "credential_rejected",
|
|
@@ -2391,18 +2499,33 @@ async function probeRunnerReadiness({
|
|
|
2391
2499
|
message: "Paired to AlgoHQ."
|
|
2392
2500
|
};
|
|
2393
2501
|
}
|
|
2502
|
+
let repositoryScope = null;
|
|
2503
|
+
try {
|
|
2504
|
+
if (!Array.isArray(repositories)) {
|
|
2505
|
+
throw new Error("VO_CODE_RUNNER_REPOS must be a repository array");
|
|
2506
|
+
}
|
|
2507
|
+
if (repositories.length > 0) repositoryScope = normalizeRunnerRepositoryScope(repositories);
|
|
2508
|
+
} catch (error) {
|
|
2509
|
+
return failed({
|
|
2510
|
+
paired: true,
|
|
2511
|
+
operatorId,
|
|
2512
|
+
tenantId,
|
|
2513
|
+
githubReady: false,
|
|
2514
|
+
error: "github_repository_scope_invalid",
|
|
2515
|
+
message: error instanceof Error ? error.message : String(error)
|
|
2516
|
+
});
|
|
2517
|
+
}
|
|
2394
2518
|
let githubResponse;
|
|
2519
|
+
let github;
|
|
2395
2520
|
try {
|
|
2396
|
-
|
|
2521
|
+
const readinessUrl = new URL(`${base}/api/v1/github/installation-readiness`);
|
|
2522
|
+
for (const repo of repositoryScope ?? []) readinessUrl.searchParams.append("repo", repo);
|
|
2523
|
+
({ response: githubResponse, body: github } = await fetchJsonWithTimeout(
|
|
2397
2524
|
fetchImpl,
|
|
2398
|
-
|
|
2399
|
-
{
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
body: "{}"
|
|
2403
|
-
},
|
|
2404
|
-
timeoutMs
|
|
2405
|
-
);
|
|
2525
|
+
readinessUrl.toString(),
|
|
2526
|
+
{ headers },
|
|
2527
|
+
githubTimeoutMs
|
|
2528
|
+
));
|
|
2406
2529
|
} catch (error) {
|
|
2407
2530
|
const detail = error instanceof Error ? error.message : String(error);
|
|
2408
2531
|
return failed({
|
|
@@ -2414,14 +2537,25 @@ async function probeRunnerReadiness({
|
|
|
2414
2537
|
message: `GitHub publication readiness could not be checked: ${detail}`
|
|
2415
2538
|
});
|
|
2416
2539
|
}
|
|
2417
|
-
const
|
|
2418
|
-
|
|
2540
|
+
const repositorySelection = github.repository_selection;
|
|
2541
|
+
const repositoriesVerified = github.repositories_verified;
|
|
2542
|
+
const returnedScope = github.repository_scope;
|
|
2543
|
+
let normalizedReturnedScope = null;
|
|
2544
|
+
try {
|
|
2545
|
+
normalizedReturnedScope = normalizeRunnerRepositoryScope(returnedScope);
|
|
2546
|
+
} catch {
|
|
2547
|
+
}
|
|
2548
|
+
const effectiveScope = repositoryScope ?? normalizedReturnedScope;
|
|
2549
|
+
const sourceVerified = repositoryScope === null ? github.repository_scope_source === "persisted_installation_singleton" && normalizedReturnedScope?.length === 1 : github.repository_scope_source === "runner_config";
|
|
2550
|
+
const repositoryScopeVerified = effectiveScope !== null && normalizedReturnedScope !== null && Array.isArray(returnedScope) && returnedScope.length === normalizedReturnedScope.length && returnedScope.every((repo, index) => repo === normalizedReturnedScope[index]) && normalizedReturnedScope.length === effectiveScope.length && normalizedReturnedScope.every((repo, index) => repo === effectiveScope[index]) && github.repository_scope_sha256 === runnerRepositoryScopeDigest(effectiveScope) && repositoriesVerified === effectiveScope.length && sourceVerified && (repositorySelection === "all" || repositorySelection === "selected");
|
|
2551
|
+
if (!githubResponse.ok || github.configured !== true || github.verified !== true || github.publication_ready !== true || !repositoryScopeVerified) {
|
|
2419
2552
|
const error = typeof github.error === "string" && github.error ? github.error : "github_not_ready";
|
|
2420
2553
|
return failed({
|
|
2421
2554
|
paired: true,
|
|
2422
2555
|
operatorId,
|
|
2423
2556
|
tenantId,
|
|
2424
2557
|
githubReady: false,
|
|
2558
|
+
retryAfterMs: responseRetryAfterMs(githubResponse, github),
|
|
2425
2559
|
error,
|
|
2426
2560
|
message: serverMessage(github, `GitHub publication preflight failed (HTTP ${githubResponse.status}).`)
|
|
2427
2561
|
});
|
|
@@ -2432,8 +2566,10 @@ async function probeRunnerReadiness({
|
|
|
2432
2566
|
operatorId,
|
|
2433
2567
|
tenantId,
|
|
2434
2568
|
githubReady: true,
|
|
2569
|
+
repositoryScope: effectiveScope,
|
|
2570
|
+
repositoryScopeSource: github.repository_scope_source,
|
|
2435
2571
|
error: null,
|
|
2436
|
-
message:
|
|
2572
|
+
message: `Paired with a verified Algosuite GitHub App installation (${repositoriesVerified} repos).`
|
|
2437
2573
|
};
|
|
2438
2574
|
}
|
|
2439
2575
|
function pairedOperatorScope(readiness) {
|
|
@@ -2472,25 +2608,51 @@ async function prepareSupervisorAuth({
|
|
|
2472
2608
|
const token = explicitAdminToken || storedCredential?.vo_credential;
|
|
2473
2609
|
if (!token) throw new Error("runner is not paired; run `vo-mcp pair` once on this host");
|
|
2474
2610
|
let operatorId = String(baseEnv.VO_CODE_RUNNER_OPERATOR_IDS || "").split(/[\s,]+/u).filter(Boolean)[0] || void 0;
|
|
2611
|
+
let repositoryScope = runnerRepositoryScopeFromEnv(baseEnv);
|
|
2612
|
+
let startupRetryAfterMs = null;
|
|
2613
|
+
let githubReady = null;
|
|
2614
|
+
let readinessError = null;
|
|
2475
2615
|
if (!explicitAdminToken) {
|
|
2476
|
-
const readiness = await probeReadiness({
|
|
2477
|
-
|
|
2478
|
-
|
|
2616
|
+
const readiness = await probeReadiness({
|
|
2617
|
+
controlPlaneUrl,
|
|
2618
|
+
token,
|
|
2619
|
+
requireGithub: true,
|
|
2620
|
+
repositories: repositoryScope
|
|
2621
|
+
});
|
|
2622
|
+
const pairedScope = pairedOperatorScope(readiness) || (readiness?.paired === true && typeof readiness.operatorId === "string" ? readiness.operatorId.trim() : "");
|
|
2623
|
+
operatorId = pairedScope || void 0;
|
|
2479
2624
|
if (!operatorId) throw new Error("runner readiness failed: paired operator scope is missing");
|
|
2625
|
+
if (readiness.ok && Array.isArray(readiness.repositoryScope)) {
|
|
2626
|
+
repositoryScope = [...readiness.repositoryScope];
|
|
2627
|
+
githubReady = true;
|
|
2628
|
+
} else {
|
|
2629
|
+
githubReady = false;
|
|
2630
|
+
readinessError = typeof readiness?.error === "string" ? readiness.error : "github_not_ready";
|
|
2631
|
+
startupRetryAfterMs = runnerReadinessRetryDelayMs(readiness);
|
|
2632
|
+
}
|
|
2480
2633
|
}
|
|
2634
|
+
const effectiveBaseEnv = repositoryScope.length > 0 ? { ...baseEnv, VO_CODE_RUNNER_REPOS: repositoryScope.join(",") } : baseEnv;
|
|
2481
2635
|
const childEnv = buildSupervisorChildEnv({
|
|
2482
|
-
baseEnv,
|
|
2636
|
+
baseEnv: effectiveBaseEnv,
|
|
2483
2637
|
controlPlaneUrl,
|
|
2484
2638
|
explicitAdminToken,
|
|
2485
2639
|
pairedOperatorId: explicitAdminToken ? null : operatorId
|
|
2486
2640
|
});
|
|
2487
2641
|
const clientEnv = {
|
|
2488
|
-
...
|
|
2642
|
+
...effectiveBaseEnv,
|
|
2489
2643
|
VO_CONTROL_PLANE_ADMIN_TOKEN: token,
|
|
2490
2644
|
VO_CONTROL_PLANE_URL: controlPlaneUrl,
|
|
2491
2645
|
...operatorId ? { VO_CODE_RUNNER_OPERATOR_IDS: operatorId } : {}
|
|
2492
2646
|
};
|
|
2493
|
-
return {
|
|
2647
|
+
return {
|
|
2648
|
+
childEnv,
|
|
2649
|
+
clientEnv,
|
|
2650
|
+
operatorId,
|
|
2651
|
+
repositoryScope,
|
|
2652
|
+
githubReady,
|
|
2653
|
+
readinessError,
|
|
2654
|
+
startupRetryAfterMs
|
|
2655
|
+
};
|
|
2494
2656
|
}
|
|
2495
2657
|
|
|
2496
2658
|
// src/runner/supervisor-credential-reader.mjs
|
|
@@ -2530,12 +2692,309 @@ function readStoredCredentialIsolated({
|
|
|
2530
2692
|
}
|
|
2531
2693
|
}
|
|
2532
2694
|
|
|
2695
|
+
// src/runner/respawn-circuit.mjs
|
|
2696
|
+
var DEFAULT_RESPAWN_CIRCUIT = Object.freeze({
|
|
2697
|
+
baseDelayMs: 2e3,
|
|
2698
|
+
maxDelayMs: 3e4,
|
|
2699
|
+
healthyResetMs: 6e4,
|
|
2700
|
+
maxRapidExits: 5
|
|
2701
|
+
});
|
|
2702
|
+
function createRespawnCircuit(options = {}) {
|
|
2703
|
+
const config = { ...DEFAULT_RESPAWN_CIRCUIT, ...options };
|
|
2704
|
+
for (const [key, value] of Object.entries(config)) {
|
|
2705
|
+
if (!Number.isInteger(value) || value < 1) throw new Error(`${key} must be a positive integer`);
|
|
2706
|
+
}
|
|
2707
|
+
let rapidExits = 0;
|
|
2708
|
+
return Object.freeze({
|
|
2709
|
+
recordExit(startedAt, exitedAt) {
|
|
2710
|
+
if (!Number.isFinite(startedAt) || !Number.isFinite(exitedAt) || exitedAt < startedAt) {
|
|
2711
|
+
throw new Error("respawn circuit timestamps are invalid");
|
|
2712
|
+
}
|
|
2713
|
+
const uptimeMs = exitedAt - startedAt;
|
|
2714
|
+
rapidExits = uptimeMs >= config.healthyResetMs ? 0 : rapidExits + 1;
|
|
2715
|
+
const tripped = rapidExits >= config.maxRapidExits;
|
|
2716
|
+
const delayMs = Math.min(
|
|
2717
|
+
config.maxDelayMs,
|
|
2718
|
+
config.baseDelayMs * 2 ** Math.max(0, rapidExits - 1)
|
|
2719
|
+
);
|
|
2720
|
+
return Object.freeze({ rapidExits, uptimeMs, delayMs, tripped });
|
|
2721
|
+
},
|
|
2722
|
+
snapshot() {
|
|
2723
|
+
return Object.freeze({ rapidExits, tripped: rapidExits >= config.maxRapidExits });
|
|
2724
|
+
}
|
|
2725
|
+
});
|
|
2726
|
+
}
|
|
2727
|
+
|
|
2728
|
+
// src/runner/supervisor-child-health.mjs
|
|
2729
|
+
var terminatedChildren = /* @__PURE__ */ new WeakSet();
|
|
2730
|
+
function attachSupervisorChildTerminationCustody(target, onTerminated, onObservedError = () => {
|
|
2731
|
+
}) {
|
|
2732
|
+
let accounted = false;
|
|
2733
|
+
const account = (kind, error = null) => {
|
|
2734
|
+
terminatedChildren.add(target);
|
|
2735
|
+
if (accounted) return false;
|
|
2736
|
+
accounted = true;
|
|
2737
|
+
onTerminated(Object.freeze({ target, kind, error }));
|
|
2738
|
+
return true;
|
|
2739
|
+
};
|
|
2740
|
+
target.on("error", (error) => {
|
|
2741
|
+
onObservedError(error);
|
|
2742
|
+
if (target.pid === void 0 || target.pid === null) account("error", error);
|
|
2743
|
+
});
|
|
2744
|
+
target.on("exit", () => {
|
|
2745
|
+
account("exit");
|
|
2746
|
+
});
|
|
2747
|
+
return Object.freeze({ accounted: () => accounted });
|
|
2748
|
+
}
|
|
2749
|
+
function supervisorChildHasExited(target) {
|
|
2750
|
+
return terminatedChildren.has(target) || target.exitCode !== null || (target.signalCode ?? null) !== null;
|
|
2751
|
+
}
|
|
2752
|
+
function supervisorChildIsRunning(target) {
|
|
2753
|
+
return !supervisorChildHasExited(target);
|
|
2754
|
+
}
|
|
2755
|
+
function shouldRespawnSupervisorChild({ stopping, degraded, handling, child }) {
|
|
2756
|
+
return !stopping && !degraded && !handling && (!child || supervisorChildHasExited(child));
|
|
2757
|
+
}
|
|
2758
|
+
function shouldDeferSupervisorReadinessExit({
|
|
2759
|
+
target,
|
|
2760
|
+
currentChild,
|
|
2761
|
+
retryAfterMs
|
|
2762
|
+
}) {
|
|
2763
|
+
return target === currentChild && target?.exitCode === 75 && Number.isFinite(retryAfterMs) && retryAfterMs > 0;
|
|
2764
|
+
}
|
|
2765
|
+
function createSupervisorReadinessDeferralTracker({
|
|
2766
|
+
currentChild,
|
|
2767
|
+
releaseCurrentChild,
|
|
2768
|
+
onCaptured
|
|
2769
|
+
}) {
|
|
2770
|
+
const pending = /* @__PURE__ */ new WeakMap();
|
|
2771
|
+
const terminal = /* @__PURE__ */ new WeakSet();
|
|
2772
|
+
const recoveryGenerations = /* @__PURE__ */ new WeakMap();
|
|
2773
|
+
return Object.freeze({
|
|
2774
|
+
recordPending(target, request) {
|
|
2775
|
+
if (!target || target !== currentChild() || terminal.has(target) || pending.has(target) || typeof request?.nonce !== "string" || !request.nonce || !Number.isFinite(request.retryAfterMs) || request.retryAfterMs <= 0) return false;
|
|
2776
|
+
pending.set(target, Object.freeze({
|
|
2777
|
+
nonce: request.nonce,
|
|
2778
|
+
retryAfterMs: Math.ceil(request.retryAfterMs)
|
|
2779
|
+
}));
|
|
2780
|
+
return true;
|
|
2781
|
+
},
|
|
2782
|
+
recordRecoveryGeneration(target, generation) {
|
|
2783
|
+
if (!target || !Number.isInteger(generation)) return false;
|
|
2784
|
+
recoveryGenerations.set(target, generation);
|
|
2785
|
+
return true;
|
|
2786
|
+
},
|
|
2787
|
+
isDeferred(target) {
|
|
2788
|
+
const request = target ? pending.get(target) : null;
|
|
2789
|
+
return Boolean(target) && (terminal.has(target) || shouldDeferSupervisorReadinessExit({
|
|
2790
|
+
target,
|
|
2791
|
+
currentChild: currentChild(),
|
|
2792
|
+
retryAfterMs: request?.retryAfterMs
|
|
2793
|
+
}));
|
|
2794
|
+
},
|
|
2795
|
+
capture(target) {
|
|
2796
|
+
if (!target) return false;
|
|
2797
|
+
if (terminal.has(target)) return true;
|
|
2798
|
+
const request = pending.get(target);
|
|
2799
|
+
if (!shouldDeferSupervisorReadinessExit({
|
|
2800
|
+
target,
|
|
2801
|
+
currentChild: currentChild(),
|
|
2802
|
+
retryAfterMs: request?.retryAfterMs
|
|
2803
|
+
})) return false;
|
|
2804
|
+
const recoveryGeneration = recoveryGenerations.get(target);
|
|
2805
|
+
terminal.add(target);
|
|
2806
|
+
pending.delete(target);
|
|
2807
|
+
recoveryGenerations.delete(target);
|
|
2808
|
+
releaseCurrentChild(target);
|
|
2809
|
+
onCaptured(Object.freeze({
|
|
2810
|
+
target,
|
|
2811
|
+
retryAfterMs: request.retryAfterMs,
|
|
2812
|
+
recoveryGeneration: Number.isInteger(recoveryGeneration) ? recoveryGeneration : null
|
|
2813
|
+
}));
|
|
2814
|
+
return true;
|
|
2815
|
+
},
|
|
2816
|
+
forget(target) {
|
|
2817
|
+
if (!target) return;
|
|
2818
|
+
pending.delete(target);
|
|
2819
|
+
recoveryGenerations.delete(target);
|
|
2820
|
+
}
|
|
2821
|
+
});
|
|
2822
|
+
}
|
|
2823
|
+
function resolveSupervisorChildStartAuthority({
|
|
2824
|
+
degradationState,
|
|
2825
|
+
deferredRecoveryGeneration
|
|
2826
|
+
}) {
|
|
2827
|
+
const snapshot = degradationState.snapshot();
|
|
2828
|
+
const exactRecovery = Number.isInteger(deferredRecoveryGeneration) && deferredRecoveryGeneration === snapshot.generation;
|
|
2829
|
+
return Object.freeze({
|
|
2830
|
+
allowed: !snapshot.degraded || exactRecovery,
|
|
2831
|
+
recoveryGeneration: exactRecovery ? deferredRecoveryGeneration : null
|
|
2832
|
+
});
|
|
2833
|
+
}
|
|
2834
|
+
function createSupervisorStartupDeferral({
|
|
2835
|
+
retryAfterMs = null,
|
|
2836
|
+
nowMs = () => Date.now()
|
|
2837
|
+
} = {}) {
|
|
2838
|
+
let started = false;
|
|
2839
|
+
let notBefore = null;
|
|
2840
|
+
const defer = (delayMs) => {
|
|
2841
|
+
if (started || !Number.isFinite(delayMs) || delayMs <= 0) return false;
|
|
2842
|
+
const candidate = nowMs() + Math.ceil(delayMs);
|
|
2843
|
+
notBefore = notBefore === null ? candidate : Math.max(notBefore, candidate);
|
|
2844
|
+
return true;
|
|
2845
|
+
};
|
|
2846
|
+
defer(retryAfterMs);
|
|
2847
|
+
return Object.freeze({
|
|
2848
|
+
defer,
|
|
2849
|
+
reopen(delayMs) {
|
|
2850
|
+
if (!Number.isFinite(delayMs) || delayMs <= 0) return false;
|
|
2851
|
+
started = false;
|
|
2852
|
+
notBefore = null;
|
|
2853
|
+
return defer(delayMs);
|
|
2854
|
+
},
|
|
2855
|
+
isPending() {
|
|
2856
|
+
return !started && notBefore !== null && nowMs() < notBefore;
|
|
2857
|
+
},
|
|
2858
|
+
remainingMs() {
|
|
2859
|
+
return started || notBefore === null ? 0 : Math.max(0, notBefore - nowMs());
|
|
2860
|
+
},
|
|
2861
|
+
consumeIfReady() {
|
|
2862
|
+
if (started || notBefore !== null && nowMs() < notBefore) return false;
|
|
2863
|
+
started = true;
|
|
2864
|
+
notBefore = null;
|
|
2865
|
+
return true;
|
|
2866
|
+
},
|
|
2867
|
+
hasStarted() {
|
|
2868
|
+
return started;
|
|
2869
|
+
}
|
|
2870
|
+
});
|
|
2871
|
+
}
|
|
2872
|
+
function createSupervisorDegradationState() {
|
|
2873
|
+
let degraded = false;
|
|
2874
|
+
let generation = 0;
|
|
2875
|
+
return {
|
|
2876
|
+
isDegraded() {
|
|
2877
|
+
return degraded;
|
|
2878
|
+
},
|
|
2879
|
+
markDegraded() {
|
|
2880
|
+
degraded = true;
|
|
2881
|
+
generation += 1;
|
|
2882
|
+
},
|
|
2883
|
+
beginHealthProof({ target, currentChild, allowRecovery, onRecovery }) {
|
|
2884
|
+
const observedGeneration = generation;
|
|
2885
|
+
return () => {
|
|
2886
|
+
if (currentChild() !== target || generation !== observedGeneration) return false;
|
|
2887
|
+
if (degraded && !allowRecovery) return false;
|
|
2888
|
+
const recovered = degraded;
|
|
2889
|
+
degraded = false;
|
|
2890
|
+
if (recovered) onRecovery?.();
|
|
2891
|
+
return true;
|
|
2892
|
+
};
|
|
2893
|
+
},
|
|
2894
|
+
beginDegradationProof({ target, currentChild, onDegraded }) {
|
|
2895
|
+
const observedGeneration = generation;
|
|
2896
|
+
return (message) => {
|
|
2897
|
+
if (currentChild() !== target || generation !== observedGeneration) return false;
|
|
2898
|
+
onDegraded(message);
|
|
2899
|
+
return true;
|
|
2900
|
+
};
|
|
2901
|
+
},
|
|
2902
|
+
snapshot() {
|
|
2903
|
+
return { degraded, generation };
|
|
2904
|
+
}
|
|
2905
|
+
};
|
|
2906
|
+
}
|
|
2907
|
+
function beginSupervisorHealthyStateCommit({
|
|
2908
|
+
degradationState,
|
|
2909
|
+
target,
|
|
2910
|
+
currentChild,
|
|
2911
|
+
allowRecovery,
|
|
2912
|
+
clearStaleExitCode
|
|
2913
|
+
}) {
|
|
2914
|
+
return degradationState.beginHealthProof({
|
|
2915
|
+
target,
|
|
2916
|
+
currentChild,
|
|
2917
|
+
allowRecovery,
|
|
2918
|
+
onRecovery: clearStaleExitCode
|
|
2919
|
+
});
|
|
2920
|
+
}
|
|
2921
|
+
var RUNNER_RECOVERY_ACTIONS = /* @__PURE__ */ new Set(["update", "reinstall", "reconnect"]);
|
|
2922
|
+
function shouldRecoverSupervisorChildAfterAction({
|
|
2923
|
+
stoppedChild,
|
|
2924
|
+
actionKind,
|
|
2925
|
+
actionSucceeded
|
|
2926
|
+
}) {
|
|
2927
|
+
return Boolean(stoppedChild) || actionSucceeded === true && RUNNER_RECOVERY_ACTIONS.has(actionKind);
|
|
2928
|
+
}
|
|
2929
|
+
function createSupervisorControlRecoveryFence() {
|
|
2930
|
+
let targetStoppedForControl = null;
|
|
2931
|
+
const consume = () => {
|
|
2932
|
+
const target = targetStoppedForControl;
|
|
2933
|
+
targetStoppedForControl = null;
|
|
2934
|
+
return target;
|
|
2935
|
+
};
|
|
2936
|
+
return {
|
|
2937
|
+
markBeforeStop(target) {
|
|
2938
|
+
targetStoppedForControl = target && supervisorChildIsRunning(target) ? target : null;
|
|
2939
|
+
return targetStoppedForControl !== null;
|
|
2940
|
+
},
|
|
2941
|
+
consume,
|
|
2942
|
+
async recoverOnce(recover) {
|
|
2943
|
+
const target = consume();
|
|
2944
|
+
if (!target) return false;
|
|
2945
|
+
await recover(target);
|
|
2946
|
+
return true;
|
|
2947
|
+
}
|
|
2948
|
+
};
|
|
2949
|
+
}
|
|
2950
|
+
async function verifySupervisorChildHealth({
|
|
2951
|
+
target,
|
|
2952
|
+
degradeOnExit,
|
|
2953
|
+
context,
|
|
2954
|
+
waitBeforeProbe,
|
|
2955
|
+
waitForLocalRunner: waitForLocalRunner2,
|
|
2956
|
+
stopExpectedly,
|
|
2957
|
+
markHealthy,
|
|
2958
|
+
markDegraded,
|
|
2959
|
+
isReadinessDeferredExit = () => false,
|
|
2960
|
+
isExpectedStop = () => false
|
|
2961
|
+
}) {
|
|
2962
|
+
try {
|
|
2963
|
+
await waitBeforeProbe();
|
|
2964
|
+
const healthy = supervisorChildIsRunning(target) && await waitForLocalRunner2(target);
|
|
2965
|
+
const exited = supervisorChildHasExited(target);
|
|
2966
|
+
if (isExpectedStop(target)) return false;
|
|
2967
|
+
if (exited && isReadinessDeferredExit(target)) return false;
|
|
2968
|
+
if (healthy && !exited) {
|
|
2969
|
+
if (markHealthy() !== false) return true;
|
|
2970
|
+
if (supervisorChildIsRunning(target)) await stopExpectedly(target);
|
|
2971
|
+
return false;
|
|
2972
|
+
}
|
|
2973
|
+
if (!exited || degradeOnExit) {
|
|
2974
|
+
const committed = markDegraded(`${context} did not become healthy; entering degraded mode`) !== false;
|
|
2975
|
+
if (!committed) return false;
|
|
2976
|
+
if (supervisorChildIsRunning(target)) await stopExpectedly(target);
|
|
2977
|
+
}
|
|
2978
|
+
return false;
|
|
2979
|
+
} catch (error) {
|
|
2980
|
+
if (isExpectedStop(target)) return false;
|
|
2981
|
+
const committed = markDegraded(
|
|
2982
|
+
`${context} health proof failed; entering degraded mode: ${error instanceof Error ? error.message : String(error)}`
|
|
2983
|
+
) !== false;
|
|
2984
|
+
if (!committed) return false;
|
|
2985
|
+
if (supervisorChildIsRunning(target)) await stopExpectedly(target).catch(() => {
|
|
2986
|
+
});
|
|
2987
|
+
return false;
|
|
2988
|
+
}
|
|
2989
|
+
}
|
|
2990
|
+
|
|
2533
2991
|
// src/runner-supervisor.mjs
|
|
2534
2992
|
var DEFAULT_CONTROL_PLANE_URL = "https://vo-control-plane-bzjphrajaq-uc.a.run.app";
|
|
2535
2993
|
var POLL_MS = 5e3;
|
|
2536
2994
|
var CHILD_START_MS = 1500;
|
|
2995
|
+
var CHILD_READINESS_TIMEOUT_MS = RUNNER_IDENTITY_READINESS_TIMEOUT_MS + RUNNER_GITHUB_READINESS_TIMEOUT_MS + 1e4;
|
|
2537
2996
|
var SUPERVISOR_CAPABILITIES = ["bundled-runtime-slots-v1", "legacy-orphan-purge-v1"];
|
|
2538
|
-
var
|
|
2997
|
+
var UUID_RE3 = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
|
|
2539
2998
|
var selfPath = fileURLToPath3(import.meta.url);
|
|
2540
2999
|
var bundledChildEntry = join5(dirname4(selfPath), "runner-cli.js");
|
|
2541
3000
|
var supervisorRuntimeRoot = runtimeRootFromEnv(process.env);
|
|
@@ -2548,7 +3007,7 @@ function packageVersion() {
|
|
|
2548
3007
|
return "unknown";
|
|
2549
3008
|
}
|
|
2550
3009
|
}
|
|
2551
|
-
var requestedSupervisorInstanceId =
|
|
3010
|
+
var requestedSupervisorInstanceId = UUID_RE3.test(String(process.env.VO_RUNNER_SUPERVISOR_INSTANCE_ID || "")) ? process.env.VO_RUNNER_SUPERVISOR_INSTANCE_ID : randomUUID4();
|
|
2552
3011
|
var supervisorInstanceId = activationSupervisorInstanceId(
|
|
2553
3012
|
runtimeRootFromEnv(process.env),
|
|
2554
3013
|
requestedSupervisorInstanceId
|
|
@@ -2572,16 +3031,27 @@ function spawnChild(childEnv) {
|
|
|
2572
3031
|
}
|
|
2573
3032
|
return spawn(process.execPath, resolved.args, {
|
|
2574
3033
|
env: childEnv,
|
|
2575
|
-
stdio: "inherit",
|
|
3034
|
+
stdio: ["inherit", "inherit", "inherit", "ipc"],
|
|
2576
3035
|
windowsHide: true
|
|
2577
3036
|
});
|
|
2578
3037
|
}
|
|
2579
3038
|
function spawnPreviousChild(entry, childEnv) {
|
|
2580
|
-
|
|
3039
|
+
const previous = spawn(process.execPath, [entry, "runner"], {
|
|
2581
3040
|
env: childEnv,
|
|
2582
|
-
stdio: "inherit",
|
|
3041
|
+
stdio: ["inherit", "inherit", "inherit", "ipc"],
|
|
2583
3042
|
windowsHide: true
|
|
2584
3043
|
});
|
|
3044
|
+
attachSupervisorChildTerminationCustody(
|
|
3045
|
+
previous,
|
|
3046
|
+
() => {
|
|
3047
|
+
},
|
|
3048
|
+
(error) => {
|
|
3049
|
+
console.error(
|
|
3050
|
+
`[vo-runner supervisor] rollback child process error: ${error instanceof Error ? error.message : String(error)}`
|
|
3051
|
+
);
|
|
3052
|
+
}
|
|
3053
|
+
);
|
|
3054
|
+
return previous;
|
|
2585
3055
|
}
|
|
2586
3056
|
async function localStatus() {
|
|
2587
3057
|
try {
|
|
@@ -2595,32 +3065,54 @@ async function localStatus() {
|
|
|
2595
3065
|
}
|
|
2596
3066
|
}
|
|
2597
3067
|
async function waitForLocalRunner(child) {
|
|
2598
|
-
const deadline = Date.now() +
|
|
3068
|
+
const deadline = Date.now() + CHILD_READINESS_TIMEOUT_MS;
|
|
2599
3069
|
while (Date.now() < deadline) {
|
|
2600
|
-
if (child
|
|
3070
|
+
if (supervisorChildHasExited(child)) return false;
|
|
2601
3071
|
const status = await localStatus();
|
|
2602
3072
|
if (status?.running === true && Number(status.pid) === Number(child.pid)) return true;
|
|
2603
3073
|
await sleep(500);
|
|
2604
3074
|
}
|
|
2605
3075
|
return false;
|
|
2606
3076
|
}
|
|
3077
|
+
async function waitForChildExit(child, timeoutMs) {
|
|
3078
|
+
if (!child || supervisorChildHasExited(child)) return true;
|
|
3079
|
+
return new Promise((resolve5) => {
|
|
3080
|
+
let settled = false;
|
|
3081
|
+
const finish = (exited) => {
|
|
3082
|
+
if (settled) return;
|
|
3083
|
+
settled = true;
|
|
3084
|
+
clearTimeout(timer);
|
|
3085
|
+
child.off("exit", onExit);
|
|
3086
|
+
resolve5(exited);
|
|
3087
|
+
};
|
|
3088
|
+
const onExit = () => finish(true);
|
|
3089
|
+
const timer = setTimeout(() => finish(false), timeoutMs);
|
|
3090
|
+
child.once("exit", onExit);
|
|
3091
|
+
if (supervisorChildHasExited(child)) finish(true);
|
|
3092
|
+
});
|
|
3093
|
+
}
|
|
2607
3094
|
async function stopChild(child) {
|
|
2608
|
-
if (!child || child
|
|
3095
|
+
if (!child || supervisorChildHasExited(child)) return;
|
|
2609
3096
|
child.kill(process.platform === "win32" ? void 0 : "SIGTERM");
|
|
2610
|
-
await
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
]);
|
|
2614
|
-
if (child.exitCode === null) child.kill("SIGKILL");
|
|
3097
|
+
if (await waitForChildExit(child, 15e3)) return;
|
|
3098
|
+
if (supervisorChildIsRunning(child)) child.kill("SIGKILL");
|
|
3099
|
+
await waitForChildExit(child, 5e3);
|
|
2615
3100
|
}
|
|
2616
3101
|
async function main() {
|
|
2617
3102
|
const stored = readStoredCredentialIsolated();
|
|
2618
3103
|
const controlPlaneUrl = process.env.VO_CONTROL_PLANE_URL || DEFAULT_CONTROL_PLANE_URL;
|
|
2619
|
-
const
|
|
3104
|
+
const pairedSupervisor = !String(process.env.VO_CONTROL_PLANE_ADMIN_TOKEN || "").trim();
|
|
3105
|
+
const supervisorAuthInput = {
|
|
2620
3106
|
baseEnv: process.env,
|
|
2621
3107
|
storedCredential: stored,
|
|
2622
3108
|
controlPlaneUrl
|
|
2623
|
-
}
|
|
3109
|
+
};
|
|
3110
|
+
const {
|
|
3111
|
+
childEnv,
|
|
3112
|
+
clientEnv,
|
|
3113
|
+
operatorId,
|
|
3114
|
+
startupRetryAfterMs
|
|
3115
|
+
} = await prepareSupervisorAuth(supervisorAuthInput);
|
|
2624
3116
|
Object.assign(childEnv, {
|
|
2625
3117
|
VO_RUNNER_SUPERVISOR_INSTANCE_ID: supervisorInstanceId,
|
|
2626
3118
|
VO_RUNNER_SUPERVISOR_VERSION: supervisorVersion,
|
|
@@ -2631,37 +3123,266 @@ async function main() {
|
|
|
2631
3123
|
let child = null;
|
|
2632
3124
|
let stopping = false;
|
|
2633
3125
|
let handling = false;
|
|
2634
|
-
|
|
3126
|
+
const expectedChildStops = /* @__PURE__ */ new WeakSet();
|
|
3127
|
+
const healthNeutralizedChildren = /* @__PURE__ */ new WeakSet();
|
|
3128
|
+
const degradationState = createSupervisorDegradationState();
|
|
3129
|
+
const markSupervisorDegraded = () => {
|
|
3130
|
+
degradationState.markDegraded();
|
|
3131
|
+
process.exitCode = 1;
|
|
3132
|
+
};
|
|
3133
|
+
const respawnCircuit = createRespawnCircuit();
|
|
3134
|
+
const controlRecoveryFence = createSupervisorControlRecoveryFence();
|
|
3135
|
+
const startupDeferral = createSupervisorStartupDeferral({
|
|
3136
|
+
retryAfterMs: startupRetryAfterMs
|
|
3137
|
+
});
|
|
3138
|
+
let startupNeedsReadinessRefresh = startupRetryAfterMs !== null;
|
|
3139
|
+
let deferredControlRecoveryGeneration = null;
|
|
3140
|
+
const stopChildExpectedly = async (target) => {
|
|
3141
|
+
if (!target) return;
|
|
3142
|
+
expectedChildStops.add(target);
|
|
3143
|
+
healthNeutralizedChildren.add(target);
|
|
3144
|
+
try {
|
|
3145
|
+
await stopChild(target);
|
|
3146
|
+
} finally {
|
|
3147
|
+
if (supervisorChildHasExited(target)) expectedChildStops.delete(target);
|
|
3148
|
+
}
|
|
3149
|
+
};
|
|
3150
|
+
const readinessDeferrals = createSupervisorReadinessDeferralTracker({
|
|
3151
|
+
currentChild: () => child,
|
|
3152
|
+
releaseCurrentChild: (target) => {
|
|
3153
|
+
if (child === target) child = null;
|
|
3154
|
+
},
|
|
3155
|
+
onCaptured: ({ retryAfterMs, recoveryGeneration }) => {
|
|
3156
|
+
startupNeedsReadinessRefresh = true;
|
|
3157
|
+
if (Number.isInteger(recoveryGeneration) && degradationState.snapshot().generation === recoveryGeneration) {
|
|
3158
|
+
deferredControlRecoveryGeneration = recoveryGeneration;
|
|
3159
|
+
}
|
|
3160
|
+
startupDeferral.reopen(retryAfterMs);
|
|
3161
|
+
console.warn(
|
|
3162
|
+
`[vo-runner supervisor] child deferred by GitHub readiness for ${retryAfterMs}ms; preserving control polling without consuming rapid-exit breaker budget`
|
|
3163
|
+
);
|
|
3164
|
+
}
|
|
3165
|
+
});
|
|
3166
|
+
const isChildReadinessDeferred = (target) => readinessDeferrals.isDeferred(target);
|
|
3167
|
+
const captureChildReadinessDeferral = (target) => readinessDeferrals.capture(target);
|
|
2635
3168
|
const respawn = () => {
|
|
2636
|
-
if (!
|
|
3169
|
+
if (!startupDeferral.hasStarted()) return;
|
|
3170
|
+
if (!shouldRespawnSupervisorChild({
|
|
3171
|
+
stopping,
|
|
3172
|
+
degraded: degradationState.isDegraded(),
|
|
3173
|
+
handling,
|
|
3174
|
+
child
|
|
3175
|
+
})) return;
|
|
3176
|
+
try {
|
|
3177
|
+
child = launchChild();
|
|
3178
|
+
void verifyChildHealth(child, false, "automatic relaunch");
|
|
3179
|
+
} catch (error) {
|
|
3180
|
+
markSupervisorDegraded();
|
|
3181
|
+
console.error(
|
|
3182
|
+
`[vo-runner supervisor] child relaunch failed; entering degraded mode: ${error instanceof Error ? error.message : String(error)}`
|
|
3183
|
+
);
|
|
3184
|
+
}
|
|
2637
3185
|
};
|
|
2638
3186
|
const launchChild = () => {
|
|
2639
3187
|
const next = spawnChild(childEnv);
|
|
2640
|
-
|
|
3188
|
+
const startedAt = Date.now();
|
|
3189
|
+
next.on("message", (message) => {
|
|
3190
|
+
const request = parseRunnerReadinessDeferralRequest(message);
|
|
3191
|
+
if (!request || !readinessDeferrals.recordPending(next, request)) return;
|
|
3192
|
+
try {
|
|
3193
|
+
next.send({
|
|
3194
|
+
type: RUNNER_READINESS_DEFERRAL_ACK_TYPE,
|
|
3195
|
+
nonce: request.nonce
|
|
3196
|
+
}, (error) => {
|
|
3197
|
+
if (error) console.warn(`[vo-runner supervisor] readiness deferral ACK failed: ${error.message}`);
|
|
3198
|
+
});
|
|
3199
|
+
} catch (error) {
|
|
3200
|
+
console.warn(
|
|
3201
|
+
`[vo-runner supervisor] readiness deferral ACK failed: ${error instanceof Error ? error.message : String(error)}`
|
|
3202
|
+
);
|
|
3203
|
+
}
|
|
3204
|
+
});
|
|
3205
|
+
attachSupervisorChildTerminationCustody(
|
|
3206
|
+
next,
|
|
3207
|
+
() => {
|
|
3208
|
+
if (stopping || expectedChildStops.delete(next)) {
|
|
3209
|
+
readinessDeferrals.forget(next);
|
|
3210
|
+
return;
|
|
3211
|
+
}
|
|
3212
|
+
if (captureChildReadinessDeferral(next)) return;
|
|
3213
|
+
readinessDeferrals.forget(next);
|
|
3214
|
+
const decision = respawnCircuit.recordExit(startedAt, Date.now());
|
|
3215
|
+
if (decision.tripped) {
|
|
3216
|
+
markSupervisorDegraded();
|
|
3217
|
+
console.error(
|
|
3218
|
+
`[vo-runner supervisor] child exited rapidly ${decision.rapidExits} times; entering degraded mode to stop startup/token churn while remote repair remains available`
|
|
3219
|
+
);
|
|
3220
|
+
return;
|
|
3221
|
+
}
|
|
3222
|
+
setTimeout(respawn, decision.delayMs);
|
|
3223
|
+
},
|
|
3224
|
+
(error) => {
|
|
3225
|
+
console.error(
|
|
3226
|
+
`[vo-runner supervisor] child process error: ${error instanceof Error ? error.message : String(error)}`
|
|
3227
|
+
);
|
|
3228
|
+
}
|
|
3229
|
+
);
|
|
2641
3230
|
return next;
|
|
2642
3231
|
};
|
|
2643
|
-
|
|
2644
|
-
|
|
2645
|
-
|
|
2646
|
-
|
|
2647
|
-
|
|
2648
|
-
|
|
2649
|
-
|
|
2650
|
-
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
|
|
3232
|
+
const verifyChildHealth = async (target, degradeOnExit, context) => {
|
|
3233
|
+
const commitSupervisorHealthyState = beginSupervisorHealthyStateCommit({
|
|
3234
|
+
degradationState,
|
|
3235
|
+
target,
|
|
3236
|
+
currentChild: () => child,
|
|
3237
|
+
allowRecovery: degradeOnExit,
|
|
3238
|
+
clearStaleExitCode: () => {
|
|
3239
|
+
process.exitCode = void 0;
|
|
3240
|
+
}
|
|
3241
|
+
});
|
|
3242
|
+
const commitSupervisorDegradedState = degradationState.beginDegradationProof({
|
|
3243
|
+
target,
|
|
3244
|
+
currentChild: () => child,
|
|
3245
|
+
onDegraded: (message) => {
|
|
3246
|
+
markSupervisorDegraded();
|
|
3247
|
+
console.error(`[vo-runner supervisor] ${message}`);
|
|
3248
|
+
}
|
|
3249
|
+
});
|
|
3250
|
+
return verifySupervisorChildHealth({
|
|
3251
|
+
target,
|
|
3252
|
+
degradeOnExit,
|
|
3253
|
+
context,
|
|
3254
|
+
waitBeforeProbe: () => sleep(CHILD_START_MS),
|
|
3255
|
+
waitForLocalRunner,
|
|
3256
|
+
stopExpectedly: stopChildExpectedly,
|
|
3257
|
+
isReadinessDeferredExit: isChildReadinessDeferred,
|
|
3258
|
+
isExpectedStop: (candidate) => healthNeutralizedChildren.has(candidate),
|
|
3259
|
+
markHealthy: commitSupervisorHealthyState,
|
|
3260
|
+
markDegraded: commitSupervisorDegradedState
|
|
3261
|
+
});
|
|
3262
|
+
};
|
|
3263
|
+
const deferChildAdmission = (retryAfterMs) => {
|
|
3264
|
+
startupNeedsReadinessRefresh = true;
|
|
3265
|
+
if (startupDeferral.hasStarted()) startupDeferral.reopen(retryAfterMs);
|
|
3266
|
+
else startupDeferral.defer(retryAfterMs);
|
|
3267
|
+
console.warn(
|
|
3268
|
+
`[vo-runner supervisor] GitHub readiness embargoed child startup for ${retryAfterMs}ms; control polling remains active`
|
|
3269
|
+
);
|
|
3270
|
+
};
|
|
3271
|
+
const refreshSupervisorChildAdmission = async () => {
|
|
3272
|
+
if (!pairedSupervisor) return true;
|
|
3273
|
+
const refreshed = await prepareSupervisorAuth({
|
|
3274
|
+
...supervisorAuthInput,
|
|
3275
|
+
baseEnv: childEnv
|
|
3276
|
+
});
|
|
3277
|
+
if (refreshed.operatorId !== operatorId) {
|
|
3278
|
+
throw new Error("runner readiness operator identity changed during supervisor admission");
|
|
3279
|
+
}
|
|
3280
|
+
if (refreshed.startupRetryAfterMs !== null) {
|
|
3281
|
+
deferChildAdmission(refreshed.startupRetryAfterMs);
|
|
3282
|
+
return false;
|
|
3283
|
+
}
|
|
3284
|
+
if (refreshed.repositoryScope.length > 0) {
|
|
3285
|
+
const repositoryScope = refreshed.repositoryScope.join(",");
|
|
3286
|
+
childEnv.VO_CODE_RUNNER_REPOS = repositoryScope;
|
|
3287
|
+
clientEnv.VO_CODE_RUNNER_REPOS = repositoryScope;
|
|
3288
|
+
}
|
|
3289
|
+
startupNeedsReadinessRefresh = false;
|
|
3290
|
+
return true;
|
|
3291
|
+
};
|
|
3292
|
+
const ensureHealthyChildAfterControl = async () => {
|
|
3293
|
+
if (!child || supervisorChildHasExited(child)) {
|
|
3294
|
+
const recoveryGeneration = degradationState.snapshot().generation;
|
|
3295
|
+
if (!startupDeferral.hasStarted() && startupDeferral.isPending()) {
|
|
3296
|
+
deferredControlRecoveryGeneration = recoveryGeneration;
|
|
3297
|
+
return false;
|
|
3298
|
+
}
|
|
3299
|
+
try {
|
|
3300
|
+
if (!await refreshSupervisorChildAdmission()) {
|
|
3301
|
+
deferredControlRecoveryGeneration = recoveryGeneration;
|
|
3302
|
+
return false;
|
|
3303
|
+
}
|
|
3304
|
+
if (!startupDeferral.hasStarted() && !startupDeferral.consumeIfReady()) return false;
|
|
3305
|
+
child = launchChild();
|
|
3306
|
+
readinessDeferrals.recordRecoveryGeneration(child, recoveryGeneration);
|
|
3307
|
+
} catch (error) {
|
|
3308
|
+
markSupervisorDegraded();
|
|
3309
|
+
console.error(
|
|
3310
|
+
`[vo-runner supervisor] control recovery relaunch failed; entering degraded mode: ${error instanceof Error ? error.message : String(error)}`
|
|
3311
|
+
);
|
|
3312
|
+
return false;
|
|
3313
|
+
}
|
|
3314
|
+
}
|
|
3315
|
+
return verifyChildHealth(child, true, "control recovery relaunch");
|
|
3316
|
+
};
|
|
3317
|
+
const startInitialChild = async (recoveryGeneration = null) => {
|
|
3318
|
+
const activationChild = launchChild();
|
|
3319
|
+
child = activationChild;
|
|
3320
|
+
if (Number.isInteger(recoveryGeneration)) {
|
|
3321
|
+
readinessDeferrals.recordRecoveryGeneration(activationChild, recoveryGeneration);
|
|
3322
|
+
}
|
|
3323
|
+
void verifyChildHealth(
|
|
3324
|
+
activationChild,
|
|
3325
|
+
Number.isInteger(recoveryGeneration),
|
|
3326
|
+
Number.isInteger(recoveryGeneration) ? "deferred control recovery" : "initial child"
|
|
3327
|
+
);
|
|
3328
|
+
if (!await finishPendingActivation({
|
|
3329
|
+
client,
|
|
3330
|
+
child: activationChild,
|
|
3331
|
+
runtimeRoot,
|
|
3332
|
+
operatorId,
|
|
3333
|
+
runnerId,
|
|
3334
|
+
selfPath,
|
|
3335
|
+
packageVersion: supervisorVersion,
|
|
3336
|
+
supervisorIdentity: supervisorControlIdentity,
|
|
3337
|
+
waitForLocalRunner,
|
|
3338
|
+
isReadinessDeferred: isChildReadinessDeferred,
|
|
3339
|
+
localStatus,
|
|
3340
|
+
stopChild: stopChildExpectedly,
|
|
3341
|
+
launchPreviousChild: (entry) => spawnPreviousChild(entry, childEnv),
|
|
3342
|
+
log: (message) => console.error(`[vo-runner supervisor] ${message}`)
|
|
3343
|
+
})) {
|
|
3344
|
+
if (captureChildReadinessDeferral(activationChild)) {
|
|
3345
|
+
console.warn("[vo-runner supervisor] activation attestation deferred with GitHub readiness; pending slot remains durable");
|
|
3346
|
+
return false;
|
|
3347
|
+
}
|
|
3348
|
+
markSupervisorDegraded();
|
|
3349
|
+
await stopChildExpectedly(activationChild);
|
|
3350
|
+
if (child === activationChild) child = null;
|
|
3351
|
+
console.error("[vo-runner supervisor] activation FAILED \u2014 entering degraded mode: not serving tasks, still polling for remote control actions");
|
|
3352
|
+
return false;
|
|
3353
|
+
}
|
|
3354
|
+
return true;
|
|
3355
|
+
};
|
|
3356
|
+
const attemptInitialChildStart = async () => {
|
|
3357
|
+
if (startupDeferral.hasStarted() || startupDeferral.isPending()) return false;
|
|
3358
|
+
const beforeRefresh = resolveSupervisorChildStartAuthority({
|
|
3359
|
+
degradationState,
|
|
3360
|
+
deferredRecoveryGeneration: deferredControlRecoveryGeneration
|
|
3361
|
+
});
|
|
3362
|
+
if (!beforeRefresh.allowed) return false;
|
|
3363
|
+
if (startupNeedsReadinessRefresh) {
|
|
3364
|
+
try {
|
|
3365
|
+
if (!await refreshSupervisorChildAdmission()) return false;
|
|
3366
|
+
} catch (error) {
|
|
3367
|
+
markSupervisorDegraded();
|
|
3368
|
+
deferredControlRecoveryGeneration = null;
|
|
3369
|
+
startupDeferral.consumeIfReady();
|
|
3370
|
+
console.error(
|
|
3371
|
+
`[vo-runner supervisor] deferred readiness refresh failed; entering degraded mode: ${error instanceof Error ? error.message : String(error)}`
|
|
3372
|
+
);
|
|
3373
|
+
return false;
|
|
3374
|
+
}
|
|
3375
|
+
}
|
|
3376
|
+
const afterRefresh = resolveSupervisorChildStartAuthority({
|
|
3377
|
+
degradationState,
|
|
3378
|
+
deferredRecoveryGeneration: deferredControlRecoveryGeneration
|
|
3379
|
+
});
|
|
3380
|
+
if (!afterRefresh.allowed) return false;
|
|
3381
|
+
if (!startupDeferral.consumeIfReady()) return false;
|
|
3382
|
+
deferredControlRecoveryGeneration = null;
|
|
3383
|
+
return startInitialChild(afterRefresh.recoveryGeneration);
|
|
3384
|
+
};
|
|
3385
|
+
await attemptInitialChildStart();
|
|
2665
3386
|
const shutdown = async () => {
|
|
2666
3387
|
if (stopping) return;
|
|
2667
3388
|
stopping = true;
|
|
@@ -2675,6 +3396,7 @@ async function main() {
|
|
|
2675
3396
|
});
|
|
2676
3397
|
while (!stopping) {
|
|
2677
3398
|
try {
|
|
3399
|
+
await attemptInitialChildStart();
|
|
2678
3400
|
const action = await client.pollRunnerControl({
|
|
2679
3401
|
runnerId,
|
|
2680
3402
|
...operatorId ? { operatorId } : {},
|
|
@@ -2684,6 +3406,7 @@ async function main() {
|
|
|
2684
3406
|
await sleep(POLL_MS);
|
|
2685
3407
|
continue;
|
|
2686
3408
|
}
|
|
3409
|
+
deferredControlRecoveryGeneration = null;
|
|
2687
3410
|
handling = true;
|
|
2688
3411
|
const beforeStop = await localStatus();
|
|
2689
3412
|
if (beforeStop && Number(beforeStop.activeTasks || 0) > 0) {
|
|
@@ -2695,9 +3418,11 @@ async function main() {
|
|
|
2695
3418
|
detail: `deferred safely: ${beforeStop.activeTasks} active task(s); retry when the runner is idle`
|
|
2696
3419
|
});
|
|
2697
3420
|
handling = false;
|
|
3421
|
+
respawn();
|
|
2698
3422
|
continue;
|
|
2699
3423
|
}
|
|
2700
|
-
|
|
3424
|
+
controlRecoveryFence.markBeforeStop(child);
|
|
3425
|
+
await stopChildExpectedly(child);
|
|
2701
3426
|
const bundledAction = action.kind === "update" || action.kind === "reinstall";
|
|
2702
3427
|
const result = bundledAction ? stageAndActivateBundledUpdate({
|
|
2703
3428
|
runtimeRoot,
|
|
@@ -2723,20 +3448,25 @@ async function main() {
|
|
|
2723
3448
|
console.warn(`[vo-runner supervisor] activated ${result.active.version}; exiting for new-process attestation`);
|
|
2724
3449
|
return;
|
|
2725
3450
|
}
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
3451
|
+
const recoveryAuthorized = shouldRecoverSupervisorChildAfterAction({
|
|
3452
|
+
stoppedChild: controlRecoveryFence.consume(),
|
|
3453
|
+
actionKind: action.kind,
|
|
3454
|
+
actionSucceeded: result.ok
|
|
3455
|
+
});
|
|
3456
|
+
const relaunchedHealthy = recoveryAuthorized ? await ensureHealthyChildAfterControl() : false;
|
|
3457
|
+
if (!relaunchedHealthy) result.ok = false;
|
|
2730
3458
|
await client.completeRunnerControl(action.actionId, {
|
|
2731
3459
|
runnerId,
|
|
2732
3460
|
...operatorId ? { operatorId } : {},
|
|
2733
3461
|
...supervisorControlIdentity,
|
|
2734
3462
|
status: result.ok ? "succeeded" : "failed",
|
|
2735
|
-
detail: result.ok ? result.detail ? `${result.detail}; runner ${packageVersion()} reconnected`.slice(0, 1e3) : `runner ${packageVersion()} reconnected` : `maintenance exited ${result.status}${result.detail ? `: ${result.detail}` : ""}`
|
|
3463
|
+
detail: result.ok ? result.detail ? `${result.detail}; runner ${packageVersion()} reconnected`.slice(0, 1e3) : `runner ${packageVersion()} reconnected` : `maintenance exited ${result.status}${result.detail ? `: ${result.detail}` : ""}${relaunchedHealthy ? "" : "; runner relaunch did not become healthy"}`
|
|
2736
3464
|
});
|
|
2737
3465
|
handling = false;
|
|
3466
|
+
respawn();
|
|
2738
3467
|
} catch (error) {
|
|
2739
3468
|
console.error(`[vo-runner supervisor] ${error instanceof Error ? error.message : String(error)}`);
|
|
3469
|
+
await controlRecoveryFence.recoverOnce(() => ensureHealthyChildAfterControl());
|
|
2740
3470
|
handling = false;
|
|
2741
3471
|
await sleep(POLL_MS);
|
|
2742
3472
|
}
|