@loucompanion/forge-bridge 0.1.3-dev.9bc5014a2270 → 0.1.3-dev.a58d7cd1092f
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +127 -0
- package/dist/bridge/bridge.base.d.ts +8 -1
- package/dist/bridge/bridge.service.d.ts +93 -3
- package/dist/bridge/bridge.service.js +1358 -47
- package/dist/bridge/bridge.types.d.ts +202 -1
- package/dist/bridge/runner-host-capabilities.d.ts +15 -0
- package/dist/bridge/runner-host-capabilities.js +93 -0
- package/dist/cleanup/cleanup.service.d.ts +14 -2
- package/dist/cleanup/cleanup.service.js +496 -3
- package/dist/cleanup/cleanup.types.d.ts +10 -0
- package/dist/cleanup/index.d.ts +1 -1
- package/dist/cleanup/index.js +1 -1
- package/dist/cleanup/internals.d.ts +2 -2
- package/dist/cleanup/internals.js +1 -1
- package/dist/cli/cli.help.js +47 -1
- package/dist/cli/cli.service.d.ts +45 -2
- package/dist/cli/cli.service.js +378 -26
- package/dist/cli/local-config.d.ts +7 -0
- package/dist/cli/local-config.js +51 -3
- package/dist/runtime/aspnet-runtime-prebuild.d.ts +36 -0
- package/dist/runtime/aspnet-runtime-prebuild.js +340 -0
- package/dist/runtime/attempt-node-runtime-launcher.d.ts +17 -0
- package/dist/runtime/attempt-node-runtime-launcher.js +105 -0
- package/dist/runtime/attempt-process-supervisor-registry.d.ts +20 -0
- package/dist/runtime/attempt-process-supervisor-registry.js +64 -0
- package/dist/runtime/attempt-process-supervisor.d.ts +97 -0
- package/dist/runtime/attempt-process-supervisor.js +386 -0
- package/dist/runtime/docker-engine-probe.d.ts +83 -0
- package/dist/runtime/docker-engine-probe.js +300 -0
- package/dist/runtime/generated-output-reset.d.ts +10 -0
- package/dist/runtime/generated-output-reset.js +103 -0
- package/dist/runtime/host-listener-inspector.d.ts +46 -0
- package/dist/runtime/host-listener-inspector.js +349 -0
- package/dist/runtime/host-port-broker.d.ts +76 -0
- package/dist/runtime/host-port-broker.js +333 -0
- package/dist/runtime/host-runtime-contract.d.ts +47 -0
- package/dist/runtime/host-runtime-contract.js +700 -0
- package/dist/runtime/host-runtime-lease.d.ts +197 -0
- package/dist/runtime/host-runtime-lease.js +2203 -0
- package/dist/runtime/host-runtime-promotion.d.ts +23 -0
- package/dist/runtime/host-runtime-promotion.js +159 -0
- package/dist/runtime/memory-capacity.d.ts +68 -0
- package/dist/runtime/memory-capacity.js +121 -0
- package/dist/runtime/node-host-runtime-adapter.d.ts +54 -0
- package/dist/runtime/node-host-runtime-adapter.js +284 -0
- package/dist/runtime/playwright-host-runtime-adapter.d.ts +21 -0
- package/dist/runtime/playwright-host-runtime-adapter.js +231 -0
- package/dist/runtime/runtime-isolation-lease.d.ts +67 -0
- package/dist/runtime/runtime-isolation-lease.js +158 -0
- package/dist/runtime/workspace-resource-estimator.d.ts +17 -0
- package/dist/runtime/workspace-resource-estimator.js +174 -0
- package/dist/sandbox/command-runner.d.ts +4 -0
- package/dist/sandbox/command-runner.js +47 -0
- package/dist/sandbox/contract.d.ts +5 -0
- package/dist/sandbox/contract.js +276 -0
- package/dist/sandbox/event-budget.d.ts +11 -0
- package/dist/sandbox/event-budget.js +27 -0
- package/dist/sandbox/image-policy.d.ts +35 -0
- package/dist/sandbox/image-policy.js +247 -0
- package/dist/sandbox/index.d.ts +5 -0
- package/dist/sandbox/index.js +5 -0
- package/dist/sandbox/lease-watchdog.d.ts +18 -0
- package/dist/sandbox/lease-watchdog.js +67 -0
- package/dist/sandbox/preflight.d.ts +84 -0
- package/dist/sandbox/preflight.js +521 -0
- package/dist/sandbox/process.d.ts +20 -0
- package/dist/sandbox/process.js +209 -0
- package/dist/sandbox/provider-auth/codex.d.ts +26 -0
- package/dist/sandbox/provider-auth/codex.js +212 -0
- package/dist/sandbox/repository-lock.d.ts +5 -0
- package/dist/sandbox/repository-lock.js +121 -0
- package/dist/sandbox/runtime.d.ts +297 -0
- package/dist/sandbox/runtime.js +2338 -0
- package/dist/sandbox/sandbox.types.d.ts +93 -0
- package/dist/sandbox/sandbox.types.js +1 -0
- package/dist/sandbox/smoke.d.ts +76 -0
- package/dist/sandbox/smoke.js +380 -0
- package/dist/sandbox/worker-supervisor.d.ts +13 -0
- package/dist/sandbox/worker-supervisor.js +36 -0
- package/dist/sandbox/worker.d.ts +1 -0
- package/dist/sandbox/worker.js +225 -0
- package/dist/sandbox/workspace.service.d.ts +13 -0
- package/dist/sandbox/workspace.service.js +72 -0
- package/dist/session/managed-attempt-tools.d.ts +21 -0
- package/dist/session/managed-attempt-tools.js +912 -0
- package/dist/session/managed-build-coordinator.d.ts +5 -0
- package/dist/session/managed-build-coordinator.js +105 -0
- package/dist/session/managed-build-receipt.d.ts +21 -0
- package/dist/session/managed-build-receipt.js +335 -0
- package/dist/session/phase-completion.artifacts.d.ts +17 -0
- package/dist/session/phase-completion.artifacts.js +256 -26
- package/dist/session/phase-completion.d.ts +12 -2
- package/dist/session/phase-completion.helpers.d.ts +5 -0
- package/dist/session/phase-completion.helpers.js +51 -10
- package/dist/session/phase-completion.js +14 -3
- package/dist/session/phase-completion.prompts.d.ts +3 -3
- package/dist/session/phase-completion.prompts.js +24 -8
- package/dist/session/phase-completion.runtime.d.ts +13 -1
- package/dist/session/phase-completion.runtime.js +189 -8
- package/dist/session/phase-completion.schemas.js +117 -4
- package/dist/session/phase-completion.validation.d.ts +4 -1
- package/dist/session/phase-completion.validation.js +153 -22
- package/dist/session/pre-publication-validation.d.ts +46 -0
- package/dist/session/pre-publication-validation.js +208 -0
- package/dist/session/provider-cli/ci-repair-readiness.d.ts +16 -0
- package/dist/session/provider-cli/ci-repair-readiness.js +110 -0
- package/dist/session/provider-cli/provider-cli.artifacts.d.ts +3 -2
- package/dist/session/provider-cli/provider-cli.artifacts.js +65 -7
- package/dist/session/provider-cli/provider-cli.base.d.ts +1 -0
- package/dist/session/provider-cli/provider-cli.deterministic.d.ts +2 -0
- package/dist/session/provider-cli/provider-cli.deterministic.js +66 -9
- package/dist/session/provider-cli/provider-cli.process.d.ts +26 -0
- package/dist/session/provider-cli/provider-cli.process.js +474 -23
- package/dist/session/provider-cli/provider-cli.prompts.d.ts +16 -2
- package/dist/session/provider-cli/provider-cli.prompts.js +224 -24
- package/dist/session/provider-cli/provider-cli.qa-sidecars.js +1 -0
- package/dist/session/provider-cli/provider-cli.runtime.d.ts +1 -1
- package/dist/session/provider-cli/provider-cli.runtime.js +6 -3
- package/dist/session/provider-cli/provider-cli.service.d.ts +26 -3
- package/dist/session/provider-cli/provider-cli.service.js +646 -90
- package/dist/session/provider-cli/provider-cli.types.d.ts +5 -0
- package/dist/session/provider-cli/provider-output-state.d.ts +2 -0
- package/dist/session/provider-cli/provider-output-state.js +91 -16
- package/dist/session/provider-cli/provider-usage.d.ts +7 -0
- package/dist/session/provider-cli/provider-usage.js +28 -5
- package/dist/session/session-redaction.d.ts +3 -0
- package/dist/session/session-redaction.js +38 -0
- package/dist/session/session.base.d.ts +6 -1
- package/dist/session/session.service.d.ts +8 -2
- package/dist/session/session.service.js +530 -42
- package/dist/session/session.types.d.ts +124 -1
- package/dist/session/session.types.js +231 -4
- package/dist/shared/format/format-bytes.d.ts +1 -0
- package/dist/shared/format/format-bytes.js +9 -0
- package/dist/shared/path/path.d.ts +1 -0
- package/dist/shared/path/path.js +21 -1
- package/dist/shared/process/dotnet-toolchain.d.ts +14 -0
- package/dist/shared/process/dotnet-toolchain.js +179 -0
- package/dist/shared/process/host-process-recovery.d.ts +12 -0
- package/dist/shared/process/host-process-recovery.js +78 -0
- package/dist/shared/process/host-process-supervisor.d.ts +59 -0
- package/dist/shared/process/host-process-supervisor.js +330 -0
- package/dist/shared/process/process.d.ts +1 -0
- package/dist/shared/process/process.js +1 -0
- package/dist/shared/process/windows-launch.d.ts +1 -1
- package/dist/shared/process/windows-launch.js +6 -3
- package/dist/shared/provider-capabilities.d.ts +1 -0
- package/dist/shared/provider-capabilities.js +3 -0
- package/dist/shared/redaction/redaction.js +16 -4
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/worktree/artifact-injector.d.ts +3 -2
- package/dist/worktree/artifact-injector.js +43 -4
- package/dist/worktree/dependency-warmer.d.ts +36 -2
- package/dist/worktree/dependency-warmer.js +347 -11
- package/dist/worktree/index.d.ts +2 -1
- package/dist/worktree/index.js +1 -0
- package/dist/worktree/prior-phase-artifact-hydrator.d.ts +2 -0
- package/dist/worktree/prior-phase-artifact-hydrator.js +136 -0
- package/dist/worktree/worktree-paths.d.ts +4 -0
- package/dist/worktree/worktree-paths.js +30 -0
- package/dist/worktree/worktree.base.d.ts +3 -0
- package/dist/worktree/worktree.service.d.ts +13 -2
- package/dist/worktree/worktree.service.js +238 -47
- package/dist/worktree/worktree.types.d.ts +5 -1
- package/package.json +8 -4
- package/runtime/windows-job-supervisor.ps1 +444 -0
- package/runtime/windows-listener-owner.ps1 +33 -0
|
@@ -0,0 +1,2203 @@
|
|
|
1
|
+
import fs from "node:fs/promises";
|
|
2
|
+
import { statfsSync } from "node:fs";
|
|
3
|
+
import os from "node:os";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { resolveExistingDirectoryPath } from "../shared/path/path.js";
|
|
6
|
+
import { StreamingProviderProcessRunner, } from "../session/provider-cli/provider-cli.process.js";
|
|
7
|
+
import { createRuntimeIsolationLease, renewRuntimeIsolationLease, transitionRuntimeIsolationLease, validRuntimeIsolationLease, withRuntimeIsolationResources, } from "./runtime-isolation-lease.js";
|
|
8
|
+
import { hostPortEnvironment, loadHostRuntimeContract, } from "./host-runtime-contract.js";
|
|
9
|
+
import { HostPortBroker, } from "./host-port-broker.js";
|
|
10
|
+
import { NodeHostRuntimeAdapter, } from "./node-host-runtime-adapter.js";
|
|
11
|
+
import { AttemptProcessSupervisorRegistry } from "./attempt-process-supervisor-registry.js";
|
|
12
|
+
import { AttemptNodeRuntimeLauncher } from "./attempt-node-runtime-launcher.js";
|
|
13
|
+
import { HostListenerInspector } from "./host-listener-inspector.js";
|
|
14
|
+
import { RecoveryStabilizedMemory, buildMemoryAdmissionDiagnostics, resolveAdaptiveMemoryReserveBytes, } from "./memory-capacity.js";
|
|
15
|
+
import { PlaywrightHostRuntimeAdapter } from "./playwright-host-runtime-adapter.js";
|
|
16
|
+
import { HostParallelPromotionStore, configuredLaunchPortNames, hostRuntimeContractFingerprint, hostRuntimeParallelEligible, } from "./host-runtime-promotion.js";
|
|
17
|
+
import { ExactHostProcessRecovery, } from "../shared/process/host-process-recovery.js";
|
|
18
|
+
import { resetGeneratedOutputs } from "./generated-output-reset.js";
|
|
19
|
+
import { architectMinimumWorkspaceEstimateBytes } from "./workspace-resource-estimator.js";
|
|
20
|
+
import { createAspNetRuntimePrebuildPlan, ensureAspNetRuntimePrebuilt, resolveWorktreeCommitSha, } from "./aspnet-runtime-prebuild.js";
|
|
21
|
+
const hostCapacityWaiterScanLimit = 64;
|
|
22
|
+
export class HostRuntimeLeaseManager {
|
|
23
|
+
stateRoot;
|
|
24
|
+
processSupervisors;
|
|
25
|
+
processRunner;
|
|
26
|
+
listenerInspector;
|
|
27
|
+
promotionStore;
|
|
28
|
+
playwrightAdapter;
|
|
29
|
+
processRecovery;
|
|
30
|
+
capacityResourceSample;
|
|
31
|
+
active = new Map();
|
|
32
|
+
pendingRenewals = new Map();
|
|
33
|
+
acquiringAdmissions = new Set();
|
|
34
|
+
admissions = new Map();
|
|
35
|
+
singletonOwners = new Map();
|
|
36
|
+
singletonWaiters = [];
|
|
37
|
+
parallelProjectOwners = new Map();
|
|
38
|
+
parallelProjectWaiters = [];
|
|
39
|
+
capacityReservations = new Map();
|
|
40
|
+
capacityWaiters = [];
|
|
41
|
+
capacityWaiterScanCursor = 0;
|
|
42
|
+
backgroundAdmissionWaiters = [];
|
|
43
|
+
degraded = new Map();
|
|
44
|
+
knownPromotions = new Map();
|
|
45
|
+
workspaceAvailableBytes = Number.MAX_SAFE_INTEGER;
|
|
46
|
+
ambientMemoryCommitmentBytes = null;
|
|
47
|
+
portBroker;
|
|
48
|
+
nodeAdapter;
|
|
49
|
+
capacityConfiguration;
|
|
50
|
+
memoryAvailability;
|
|
51
|
+
constructor(stateRoot, portBroker = new HostPortBroker(), nodeAdapter, processSupervisors = new AttemptProcessSupervisorRegistry(), processRunner = new StreamingProviderProcessRunner(), listenerInspector = new HostListenerInspector(), promotionStore = new HostParallelPromotionStore(stateRoot), playwrightAdapter = new PlaywrightHostRuntimeAdapter(), processRecovery = new ExactHostProcessRecovery(), capacityResourceSample) {
|
|
52
|
+
this.stateRoot = stateRoot;
|
|
53
|
+
this.processSupervisors = processSupervisors;
|
|
54
|
+
this.processRunner = processRunner;
|
|
55
|
+
this.listenerInspector = listenerInspector;
|
|
56
|
+
this.promotionStore = promotionStore;
|
|
57
|
+
this.playwrightAdapter = playwrightAdapter;
|
|
58
|
+
this.processRecovery = processRecovery;
|
|
59
|
+
this.capacityResourceSample = capacityResourceSample;
|
|
60
|
+
this.portBroker = portBroker;
|
|
61
|
+
this.nodeAdapter = nodeAdapter ?? new NodeHostRuntimeAdapter(portBroker);
|
|
62
|
+
this.capacityConfiguration = loadHostCapacityConfiguration(capacityResourceSample);
|
|
63
|
+
this.memoryAvailability = new RecoveryStabilizedMemory(this.capacityConfiguration.memoryRecoverySamples, this.capacityConfiguration.memoryRecoverySampleIntervalMs);
|
|
64
|
+
}
|
|
65
|
+
async reap(options) {
|
|
66
|
+
await this.refreshKnownPromotions();
|
|
67
|
+
const report = {
|
|
68
|
+
examined: 0,
|
|
69
|
+
reaped: 0,
|
|
70
|
+
processesTerminated: 0,
|
|
71
|
+
reusedPidsProtected: 0,
|
|
72
|
+
degraded: 0,
|
|
73
|
+
};
|
|
74
|
+
for (const ledger of await this.readLedgers()) {
|
|
75
|
+
const lease = ledger.runtimeLease;
|
|
76
|
+
if (lease.runnerId !== options.runnerId)
|
|
77
|
+
continue;
|
|
78
|
+
const active = this.active.get(lease.dispatchAttemptId);
|
|
79
|
+
if (active) {
|
|
80
|
+
if (active.lease.state !== "degraded")
|
|
81
|
+
continue;
|
|
82
|
+
report.examined += 1;
|
|
83
|
+
try {
|
|
84
|
+
const released = await this.release(lease.dispatchAttemptId, lease.fencingToken);
|
|
85
|
+
if (released?.state === "released")
|
|
86
|
+
report.reaped += 1;
|
|
87
|
+
else
|
|
88
|
+
report.degraded += 1;
|
|
89
|
+
}
|
|
90
|
+
catch {
|
|
91
|
+
report.degraded += 1;
|
|
92
|
+
}
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
95
|
+
const staleInstance = lease.runnerInstanceId !== options.runnerInstanceId;
|
|
96
|
+
const expired = Date.parse(lease.expiresAt) <= (options.nowMs ?? Date.now());
|
|
97
|
+
if (!staleInstance && !expired && lease.state !== "released" && lease.state !== "degraded")
|
|
98
|
+
continue;
|
|
99
|
+
report.examined += 1;
|
|
100
|
+
const attemptRoot = this.attemptRoot(lease.dispatchAttemptId);
|
|
101
|
+
let outcomes;
|
|
102
|
+
try {
|
|
103
|
+
outcomes = await this.recoverPersistedProcesses(lease, attemptRoot);
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
await this.markLedgerDegraded(ledger);
|
|
107
|
+
report.degraded += 1;
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
report.processesTerminated += outcomes.filter(outcome => outcome === "terminated").length;
|
|
111
|
+
report.reusedPidsProtected += outcomes.filter(outcome => outcome === "pid_reused").length;
|
|
112
|
+
if (outcomes.includes("unverifiable")) {
|
|
113
|
+
await this.markLedgerDegraded(ledger);
|
|
114
|
+
report.degraded += 1;
|
|
115
|
+
continue;
|
|
116
|
+
}
|
|
117
|
+
try {
|
|
118
|
+
const releasing = transitionHostLeaseForCleanup(lease, "releasing");
|
|
119
|
+
await this.writeLedger(releasing, ledger.repoId);
|
|
120
|
+
await fs.rm(attemptRoot, { recursive: true, force: true });
|
|
121
|
+
await fs.rm(this.ledgerPath(lease.dispatchAttemptId), { force: true });
|
|
122
|
+
this.forgetDegraded(lease.dispatchAttemptId, lease.fencingToken);
|
|
123
|
+
report.reaped += 1;
|
|
124
|
+
}
|
|
125
|
+
catch {
|
|
126
|
+
await this.markLedgerDegraded(ledger);
|
|
127
|
+
report.degraded += 1;
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
return report;
|
|
131
|
+
}
|
|
132
|
+
snapshot() {
|
|
133
|
+
this.reconcileReservationOwners();
|
|
134
|
+
const projects = new Map();
|
|
135
|
+
for (const promotion of this.knownPromotions.values()) {
|
|
136
|
+
const project = capacityProject(projects, promotion.repoId);
|
|
137
|
+
project.safetyClass = conservativeSafetyClass(project.safetyClass, "host_parallel_safe");
|
|
138
|
+
project.runtimeContractFingerprint = promotion.runtimeContractFingerprint;
|
|
139
|
+
project.runtimeContractSource = promotion.runtimeContractSource ?? null;
|
|
140
|
+
project.promotedAt = promotion.observedAt;
|
|
141
|
+
}
|
|
142
|
+
for (const [dispatchAttemptId, admission] of this.admissions) {
|
|
143
|
+
const active = this.active.get(dispatchAttemptId);
|
|
144
|
+
const project = capacityProject(projects, admission.repoId);
|
|
145
|
+
if (active) {
|
|
146
|
+
if (active.lease.state === "degraded") {
|
|
147
|
+
project.degradedAttempts += 1;
|
|
148
|
+
project.safetyClass = conservativeSafetyClass(project.safetyClass, "host_serial_only");
|
|
149
|
+
}
|
|
150
|
+
else {
|
|
151
|
+
project.activeAttempts += 1;
|
|
152
|
+
const effectiveConcurrency = effectiveHostConcurrency(active.lease, active.runtimeContract, active.executionClass);
|
|
153
|
+
project.safetyClass = conservativeSafetyClass(project.safetyClass, effectiveConcurrency === "host_parallel_safe"
|
|
154
|
+
? effectiveConcurrency
|
|
155
|
+
: admission.executionClass === "analysis_only"
|
|
156
|
+
? "host_parallel_safe"
|
|
157
|
+
: admission.parallelProjectKey && admission.provisionalPromotionFingerprint
|
|
158
|
+
? "host_parallel_candidate"
|
|
159
|
+
: "host_serial_only");
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
else {
|
|
163
|
+
project.reservedAttempts += 1;
|
|
164
|
+
project.safetyClass = conservativeSafetyClass(project.safetyClass, admission.executionClass === "analysis_only"
|
|
165
|
+
? "host_parallel_safe"
|
|
166
|
+
: admission.provisionalPromotionFingerprint ? "host_parallel_candidate" : "host_serial_only");
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
for (const [dispatchAttemptId, degraded] of this.degraded) {
|
|
170
|
+
if (this.active.has(dispatchAttemptId) || !degraded.repoId)
|
|
171
|
+
continue;
|
|
172
|
+
const project = capacityProject(projects, degraded.repoId);
|
|
173
|
+
project.degradedAttempts += 1;
|
|
174
|
+
project.safetyClass = conservativeSafetyClass(project.safetyClass, "host_serial_only");
|
|
175
|
+
}
|
|
176
|
+
for (const waiter of this.singletonWaiters) {
|
|
177
|
+
const repoId = repoIdFromProjectSingletonKeys(waiter.keys);
|
|
178
|
+
if (!repoId)
|
|
179
|
+
continue;
|
|
180
|
+
const project = capacityProject(projects, repoId);
|
|
181
|
+
project.waitingAttempts += 1;
|
|
182
|
+
project.safetyClass = conservativeSafetyClass(project.safetyClass, "host_serial_only");
|
|
183
|
+
}
|
|
184
|
+
for (const waiter of this.parallelProjectWaiters) {
|
|
185
|
+
const repoId = repoIdFromProjectSingletonKey(waiter.key);
|
|
186
|
+
if (!repoId)
|
|
187
|
+
continue;
|
|
188
|
+
const project = capacityProject(projects, repoId);
|
|
189
|
+
project.waitingAttempts += 1;
|
|
190
|
+
project.safetyClass = conservativeSafetyClass(project.safetyClass, "host_parallel_candidate");
|
|
191
|
+
}
|
|
192
|
+
for (const waiter of this.backgroundAdmissionWaiters) {
|
|
193
|
+
const repoId = repoIdFromProjectSingletonKeys(waiter.keys);
|
|
194
|
+
if (!repoId)
|
|
195
|
+
continue;
|
|
196
|
+
const project = capacityProject(projects, repoId);
|
|
197
|
+
project.waitingAttempts += 1;
|
|
198
|
+
project.safetyClass = conservativeSafetyClass(project.safetyClass, "host_serial_only");
|
|
199
|
+
}
|
|
200
|
+
const activeAttempts = [...this.active.values()]
|
|
201
|
+
.filter(active => active.lease.state !== "degraded").length;
|
|
202
|
+
const degradedAttempts = new Set([
|
|
203
|
+
...this.degraded.keys(),
|
|
204
|
+
...[...this.active.entries()]
|
|
205
|
+
.filter(([, active]) => active.lease.state === "degraded")
|
|
206
|
+
.map(([dispatchAttemptId]) => dispatchAttemptId),
|
|
207
|
+
]).size;
|
|
208
|
+
const hostResources = this.hostResourceSnapshot();
|
|
209
|
+
return {
|
|
210
|
+
activeAttempts,
|
|
211
|
+
reservedAttempts: Math.max(0, this.admissions.size - this.active.size),
|
|
212
|
+
waitingAttempts: this.singletonWaiters.length
|
|
213
|
+
+ this.parallelProjectWaiters.length
|
|
214
|
+
+ this.capacityWaiters.length
|
|
215
|
+
+ this.backgroundAdmissionWaiters.length,
|
|
216
|
+
degradedAttempts,
|
|
217
|
+
exclusiveReservations: this.singletonOwners.size,
|
|
218
|
+
sharedReservations: [...this.parallelProjectOwners.values()]
|
|
219
|
+
.reduce((count, owners) => count + owners.size, 0),
|
|
220
|
+
...hostResources,
|
|
221
|
+
projects: [...projects.values()]
|
|
222
|
+
.map(project => ({
|
|
223
|
+
...project,
|
|
224
|
+
safetyClass: project.safetyClass ?? "host_serial_only",
|
|
225
|
+
}))
|
|
226
|
+
.sort((left, right) => left.repoId.localeCompare(right.repoId)),
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
hasForegroundDemand() {
|
|
230
|
+
return [...this.admissions.values()].some(admission => admission.priority === "foreground")
|
|
231
|
+
|| this.singletonWaiters.some(waiter => waiter.priority === "foreground")
|
|
232
|
+
|| this.parallelProjectWaiters.some(waiter => waiter.priority === "foreground")
|
|
233
|
+
|| this.capacityWaiters.some(waiter => waiter.priority === "foreground");
|
|
234
|
+
}
|
|
235
|
+
async acquire(dispatch, signal) {
|
|
236
|
+
const sandbox = dispatch.sandbox;
|
|
237
|
+
if (!sandbox || sandbox.executionMode !== "host") {
|
|
238
|
+
throw new Error("host_runtime_lease_invalid: managed host sandbox context is required.");
|
|
239
|
+
}
|
|
240
|
+
uuidHex(sandbox.dispatchAttemptId, "dispatch attempt");
|
|
241
|
+
uuidHex(dispatch.lease.fencingToken, "fencing token");
|
|
242
|
+
const owner = {
|
|
243
|
+
dispatchAttemptId: sandbox.dispatchAttemptId,
|
|
244
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
245
|
+
};
|
|
246
|
+
const existing = this.admissions.get(sandbox.dispatchAttemptId);
|
|
247
|
+
if (existing) {
|
|
248
|
+
if (existing.fencingToken !== owner.fencingToken) {
|
|
249
|
+
throw new Error("host_runtime_singleton_conflict: dispatch attempt already holds admission under a different fence.");
|
|
250
|
+
}
|
|
251
|
+
return existing.keys;
|
|
252
|
+
}
|
|
253
|
+
const executionClass = runtimeExecutionClass(dispatch);
|
|
254
|
+
const priority = runtimeAdmissionPriority(dispatch);
|
|
255
|
+
const promotion = executionClass === "managed_runtime"
|
|
256
|
+
? await this.promotionStore.load(dispatch.repo.id)
|
|
257
|
+
: null;
|
|
258
|
+
if (promotion)
|
|
259
|
+
this.knownPromotions.set(promotion.repoId, promotion);
|
|
260
|
+
const projectKey = executionClass === "analysis_only"
|
|
261
|
+
? projectHostAnalysisKey(dispatch.repo.id)
|
|
262
|
+
: projectHostRuntimeSingletonKey(dispatch.repo.id);
|
|
263
|
+
const useParallelAdmission = priority === "foreground"
|
|
264
|
+
&& (promotion !== null || executionClass === "analysis_only");
|
|
265
|
+
let keys = useParallelAdmission ? [] : [projectKey];
|
|
266
|
+
const resources = normalizedHostResourceRequest(sandbox.requestedResources);
|
|
267
|
+
const expiresAt = this.pendingRenewals.get(sandbox.dispatchAttemptId)?.fencingToken === owner.fencingToken
|
|
268
|
+
? this.pendingRenewals.get(sandbox.dispatchAttemptId).expiresAt
|
|
269
|
+
: dispatch.lease.expiresAt;
|
|
270
|
+
const acquisitionKey = hostRuntimeOwnerKey(owner);
|
|
271
|
+
this.acquiringAdmissions.add(acquisitionKey);
|
|
272
|
+
try {
|
|
273
|
+
try {
|
|
274
|
+
if (priority === "background") {
|
|
275
|
+
await this.refreshWorkspaceCapacity();
|
|
276
|
+
keys = await this.reserveBackgroundAdmission(keys, owner, resources, expiresAt, signal);
|
|
277
|
+
}
|
|
278
|
+
else if (useParallelAdmission) {
|
|
279
|
+
await this.reserveParallelProject(projectKey, owner, expiresAt, signal, priority);
|
|
280
|
+
}
|
|
281
|
+
else {
|
|
282
|
+
keys = await this.reserveSingletonKeys(keys, owner, expiresAt, signal, priority);
|
|
283
|
+
}
|
|
284
|
+
if (priority === "foreground") {
|
|
285
|
+
await this.refreshWorkspaceCapacity();
|
|
286
|
+
const capacityExpiresAt = this.pendingRenewals.get(sandbox.dispatchAttemptId)?.fencingToken === owner.fencingToken
|
|
287
|
+
? this.pendingRenewals.get(sandbox.dispatchAttemptId).expiresAt
|
|
288
|
+
: expiresAt;
|
|
289
|
+
await this.reserveHostCapacity(owner, resources, capacityExpiresAt, signal, priority);
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
catch (error) {
|
|
293
|
+
this.releaseHostCapacity(owner.dispatchAttemptId, owner.fencingToken);
|
|
294
|
+
this.releaseSingletonKeys(owner.dispatchAttemptId, owner.fencingToken);
|
|
295
|
+
this.releaseParallelProjects(owner.dispatchAttemptId, owner.fencingToken);
|
|
296
|
+
this.clearPendingRenewal(owner);
|
|
297
|
+
throw error;
|
|
298
|
+
}
|
|
299
|
+
const admittedPromotion = useParallelAdmission
|
|
300
|
+
? promotion ?? (executionClass === "managed_runtime" && keys.length === 0
|
|
301
|
+
? this.knownPromotions.get(dispatch.repo.id) ?? null
|
|
302
|
+
: null)
|
|
303
|
+
: null;
|
|
304
|
+
this.admissions.set(sandbox.dispatchAttemptId, {
|
|
305
|
+
repoId: dispatch.repo.id,
|
|
306
|
+
fencingToken: owner.fencingToken,
|
|
307
|
+
executionClass,
|
|
308
|
+
priority,
|
|
309
|
+
provisionalPromotionFingerprint: admittedPromotion?.runtimeContractFingerprint ?? null,
|
|
310
|
+
keys,
|
|
311
|
+
parallelProjectKey: useParallelAdmission ? projectKey : null,
|
|
312
|
+
});
|
|
313
|
+
return keys;
|
|
314
|
+
}
|
|
315
|
+
finally {
|
|
316
|
+
this.acquiringAdmissions.delete(acquisitionKey);
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
async reserveHostCapacity(owner, resources, expiresAt, signal, priority = "foreground") {
|
|
320
|
+
const existing = this.capacityReservations.get(owner.dispatchAttemptId);
|
|
321
|
+
if (existing) {
|
|
322
|
+
if (!sameSingletonOwner(existing.owner, owner)) {
|
|
323
|
+
throw new Error("host_runtime_capacity_conflict: dispatch attempt already holds capacity under a different fence.");
|
|
324
|
+
}
|
|
325
|
+
return;
|
|
326
|
+
}
|
|
327
|
+
if (signal?.aborted)
|
|
328
|
+
throw cancellationError(signal);
|
|
329
|
+
throwIfAdmissionExpired(expiresAt);
|
|
330
|
+
const impossibleReason = this.hostCapacityImpossibleReason(resources);
|
|
331
|
+
if (impossibleReason)
|
|
332
|
+
throw new HostRuntimeCapacityRequestImpossibleError(impossibleReason);
|
|
333
|
+
if (this.capacityWaiters.length === 0 && this.hostCapacityAvailable(resources)) {
|
|
334
|
+
this.capacityReservations.set(owner.dispatchAttemptId, { owner, resources, state: "pending" });
|
|
335
|
+
return;
|
|
336
|
+
}
|
|
337
|
+
return new Promise((resolve, reject) => {
|
|
338
|
+
const waiter = {
|
|
339
|
+
owner,
|
|
340
|
+
resources,
|
|
341
|
+
priority,
|
|
342
|
+
resolve,
|
|
343
|
+
reject,
|
|
344
|
+
signal,
|
|
345
|
+
expiresAt,
|
|
346
|
+
deadlineTimer: null,
|
|
347
|
+
capacityRecheckTimer: null,
|
|
348
|
+
abort: () => {
|
|
349
|
+
if (!this.detachCapacityWaiter(waiter))
|
|
350
|
+
return;
|
|
351
|
+
this.clearPendingRenewal(owner);
|
|
352
|
+
reject(cancellationError(signal));
|
|
353
|
+
this.drainAdmissionWaiters();
|
|
354
|
+
},
|
|
355
|
+
expire: () => {
|
|
356
|
+
if (!this.detachCapacityWaiter(waiter))
|
|
357
|
+
return;
|
|
358
|
+
this.clearPendingRenewal(owner);
|
|
359
|
+
reject(new HostRuntimeAdmissionExpiredError());
|
|
360
|
+
this.drainAdmissionWaiters();
|
|
361
|
+
},
|
|
362
|
+
};
|
|
363
|
+
waiter.deadlineTimer = scheduleAdmissionDeadline(expiresAt, waiter.expire);
|
|
364
|
+
// Memory, CPU and filesystem capacity can recover without one of our leases
|
|
365
|
+
// being released (for example when an unrelated host process exits). Recheck
|
|
366
|
+
// ambient capacity while queued so readiness probes and attempts do not remain
|
|
367
|
+
// stranded behind a stale low-resource sample.
|
|
368
|
+
waiter.capacityRecheckTimer = setInterval(() => this.drainCapacityWaiters(), 1_000);
|
|
369
|
+
waiter.capacityRecheckTimer.unref?.();
|
|
370
|
+
signal?.addEventListener("abort", waiter.abort, { once: true });
|
|
371
|
+
enqueueAdmissionWaiter(this.capacityWaiters, waiter);
|
|
372
|
+
this.drainCapacityWaiters();
|
|
373
|
+
});
|
|
374
|
+
}
|
|
375
|
+
async reserveBackgroundAdmission(keys, owner, resources, expiresAt, signal) {
|
|
376
|
+
if (signal?.aborted)
|
|
377
|
+
throw cancellationError(signal);
|
|
378
|
+
throwIfAdmissionExpired(expiresAt);
|
|
379
|
+
return await new Promise((resolve, reject) => {
|
|
380
|
+
const waiter = {
|
|
381
|
+
owner,
|
|
382
|
+
keys,
|
|
383
|
+
resources,
|
|
384
|
+
priority: "background",
|
|
385
|
+
resolve,
|
|
386
|
+
reject,
|
|
387
|
+
signal,
|
|
388
|
+
expiresAt,
|
|
389
|
+
deadlineTimer: null,
|
|
390
|
+
capacityRecheckTimer: null,
|
|
391
|
+
abort: () => {
|
|
392
|
+
removeWaiter(this.backgroundAdmissionWaiters, waiter);
|
|
393
|
+
clearAdmissionDeadline(waiter);
|
|
394
|
+
clearCapacityRecheck(waiter);
|
|
395
|
+
this.clearPendingRenewal(owner);
|
|
396
|
+
reject(cancellationError(signal));
|
|
397
|
+
this.drainAdmissionWaiters();
|
|
398
|
+
},
|
|
399
|
+
expire: () => {
|
|
400
|
+
removeWaiter(this.backgroundAdmissionWaiters, waiter);
|
|
401
|
+
clearCapacityRecheck(waiter);
|
|
402
|
+
signal?.removeEventListener("abort", waiter.abort);
|
|
403
|
+
this.clearPendingRenewal(owner);
|
|
404
|
+
reject(new HostRuntimeAdmissionExpiredError());
|
|
405
|
+
this.drainAdmissionWaiters();
|
|
406
|
+
},
|
|
407
|
+
};
|
|
408
|
+
waiter.deadlineTimer = scheduleAdmissionDeadline(expiresAt, waiter.expire);
|
|
409
|
+
waiter.capacityRecheckTimer = setInterval(() => this.drainAdmissionWaiters(), 1_000);
|
|
410
|
+
waiter.capacityRecheckTimer.unref?.();
|
|
411
|
+
signal?.addEventListener("abort", waiter.abort, { once: true });
|
|
412
|
+
this.backgroundAdmissionWaiters.push(waiter);
|
|
413
|
+
this.drainAdmissionWaiters();
|
|
414
|
+
});
|
|
415
|
+
}
|
|
416
|
+
hostCapacityAvailable(resources) {
|
|
417
|
+
const capacity = this.hostResourceSnapshot();
|
|
418
|
+
return capacity.availableAttempts >= resources.sandboxSlots
|
|
419
|
+
&& capacity.availableMemoryBytes >= resources.memoryBytes
|
|
420
|
+
&& capacity.availableCpuMillis >= resources.cpuMillis
|
|
421
|
+
&& capacity.availableWorkspaceBytes >= resources.workspaceBytes;
|
|
422
|
+
}
|
|
423
|
+
hostCapacityImpossibleReason(resources) {
|
|
424
|
+
const { hardMaxAttempts, idleHostReserveBytes, cpuReserveMillis } = this.capacityConfiguration;
|
|
425
|
+
if (resources.sandboxSlots > hardMaxAttempts) {
|
|
426
|
+
return `requested ${resources.sandboxSlots} sandbox slots exceed the host maximum of ${hardMaxAttempts}.`;
|
|
427
|
+
}
|
|
428
|
+
const totalMemoryBytes = this.capacityResourceSample?.totalMemoryBytes ?? os.totalmem();
|
|
429
|
+
const maximumMemoryBytes = Math.max(0, totalMemoryBytes - idleHostReserveBytes);
|
|
430
|
+
if (resources.memoryBytes > maximumMemoryBytes) {
|
|
431
|
+
return `requested memory ${resources.memoryBytes} exceeds the host maximum of ${maximumMemoryBytes} bytes after reserve.`;
|
|
432
|
+
}
|
|
433
|
+
const totalCpuMillis = this.capacityResourceSample?.totalCpuMillis ?? os.cpus().length * 1_000;
|
|
434
|
+
const maximumCpuMillis = Math.max(0, totalCpuMillis - cpuReserveMillis);
|
|
435
|
+
if (resources.cpuMillis > maximumCpuMillis) {
|
|
436
|
+
return `requested CPU ${resources.cpuMillis} millis exceeds the host maximum of ${maximumCpuMillis} millis after reserve.`;
|
|
437
|
+
}
|
|
438
|
+
return null;
|
|
439
|
+
}
|
|
440
|
+
hostResourceSnapshot() {
|
|
441
|
+
if (!this.capacityResourceSample) {
|
|
442
|
+
try {
|
|
443
|
+
const stats = statfsSync(this.stateRoot);
|
|
444
|
+
this.workspaceAvailableBytes = Math.max(0, stats.bavail * stats.bsize);
|
|
445
|
+
}
|
|
446
|
+
catch {
|
|
447
|
+
// State root may not exist during the first capability heartbeat. The first
|
|
448
|
+
// admission creates it and refreshes the sample before reserving resources.
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
const { hardMaxAttempts, cpuReserveMillis, workspaceReserveBytes } = this.capacityConfiguration;
|
|
452
|
+
const hostReserveBytes = this.capacityReservations.size === 0
|
|
453
|
+
? this.capacityConfiguration.idleHostReserveBytes
|
|
454
|
+
: this.capacityConfiguration.hostReserveBytes;
|
|
455
|
+
const activeReservationMemoryBytes = [...this.capacityReservations.values()]
|
|
456
|
+
.reduce((total, reservation) => total + reservation.resources.memoryBytes, 0);
|
|
457
|
+
const pendingReservationMemoryBytes = [...this.capacityReservations.values()]
|
|
458
|
+
.filter(reservation => reservation.state === "pending")
|
|
459
|
+
.reduce((total, reservation) => total + reservation.resources.memoryBytes, 0);
|
|
460
|
+
const activeReservationCpuMillis = [...this.capacityReservations.values()]
|
|
461
|
+
.reduce((total, reservation) => total + reservation.resources.cpuMillis, 0);
|
|
462
|
+
const activeReservationWorkspaceBytes = [...this.capacityReservations.values()]
|
|
463
|
+
.reduce((total, reservation) => total + reservation.resources.workspaceBytes, 0);
|
|
464
|
+
const sampledAt = new Date().toISOString();
|
|
465
|
+
const totalMemoryBytes = this.capacityResourceSample?.totalMemoryBytes ?? os.totalmem();
|
|
466
|
+
const rawAvailableMemoryBytes = this.capacityResourceSample?.availableMemoryBytes ?? os.freemem();
|
|
467
|
+
const sampledAvailableMemoryBytes = this.memoryAvailability.observe(rawAvailableMemoryBytes);
|
|
468
|
+
if (this.capacityReservations.size === 0) {
|
|
469
|
+
// Capture non-Forge resident use before the first reservation. Refreshing
|
|
470
|
+
// while idle lets the baseline recover when an unrelated process exits;
|
|
471
|
+
// freezing it while leases exist prevents pending/low-RSS workers from
|
|
472
|
+
// borrowing memory already occupied by the host and control plane.
|
|
473
|
+
this.ambientMemoryCommitmentBytes = Math.max(0, totalMemoryBytes - rawAvailableMemoryBytes);
|
|
474
|
+
}
|
|
475
|
+
const committedMemoryBytes = Math.min(totalMemoryBytes, (this.ambientMemoryCommitmentBytes ?? 0) + activeReservationMemoryBytes);
|
|
476
|
+
const totalCpuMillis = this.capacityResourceSample?.totalCpuMillis ?? os.cpus().length * 1000;
|
|
477
|
+
const sampledWorkspaceBytes = this.capacityResourceSample?.availableWorkspaceBytes ?? this.workspaceAvailableBytes;
|
|
478
|
+
const memoryAdmission = buildMemoryAdmissionDiagnostics({
|
|
479
|
+
sampledAt,
|
|
480
|
+
totalMemoryBytes,
|
|
481
|
+
rawAvailableMemoryBytes,
|
|
482
|
+
stabilizedAvailableMemoryBytes: sampledAvailableMemoryBytes,
|
|
483
|
+
controlPlaneMemoryBytes: process.memoryUsage().rss,
|
|
484
|
+
reserveMemoryBytes: hostReserveBytes,
|
|
485
|
+
committedMemoryBytes,
|
|
486
|
+
pendingCommitmentMemoryBytes: pendingReservationMemoryBytes,
|
|
487
|
+
});
|
|
488
|
+
const availableMemoryBytes = memoryAdmission.availableForAdmissionBytes;
|
|
489
|
+
const availableCpuMillis = Math.max(0, totalCpuMillis - cpuReserveMillis - activeReservationCpuMillis);
|
|
490
|
+
const availableWorkspaceBytes = Math.max(0, sampledWorkspaceBytes - workspaceReserveBytes - activeReservationWorkspaceBytes);
|
|
491
|
+
const slotAvailability = Math.max(0, hardMaxAttempts - this.capacityReservations.size);
|
|
492
|
+
// Capability snapshots advertise whether any supported attempt can fit.
|
|
493
|
+
// The queue/admission path still checks the exact project-derived request.
|
|
494
|
+
const minimumRequest = minimumSupportedHostResourceRequest();
|
|
495
|
+
const resourceAvailability = Math.min(resourceSlots(availableMemoryBytes, minimumRequest.memoryBytes), resourceSlots(availableCpuMillis, minimumRequest.cpuMillis), resourceSlots(availableWorkspaceBytes, minimumRequest.workspaceBytes));
|
|
496
|
+
const availableAttempts = Math.max(0, Math.min(slotAvailability, resourceAvailability));
|
|
497
|
+
const unavailableReason = slotAvailability <= 0
|
|
498
|
+
? `Host attempt capacity is full (${this.capacityReservations.size}/${hardMaxAttempts}).`
|
|
499
|
+
: availableMemoryBytes < minimumRequest.memoryBytes
|
|
500
|
+
? "Host memory is below the admission reserve."
|
|
501
|
+
: availableCpuMillis < minimumRequest.cpuMillis
|
|
502
|
+
? "Host CPU capacity is below the admission reserve."
|
|
503
|
+
: availableWorkspaceBytes < minimumRequest.workspaceBytes
|
|
504
|
+
? "Host workspace capacity is below the admission reserve."
|
|
505
|
+
: null;
|
|
506
|
+
return {
|
|
507
|
+
hardMaxAttempts,
|
|
508
|
+
effectiveMaxAttempts: this.capacityReservations.size + availableAttempts,
|
|
509
|
+
availableAttempts,
|
|
510
|
+
totalMemoryBytes,
|
|
511
|
+
availableMemoryBytes,
|
|
512
|
+
hostReserveBytes,
|
|
513
|
+
availableCpuMillis,
|
|
514
|
+
availableWorkspaceBytes,
|
|
515
|
+
activeReservationMemoryBytes,
|
|
516
|
+
memoryAdmission,
|
|
517
|
+
unavailableCode: unavailableReason ? "capacity_unavailable" : null,
|
|
518
|
+
unavailableReason,
|
|
519
|
+
};
|
|
520
|
+
}
|
|
521
|
+
async refreshWorkspaceCapacity() {
|
|
522
|
+
if (this.capacityResourceSample)
|
|
523
|
+
return;
|
|
524
|
+
try {
|
|
525
|
+
await fs.mkdir(this.stateRoot, { recursive: true });
|
|
526
|
+
const stats = await fs.statfs(this.stateRoot);
|
|
527
|
+
this.workspaceAvailableBytes = Math.max(0, stats.bavail * stats.bsize);
|
|
528
|
+
}
|
|
529
|
+
catch {
|
|
530
|
+
// Keep the last trustworthy sample. Memory/CPU and the hard ceiling still
|
|
531
|
+
// protect admission when the filesystem cannot be sampled temporarily.
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
releaseHostCapacity(dispatchAttemptId, fencingToken) {
|
|
535
|
+
const reservation = this.capacityReservations.get(dispatchAttemptId);
|
|
536
|
+
if (!reservation || reservation.owner.fencingToken !== fencingToken)
|
|
537
|
+
return;
|
|
538
|
+
this.capacityReservations.delete(dispatchAttemptId);
|
|
539
|
+
this.drainAdmissionWaiters();
|
|
540
|
+
}
|
|
541
|
+
drainCapacityWaiters() {
|
|
542
|
+
for (const waiter of this.capacityWaitersForScan()) {
|
|
543
|
+
if (!this.capacityWaiters.includes(waiter))
|
|
544
|
+
continue;
|
|
545
|
+
if (waiter.signal?.aborted) {
|
|
546
|
+
this.detachCapacityWaiter(waiter);
|
|
547
|
+
this.clearPendingRenewal(waiter.owner);
|
|
548
|
+
waiter.reject(cancellationError(waiter.signal));
|
|
549
|
+
continue;
|
|
550
|
+
}
|
|
551
|
+
if (admissionExpired(waiter.expiresAt)) {
|
|
552
|
+
this.detachCapacityWaiter(waiter);
|
|
553
|
+
this.clearPendingRenewal(waiter.owner);
|
|
554
|
+
waiter.reject(new HostRuntimeAdmissionExpiredError());
|
|
555
|
+
continue;
|
|
556
|
+
}
|
|
557
|
+
const impossibleReason = this.hostCapacityImpossibleReason(waiter.resources);
|
|
558
|
+
if (impossibleReason) {
|
|
559
|
+
this.detachCapacityWaiter(waiter);
|
|
560
|
+
this.clearPendingRenewal(waiter.owner);
|
|
561
|
+
waiter.reject(new HostRuntimeCapacityRequestImpossibleError(impossibleReason));
|
|
562
|
+
continue;
|
|
563
|
+
}
|
|
564
|
+
if (!this.hostCapacityAvailable(waiter.resources))
|
|
565
|
+
continue;
|
|
566
|
+
this.detachCapacityWaiter(waiter);
|
|
567
|
+
this.capacityReservations.set(waiter.owner.dispatchAttemptId, {
|
|
568
|
+
owner: waiter.owner,
|
|
569
|
+
resources: waiter.resources,
|
|
570
|
+
state: "pending",
|
|
571
|
+
});
|
|
572
|
+
waiter.resolve();
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
capacityWaitersForScan() {
|
|
576
|
+
const head = this.capacityWaiters[0];
|
|
577
|
+
if (!head) {
|
|
578
|
+
this.capacityWaiterScanCursor = 0;
|
|
579
|
+
return [];
|
|
580
|
+
}
|
|
581
|
+
const scanCount = Math.min(hostCapacityWaiterScanLimit, this.capacityWaiters.length);
|
|
582
|
+
if (scanCount === this.capacityWaiters.length) {
|
|
583
|
+
this.capacityWaiterScanCursor = 0;
|
|
584
|
+
return this.capacityWaiters.slice();
|
|
585
|
+
}
|
|
586
|
+
// The head always gets the first opportunity, while a rotating bounded
|
|
587
|
+
// window prevents an unfit renewed head (or a very deep queue) from
|
|
588
|
+
// indefinitely hiding a later request that fits current capacity.
|
|
589
|
+
const laterCount = this.capacityWaiters.length - 1;
|
|
590
|
+
const laterScanCount = scanCount - 1;
|
|
591
|
+
const start = this.capacityWaiterScanCursor % laterCount;
|
|
592
|
+
const result = [head];
|
|
593
|
+
for (let offset = 0; offset < laterScanCount; offset += 1) {
|
|
594
|
+
result.push(this.capacityWaiters[1 + ((start + offset) % laterCount)]);
|
|
595
|
+
}
|
|
596
|
+
this.capacityWaiterScanCursor = (start + laterScanCount) % laterCount;
|
|
597
|
+
return result;
|
|
598
|
+
}
|
|
599
|
+
detachCapacityWaiter(waiter) {
|
|
600
|
+
const index = this.capacityWaiters.indexOf(waiter);
|
|
601
|
+
if (index < 0)
|
|
602
|
+
return false;
|
|
603
|
+
this.capacityWaiters.splice(index, 1);
|
|
604
|
+
clearAdmissionDeadline(waiter);
|
|
605
|
+
clearCapacityRecheck(waiter);
|
|
606
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
607
|
+
if (this.capacityWaiters.length === 0)
|
|
608
|
+
this.capacityWaiterScanCursor = 0;
|
|
609
|
+
return true;
|
|
610
|
+
}
|
|
611
|
+
async reserveSingletonKeys(keys, owner, expiresAt, signal, priority = "foreground") {
|
|
612
|
+
const owned = keys.every(key => sameSingletonOwner(this.singletonOwners.get(key), owner));
|
|
613
|
+
if (owned)
|
|
614
|
+
return keys;
|
|
615
|
+
const staleFence = [...this.singletonOwners.values()].find(existing => existing.dispatchAttemptId === owner.dispatchAttemptId
|
|
616
|
+
&& existing.fencingToken !== owner.fencingToken);
|
|
617
|
+
if (staleFence) {
|
|
618
|
+
throw new Error("host_runtime_singleton_conflict: dispatch attempt already holds the singleton under a different fence.");
|
|
619
|
+
}
|
|
620
|
+
if (signal?.aborted)
|
|
621
|
+
throw cancellationError(signal);
|
|
622
|
+
throwIfAdmissionExpired(expiresAt);
|
|
623
|
+
if (this.singletonKeysAvailable(keys)) {
|
|
624
|
+
this.assignSingletonKeys(keys, owner);
|
|
625
|
+
return keys;
|
|
626
|
+
}
|
|
627
|
+
return new Promise((resolve, reject) => {
|
|
628
|
+
const waiter = {
|
|
629
|
+
owner,
|
|
630
|
+
keys,
|
|
631
|
+
priority,
|
|
632
|
+
resolve,
|
|
633
|
+
reject,
|
|
634
|
+
signal,
|
|
635
|
+
expiresAt,
|
|
636
|
+
deadlineTimer: null,
|
|
637
|
+
abort: () => {
|
|
638
|
+
const index = this.singletonWaiters.indexOf(waiter);
|
|
639
|
+
if (index >= 0)
|
|
640
|
+
this.singletonWaiters.splice(index, 1);
|
|
641
|
+
clearAdmissionDeadline(waiter);
|
|
642
|
+
this.clearPendingRenewal(owner);
|
|
643
|
+
reject(cancellationError(signal));
|
|
644
|
+
this.drainAdmissionWaiters();
|
|
645
|
+
},
|
|
646
|
+
expire: () => {
|
|
647
|
+
const index = this.singletonWaiters.indexOf(waiter);
|
|
648
|
+
if (index < 0)
|
|
649
|
+
return;
|
|
650
|
+
this.singletonWaiters.splice(index, 1);
|
|
651
|
+
clearAdmissionDeadline(waiter);
|
|
652
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
653
|
+
this.clearPendingRenewal(owner);
|
|
654
|
+
reject(new HostRuntimeAdmissionExpiredError());
|
|
655
|
+
this.drainAdmissionWaiters();
|
|
656
|
+
},
|
|
657
|
+
};
|
|
658
|
+
enqueueAdmissionWaiter(this.singletonWaiters, waiter);
|
|
659
|
+
signal?.addEventListener("abort", waiter.abort, { once: true });
|
|
660
|
+
waiter.deadlineTimer = scheduleAdmissionDeadline(waiter.expiresAt, waiter.expire);
|
|
661
|
+
this.drainAdmissionWaiters();
|
|
662
|
+
});
|
|
663
|
+
}
|
|
664
|
+
async reserveParallelProject(key, owner, expiresAt, signal, priority = "foreground") {
|
|
665
|
+
if (this.parallelProjectOwned(key, owner))
|
|
666
|
+
return;
|
|
667
|
+
const staleFence = [...(this.parallelProjectOwners.get(key)?.values() ?? [])]
|
|
668
|
+
.find(existing => existing.dispatchAttemptId === owner.dispatchAttemptId
|
|
669
|
+
&& existing.fencingToken !== owner.fencingToken);
|
|
670
|
+
if (staleFence) {
|
|
671
|
+
throw new Error("host_runtime_parallel_conflict: dispatch attempt already holds shared admission under a different fence.");
|
|
672
|
+
}
|
|
673
|
+
if (signal?.aborted)
|
|
674
|
+
throw cancellationError(signal);
|
|
675
|
+
throwIfAdmissionExpired(expiresAt);
|
|
676
|
+
if (this.parallelProjectAvailable(key, priority)) {
|
|
677
|
+
this.assignParallelProject(key, owner);
|
|
678
|
+
return;
|
|
679
|
+
}
|
|
680
|
+
await new Promise((resolve, reject) => {
|
|
681
|
+
const waiter = {
|
|
682
|
+
owner,
|
|
683
|
+
key,
|
|
684
|
+
priority,
|
|
685
|
+
resolve,
|
|
686
|
+
reject,
|
|
687
|
+
signal,
|
|
688
|
+
expiresAt,
|
|
689
|
+
deadlineTimer: null,
|
|
690
|
+
abort: () => {
|
|
691
|
+
const index = this.parallelProjectWaiters.indexOf(waiter);
|
|
692
|
+
if (index >= 0)
|
|
693
|
+
this.parallelProjectWaiters.splice(index, 1);
|
|
694
|
+
clearAdmissionDeadline(waiter);
|
|
695
|
+
this.clearPendingRenewal(owner);
|
|
696
|
+
reject(cancellationError(signal));
|
|
697
|
+
this.drainAdmissionWaiters();
|
|
698
|
+
},
|
|
699
|
+
expire: () => {
|
|
700
|
+
const index = this.parallelProjectWaiters.indexOf(waiter);
|
|
701
|
+
if (index < 0)
|
|
702
|
+
return;
|
|
703
|
+
this.parallelProjectWaiters.splice(index, 1);
|
|
704
|
+
clearAdmissionDeadline(waiter);
|
|
705
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
706
|
+
this.clearPendingRenewal(owner);
|
|
707
|
+
reject(new HostRuntimeAdmissionExpiredError());
|
|
708
|
+
this.drainAdmissionWaiters();
|
|
709
|
+
},
|
|
710
|
+
};
|
|
711
|
+
enqueueAdmissionWaiter(this.parallelProjectWaiters, waiter);
|
|
712
|
+
signal?.addEventListener("abort", waiter.abort, { once: true });
|
|
713
|
+
waiter.deadlineTimer = scheduleAdmissionDeadline(waiter.expiresAt, waiter.expire);
|
|
714
|
+
this.drainAdmissionWaiters();
|
|
715
|
+
});
|
|
716
|
+
}
|
|
717
|
+
async upgradeParallelProjectToSingleton(keys, parallelProjectKey, owner, expiresAt, signal, priority = "foreground") {
|
|
718
|
+
if (!this.parallelProjectOwned(parallelProjectKey, owner)) {
|
|
719
|
+
throw new Error("host_runtime_parallel_unreserved: matching shared project admission is required before serialization.");
|
|
720
|
+
}
|
|
721
|
+
if (signal?.aborted)
|
|
722
|
+
throw cancellationError(signal);
|
|
723
|
+
throwIfAdmissionExpired(expiresAt);
|
|
724
|
+
return await new Promise((resolve, reject) => {
|
|
725
|
+
const waiter = {
|
|
726
|
+
owner,
|
|
727
|
+
keys,
|
|
728
|
+
priority,
|
|
729
|
+
resolve,
|
|
730
|
+
reject,
|
|
731
|
+
signal,
|
|
732
|
+
expiresAt,
|
|
733
|
+
deadlineTimer: null,
|
|
734
|
+
abort: () => {
|
|
735
|
+
const index = this.singletonWaiters.indexOf(waiter);
|
|
736
|
+
if (index >= 0)
|
|
737
|
+
this.singletonWaiters.splice(index, 1);
|
|
738
|
+
clearAdmissionDeadline(waiter);
|
|
739
|
+
this.clearPendingRenewal(owner);
|
|
740
|
+
reject(cancellationError(signal));
|
|
741
|
+
this.drainAdmissionWaiters();
|
|
742
|
+
},
|
|
743
|
+
expire: () => {
|
|
744
|
+
const index = this.singletonWaiters.indexOf(waiter);
|
|
745
|
+
if (index < 0)
|
|
746
|
+
return;
|
|
747
|
+
this.singletonWaiters.splice(index, 1);
|
|
748
|
+
clearAdmissionDeadline(waiter);
|
|
749
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
750
|
+
this.clearPendingRenewal(owner);
|
|
751
|
+
reject(new HostRuntimeAdmissionExpiredError());
|
|
752
|
+
this.drainAdmissionWaiters();
|
|
753
|
+
},
|
|
754
|
+
};
|
|
755
|
+
// Queue the exclusive claim before relinquishing the shared claim. This
|
|
756
|
+
// makes the promotion downgrade atomic from the broker's perspective and
|
|
757
|
+
// prevents a background readiness probe from jumping ahead in between.
|
|
758
|
+
enqueueAdmissionWaiter(this.singletonWaiters, waiter);
|
|
759
|
+
signal?.addEventListener("abort", waiter.abort, { once: true });
|
|
760
|
+
waiter.deadlineTimer = scheduleAdmissionDeadline(waiter.expiresAt, waiter.expire);
|
|
761
|
+
this.releaseParallelProjectNoDrain(parallelProjectKey, owner);
|
|
762
|
+
this.drainAdmissionWaiters();
|
|
763
|
+
});
|
|
764
|
+
}
|
|
765
|
+
async start(dispatch, worktreePath, signal, onAdmissionCheckpoint) {
|
|
766
|
+
const sandbox = dispatch.sandbox;
|
|
767
|
+
if (!sandbox || sandbox.executionMode !== "host") {
|
|
768
|
+
throw new Error("host_runtime_lease_invalid: managed host sandbox context is required.");
|
|
769
|
+
}
|
|
770
|
+
const existing = this.active.get(sandbox.dispatchAttemptId);
|
|
771
|
+
if (existing) {
|
|
772
|
+
if (existing.lease.fencingToken !== dispatch.lease.fencingToken) {
|
|
773
|
+
throw new Error("host_runtime_lease_conflict: an active host lease has a different fence.");
|
|
774
|
+
}
|
|
775
|
+
return {
|
|
776
|
+
lease: existing.lease,
|
|
777
|
+
environment: hostRuntimeEnvironment(dispatch.session.id, existing.lease, existing.runtimeContract, existing.executionClass),
|
|
778
|
+
runtimeContract: existing.runtimeContract,
|
|
779
|
+
executionClass: existing.executionClass,
|
|
780
|
+
};
|
|
781
|
+
}
|
|
782
|
+
const admission = this.admissions.get(sandbox.dispatchAttemptId);
|
|
783
|
+
const owner = { dispatchAttemptId: sandbox.dispatchAttemptId, fencingToken: dispatch.lease.fencingToken };
|
|
784
|
+
if (!admission || admission.fencingToken !== owner.fencingToken) {
|
|
785
|
+
throw new Error("host_runtime_singleton_unreserved: managed host admission is required before start.");
|
|
786
|
+
}
|
|
787
|
+
const attemptHex = uuidHex(sandbox.dispatchAttemptId, "dispatch attempt");
|
|
788
|
+
const normalizedWorktreePath = await resolveExistingDirectoryPath(worktreePath);
|
|
789
|
+
const attemptRoot = path.join(this.stateRoot, "host-attempts", attemptHex);
|
|
790
|
+
const temporaryPath = path.join(attemptRoot, "tmp");
|
|
791
|
+
const logPath = path.join(attemptRoot, "logs");
|
|
792
|
+
const cachePath = path.join(attemptRoot, "cache");
|
|
793
|
+
const browserProfilePath = path.join(attemptRoot, "browser-profile");
|
|
794
|
+
const testOutputPath = path.join(attemptRoot, "test-output");
|
|
795
|
+
const processSupervisorMetadataPath = path.join(logPath, "attempt-process-supervisor.json");
|
|
796
|
+
const processSupervisorId = `host-${attemptHex}`;
|
|
797
|
+
const renewedExpiry = this.pendingRenewals.get(sandbox.dispatchAttemptId);
|
|
798
|
+
const expiresAt = renewedExpiry?.fencingToken === dispatch.lease.fencingToken
|
|
799
|
+
? renewedExpiry.expiresAt
|
|
800
|
+
: dispatch.lease.expiresAt;
|
|
801
|
+
let processSupervisor = null;
|
|
802
|
+
try {
|
|
803
|
+
let runtimeContract;
|
|
804
|
+
try {
|
|
805
|
+
runtimeContract = await loadHostRuntimeContract(normalizedWorktreePath);
|
|
806
|
+
}
|
|
807
|
+
catch (error) {
|
|
808
|
+
if (admission.executionClass === "managed_runtime" && admission.provisionalPromotionFingerprint) {
|
|
809
|
+
await this.invalidatePromotion(dispatch.repo.id).catch(() => undefined);
|
|
810
|
+
}
|
|
811
|
+
throw error;
|
|
812
|
+
}
|
|
813
|
+
if (runtimeContract.contract.runtime.hostConcurrency === "disabled") {
|
|
814
|
+
throw new Error("host_runtime_disabled: project runtime configuration requires Docker execution.");
|
|
815
|
+
}
|
|
816
|
+
const runtimeContractFingerprint = hostRuntimeContractFingerprint(runtimeContract);
|
|
817
|
+
const promotion = admission.executionClass === "managed_runtime"
|
|
818
|
+
? await this.promotionStore.load(dispatch.repo.id)
|
|
819
|
+
: null;
|
|
820
|
+
const promotionMatches = hostRuntimeParallelEligible(runtimeContract)
|
|
821
|
+
&& promotion?.runtimeContractFingerprint === runtimeContractFingerprint;
|
|
822
|
+
if (admission.executionClass === "managed_runtime" && admission.keys.length === 0 && !promotionMatches) {
|
|
823
|
+
await this.invalidatePromotion(dispatch.repo.id);
|
|
824
|
+
const projectKeys = hostRuntimeSingletonKeys(dispatch);
|
|
825
|
+
const expiresAt = this.pendingRenewals.get(sandbox.dispatchAttemptId)?.fencingToken === owner.fencingToken
|
|
826
|
+
? this.pendingRenewals.get(sandbox.dispatchAttemptId).expiresAt
|
|
827
|
+
: dispatch.lease.expiresAt;
|
|
828
|
+
const parallelProjectKey = admission.parallelProjectKey;
|
|
829
|
+
admission.parallelProjectKey = null;
|
|
830
|
+
const serializedAdmission = parallelProjectKey
|
|
831
|
+
? this.upgradeParallelProjectToSingleton(projectKeys, parallelProjectKey, owner, expiresAt, signal, admission.priority)
|
|
832
|
+
: this.reserveSingletonKeys(projectKeys, owner, expiresAt, signal, admission.priority);
|
|
833
|
+
// The checkpoint is allowed to do I/O. Observe an early admission
|
|
834
|
+
// failure now so a slow acknowledgement cannot create an unhandled
|
|
835
|
+
// rejection before the operational promise is awaited below.
|
|
836
|
+
void serializedAdmission.catch(() => undefined);
|
|
837
|
+
// Begin the operational downgrade before publishing telemetry. A lost
|
|
838
|
+
// callback acknowledgement must not leave the attempt holding its old
|
|
839
|
+
// shared admission while every recovery probe waits behind it.
|
|
840
|
+
await onAdmissionCheckpoint?.({
|
|
841
|
+
code: "host_runtime_serial_admission_waiting",
|
|
842
|
+
message: "The prepared runtime contract no longer matches the parallel-readiness proof. Waiting for foreground-priority serialized admission before launch.",
|
|
843
|
+
expiresAt,
|
|
844
|
+
});
|
|
845
|
+
admission.keys = await serializedAdmission;
|
|
846
|
+
while (admission.keys.length === 0) {
|
|
847
|
+
const promoted = this.knownPromotions.get(dispatch.repo.id);
|
|
848
|
+
if (!promoted) {
|
|
849
|
+
throw new Error("host_runtime_parallel_promotion_missing: shared admission requires matching promotion evidence.");
|
|
850
|
+
}
|
|
851
|
+
if (promoted.runtimeContractFingerprint !== runtimeContractFingerprint) {
|
|
852
|
+
this.releaseParallelProjectNoDrain(projectKeys[0], owner);
|
|
853
|
+
await this.invalidatePromotion(dispatch.repo.id);
|
|
854
|
+
admission.keys = await this.reserveSingletonKeys(projectKeys, owner, expiresAt, signal, admission.priority);
|
|
855
|
+
continue;
|
|
856
|
+
}
|
|
857
|
+
admission.parallelProjectKey = projectKeys[0] ?? null;
|
|
858
|
+
admission.provisionalPromotionFingerprint = promoted.runtimeContractFingerprint;
|
|
859
|
+
}
|
|
860
|
+
}
|
|
861
|
+
const declaredSingletonKeys = runtimeContract.contract.runtime.singletonResources
|
|
862
|
+
.map(hostRuntimeDeclaredSingletonKey);
|
|
863
|
+
const missingSingletonKeys = declaredSingletonKeys
|
|
864
|
+
.filter(key => !admission.keys.includes(key));
|
|
865
|
+
if (missingSingletonKeys.length > 0) {
|
|
866
|
+
const expiresAt = this.pendingRenewals.get(sandbox.dispatchAttemptId)?.fencingToken === owner.fencingToken
|
|
867
|
+
? this.pendingRenewals.get(sandbox.dispatchAttemptId).expiresAt
|
|
868
|
+
: dispatch.lease.expiresAt;
|
|
869
|
+
const reservedSingletonKeys = await this.reserveSingletonKeys(missingSingletonKeys, owner, expiresAt, signal, admission.priority);
|
|
870
|
+
admission.keys = [...admission.keys, ...reservedSingletonKeys]
|
|
871
|
+
.sort((left, right) => left.localeCompare(right));
|
|
872
|
+
}
|
|
873
|
+
const singletonKeys = admission.keys;
|
|
874
|
+
if (!singletonKeys.every(key => sameSingletonOwner(this.singletonOwners.get(key), owner))) {
|
|
875
|
+
throw new Error("host_runtime_singleton_unreserved: managed host singleton ownership is required before start.");
|
|
876
|
+
}
|
|
877
|
+
if (admission.parallelProjectKey && !this.parallelProjectOwned(admission.parallelProjectKey, owner)) {
|
|
878
|
+
throw new Error("host_runtime_parallel_unreserved: matching shared project admission is required before start.");
|
|
879
|
+
}
|
|
880
|
+
await Promise.all([
|
|
881
|
+
ensurePrivateDirectory(temporaryPath),
|
|
882
|
+
ensurePrivateDirectory(logPath),
|
|
883
|
+
ensurePrivateDirectory(cachePath),
|
|
884
|
+
ensurePrivateDirectory(browserProfilePath),
|
|
885
|
+
ensurePrivateDirectory(testOutputPath),
|
|
886
|
+
]);
|
|
887
|
+
await resetGeneratedOutputs(normalizedWorktreePath, testOutputPath);
|
|
888
|
+
const inputCommitSha = await resolveWorktreeCommitSha(normalizedWorktreePath, sandbox.inputCommitSha);
|
|
889
|
+
let lease = createRuntimeIsolationLease({
|
|
890
|
+
leaseId: sandbox.executionLeaseId,
|
|
891
|
+
dispatchAttemptId: sandbox.dispatchAttemptId,
|
|
892
|
+
executionLeaseId: sandbox.executionLeaseId,
|
|
893
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
894
|
+
runnerId: dispatch.execution.runnerId ?? "",
|
|
895
|
+
runnerInstanceId: sandbox.runnerInstanceId,
|
|
896
|
+
worktreeKey: dispatch.lease.worktreeKey,
|
|
897
|
+
worktreePath: normalizedWorktreePath,
|
|
898
|
+
mode: "host",
|
|
899
|
+
expiresAt,
|
|
900
|
+
resources: {
|
|
901
|
+
memoryReservationBytes: 0,
|
|
902
|
+
cpuMillis: 0,
|
|
903
|
+
pids: 0,
|
|
904
|
+
ports: [],
|
|
905
|
+
paths: [
|
|
906
|
+
{ path: normalizedWorktreePath, role: "worktree" },
|
|
907
|
+
{ path: temporaryPath, role: "temporary" },
|
|
908
|
+
{ path: logPath, role: "logs" },
|
|
909
|
+
{ path: cachePath, role: "cache" },
|
|
910
|
+
{ path: browserProfilePath, role: "browser_profile" },
|
|
911
|
+
{ path: testOutputPath, role: "test_output" },
|
|
912
|
+
{ path: processSupervisorMetadataPath, role: "process_supervisor_metadata" },
|
|
913
|
+
],
|
|
914
|
+
processSupervisorId,
|
|
915
|
+
dockerResources: [],
|
|
916
|
+
singletonKeys,
|
|
917
|
+
},
|
|
918
|
+
});
|
|
919
|
+
await this.writeLedger(lease);
|
|
920
|
+
lease = transitionRuntimeIsolationLease(lease, "starting");
|
|
921
|
+
await this.writeLedger(lease);
|
|
922
|
+
lease = transitionRuntimeIsolationLease(lease, "active");
|
|
923
|
+
await this.writeLedger(lease);
|
|
924
|
+
processSupervisor = this.processSupervisors.register({
|
|
925
|
+
processSupervisorId,
|
|
926
|
+
dispatchAttemptId: lease.dispatchAttemptId,
|
|
927
|
+
executionLeaseId: lease.executionLeaseId,
|
|
928
|
+
fencingToken: lease.fencingToken,
|
|
929
|
+
logDirectory: logPath,
|
|
930
|
+
}, this.processRunner, { signal });
|
|
931
|
+
this.active.set(sandbox.dispatchAttemptId, {
|
|
932
|
+
lease,
|
|
933
|
+
repoId: dispatch.repo.id,
|
|
934
|
+
sessionId: dispatch.session.id,
|
|
935
|
+
attemptRoot,
|
|
936
|
+
runtimeContract,
|
|
937
|
+
runtimeContractFingerprint,
|
|
938
|
+
inputCommitSha,
|
|
939
|
+
executionClass: admission.executionClass,
|
|
940
|
+
parallelPromotionEligible: hostRuntimeParallelEligible(runtimeContract),
|
|
941
|
+
processSupervisor,
|
|
942
|
+
adapterCleanups: [],
|
|
943
|
+
pendingAdapterLaunches: new Map(),
|
|
944
|
+
releasing: false,
|
|
945
|
+
});
|
|
946
|
+
this.pendingRenewals.delete(sandbox.dispatchAttemptId);
|
|
947
|
+
return {
|
|
948
|
+
lease,
|
|
949
|
+
environment: hostRuntimeEnvironment(dispatch.session.id, lease, runtimeContract, admission.executionClass),
|
|
950
|
+
runtimeContract,
|
|
951
|
+
executionClass: admission.executionClass,
|
|
952
|
+
};
|
|
953
|
+
}
|
|
954
|
+
catch (error) {
|
|
955
|
+
this.pendingRenewals.delete(sandbox.dispatchAttemptId);
|
|
956
|
+
if (processSupervisor) {
|
|
957
|
+
await processSupervisor.stopAll(dispatch.lease.fencingToken, "Host runtime start rolled back.").catch(() => undefined);
|
|
958
|
+
this.processSupervisors.unregister(sandbox.dispatchAttemptId, dispatch.lease.fencingToken);
|
|
959
|
+
}
|
|
960
|
+
await fs.rm(attemptRoot, { recursive: true, force: true }).catch(() => undefined);
|
|
961
|
+
await fs.rm(this.ledgerPath(sandbox.dispatchAttemptId), { force: true }).catch(() => undefined);
|
|
962
|
+
this.releaseSingletonKeys(sandbox.dispatchAttemptId, dispatch.lease.fencingToken);
|
|
963
|
+
this.releaseParallelProjects(sandbox.dispatchAttemptId, dispatch.lease.fencingToken);
|
|
964
|
+
this.deleteAdmission(sandbox.dispatchAttemptId, dispatch.lease.fencingToken);
|
|
965
|
+
throw error;
|
|
966
|
+
}
|
|
967
|
+
}
|
|
968
|
+
async updateLease(dispatchAttemptId, fencingToken, expiresAt) {
|
|
969
|
+
try {
|
|
970
|
+
uuidHex(dispatchAttemptId, "dispatch attempt");
|
|
971
|
+
uuidHex(fencingToken, "fencing token");
|
|
972
|
+
}
|
|
973
|
+
catch {
|
|
974
|
+
return;
|
|
975
|
+
}
|
|
976
|
+
const active = this.active.get(dispatchAttemptId);
|
|
977
|
+
if (!active) {
|
|
978
|
+
if (Number.isFinite(Date.parse(expiresAt))) {
|
|
979
|
+
const pending = this.pendingRenewals.get(dispatchAttemptId);
|
|
980
|
+
if (!pending
|
|
981
|
+
|| pending.fencingToken !== fencingToken
|
|
982
|
+
|| Date.parse(expiresAt) > Date.parse(pending.expiresAt)) {
|
|
983
|
+
this.pendingRenewals.set(dispatchAttemptId, { fencingToken, expiresAt });
|
|
984
|
+
}
|
|
985
|
+
this.renewAdmissionWaiterDeadline(dispatchAttemptId, fencingToken, expiresAt);
|
|
986
|
+
}
|
|
987
|
+
return;
|
|
988
|
+
}
|
|
989
|
+
const renewed = renewRuntimeIsolationLease(active.lease, fencingToken, expiresAt);
|
|
990
|
+
if (!renewed)
|
|
991
|
+
return;
|
|
992
|
+
active.lease = renewed;
|
|
993
|
+
await this.writeLedger(renewed);
|
|
994
|
+
}
|
|
995
|
+
async startConfiguredProject(dispatchAttemptId, fencingToken, signal, onProjectCheckpoint) {
|
|
996
|
+
const active = this.active.get(dispatchAttemptId);
|
|
997
|
+
if (!active || active.lease.fencingToken !== fencingToken || active.lease.state !== "active" || active.releasing) {
|
|
998
|
+
throw new Error("host_runtime_project_unavailable: an active matching host runtime lease is required.");
|
|
999
|
+
}
|
|
1000
|
+
if (active.executionClass === "analysis_only") {
|
|
1001
|
+
return {
|
|
1002
|
+
lease: active.lease,
|
|
1003
|
+
environment: hostRuntimeEnvironment(active.sessionId, active.lease, active.runtimeContract, active.executionClass),
|
|
1004
|
+
runtimeContract: active.runtimeContract,
|
|
1005
|
+
executionClass: active.executionClass,
|
|
1006
|
+
};
|
|
1007
|
+
}
|
|
1008
|
+
const configuredPorts = active.runtimeContract.contract.runtime.ports.filter((port) => port.launch !== null);
|
|
1009
|
+
if (configuredPorts.length === 0) {
|
|
1010
|
+
return {
|
|
1011
|
+
lease: active.lease,
|
|
1012
|
+
environment: hostRuntimeEnvironment(active.sessionId, active.lease, active.runtimeContract, active.executionClass),
|
|
1013
|
+
runtimeContract: active.runtimeContract,
|
|
1014
|
+
executionClass: active.executionClass,
|
|
1015
|
+
};
|
|
1016
|
+
}
|
|
1017
|
+
const owner = hostPortOwner(active.lease);
|
|
1018
|
+
const logDirectory = active.lease.resources.paths.find(entry => entry.role === "logs")?.path;
|
|
1019
|
+
if (!logDirectory)
|
|
1020
|
+
throw new Error("host_runtime_project_invalid: attempt log directory is unavailable.");
|
|
1021
|
+
const launcher = new AttemptNodeRuntimeLauncher(active.processSupervisor, owner, logDirectory, this.listenerInspector);
|
|
1022
|
+
for (const configuredPort of configuredPorts) {
|
|
1023
|
+
const workingDirectory = await containedWorkingDirectory(active.lease.worktreePath, configuredPort.launch.workingDirectory);
|
|
1024
|
+
const launchEnvironment = {
|
|
1025
|
+
...configuredPort.launch.environment,
|
|
1026
|
+
...hostRuntimeEnvironment(active.sessionId, active.lease, active.runtimeContract, active.executionClass),
|
|
1027
|
+
};
|
|
1028
|
+
const prebuild = createAspNetRuntimePrebuildPlan({
|
|
1029
|
+
command: configuredPort.launch.command,
|
|
1030
|
+
args: configuredPort.launch.args,
|
|
1031
|
+
cwd: workingDirectory,
|
|
1032
|
+
worktreePath: active.lease.worktreePath,
|
|
1033
|
+
environment: launchEnvironment,
|
|
1034
|
+
inputCommitSha: active.inputCommitSha,
|
|
1035
|
+
runtimeContractFingerprint: active.runtimeContractFingerprint,
|
|
1036
|
+
});
|
|
1037
|
+
if (prebuild) {
|
|
1038
|
+
await onProjectCheckpoint?.({
|
|
1039
|
+
code: "host_runtime_build_started",
|
|
1040
|
+
message: `Preparing the ASP.NET runtime for commit ${active.inputCommitSha.slice(0, 12)} before binding its assigned port.`,
|
|
1041
|
+
data: { input_commit_sha: active.inputCommitSha, project_path: prebuild.projectPath },
|
|
1042
|
+
});
|
|
1043
|
+
const build = await ensureAspNetRuntimePrebuilt(prebuild, async (plan, buildSignal) => await executeAspNetRuntimeBuild(active.processSupervisor, logDirectory, plan, buildSignal), signal);
|
|
1044
|
+
await onProjectCheckpoint?.({
|
|
1045
|
+
code: build.cacheHit ? "host_runtime_build_cached" : "host_runtime_build_completed",
|
|
1046
|
+
message: build.cacheHit
|
|
1047
|
+
? `Reused the verified ASP.NET build for commit ${active.inputCommitSha.slice(0, 12)}.`
|
|
1048
|
+
: `ASP.NET runtime build completed for commit ${active.inputCommitSha.slice(0, 12)}; launching without another build.`,
|
|
1049
|
+
data: {
|
|
1050
|
+
input_commit_sha: active.inputCommitSha,
|
|
1051
|
+
build_fingerprint: build.fingerprint,
|
|
1052
|
+
receipt_path: build.receiptPath,
|
|
1053
|
+
},
|
|
1054
|
+
});
|
|
1055
|
+
}
|
|
1056
|
+
await this.launchNodeRuntime(dispatchAttemptId, fencingToken, configuredPort.name, {
|
|
1057
|
+
command: configuredPort.launch.command,
|
|
1058
|
+
args: configuredPort.launch.args,
|
|
1059
|
+
cwd: workingDirectory,
|
|
1060
|
+
environment: launchEnvironment,
|
|
1061
|
+
}, launcher, signal);
|
|
1062
|
+
}
|
|
1063
|
+
const promotedLease = await this.promoteVerifiedRuntime(active, active.lease);
|
|
1064
|
+
const runtimeEnvironment = hostRuntimeEnvironment(active.sessionId, promotedLease, active.runtimeContract, active.executionClass);
|
|
1065
|
+
const playwright = active.runtimeContract.contract.runtime.playwright;
|
|
1066
|
+
if (!playwright) {
|
|
1067
|
+
return {
|
|
1068
|
+
lease: promotedLease,
|
|
1069
|
+
environment: runtimeEnvironment,
|
|
1070
|
+
runtimeContract: active.runtimeContract,
|
|
1071
|
+
executionClass: active.executionClass,
|
|
1072
|
+
};
|
|
1073
|
+
}
|
|
1074
|
+
const playwrightPort = promotedLease.resources.ports.find(port => port.name === playwright.baseUrlPort);
|
|
1075
|
+
const playwrightPortConfig = active.runtimeContract.contract.runtime.ports
|
|
1076
|
+
.find(port => port.name === playwright.baseUrlPort);
|
|
1077
|
+
if (!playwrightPort || !playwrightPortConfig?.baseUrlEnv) {
|
|
1078
|
+
throw new Error("host_runtime_playwright_unavailable: the configured verified base URL is unavailable.");
|
|
1079
|
+
}
|
|
1080
|
+
if (playwrightPort.hostIp !== "127.0.0.1") {
|
|
1081
|
+
throw new Error("host_runtime_playwright_unavailable: Playwright requires an IPv4 loopback listener.");
|
|
1082
|
+
}
|
|
1083
|
+
const baseUrl = hostPortEnvironment(playwrightPortConfig, playwrightPort.hostIp, playwrightPort.hostPort)[playwrightPortConfig.baseUrlEnv];
|
|
1084
|
+
if (!baseUrl)
|
|
1085
|
+
throw new Error("host_runtime_playwright_unavailable: the configured base URL could not be rendered.");
|
|
1086
|
+
const pathFor = (role) => promotedLease.resources.paths.find(entry => entry.role === role)?.path ?? "";
|
|
1087
|
+
const preparedPlaywright = await this.playwrightAdapter.prepare(playwright, {
|
|
1088
|
+
worktreePath: promotedLease.worktreePath,
|
|
1089
|
+
temporaryDirectory: pathFor("temporary"),
|
|
1090
|
+
testOutputDirectory: pathFor("test_output"),
|
|
1091
|
+
browserProfileDirectory: pathFor("browser_profile"),
|
|
1092
|
+
baseUrl,
|
|
1093
|
+
ownershipToken: uuidHex(promotedLease.dispatchAttemptId, "dispatch attempt"),
|
|
1094
|
+
});
|
|
1095
|
+
return {
|
|
1096
|
+
lease: promotedLease,
|
|
1097
|
+
environment: { ...runtimeEnvironment, ...preparedPlaywright.environment },
|
|
1098
|
+
runtimeContract: active.runtimeContract,
|
|
1099
|
+
executionClass: active.executionClass,
|
|
1100
|
+
};
|
|
1101
|
+
}
|
|
1102
|
+
providerProcessRunner(dispatchAttemptId, fencingToken) {
|
|
1103
|
+
const active = this.active.get(dispatchAttemptId);
|
|
1104
|
+
if (!active || active.lease.fencingToken !== fencingToken || active.releasing)
|
|
1105
|
+
return null;
|
|
1106
|
+
return new AttemptSupervisedProviderProcessRunner(active.processSupervisor, hostPortOwner(active.lease), this.listenerInspector, active.executionClass === "analysis_only", () => this.markCapacityReservationRunning(dispatchAttemptId, fencingToken));
|
|
1107
|
+
}
|
|
1108
|
+
markCapacityReservationRunning(dispatchAttemptId, fencingToken) {
|
|
1109
|
+
const reservation = this.capacityReservations.get(dispatchAttemptId);
|
|
1110
|
+
if (reservation?.owner.fencingToken === fencingToken)
|
|
1111
|
+
reservation.state = "running";
|
|
1112
|
+
}
|
|
1113
|
+
providerMemoryTelemetry(dispatchAttemptId, fencingToken) {
|
|
1114
|
+
const active = this.active.get(dispatchAttemptId);
|
|
1115
|
+
if (!active || active.lease.fencingToken !== fencingToken || active.releasing)
|
|
1116
|
+
return null;
|
|
1117
|
+
return active.processSupervisor.providerMemoryTelemetry();
|
|
1118
|
+
}
|
|
1119
|
+
async promoteVerifiedRuntime(active, runtimeLease) {
|
|
1120
|
+
if (!active.parallelPromotionEligible)
|
|
1121
|
+
return runtimeLease;
|
|
1122
|
+
const verifiedPortNames = runtimeLease.resources.ports.map(port => port.name).sort((left, right) => left.localeCompare(right));
|
|
1123
|
+
const requiredPortNames = configuredLaunchPortNames(active.runtimeContract);
|
|
1124
|
+
if (requiredPortNames.length !== verifiedPortNames.length
|
|
1125
|
+
|| requiredPortNames.some((name, index) => name !== verifiedPortNames[index])) {
|
|
1126
|
+
return runtimeLease;
|
|
1127
|
+
}
|
|
1128
|
+
const projectKey = projectHostRuntimeSingletonKey(active.repoId);
|
|
1129
|
+
const promotedLease = withRuntimeIsolationResources(runtimeLease, {
|
|
1130
|
+
...runtimeLease.resources,
|
|
1131
|
+
singletonKeys: runtimeLease.resources.singletonKeys.filter(key => key !== projectKey),
|
|
1132
|
+
});
|
|
1133
|
+
await this.writeLedger(promotedLease);
|
|
1134
|
+
active.lease = promotedLease;
|
|
1135
|
+
try {
|
|
1136
|
+
const promotion = await this.promotionStore.observe(active.repoId, active.runtimeContract, runtimeLease.dispatchAttemptId, verifiedPortNames);
|
|
1137
|
+
this.knownPromotions.set(promotion.repoId, promotion);
|
|
1138
|
+
this.convertSingletonToParallelProject(projectKey, {
|
|
1139
|
+
dispatchAttemptId: runtimeLease.dispatchAttemptId,
|
|
1140
|
+
fencingToken: runtimeLease.fencingToken,
|
|
1141
|
+
}, promotion);
|
|
1142
|
+
const admission = this.admissions.get(runtimeLease.dispatchAttemptId);
|
|
1143
|
+
if (admission?.fencingToken === runtimeLease.fencingToken) {
|
|
1144
|
+
this.admissions.set(runtimeLease.dispatchAttemptId, {
|
|
1145
|
+
...admission,
|
|
1146
|
+
keys: promotedLease.resources.singletonKeys,
|
|
1147
|
+
provisionalPromotionFingerprint: promotion.runtimeContractFingerprint,
|
|
1148
|
+
parallelProjectKey: projectKey,
|
|
1149
|
+
});
|
|
1150
|
+
}
|
|
1151
|
+
return promotedLease;
|
|
1152
|
+
}
|
|
1153
|
+
catch {
|
|
1154
|
+
await this.invalidatePromotion(active.repoId).catch(() => undefined);
|
|
1155
|
+
active.lease = runtimeLease;
|
|
1156
|
+
await this.writeLedger(runtimeLease);
|
|
1157
|
+
return runtimeLease;
|
|
1158
|
+
}
|
|
1159
|
+
}
|
|
1160
|
+
async launchNodeRuntime(dispatchAttemptId, fencingToken, portName, command, launcher, signal) {
|
|
1161
|
+
const active = this.active.get(dispatchAttemptId);
|
|
1162
|
+
if (!active || active.lease.fencingToken !== fencingToken || active.lease.state !== "active" || active.releasing) {
|
|
1163
|
+
throw new Error("host_runtime_adapter_unavailable: an active matching host runtime lease is required.");
|
|
1164
|
+
}
|
|
1165
|
+
const portConfig = active.runtimeContract.contract.runtime.ports.find(port => port.name === portName);
|
|
1166
|
+
if (!portConfig)
|
|
1167
|
+
throw new Error(`host_runtime_adapter_invalid: configured port '${portName}' was not found.`);
|
|
1168
|
+
if (active.lease.resources.ports.some(port => port.name === portName) || active.pendingAdapterLaunches.has(portName)) {
|
|
1169
|
+
throw new Error(`host_runtime_adapter_conflict: configured port '${portName}' is already active.`);
|
|
1170
|
+
}
|
|
1171
|
+
const controller = linkedAbortController(signal);
|
|
1172
|
+
let retainedByActiveRuntime = false;
|
|
1173
|
+
const operation = (async () => {
|
|
1174
|
+
const owner = hostPortOwner(active.lease);
|
|
1175
|
+
const launched = await this.nodeAdapter.launch(owner, portConfig, command, launcher, controller.signal);
|
|
1176
|
+
let cleaned = false;
|
|
1177
|
+
const cleanup = async () => {
|
|
1178
|
+
if (cleaned)
|
|
1179
|
+
return;
|
|
1180
|
+
await launcher.stop(launched.value);
|
|
1181
|
+
await this.portBroker.release(launched.reservation.reservationId, owner);
|
|
1182
|
+
cleaned = true;
|
|
1183
|
+
unlinkAbortController(controller, signal);
|
|
1184
|
+
};
|
|
1185
|
+
active.adapterCleanups.push(cleanup);
|
|
1186
|
+
try {
|
|
1187
|
+
if (active.releasing)
|
|
1188
|
+
throw new Error("host_runtime_adapter_cancelled: host runtime release started during launch.");
|
|
1189
|
+
const runtimeLease = withRuntimeIsolationResources(active.lease, {
|
|
1190
|
+
...active.lease.resources,
|
|
1191
|
+
ports: [...active.lease.resources.ports, {
|
|
1192
|
+
name: launched.reservation.name,
|
|
1193
|
+
protocol: launched.reservation.protocol,
|
|
1194
|
+
hostIp: launched.reservation.hostIp,
|
|
1195
|
+
hostPort: launched.reservation.hostPort,
|
|
1196
|
+
targetPort: launched.reservation.targetPort,
|
|
1197
|
+
}],
|
|
1198
|
+
});
|
|
1199
|
+
await this.writeLedger(runtimeLease);
|
|
1200
|
+
active.lease = runtimeLease;
|
|
1201
|
+
retainedByActiveRuntime = true;
|
|
1202
|
+
return { ...launched, runtimeLease };
|
|
1203
|
+
}
|
|
1204
|
+
catch (error) {
|
|
1205
|
+
try {
|
|
1206
|
+
await cleanup();
|
|
1207
|
+
removeCleanup(active.adapterCleanups, cleanup);
|
|
1208
|
+
}
|
|
1209
|
+
catch {
|
|
1210
|
+
// Preserve the cleanup and broker claim for the fenced release retry.
|
|
1211
|
+
}
|
|
1212
|
+
throw error;
|
|
1213
|
+
}
|
|
1214
|
+
})();
|
|
1215
|
+
active.pendingAdapterLaunches.set(portName, { controller, promise: operation });
|
|
1216
|
+
try {
|
|
1217
|
+
return await operation;
|
|
1218
|
+
}
|
|
1219
|
+
finally {
|
|
1220
|
+
active.pendingAdapterLaunches.delete(portName);
|
|
1221
|
+
if (!retainedByActiveRuntime)
|
|
1222
|
+
unlinkAbortController(controller, signal);
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
async release(dispatchAttemptId, fencingToken) {
|
|
1226
|
+
const active = this.active.get(dispatchAttemptId);
|
|
1227
|
+
this.pendingRenewals.delete(dispatchAttemptId);
|
|
1228
|
+
if (!active) {
|
|
1229
|
+
this.cancelAdmissionWaiters(dispatchAttemptId, fencingToken, "Managed host admission was released before acquisition.");
|
|
1230
|
+
this.releaseSingletonKeys(dispatchAttemptId, fencingToken);
|
|
1231
|
+
this.releaseParallelProjects(dispatchAttemptId, fencingToken);
|
|
1232
|
+
this.deleteAdmission(dispatchAttemptId, fencingToken);
|
|
1233
|
+
return null;
|
|
1234
|
+
}
|
|
1235
|
+
if (active.lease.fencingToken !== fencingToken)
|
|
1236
|
+
return null;
|
|
1237
|
+
active.releasing = true;
|
|
1238
|
+
for (const pending of active.pendingAdapterLaunches.values()) {
|
|
1239
|
+
pending.controller.abort("Host runtime release cancelled the pending adapter launch.");
|
|
1240
|
+
}
|
|
1241
|
+
await Promise.allSettled([...active.pendingAdapterLaunches.values()].map(pending => pending.promise));
|
|
1242
|
+
let releasing = transitionRuntimeIsolationLease(active.lease, "releasing");
|
|
1243
|
+
active.lease = releasing;
|
|
1244
|
+
await this.writeLedger(releasing);
|
|
1245
|
+
try {
|
|
1246
|
+
await active.processSupervisor.stopAll(fencingToken, "Host runtime lease released.");
|
|
1247
|
+
await releaseAdapterResources(active.adapterCleanups);
|
|
1248
|
+
await fs.rm(active.attemptRoot, { recursive: true, force: true });
|
|
1249
|
+
releasing = transitionRuntimeIsolationLease(releasing, "released");
|
|
1250
|
+
active.lease = releasing;
|
|
1251
|
+
await this.writeLedger(releasing);
|
|
1252
|
+
await fs.rm(this.ledgerPath(dispatchAttemptId), { force: true });
|
|
1253
|
+
this.active.delete(dispatchAttemptId);
|
|
1254
|
+
this.processSupervisors.unregister(dispatchAttemptId, fencingToken);
|
|
1255
|
+
this.releaseSingletonKeys(dispatchAttemptId, fencingToken);
|
|
1256
|
+
this.releaseParallelProjects(dispatchAttemptId, fencingToken);
|
|
1257
|
+
this.deleteAdmission(dispatchAttemptId, fencingToken);
|
|
1258
|
+
return releasing;
|
|
1259
|
+
}
|
|
1260
|
+
catch (error) {
|
|
1261
|
+
const degraded = transitionRuntimeIsolationLease(releasing, "degraded");
|
|
1262
|
+
active.lease = degraded;
|
|
1263
|
+
await this.writeLedger(degraded).catch(() => undefined);
|
|
1264
|
+
throw new HostRuntimeLeaseCleanupError(degraded, error);
|
|
1265
|
+
}
|
|
1266
|
+
}
|
|
1267
|
+
async readLedgers() {
|
|
1268
|
+
const root = path.join(this.stateRoot, "runtime-ledger", "host");
|
|
1269
|
+
const entries = await fs.readdir(root, { withFileTypes: true }).catch(error => {
|
|
1270
|
+
if (error.code === "ENOENT")
|
|
1271
|
+
return [];
|
|
1272
|
+
throw error;
|
|
1273
|
+
});
|
|
1274
|
+
const ledgers = [];
|
|
1275
|
+
for (const entry of entries) {
|
|
1276
|
+
if (!entry.isFile() || !/^[0-9a-f]{32}\.json$/u.test(entry.name))
|
|
1277
|
+
continue;
|
|
1278
|
+
try {
|
|
1279
|
+
const value = JSON.parse(await fs.readFile(path.join(root, entry.name), "utf8"));
|
|
1280
|
+
if (!validHostRuntimeLedger(value))
|
|
1281
|
+
continue;
|
|
1282
|
+
const expectedName = `${uuidHex(value.runtimeLease.dispatchAttemptId, "dispatch attempt")}.json`;
|
|
1283
|
+
if (entry.name !== expectedName)
|
|
1284
|
+
continue;
|
|
1285
|
+
ledgers.push(value);
|
|
1286
|
+
}
|
|
1287
|
+
catch {
|
|
1288
|
+
// An invalid ledger has no trustworthy ownership identity and is never reaped.
|
|
1289
|
+
}
|
|
1290
|
+
}
|
|
1291
|
+
return ledgers;
|
|
1292
|
+
}
|
|
1293
|
+
async recoverPersistedProcesses(lease, attemptRoot) {
|
|
1294
|
+
const logsRoot = path.join(attemptRoot, "logs");
|
|
1295
|
+
const expectedAttemptRoot = path.resolve(this.attemptRoot(lease.dispatchAttemptId));
|
|
1296
|
+
if (path.resolve(attemptRoot) !== expectedAttemptRoot) {
|
|
1297
|
+
throw new Error("host_runtime_reaper_invalid: attempt path is outside the fenced attempt root.");
|
|
1298
|
+
}
|
|
1299
|
+
const entries = await fs.readdir(logsRoot, { withFileTypes: true }).catch(error => {
|
|
1300
|
+
if (error.code === "ENOENT")
|
|
1301
|
+
return [];
|
|
1302
|
+
throw error;
|
|
1303
|
+
});
|
|
1304
|
+
const outcomes = [];
|
|
1305
|
+
for (const entry of entries) {
|
|
1306
|
+
if (!entry.isFile() || !/^process-supervisor\.[a-z][a-z0-9_-]{0,31}\.\d+\.json$/u.test(entry.name))
|
|
1307
|
+
continue;
|
|
1308
|
+
const metadataPath = path.join(logsRoot, entry.name);
|
|
1309
|
+
const value = JSON.parse(await fs.readFile(metadataPath, "utf8"));
|
|
1310
|
+
if (!validHostProcessSupervisorMetadata(value, lease)) {
|
|
1311
|
+
throw new Error("host_runtime_reaper_invalid: process ownership metadata did not match the runtime lease.");
|
|
1312
|
+
}
|
|
1313
|
+
outcomes.push(await this.processRecovery.recover(value));
|
|
1314
|
+
}
|
|
1315
|
+
return outcomes;
|
|
1316
|
+
}
|
|
1317
|
+
async markLedgerDegraded(ledger) {
|
|
1318
|
+
const lease = ledger.runtimeLease;
|
|
1319
|
+
if (lease.state === "released")
|
|
1320
|
+
return;
|
|
1321
|
+
const degraded = transitionHostLeaseForCleanup(lease, "degraded");
|
|
1322
|
+
const repoId = ledger.repoId ?? repoIdFromProjectSingletonKeys(lease.resources.singletonKeys);
|
|
1323
|
+
this.rememberDegraded(degraded, repoId);
|
|
1324
|
+
await this.writeLedger(degraded, repoId ?? undefined).catch(() => undefined);
|
|
1325
|
+
}
|
|
1326
|
+
rememberDegraded(lease, repoId) {
|
|
1327
|
+
this.degraded.set(lease.dispatchAttemptId, { lease, repoId });
|
|
1328
|
+
if (!repoId)
|
|
1329
|
+
return;
|
|
1330
|
+
const owner = { dispatchAttemptId: lease.dispatchAttemptId, fencingToken: lease.fencingToken };
|
|
1331
|
+
const projectKey = projectHostRuntimeSingletonKey(repoId);
|
|
1332
|
+
const existing = this.singletonOwners.get(projectKey);
|
|
1333
|
+
if (!existing || sameSingletonOwner(existing, owner))
|
|
1334
|
+
this.singletonOwners.set(projectKey, owner);
|
|
1335
|
+
}
|
|
1336
|
+
forgetDegraded(dispatchAttemptId, fencingToken) {
|
|
1337
|
+
if (this.degraded.get(dispatchAttemptId)?.lease.fencingToken !== fencingToken)
|
|
1338
|
+
return;
|
|
1339
|
+
this.degraded.delete(dispatchAttemptId);
|
|
1340
|
+
this.releaseSingletonKeys(dispatchAttemptId, fencingToken);
|
|
1341
|
+
this.releaseParallelProjects(dispatchAttemptId, fencingToken);
|
|
1342
|
+
}
|
|
1343
|
+
attemptRoot(dispatchAttemptId) {
|
|
1344
|
+
return path.join(this.stateRoot, "host-attempts", uuidHex(dispatchAttemptId, "dispatch attempt"));
|
|
1345
|
+
}
|
|
1346
|
+
async writeLedger(runtimeLease, explicitRepoId) {
|
|
1347
|
+
const root = path.join(this.stateRoot, "runtime-ledger", "host");
|
|
1348
|
+
await ensurePrivateDirectory(root);
|
|
1349
|
+
const target = this.ledgerPath(runtimeLease.dispatchAttemptId);
|
|
1350
|
+
const temporary = `${target}.${process.pid}.${Date.now()}.tmp`;
|
|
1351
|
+
try {
|
|
1352
|
+
const handle = await fs.open(temporary, "wx", 0o600);
|
|
1353
|
+
try {
|
|
1354
|
+
const repoId = explicitRepoId
|
|
1355
|
+
?? this.active.get(runtimeLease.dispatchAttemptId)?.repoId
|
|
1356
|
+
?? this.admissions.get(runtimeLease.dispatchAttemptId)?.repoId
|
|
1357
|
+
?? this.degraded.get(runtimeLease.dispatchAttemptId)?.repoId
|
|
1358
|
+
?? undefined;
|
|
1359
|
+
const ledger = {
|
|
1360
|
+
schemaVersion: 1,
|
|
1361
|
+
...(repoId ? { repoId } : {}),
|
|
1362
|
+
runtimeLease,
|
|
1363
|
+
};
|
|
1364
|
+
await handle.writeFile(`${JSON.stringify(ledger)}\n`);
|
|
1365
|
+
await handle.sync();
|
|
1366
|
+
}
|
|
1367
|
+
finally {
|
|
1368
|
+
await handle.close();
|
|
1369
|
+
}
|
|
1370
|
+
await fs.rename(temporary, target);
|
|
1371
|
+
}
|
|
1372
|
+
catch (error) {
|
|
1373
|
+
await fs.rm(temporary, { force: true }).catch(() => undefined);
|
|
1374
|
+
throw error;
|
|
1375
|
+
}
|
|
1376
|
+
}
|
|
1377
|
+
ledgerPath(dispatchAttemptId) {
|
|
1378
|
+
const attemptHex = uuidHex(dispatchAttemptId, "dispatch attempt");
|
|
1379
|
+
return path.join(this.stateRoot, "runtime-ledger", "host", `${attemptHex}.json`);
|
|
1380
|
+
}
|
|
1381
|
+
singletonKeysAvailable(keys) {
|
|
1382
|
+
return keys.every(key => !this.singletonOwners.has(key) && !this.parallelProjectOwners.has(key));
|
|
1383
|
+
}
|
|
1384
|
+
assignSingletonKeys(keys, owner) {
|
|
1385
|
+
for (const key of keys)
|
|
1386
|
+
this.singletonOwners.set(key, owner);
|
|
1387
|
+
}
|
|
1388
|
+
releaseSingletonKeys(dispatchAttemptId, fencingToken) {
|
|
1389
|
+
for (const [key, owner] of this.singletonOwners) {
|
|
1390
|
+
if (owner.dispatchAttemptId === dispatchAttemptId && owner.fencingToken === fencingToken) {
|
|
1391
|
+
this.singletonOwners.delete(key);
|
|
1392
|
+
}
|
|
1393
|
+
}
|
|
1394
|
+
this.drainAdmissionWaiters();
|
|
1395
|
+
}
|
|
1396
|
+
convertSingletonToParallelProject(key, owner, promotion) {
|
|
1397
|
+
if (!sameSingletonOwner(this.singletonOwners.get(key), owner)) {
|
|
1398
|
+
throw new Error("host_runtime_parallel_promotion_stale: matching exclusive project admission is required.");
|
|
1399
|
+
}
|
|
1400
|
+
this.assignParallelProject(key, owner);
|
|
1401
|
+
this.singletonOwners.delete(key);
|
|
1402
|
+
for (let index = 0; index < this.singletonWaiters.length;) {
|
|
1403
|
+
const waiter = this.singletonWaiters[index];
|
|
1404
|
+
if (waiter.priority !== "foreground" || waiter.keys.length !== 1 || waiter.keys[0] !== key) {
|
|
1405
|
+
index += 1;
|
|
1406
|
+
continue;
|
|
1407
|
+
}
|
|
1408
|
+
this.singletonWaiters.splice(index, 1);
|
|
1409
|
+
clearAdmissionDeadline(waiter);
|
|
1410
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
1411
|
+
this.assignParallelProject(key, waiter.owner);
|
|
1412
|
+
this.knownPromotions.set(promotion.repoId, promotion);
|
|
1413
|
+
waiter.resolve([]);
|
|
1414
|
+
}
|
|
1415
|
+
this.drainAdmissionWaiters();
|
|
1416
|
+
}
|
|
1417
|
+
parallelProjectOwned(key, owner) {
|
|
1418
|
+
return sameSingletonOwner(this.parallelProjectOwners.get(key)?.get(owner.dispatchAttemptId), owner);
|
|
1419
|
+
}
|
|
1420
|
+
parallelProjectAvailable(key, priority = "foreground") {
|
|
1421
|
+
return !this.singletonOwners.has(key)
|
|
1422
|
+
&& !this.singletonWaiters.some(waiter => waiter.keys.includes(key)
|
|
1423
|
+
&& (priority === "background" || waiter.priority === "foreground"));
|
|
1424
|
+
}
|
|
1425
|
+
assignParallelProject(key, owner) {
|
|
1426
|
+
const owners = this.parallelProjectOwners.get(key) ?? new Map();
|
|
1427
|
+
owners.set(owner.dispatchAttemptId, owner);
|
|
1428
|
+
this.parallelProjectOwners.set(key, owners);
|
|
1429
|
+
}
|
|
1430
|
+
releaseParallelProjectNoDrain(key, owner) {
|
|
1431
|
+
const owners = this.parallelProjectOwners.get(key);
|
|
1432
|
+
if (!sameSingletonOwner(owners?.get(owner.dispatchAttemptId), owner))
|
|
1433
|
+
return;
|
|
1434
|
+
owners.delete(owner.dispatchAttemptId);
|
|
1435
|
+
if (owners.size === 0)
|
|
1436
|
+
this.parallelProjectOwners.delete(key);
|
|
1437
|
+
}
|
|
1438
|
+
releaseParallelProjects(dispatchAttemptId, fencingToken) {
|
|
1439
|
+
const owner = { dispatchAttemptId, fencingToken };
|
|
1440
|
+
for (const key of [...this.parallelProjectOwners.keys()]) {
|
|
1441
|
+
this.releaseParallelProjectNoDrain(key, owner);
|
|
1442
|
+
}
|
|
1443
|
+
this.drainAdmissionWaiters();
|
|
1444
|
+
}
|
|
1445
|
+
deleteAdmission(dispatchAttemptId, fencingToken) {
|
|
1446
|
+
if (this.admissions.get(dispatchAttemptId)?.fencingToken === fencingToken) {
|
|
1447
|
+
this.admissions.delete(dispatchAttemptId);
|
|
1448
|
+
}
|
|
1449
|
+
this.releaseHostCapacity(dispatchAttemptId, fencingToken);
|
|
1450
|
+
}
|
|
1451
|
+
renewAdmissionWaiterDeadline(dispatchAttemptId, fencingToken, expiresAt) {
|
|
1452
|
+
for (const waiter of [
|
|
1453
|
+
...this.singletonWaiters,
|
|
1454
|
+
...this.parallelProjectWaiters,
|
|
1455
|
+
...this.capacityWaiters,
|
|
1456
|
+
...this.backgroundAdmissionWaiters,
|
|
1457
|
+
]) {
|
|
1458
|
+
if (!sameSingletonOwner(waiter.owner, { dispatchAttemptId, fencingToken })
|
|
1459
|
+
|| Date.parse(expiresAt) <= Date.parse(waiter.expiresAt))
|
|
1460
|
+
continue;
|
|
1461
|
+
waiter.expiresAt = expiresAt;
|
|
1462
|
+
clearAdmissionDeadline(waiter);
|
|
1463
|
+
waiter.deadlineTimer = scheduleAdmissionDeadline(waiter.expiresAt, waiter.expire);
|
|
1464
|
+
}
|
|
1465
|
+
}
|
|
1466
|
+
clearPendingRenewal(owner) {
|
|
1467
|
+
if (this.pendingRenewals.get(owner.dispatchAttemptId)?.fencingToken === owner.fencingToken) {
|
|
1468
|
+
this.pendingRenewals.delete(owner.dispatchAttemptId);
|
|
1469
|
+
}
|
|
1470
|
+
}
|
|
1471
|
+
cancelAdmissionWaiters(dispatchAttemptId, fencingToken, reason) {
|
|
1472
|
+
const owner = { dispatchAttemptId, fencingToken };
|
|
1473
|
+
for (const waiters of [
|
|
1474
|
+
this.singletonWaiters,
|
|
1475
|
+
this.parallelProjectWaiters,
|
|
1476
|
+
this.capacityWaiters,
|
|
1477
|
+
this.backgroundAdmissionWaiters,
|
|
1478
|
+
]) {
|
|
1479
|
+
for (let index = waiters.length - 1; index >= 0; index -= 1) {
|
|
1480
|
+
const waiter = waiters[index];
|
|
1481
|
+
if (!sameSingletonOwner(waiter.owner, owner))
|
|
1482
|
+
continue;
|
|
1483
|
+
waiters.splice(index, 1);
|
|
1484
|
+
clearAdmissionDeadline(waiter);
|
|
1485
|
+
if ("capacityRecheckTimer" in waiter)
|
|
1486
|
+
clearCapacityRecheck(waiter);
|
|
1487
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
1488
|
+
waiter.reject(new Error(`host_runtime_admission_cancelled: ${reason}`));
|
|
1489
|
+
}
|
|
1490
|
+
}
|
|
1491
|
+
this.drainAdmissionWaiters();
|
|
1492
|
+
}
|
|
1493
|
+
drainSingletonWaiters() {
|
|
1494
|
+
for (let index = 0; index < this.singletonWaiters.length;) {
|
|
1495
|
+
const waiter = this.singletonWaiters[index];
|
|
1496
|
+
if (waiter.signal?.aborted) {
|
|
1497
|
+
this.singletonWaiters.splice(index, 1);
|
|
1498
|
+
clearAdmissionDeadline(waiter);
|
|
1499
|
+
waiter.signal.removeEventListener("abort", waiter.abort);
|
|
1500
|
+
this.clearPendingRenewal(waiter.owner);
|
|
1501
|
+
waiter.reject(cancellationError(waiter.signal));
|
|
1502
|
+
continue;
|
|
1503
|
+
}
|
|
1504
|
+
if (admissionExpired(waiter.expiresAt)) {
|
|
1505
|
+
this.singletonWaiters.splice(index, 1);
|
|
1506
|
+
clearAdmissionDeadline(waiter);
|
|
1507
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
1508
|
+
this.clearPendingRenewal(waiter.owner);
|
|
1509
|
+
waiter.reject(new HostRuntimeAdmissionExpiredError());
|
|
1510
|
+
continue;
|
|
1511
|
+
}
|
|
1512
|
+
if (!this.singletonKeysAvailable(waiter.keys)) {
|
|
1513
|
+
index += 1;
|
|
1514
|
+
continue;
|
|
1515
|
+
}
|
|
1516
|
+
this.singletonWaiters.splice(index, 1);
|
|
1517
|
+
clearAdmissionDeadline(waiter);
|
|
1518
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
1519
|
+
this.assignSingletonKeys(waiter.keys, waiter.owner);
|
|
1520
|
+
waiter.resolve(waiter.keys);
|
|
1521
|
+
}
|
|
1522
|
+
}
|
|
1523
|
+
drainParallelProjectWaiters() {
|
|
1524
|
+
for (let index = 0; index < this.parallelProjectWaiters.length;) {
|
|
1525
|
+
const waiter = this.parallelProjectWaiters[index];
|
|
1526
|
+
if (waiter.signal?.aborted) {
|
|
1527
|
+
this.parallelProjectWaiters.splice(index, 1);
|
|
1528
|
+
clearAdmissionDeadline(waiter);
|
|
1529
|
+
waiter.signal.removeEventListener("abort", waiter.abort);
|
|
1530
|
+
this.clearPendingRenewal(waiter.owner);
|
|
1531
|
+
waiter.reject(cancellationError(waiter.signal));
|
|
1532
|
+
continue;
|
|
1533
|
+
}
|
|
1534
|
+
if (admissionExpired(waiter.expiresAt)) {
|
|
1535
|
+
this.parallelProjectWaiters.splice(index, 1);
|
|
1536
|
+
clearAdmissionDeadline(waiter);
|
|
1537
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
1538
|
+
this.clearPendingRenewal(waiter.owner);
|
|
1539
|
+
waiter.reject(new HostRuntimeAdmissionExpiredError());
|
|
1540
|
+
continue;
|
|
1541
|
+
}
|
|
1542
|
+
if (!this.parallelProjectAvailable(waiter.key, waiter.priority)) {
|
|
1543
|
+
index += 1;
|
|
1544
|
+
continue;
|
|
1545
|
+
}
|
|
1546
|
+
this.parallelProjectWaiters.splice(index, 1);
|
|
1547
|
+
clearAdmissionDeadline(waiter);
|
|
1548
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
1549
|
+
this.assignParallelProject(waiter.key, waiter.owner);
|
|
1550
|
+
waiter.resolve();
|
|
1551
|
+
}
|
|
1552
|
+
}
|
|
1553
|
+
drainBackgroundAdmissionWaiters() {
|
|
1554
|
+
if (this.hasForegroundDemand())
|
|
1555
|
+
return;
|
|
1556
|
+
for (let index = 0; index < this.backgroundAdmissionWaiters.length;) {
|
|
1557
|
+
const waiter = this.backgroundAdmissionWaiters[index];
|
|
1558
|
+
if (waiter.signal?.aborted) {
|
|
1559
|
+
this.backgroundAdmissionWaiters.splice(index, 1);
|
|
1560
|
+
clearAdmissionDeadline(waiter);
|
|
1561
|
+
clearCapacityRecheck(waiter);
|
|
1562
|
+
waiter.signal.removeEventListener("abort", waiter.abort);
|
|
1563
|
+
this.clearPendingRenewal(waiter.owner);
|
|
1564
|
+
waiter.reject(cancellationError(waiter.signal));
|
|
1565
|
+
continue;
|
|
1566
|
+
}
|
|
1567
|
+
if (admissionExpired(waiter.expiresAt)) {
|
|
1568
|
+
this.backgroundAdmissionWaiters.splice(index, 1);
|
|
1569
|
+
clearAdmissionDeadline(waiter);
|
|
1570
|
+
clearCapacityRecheck(waiter);
|
|
1571
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
1572
|
+
this.clearPendingRenewal(waiter.owner);
|
|
1573
|
+
waiter.reject(new HostRuntimeAdmissionExpiredError());
|
|
1574
|
+
continue;
|
|
1575
|
+
}
|
|
1576
|
+
if (!this.singletonKeysAvailable(waiter.keys) || !this.hostCapacityAvailable(waiter.resources)) {
|
|
1577
|
+
index += 1;
|
|
1578
|
+
continue;
|
|
1579
|
+
}
|
|
1580
|
+
this.backgroundAdmissionWaiters.splice(index, 1);
|
|
1581
|
+
clearAdmissionDeadline(waiter);
|
|
1582
|
+
clearCapacityRecheck(waiter);
|
|
1583
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
1584
|
+
this.assignSingletonKeys(waiter.keys, waiter.owner);
|
|
1585
|
+
this.capacityReservations.set(waiter.owner.dispatchAttemptId, {
|
|
1586
|
+
owner: waiter.owner,
|
|
1587
|
+
resources: waiter.resources,
|
|
1588
|
+
state: "pending",
|
|
1589
|
+
});
|
|
1590
|
+
waiter.resolve(waiter.keys);
|
|
1591
|
+
}
|
|
1592
|
+
}
|
|
1593
|
+
async refreshKnownPromotions() {
|
|
1594
|
+
const promotions = await this.promotionStore.list();
|
|
1595
|
+
this.knownPromotions.clear();
|
|
1596
|
+
for (const promotion of promotions)
|
|
1597
|
+
this.knownPromotions.set(promotion.repoId, promotion);
|
|
1598
|
+
}
|
|
1599
|
+
async invalidatePromotion(repoId) {
|
|
1600
|
+
await this.promotionStore.invalidate(repoId);
|
|
1601
|
+
this.knownPromotions.delete(repoId.trim().toLowerCase());
|
|
1602
|
+
}
|
|
1603
|
+
drainAdmissionWaiters() {
|
|
1604
|
+
this.reconcileReservationOwners();
|
|
1605
|
+
this.drainSingletonWaiters();
|
|
1606
|
+
this.drainParallelProjectWaiters();
|
|
1607
|
+
this.drainCapacityWaiters();
|
|
1608
|
+
this.drainBackgroundAdmissionWaiters();
|
|
1609
|
+
}
|
|
1610
|
+
reconcileReservationOwners() {
|
|
1611
|
+
for (const [key, owner] of this.singletonOwners) {
|
|
1612
|
+
if (!this.reservationOwnerIsLive(owner))
|
|
1613
|
+
this.singletonOwners.delete(key);
|
|
1614
|
+
}
|
|
1615
|
+
for (const [key, owners] of this.parallelProjectOwners) {
|
|
1616
|
+
for (const [dispatchAttemptId, owner] of owners) {
|
|
1617
|
+
const awaitingExclusiveUpgrade = this.singletonWaiters.some(waiter => waiter.keys.includes(key) && sameSingletonOwner(waiter.owner, owner));
|
|
1618
|
+
// A parallel owner cannot legitimately keep the shared claim after the
|
|
1619
|
+
// same fenced attempt has queued its exclusive downgrade. Reconcile the
|
|
1620
|
+
// half-upgraded state before testing singleton availability so one missed
|
|
1621
|
+
// release cannot make an attempt wait behind itself until lease expiry.
|
|
1622
|
+
if (awaitingExclusiveUpgrade || !this.reservationOwnerIsLive(owner)) {
|
|
1623
|
+
owners.delete(dispatchAttemptId);
|
|
1624
|
+
}
|
|
1625
|
+
}
|
|
1626
|
+
if (owners.size === 0)
|
|
1627
|
+
this.parallelProjectOwners.delete(key);
|
|
1628
|
+
}
|
|
1629
|
+
}
|
|
1630
|
+
reservationOwnerIsLive(owner) {
|
|
1631
|
+
if (this.acquiringAdmissions.has(hostRuntimeOwnerKey(owner)))
|
|
1632
|
+
return true;
|
|
1633
|
+
if (this.admissions.get(owner.dispatchAttemptId)?.fencingToken === owner.fencingToken)
|
|
1634
|
+
return true;
|
|
1635
|
+
if (this.active.get(owner.dispatchAttemptId)?.lease.fencingToken === owner.fencingToken)
|
|
1636
|
+
return true;
|
|
1637
|
+
if (this.degraded.get(owner.dispatchAttemptId)?.lease.fencingToken === owner.fencingToken)
|
|
1638
|
+
return true;
|
|
1639
|
+
if (sameSingletonOwner(this.capacityReservations.get(owner.dispatchAttemptId)?.owner, owner))
|
|
1640
|
+
return true;
|
|
1641
|
+
return [
|
|
1642
|
+
...this.singletonWaiters,
|
|
1643
|
+
...this.parallelProjectWaiters,
|
|
1644
|
+
...this.capacityWaiters,
|
|
1645
|
+
...this.backgroundAdmissionWaiters,
|
|
1646
|
+
].some(waiter => sameSingletonOwner(waiter.owner, owner));
|
|
1647
|
+
}
|
|
1648
|
+
}
|
|
1649
|
+
class AttemptSupervisedProviderProcessRunner {
|
|
1650
|
+
supervisor;
|
|
1651
|
+
owner;
|
|
1652
|
+
listenerInspector;
|
|
1653
|
+
enforceNoListeners;
|
|
1654
|
+
onProcessTreeStarted;
|
|
1655
|
+
constructor(supervisor, owner, listenerInspector, enforceNoListeners, onProcessTreeStarted) {
|
|
1656
|
+
this.supervisor = supervisor;
|
|
1657
|
+
this.owner = owner;
|
|
1658
|
+
this.listenerInspector = listenerInspector;
|
|
1659
|
+
this.enforceNoListeners = enforceNoListeners;
|
|
1660
|
+
this.onProcessTreeStarted = onProcessTreeStarted;
|
|
1661
|
+
}
|
|
1662
|
+
async run(request, handlers = {}, signal) {
|
|
1663
|
+
const baselineListeners = this.enforceNoListeners
|
|
1664
|
+
? new Set(await this.listenerInspector.inspectLoopbackListeners())
|
|
1665
|
+
: new Set();
|
|
1666
|
+
const child = await this.supervisor.startChildReady("provider", request, handlers, signal, { maxStartAttempts: request.stdin === undefined ? 1 : 3, retryDelayMs: 100 });
|
|
1667
|
+
this.onProcessTreeStarted();
|
|
1668
|
+
if (!this.enforceNoListeners)
|
|
1669
|
+
return await child.result;
|
|
1670
|
+
let settled = false;
|
|
1671
|
+
let inspectionFailure;
|
|
1672
|
+
const monitor = (async () => {
|
|
1673
|
+
while (!settled) {
|
|
1674
|
+
try {
|
|
1675
|
+
const ports = (await this.listenerInspector.inspectOwnedLoopbackListeners(child, this.owner))
|
|
1676
|
+
.filter(port => !baselineListeners.has(port));
|
|
1677
|
+
if (ports.length > 0) {
|
|
1678
|
+
inspectionFailure = new Error(`host_analysis_listener_violation: analysis-only provider opened undeclared loopback listener(s): ${ports.join(", ")}.`);
|
|
1679
|
+
await child.stop(this.owner.fencingToken).catch(() => undefined);
|
|
1680
|
+
return;
|
|
1681
|
+
}
|
|
1682
|
+
}
|
|
1683
|
+
catch (error) {
|
|
1684
|
+
inspectionFailure = error;
|
|
1685
|
+
await child.stop(this.owner.fencingToken).catch(() => undefined);
|
|
1686
|
+
return;
|
|
1687
|
+
}
|
|
1688
|
+
await waitForAnalysisInspection(250, signal).catch(error => {
|
|
1689
|
+
if (!settled && !signal?.aborted)
|
|
1690
|
+
inspectionFailure = error;
|
|
1691
|
+
});
|
|
1692
|
+
}
|
|
1693
|
+
})();
|
|
1694
|
+
try {
|
|
1695
|
+
const result = await child.result;
|
|
1696
|
+
settled = true;
|
|
1697
|
+
await monitor;
|
|
1698
|
+
if (inspectionFailure)
|
|
1699
|
+
throw inspectionFailure;
|
|
1700
|
+
return result;
|
|
1701
|
+
}
|
|
1702
|
+
catch (error) {
|
|
1703
|
+
settled = true;
|
|
1704
|
+
await monitor.catch(() => undefined);
|
|
1705
|
+
if (inspectionFailure)
|
|
1706
|
+
throw inspectionFailure;
|
|
1707
|
+
throw error;
|
|
1708
|
+
}
|
|
1709
|
+
}
|
|
1710
|
+
}
|
|
1711
|
+
function waitForAnalysisInspection(milliseconds, signal) {
|
|
1712
|
+
if (signal?.aborted)
|
|
1713
|
+
return Promise.reject(cancellationError(signal));
|
|
1714
|
+
return new Promise((resolve, reject) => {
|
|
1715
|
+
const timer = setTimeout(() => {
|
|
1716
|
+
signal?.removeEventListener("abort", abort);
|
|
1717
|
+
resolve();
|
|
1718
|
+
}, milliseconds);
|
|
1719
|
+
const abort = () => {
|
|
1720
|
+
clearTimeout(timer);
|
|
1721
|
+
reject(cancellationError(signal));
|
|
1722
|
+
};
|
|
1723
|
+
signal?.addEventListener("abort", abort, { once: true });
|
|
1724
|
+
});
|
|
1725
|
+
}
|
|
1726
|
+
export class HostRuntimeLeaseCleanupError extends Error {
|
|
1727
|
+
lease;
|
|
1728
|
+
constructor(lease, cause) {
|
|
1729
|
+
super(`host_runtime_cleanup_degraded: ${cause instanceof Error ? cause.message : String(cause)}`, { cause });
|
|
1730
|
+
this.lease = lease;
|
|
1731
|
+
this.name = "HostRuntimeLeaseCleanupError";
|
|
1732
|
+
}
|
|
1733
|
+
}
|
|
1734
|
+
export class HostRuntimeAdmissionExpiredError extends Error {
|
|
1735
|
+
constructor() {
|
|
1736
|
+
super("host_runtime_admission_expired: dispatch lease expired while waiting for managed host admission.");
|
|
1737
|
+
this.name = "HostRuntimeAdmissionExpiredError";
|
|
1738
|
+
}
|
|
1739
|
+
}
|
|
1740
|
+
export class HostRuntimeCapacityRequestImpossibleError extends Error {
|
|
1741
|
+
constructor(reason) {
|
|
1742
|
+
super(`host_runtime_capacity_request_impossible: ${reason}`);
|
|
1743
|
+
this.name = "HostRuntimeCapacityRequestImpossibleError";
|
|
1744
|
+
}
|
|
1745
|
+
}
|
|
1746
|
+
export function hostRuntimeEventResources(lease, runtimeContract, executionClass = "managed_runtime") {
|
|
1747
|
+
const requestedHostConcurrency = runtimeContract?.contract.runtime.hostConcurrency ?? "serial";
|
|
1748
|
+
const configuredLaunches = runtimeContract?.contract.runtime.ports.filter(port => port.launch !== null) ?? [];
|
|
1749
|
+
const verifiedLaunches = configuredLaunches.filter(configured => lease.resources.ports.some(port => port.name === configured.name));
|
|
1750
|
+
const effectiveConcurrency = effectiveHostConcurrency(lease, runtimeContract, executionClass);
|
|
1751
|
+
return {
|
|
1752
|
+
agentContainer: null,
|
|
1753
|
+
sidecarContainers: [],
|
|
1754
|
+
networks: [],
|
|
1755
|
+
volumes: [],
|
|
1756
|
+
publishedPorts: lease.resources.ports,
|
|
1757
|
+
hostConcurrency: effectiveConcurrency,
|
|
1758
|
+
requestedHostConcurrency,
|
|
1759
|
+
hostParallelPromotion: effectiveConcurrency === "host_parallel_safe"
|
|
1760
|
+
? executionClass === "analysis_only" ? "analysis_profile_enforced" : "observed_probe_passed"
|
|
1761
|
+
: requestedHostConcurrency !== "parallel"
|
|
1762
|
+
? "not_requested"
|
|
1763
|
+
: configuredLaunches.length > 0 && verifiedLaunches.length === configuredLaunches.length
|
|
1764
|
+
? "pending_live_parallel_proof"
|
|
1765
|
+
: "pending_adapter_probe",
|
|
1766
|
+
verifiedRuntimePorts: verifiedLaunches.map(port => port.name),
|
|
1767
|
+
runtimeContractSource: runtimeContract?.source ?? "default",
|
|
1768
|
+
executionClass,
|
|
1769
|
+
requestedPorts: runtimeContract?.contract.runtime.ports.map(port => port.name) ?? [],
|
|
1770
|
+
singletonKeys: lease.resources.singletonKeys,
|
|
1771
|
+
processSupervisorId: lease.resources.processSupervisorId,
|
|
1772
|
+
runtimeLease: lease,
|
|
1773
|
+
};
|
|
1774
|
+
}
|
|
1775
|
+
export function hostRuntimeSingletonKeys(dispatch) {
|
|
1776
|
+
return [projectHostRuntimeSingletonKey(dispatch.repo.id)];
|
|
1777
|
+
}
|
|
1778
|
+
function hostRuntimeEnvironment(sessionId, lease, runtimeContract, executionClass) {
|
|
1779
|
+
const pathFor = (role) => lease.resources.paths.find(entry => entry.role === role)?.path ?? "";
|
|
1780
|
+
const environment = {
|
|
1781
|
+
FORGE_SESSION_ID: sessionId,
|
|
1782
|
+
FORGE_RUNTIME_MODE: "host",
|
|
1783
|
+
FORGE_RESOURCE_LEASE_ID: lease.leaseId,
|
|
1784
|
+
FORGE_DISPATCH_ATTEMPT_ID: lease.dispatchAttemptId,
|
|
1785
|
+
FORGE_E2E_OWNERSHIP_MODE: "attempt",
|
|
1786
|
+
FORGE_E2E_OWNERSHIP_TOKEN: uuidHex(lease.dispatchAttemptId, "dispatch attempt"),
|
|
1787
|
+
FORGE_EXECUTION_LEASE_ID: lease.executionLeaseId,
|
|
1788
|
+
FORGE_FENCING_TOKEN: lease.fencingToken,
|
|
1789
|
+
FORGE_WORKSPACE: lease.worktreePath,
|
|
1790
|
+
FORGE_WORKTREE_PATH: lease.worktreePath,
|
|
1791
|
+
FORGE_TMPDIR: pathFor("temporary"),
|
|
1792
|
+
FORGE_LOG_DIR: pathFor("logs"),
|
|
1793
|
+
FORGE_CACHE_DIR: pathFor("cache"),
|
|
1794
|
+
FORGE_BROWSER_PROFILE_DIR: pathFor("browser_profile"),
|
|
1795
|
+
FORGE_TEST_OUTPUT_DIR: pathFor("test_output"),
|
|
1796
|
+
FORGE_ATTEMPT_DATABASE_SCHEMA: `forge_attempt_${uuidHex(lease.dispatchAttemptId, "dispatch attempt")}`,
|
|
1797
|
+
FORGE_ATTEMPT_DATABASE_READINESS_PATH: path.join(pathFor("logs"), "attempt-database-readiness.json"),
|
|
1798
|
+
FORGE_PROCESS_SUPERVISOR_ID: lease.resources.processSupervisorId ?? "",
|
|
1799
|
+
FORGE_PROCESS_SUPERVISOR_METADATA_PATH: pathFor("process_supervisor_metadata"),
|
|
1800
|
+
FORGE_HOST_CONCURRENCY: effectiveHostConcurrency(lease, runtimeContract, executionClass),
|
|
1801
|
+
FORGE_HOST_CONCURRENCY_REQUESTED: runtimeContract.contract.runtime.hostConcurrency,
|
|
1802
|
+
FORGE_RUNTIME_EXECUTION_CLASS: executionClass,
|
|
1803
|
+
FORGE_HOST_RUNTIME_CONTRACT_SOURCE: runtimeContract.source,
|
|
1804
|
+
FORGE_SINGLETON_KEYS: lease.resources.singletonKeys.join(","),
|
|
1805
|
+
TMPDIR: pathFor("temporary"),
|
|
1806
|
+
TMP: pathFor("temporary"),
|
|
1807
|
+
TEMP: pathFor("temporary"),
|
|
1808
|
+
XDG_CACHE_HOME: pathFor("cache"),
|
|
1809
|
+
NPM_CONFIG_CACHE: path.join(pathFor("cache"), "npm"),
|
|
1810
|
+
};
|
|
1811
|
+
if (lease.resources.singletonKeys.length === 0)
|
|
1812
|
+
delete environment.FORGE_SINGLETON_KEYS;
|
|
1813
|
+
const resolvedBaseUrls = [];
|
|
1814
|
+
for (const assigned of lease.resources.ports) {
|
|
1815
|
+
const configured = runtimeContract.contract.runtime.ports.find(port => port.name === assigned.name);
|
|
1816
|
+
if (!configured || assigned.hostIp !== "127.0.0.1")
|
|
1817
|
+
continue;
|
|
1818
|
+
Object.assign(environment, hostPortEnvironment(configured, assigned.hostIp, assigned.hostPort));
|
|
1819
|
+
const normalizedName = assigned.name.toUpperCase().replaceAll(/[^A-Z0-9]/gu, "_");
|
|
1820
|
+
environment[`FORGE_${normalizedName}_PORT`] = String(assigned.hostPort);
|
|
1821
|
+
if (configured.baseUrlEnv && configured.format) {
|
|
1822
|
+
const baseUrl = hostPortEnvironment(configured, assigned.hostIp, assigned.hostPort)[configured.baseUrlEnv];
|
|
1823
|
+
if (baseUrl) {
|
|
1824
|
+
environment[`FORGE_${normalizedName}_URL`] = baseUrl;
|
|
1825
|
+
resolvedBaseUrls.push(baseUrl);
|
|
1826
|
+
}
|
|
1827
|
+
}
|
|
1828
|
+
}
|
|
1829
|
+
if (environment.FORGE_API_URL)
|
|
1830
|
+
environment.FORGE_E2E_API_URL = environment.FORGE_API_URL;
|
|
1831
|
+
if (environment.FORGE_WEB_URL)
|
|
1832
|
+
environment.FORGE_E2E_WEB_URL = environment.FORGE_WEB_URL;
|
|
1833
|
+
if (resolvedBaseUrls.length === 1)
|
|
1834
|
+
environment.FORGE_BASE_URL = resolvedBaseUrls[0];
|
|
1835
|
+
return environment;
|
|
1836
|
+
}
|
|
1837
|
+
async function executeAspNetRuntimeBuild(supervisor, logDirectory, plan, signal) {
|
|
1838
|
+
const childName = `runtime-build-${plan.fingerprint.slice(-12)}`;
|
|
1839
|
+
const stdoutLogPath = path.join(logDirectory, `${childName}.stdout.log`);
|
|
1840
|
+
const stderrLogPath = path.join(logDirectory, `${childName}.stderr.log`);
|
|
1841
|
+
let stdoutTail = "";
|
|
1842
|
+
let stderrTail = "";
|
|
1843
|
+
const append = async (target, line) => {
|
|
1844
|
+
await fs.appendFile(target, `${line}\n`, { encoding: "utf8", mode: 0o600 });
|
|
1845
|
+
};
|
|
1846
|
+
const child = await supervisor.startChildReady(childName, {
|
|
1847
|
+
command: plan.command,
|
|
1848
|
+
args: plan.args,
|
|
1849
|
+
cwd: plan.cwd,
|
|
1850
|
+
env: plan.environment,
|
|
1851
|
+
captureOutput: false,
|
|
1852
|
+
}, {
|
|
1853
|
+
onStdoutLine: async (line) => {
|
|
1854
|
+
stdoutTail = appendRuntimeBuildTail(stdoutTail, line);
|
|
1855
|
+
await append(stdoutLogPath, line);
|
|
1856
|
+
},
|
|
1857
|
+
onStderrLine: async (line) => {
|
|
1858
|
+
stderrTail = appendRuntimeBuildTail(stderrTail, line);
|
|
1859
|
+
await append(stderrLogPath, line);
|
|
1860
|
+
},
|
|
1861
|
+
}, signal, { maxStartAttempts: 2, retryDelayMs: 250 });
|
|
1862
|
+
const result = await child.result;
|
|
1863
|
+
return {
|
|
1864
|
+
exitCode: result.exitCode,
|
|
1865
|
+
stdoutTail: appendRuntimeBuildTail(stdoutTail, result.stdout),
|
|
1866
|
+
stderrTail: appendRuntimeBuildTail(stderrTail, result.stderr),
|
|
1867
|
+
stdoutLogPath,
|
|
1868
|
+
stderrLogPath,
|
|
1869
|
+
};
|
|
1870
|
+
}
|
|
1871
|
+
function appendRuntimeBuildTail(current, value) {
|
|
1872
|
+
if (!value)
|
|
1873
|
+
return current;
|
|
1874
|
+
return `${current}${current ? "\n" : ""}${value}`.slice(-4_000);
|
|
1875
|
+
}
|
|
1876
|
+
function effectiveHostConcurrency(lease, runtimeContract, executionClass = "managed_runtime") {
|
|
1877
|
+
if (executionClass === "analysis_only") {
|
|
1878
|
+
const analysisSingletonHeld = lease.resources.singletonKeys.length > 0;
|
|
1879
|
+
return analysisSingletonHeld ? "host_serial_only" : "host_parallel_safe";
|
|
1880
|
+
}
|
|
1881
|
+
if (!runtimeContract || !hostRuntimeParallelEligible(runtimeContract))
|
|
1882
|
+
return "host_serial_only";
|
|
1883
|
+
const requiredPorts = configuredLaunchPortNames(runtimeContract);
|
|
1884
|
+
const everyPortVerified = requiredPorts.length > 0
|
|
1885
|
+
&& requiredPorts.every(name => lease.resources.ports.some(port => port.name === name));
|
|
1886
|
+
const projectSingletonHeld = lease.resources.singletonKeys.some(key => key.endsWith(":host-runtime"));
|
|
1887
|
+
return everyPortVerified && !projectSingletonHeld ? "host_parallel_safe" : "host_serial_only";
|
|
1888
|
+
}
|
|
1889
|
+
function projectHostRuntimeSingletonKey(repoId) {
|
|
1890
|
+
return `project:${repoId.toLowerCase()}:host-runtime`;
|
|
1891
|
+
}
|
|
1892
|
+
function projectHostAnalysisKey(repoId) {
|
|
1893
|
+
return `project:${repoId.toLowerCase()}:host-analysis`;
|
|
1894
|
+
}
|
|
1895
|
+
function hostRuntimeDeclaredSingletonKey(resource) {
|
|
1896
|
+
return `runtime:${resource}`;
|
|
1897
|
+
}
|
|
1898
|
+
function runtimeExecutionClass(dispatch) {
|
|
1899
|
+
return dispatch.runtimeIntent?.executionClass === "analysis_only"
|
|
1900
|
+
&& dispatch.runtimeIntent.applicationRuntimeRequired === false
|
|
1901
|
+
? "analysis_only"
|
|
1902
|
+
: "managed_runtime";
|
|
1903
|
+
}
|
|
1904
|
+
function runtimeAdmissionPriority(dispatch) {
|
|
1905
|
+
return dispatch.execution.lane.trim().toLowerCase() === "runtime_readiness"
|
|
1906
|
+
|| dispatch.cycleRun.phase.trim().toLowerCase() === "readiness_probe"
|
|
1907
|
+
? "background"
|
|
1908
|
+
: "foreground";
|
|
1909
|
+
}
|
|
1910
|
+
function enqueueAdmissionWaiter(waiters, waiter) {
|
|
1911
|
+
if (waiter.priority === "background") {
|
|
1912
|
+
waiters.push(waiter);
|
|
1913
|
+
return;
|
|
1914
|
+
}
|
|
1915
|
+
const backgroundIndex = waiters.findIndex(candidate => candidate.priority === "background");
|
|
1916
|
+
if (backgroundIndex < 0)
|
|
1917
|
+
waiters.push(waiter);
|
|
1918
|
+
else
|
|
1919
|
+
waiters.splice(backgroundIndex, 0, waiter);
|
|
1920
|
+
}
|
|
1921
|
+
function defaultHostResourceRequest() {
|
|
1922
|
+
const workerMemoryLimitBytes = configuredInteger("FORGE_HOST_DEFAULT_WORKER_MEMORY_LIMIT_BYTES", 2 * 1024 ** 3, 128 * 1024 ** 2, 512 * 1024 ** 3);
|
|
1923
|
+
const runtimeOverheadBytes = 128 * 1024 ** 2;
|
|
1924
|
+
return {
|
|
1925
|
+
sandboxSlots: 1,
|
|
1926
|
+
cpuMillis: 2000,
|
|
1927
|
+
workerMemoryLimitBytes,
|
|
1928
|
+
serviceMemoryLimitBytes: 0,
|
|
1929
|
+
runtimeOverheadBytes,
|
|
1930
|
+
memoryBytes: workerMemoryLimitBytes + runtimeOverheadBytes,
|
|
1931
|
+
workspaceBytes: 10 * 1024 ** 3,
|
|
1932
|
+
pids: 512,
|
|
1933
|
+
buildSlots: 1,
|
|
1934
|
+
};
|
|
1935
|
+
}
|
|
1936
|
+
function minimumSupportedHostResourceRequest() {
|
|
1937
|
+
return {
|
|
1938
|
+
sandboxSlots: 1,
|
|
1939
|
+
cpuMillis: 1000,
|
|
1940
|
+
workerMemoryLimitBytes: 1024 ** 3,
|
|
1941
|
+
serviceMemoryLimitBytes: 0,
|
|
1942
|
+
runtimeOverheadBytes: 128 * 1024 ** 2,
|
|
1943
|
+
memoryBytes: 1152 * 1024 ** 2,
|
|
1944
|
+
workspaceBytes: architectMinimumWorkspaceEstimateBytes,
|
|
1945
|
+
pids: 256,
|
|
1946
|
+
buildSlots: 0,
|
|
1947
|
+
};
|
|
1948
|
+
}
|
|
1949
|
+
function normalizedHostResourceRequest(resources) {
|
|
1950
|
+
const defaults = defaultHostResourceRequest();
|
|
1951
|
+
const workerMemoryLimitBytes = positiveResourceValue(resources?.workerMemoryLimitBytes, defaults.workerMemoryLimitBytes);
|
|
1952
|
+
const serviceMemoryLimitBytes = nonNegativeResourceValue(resources?.serviceMemoryLimitBytes, defaults.serviceMemoryLimitBytes);
|
|
1953
|
+
const runtimeOverheadBytes = nonNegativeResourceValue(resources?.runtimeOverheadBytes, defaults.runtimeOverheadBytes);
|
|
1954
|
+
return {
|
|
1955
|
+
sandboxSlots: positiveResourceValue(resources?.sandboxSlots, defaults.sandboxSlots),
|
|
1956
|
+
cpuMillis: positiveResourceValue(resources?.cpuMillis, defaults.cpuMillis),
|
|
1957
|
+
workerMemoryLimitBytes,
|
|
1958
|
+
serviceMemoryLimitBytes,
|
|
1959
|
+
runtimeOverheadBytes,
|
|
1960
|
+
memoryBytes: workerMemoryLimitBytes + serviceMemoryLimitBytes + runtimeOverheadBytes,
|
|
1961
|
+
workspaceBytes: positiveResourceValue(resources?.workspaceBytes, defaults.workspaceBytes),
|
|
1962
|
+
pids: positiveResourceValue(resources?.pids, defaults.pids),
|
|
1963
|
+
buildSlots: nonNegativeResourceValue(resources?.buildSlots, defaults.buildSlots),
|
|
1964
|
+
};
|
|
1965
|
+
}
|
|
1966
|
+
function positiveResourceValue(value, fallback) {
|
|
1967
|
+
return Number.isFinite(value) && value > 0 ? Math.floor(value) : fallback;
|
|
1968
|
+
}
|
|
1969
|
+
function nonNegativeResourceValue(value, fallback) {
|
|
1970
|
+
return Number.isFinite(value) && value >= 0 ? Math.floor(value) : fallback;
|
|
1971
|
+
}
|
|
1972
|
+
function configuredInteger(name, fallback, minimum, maximum) {
|
|
1973
|
+
const parsed = Number.parseInt(process.env[name] ?? "", 10);
|
|
1974
|
+
return Number.isFinite(parsed) ? Math.max(minimum, Math.min(maximum, parsed)) : fallback;
|
|
1975
|
+
}
|
|
1976
|
+
function loadHostCapacityConfiguration(capacityResourceSample) {
|
|
1977
|
+
const sampledCpuCount = capacityResourceSample
|
|
1978
|
+
? Math.max(1, Math.floor(capacityResourceSample.totalCpuMillis / 1_000))
|
|
1979
|
+
: os.cpus().length;
|
|
1980
|
+
const totalMemoryBytes = capacityResourceSample?.totalMemoryBytes ?? os.totalmem();
|
|
1981
|
+
const reserveFloorBytes = configuredInteger("FORGE_HOST_MEMORY_RESERVE_FLOOR_BYTES", 1024 ** 3, 512 * 1024 ** 2, 128 * 1024 ** 3);
|
|
1982
|
+
const reserveCeilingBytes = configuredInteger("FORGE_HOST_MEMORY_RESERVE_CEILING_BYTES", 4 * 1024 ** 3, reserveFloorBytes, 128 * 1024 ** 3);
|
|
1983
|
+
const hostReserveBytes = resolveAdaptiveMemoryReserveBytes(totalMemoryBytes, {
|
|
1984
|
+
explicitBytes: configuredOptionalInteger("FORGE_HOST_MEMORY_RESERVE_BYTES", 512 * 1024 ** 2, 128 * 1024 ** 3),
|
|
1985
|
+
floorBytes: reserveFloorBytes,
|
|
1986
|
+
ceilingBytes: reserveCeilingBytes,
|
|
1987
|
+
percent: configuredNumber("FORGE_HOST_MEMORY_RESERVE_PERCENT", 12.5, 1, 50),
|
|
1988
|
+
});
|
|
1989
|
+
const minimumIdleReserveBytes = Math.min(hostReserveBytes, reserveFloorBytes);
|
|
1990
|
+
return {
|
|
1991
|
+
hardMaxAttempts: configuredInteger("FORGE_HOST_MAX_CONCURRENT_ATTEMPTS", Math.max(1, Math.min(8, Math.floor(sampledCpuCount / 2))), 1, 32),
|
|
1992
|
+
hostReserveBytes,
|
|
1993
|
+
idleHostReserveBytes: configuredInteger("FORGE_HOST_IDLE_MEMORY_RESERVE_BYTES", minimumIdleReserveBytes, 512 * 1024 ** 2, hostReserveBytes),
|
|
1994
|
+
cpuReserveMillis: configuredInteger("FORGE_HOST_CPU_RESERVE_MILLIS", 1000, 0, 32_000),
|
|
1995
|
+
workspaceReserveBytes: configuredInteger("FORGE_HOST_WORKSPACE_RESERVE_BYTES", 10 * 1024 ** 3, 1024 ** 3, Number.MAX_SAFE_INTEGER),
|
|
1996
|
+
memoryRecoverySamples: configuredInteger("FORGE_HOST_MEMORY_RECOVERY_SAMPLES", 2, 1, 10),
|
|
1997
|
+
memoryRecoverySampleIntervalMs: configuredInteger("FORGE_HOST_MEMORY_RECOVERY_SAMPLE_INTERVAL_MS", 1_000, 100, 30_000),
|
|
1998
|
+
};
|
|
1999
|
+
}
|
|
2000
|
+
function configuredOptionalInteger(name, minimum, maximum) {
|
|
2001
|
+
const raw = process.env[name];
|
|
2002
|
+
if (raw === undefined || raw.trim().length === 0)
|
|
2003
|
+
return null;
|
|
2004
|
+
const parsed = Number.parseInt(raw, 10);
|
|
2005
|
+
return Number.isFinite(parsed) ? Math.max(minimum, Math.min(maximum, parsed)) : null;
|
|
2006
|
+
}
|
|
2007
|
+
function configuredNumber(name, fallback, minimum, maximum) {
|
|
2008
|
+
const parsed = Number(process.env[name] ?? "");
|
|
2009
|
+
return Number.isFinite(parsed) && parsed > 0
|
|
2010
|
+
? Math.max(minimum, Math.min(maximum, parsed))
|
|
2011
|
+
: fallback;
|
|
2012
|
+
}
|
|
2013
|
+
function resourceSlots(available, requested) {
|
|
2014
|
+
if (requested <= 0)
|
|
2015
|
+
return Number.MAX_SAFE_INTEGER;
|
|
2016
|
+
return Math.max(0, Math.floor(available / requested));
|
|
2017
|
+
}
|
|
2018
|
+
function removeWaiter(waiters, waiter) {
|
|
2019
|
+
const index = waiters.indexOf(waiter);
|
|
2020
|
+
if (index >= 0)
|
|
2021
|
+
waiters.splice(index, 1);
|
|
2022
|
+
}
|
|
2023
|
+
function capacityProject(projects, repoId) {
|
|
2024
|
+
const normalized = repoId.trim().toLowerCase();
|
|
2025
|
+
const existing = projects.get(normalized);
|
|
2026
|
+
if (existing)
|
|
2027
|
+
return existing;
|
|
2028
|
+
const created = {
|
|
2029
|
+
repoId: normalized,
|
|
2030
|
+
safetyClass: null,
|
|
2031
|
+
activeAttempts: 0,
|
|
2032
|
+
reservedAttempts: 0,
|
|
2033
|
+
waitingAttempts: 0,
|
|
2034
|
+
degradedAttempts: 0,
|
|
2035
|
+
runtimeContractFingerprint: null,
|
|
2036
|
+
runtimeContractSource: null,
|
|
2037
|
+
promotedAt: null,
|
|
2038
|
+
};
|
|
2039
|
+
projects.set(normalized, created);
|
|
2040
|
+
return created;
|
|
2041
|
+
}
|
|
2042
|
+
function conservativeSafetyClass(left, right) {
|
|
2043
|
+
if (!left)
|
|
2044
|
+
return right;
|
|
2045
|
+
const rank = {
|
|
2046
|
+
host_serial_only: 0,
|
|
2047
|
+
host_parallel_candidate: 1,
|
|
2048
|
+
host_parallel_safe: 2,
|
|
2049
|
+
};
|
|
2050
|
+
return rank[right] < rank[left] ? right : left;
|
|
2051
|
+
}
|
|
2052
|
+
function repoIdFromProjectSingletonKeys(keys) {
|
|
2053
|
+
return keys.map(repoIdFromProjectSingletonKey).find((value) => value !== null) ?? null;
|
|
2054
|
+
}
|
|
2055
|
+
function repoIdFromProjectSingletonKey(key) {
|
|
2056
|
+
const prefix = "project:";
|
|
2057
|
+
const suffix = [":host-runtime", ":host-analysis"].find(candidate => key.endsWith(candidate));
|
|
2058
|
+
return key.startsWith(prefix) && suffix
|
|
2059
|
+
? key.slice(prefix.length, -suffix.length)
|
|
2060
|
+
: null;
|
|
2061
|
+
}
|
|
2062
|
+
function sameSingletonOwner(left, right) {
|
|
2063
|
+
return left?.dispatchAttemptId === right.dispatchAttemptId && left.fencingToken === right.fencingToken;
|
|
2064
|
+
}
|
|
2065
|
+
function hostRuntimeOwnerKey(owner) {
|
|
2066
|
+
return `${owner.dispatchAttemptId}:${owner.fencingToken}`;
|
|
2067
|
+
}
|
|
2068
|
+
function hostPortOwner(lease) {
|
|
2069
|
+
const processSupervisorId = lease.resources.processSupervisorId;
|
|
2070
|
+
if (!processSupervisorId) {
|
|
2071
|
+
throw new Error("host_runtime_adapter_invalid: process supervisor identity is required.");
|
|
2072
|
+
}
|
|
2073
|
+
return {
|
|
2074
|
+
dispatchAttemptId: lease.dispatchAttemptId,
|
|
2075
|
+
executionLeaseId: lease.executionLeaseId,
|
|
2076
|
+
fencingToken: lease.fencingToken,
|
|
2077
|
+
processSupervisorId,
|
|
2078
|
+
};
|
|
2079
|
+
}
|
|
2080
|
+
function linkedAbortController(signal) {
|
|
2081
|
+
const controller = new AbortController();
|
|
2082
|
+
if (signal?.aborted) {
|
|
2083
|
+
controller.abort(signal.reason);
|
|
2084
|
+
}
|
|
2085
|
+
else {
|
|
2086
|
+
signal?.addEventListener("abort", () => controller.abort(signal.reason), { once: true, signal: controller.signal });
|
|
2087
|
+
}
|
|
2088
|
+
return controller;
|
|
2089
|
+
}
|
|
2090
|
+
function unlinkAbortController(controller, _signal) {
|
|
2091
|
+
if (!controller.signal.aborted)
|
|
2092
|
+
controller.abort("Host adapter launch settled.");
|
|
2093
|
+
}
|
|
2094
|
+
function removeCleanup(cleanups, cleanup) {
|
|
2095
|
+
const index = cleanups.indexOf(cleanup);
|
|
2096
|
+
if (index >= 0)
|
|
2097
|
+
cleanups.splice(index, 1);
|
|
2098
|
+
}
|
|
2099
|
+
async function releaseAdapterResources(cleanups) {
|
|
2100
|
+
const pending = cleanups.splice(0);
|
|
2101
|
+
const failed = [];
|
|
2102
|
+
let firstError;
|
|
2103
|
+
for (const cleanup of pending.reverse()) {
|
|
2104
|
+
try {
|
|
2105
|
+
await cleanup();
|
|
2106
|
+
}
|
|
2107
|
+
catch (error) {
|
|
2108
|
+
firstError ??= error;
|
|
2109
|
+
failed.unshift(cleanup);
|
|
2110
|
+
}
|
|
2111
|
+
}
|
|
2112
|
+
cleanups.push(...failed);
|
|
2113
|
+
if (firstError !== undefined)
|
|
2114
|
+
throw firstError;
|
|
2115
|
+
}
|
|
2116
|
+
function cancellationError(signal) {
|
|
2117
|
+
return new Error(String(signal?.reason ?? "Dispatch cancelled while waiting for host runtime singleton ownership."));
|
|
2118
|
+
}
|
|
2119
|
+
function validHostRuntimeLedger(value) {
|
|
2120
|
+
if (typeof value !== "object" || value === null)
|
|
2121
|
+
return false;
|
|
2122
|
+
const ledger = value;
|
|
2123
|
+
return ledger.schemaVersion === 1
|
|
2124
|
+
&& (ledger.repoId === undefined || /^[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}$/iu.test(ledger.repoId))
|
|
2125
|
+
&& validRuntimeIsolationLease(ledger.runtimeLease)
|
|
2126
|
+
&& ledger.runtimeLease.mode === "host";
|
|
2127
|
+
}
|
|
2128
|
+
function validHostProcessSupervisorMetadata(value, lease) {
|
|
2129
|
+
if (typeof value !== "object" || value === null)
|
|
2130
|
+
return false;
|
|
2131
|
+
const metadata = value;
|
|
2132
|
+
return metadata.schemaVersion === 1
|
|
2133
|
+
&& /^host-[0-9a-f]{32}$/u.test(metadata.supervisorId)
|
|
2134
|
+
&& (metadata.strategy === "windows_job_object" || metadata.strategy === "unix_process_group")
|
|
2135
|
+
&& Number.isSafeInteger(metadata.rootPid)
|
|
2136
|
+
&& metadata.rootPid > 0
|
|
2137
|
+
&& Number.isFinite(Date.parse(metadata.rootCreatedAt))
|
|
2138
|
+
&& (metadata.rootIdentity == null
|
|
2139
|
+
|| (typeof metadata.rootIdentity === "string" && metadata.rootIdentity.length <= 256))
|
|
2140
|
+
&& /^[0-9a-f]{64}$/u.test(metadata.commandDigest)
|
|
2141
|
+
&& containedMetadataWorktreePath(lease.worktreePath, metadata.worktreePath)
|
|
2142
|
+
&& metadata.runtimeLeaseId === lease.executionLeaseId
|
|
2143
|
+
&& metadata.dispatchAttemptId === lease.dispatchAttemptId
|
|
2144
|
+
&& metadata.fencingToken === lease.fencingToken
|
|
2145
|
+
&& Number.isSafeInteger(metadata.bridgePid)
|
|
2146
|
+
&& metadata.bridgePid > 0;
|
|
2147
|
+
}
|
|
2148
|
+
function containedMetadataWorktreePath(worktreePath, metadataWorktreePath) {
|
|
2149
|
+
const relative = path.relative(path.resolve(worktreePath), path.resolve(metadataWorktreePath));
|
|
2150
|
+
return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative));
|
|
2151
|
+
}
|
|
2152
|
+
function transitionHostLeaseForCleanup(lease, state) {
|
|
2153
|
+
if (lease.state === state || lease.state === "released")
|
|
2154
|
+
return lease;
|
|
2155
|
+
return transitionRuntimeIsolationLease(lease, state);
|
|
2156
|
+
}
|
|
2157
|
+
function throwIfAdmissionExpired(expiresAt) {
|
|
2158
|
+
if (admissionExpired(expiresAt))
|
|
2159
|
+
throw new HostRuntimeAdmissionExpiredError();
|
|
2160
|
+
}
|
|
2161
|
+
function admissionExpired(expiresAt) {
|
|
2162
|
+
const deadline = Date.parse(expiresAt);
|
|
2163
|
+
return !Number.isFinite(deadline) || deadline <= Date.now();
|
|
2164
|
+
}
|
|
2165
|
+
function scheduleAdmissionDeadline(expiresAt, expire) {
|
|
2166
|
+
const delay = Math.max(0, Date.parse(expiresAt) - Date.now());
|
|
2167
|
+
const timer = setTimeout(expire, Math.min(delay, 2_147_483_647));
|
|
2168
|
+
timer.unref?.();
|
|
2169
|
+
return timer;
|
|
2170
|
+
}
|
|
2171
|
+
function clearAdmissionDeadline(waiter) {
|
|
2172
|
+
if (waiter.deadlineTimer)
|
|
2173
|
+
clearTimeout(waiter.deadlineTimer);
|
|
2174
|
+
waiter.deadlineTimer = null;
|
|
2175
|
+
}
|
|
2176
|
+
function clearCapacityRecheck(waiter) {
|
|
2177
|
+
if (waiter.capacityRecheckTimer)
|
|
2178
|
+
clearInterval(waiter.capacityRecheckTimer);
|
|
2179
|
+
waiter.capacityRecheckTimer = null;
|
|
2180
|
+
}
|
|
2181
|
+
async function ensurePrivateDirectory(directory) {
|
|
2182
|
+
await fs.mkdir(directory, { recursive: true, mode: 0o700 });
|
|
2183
|
+
if (process.platform !== "win32")
|
|
2184
|
+
await fs.chmod(directory, 0o700);
|
|
2185
|
+
}
|
|
2186
|
+
async function containedWorkingDirectory(worktreePath, configuredDirectory) {
|
|
2187
|
+
const [realWorktree, realDirectory] = await Promise.all([
|
|
2188
|
+
fs.realpath(worktreePath),
|
|
2189
|
+
fs.realpath(path.resolve(worktreePath, configuredDirectory)),
|
|
2190
|
+
]);
|
|
2191
|
+
const relative = path.relative(realWorktree, realDirectory);
|
|
2192
|
+
if (relative.startsWith("..") || path.isAbsolute(relative)) {
|
|
2193
|
+
throw new Error("host_runtime_project_invalid: launch working directory must remain inside the prepared worktree.");
|
|
2194
|
+
}
|
|
2195
|
+
return realDirectory;
|
|
2196
|
+
}
|
|
2197
|
+
function uuidHex(value, label) {
|
|
2198
|
+
const normalized = value.replaceAll("-", "").toLowerCase();
|
|
2199
|
+
if (!/^[0-9a-f]{32}$/u.test(normalized)) {
|
|
2200
|
+
throw new Error(`host_runtime_lease_invalid: ${label} id must be a full UUID.`);
|
|
2201
|
+
}
|
|
2202
|
+
return normalized;
|
|
2203
|
+
}
|