@loucompanion/forge-bridge 0.1.3-dev.b6be5e7f9668 → 0.1.3-dev.ba7d36726eda
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +78 -0
- package/dist/bridge/bridge.base.d.ts +5 -1
- package/dist/bridge/bridge.service.d.ts +52 -3
- package/dist/bridge/bridge.service.js +796 -22
- package/dist/bridge/bridge.types.d.ts +135 -1
- package/dist/cleanup/cleanup.service.d.ts +13 -2
- package/dist/cleanup/cleanup.service.js +435 -3
- package/dist/cleanup/cleanup.types.d.ts +10 -0
- package/dist/cleanup/index.d.ts +1 -1
- package/dist/cleanup/index.js +1 -1
- package/dist/cleanup/internals.d.ts +2 -2
- package/dist/cleanup/internals.js +1 -1
- package/dist/cli/cli.help.d.ts +2 -0
- package/dist/cli/cli.help.js +217 -0
- package/dist/cli/cli.service.d.ts +42 -2
- package/dist/cli/cli.service.js +356 -205
- package/dist/cli/local-config.d.ts +6 -0
- package/dist/cli/local-config.js +45 -2
- package/dist/runtime/attempt-node-runtime-launcher.d.ts +15 -0
- package/dist/runtime/attempt-node-runtime-launcher.js +50 -0
- package/dist/runtime/attempt-process-supervisor-registry.d.ts +20 -0
- package/dist/runtime/attempt-process-supervisor-registry.js +64 -0
- package/dist/runtime/attempt-process-supervisor.d.ts +74 -0
- package/dist/runtime/attempt-process-supervisor.js +290 -0
- package/dist/runtime/host-listener-inspector.d.ts +28 -0
- package/dist/runtime/host-listener-inspector.js +185 -0
- package/dist/runtime/host-port-broker.d.ts +76 -0
- package/dist/runtime/host-port-broker.js +333 -0
- package/dist/runtime/host-runtime-contract.d.ts +39 -0
- package/dist/runtime/host-runtime-contract.js +295 -0
- package/dist/runtime/host-runtime-lease.d.ts +122 -0
- package/dist/runtime/host-runtime-lease.js +1241 -0
- package/dist/runtime/host-runtime-promotion.d.ts +21 -0
- package/dist/runtime/host-runtime-promotion.js +128 -0
- package/dist/runtime/node-host-runtime-adapter.d.ts +44 -0
- package/dist/runtime/node-host-runtime-adapter.js +205 -0
- package/dist/runtime/playwright-host-runtime-adapter.d.ts +20 -0
- package/dist/runtime/playwright-host-runtime-adapter.js +136 -0
- package/dist/runtime/runtime-isolation-lease.d.ts +67 -0
- package/dist/runtime/runtime-isolation-lease.js +158 -0
- package/dist/sandbox/command-runner.d.ts +4 -0
- package/dist/sandbox/command-runner.js +47 -0
- package/dist/sandbox/contract.d.ts +5 -0
- package/dist/sandbox/contract.js +276 -0
- package/dist/sandbox/event-budget.d.ts +11 -0
- package/dist/sandbox/event-budget.js +27 -0
- package/dist/sandbox/image-policy.d.ts +35 -0
- package/dist/sandbox/image-policy.js +247 -0
- package/dist/sandbox/index.d.ts +5 -0
- package/dist/sandbox/index.js +5 -0
- package/dist/sandbox/lease-watchdog.d.ts +18 -0
- package/dist/sandbox/lease-watchdog.js +67 -0
- package/dist/sandbox/preflight.d.ts +45 -0
- package/dist/sandbox/preflight.js +300 -0
- package/dist/sandbox/process.d.ts +20 -0
- package/dist/sandbox/process.js +209 -0
- package/dist/sandbox/provider-auth/codex.d.ts +26 -0
- package/dist/sandbox/provider-auth/codex.js +212 -0
- package/dist/sandbox/repository-lock.d.ts +5 -0
- package/dist/sandbox/repository-lock.js +119 -0
- package/dist/sandbox/runtime.d.ts +297 -0
- package/dist/sandbox/runtime.js +2338 -0
- package/dist/sandbox/sandbox.types.d.ts +86 -0
- package/dist/sandbox/sandbox.types.js +1 -0
- package/dist/sandbox/smoke.d.ts +76 -0
- package/dist/sandbox/smoke.js +380 -0
- package/dist/sandbox/worker-supervisor.d.ts +13 -0
- package/dist/sandbox/worker-supervisor.js +36 -0
- package/dist/sandbox/worker.d.ts +1 -0
- package/dist/sandbox/worker.js +225 -0
- package/dist/sandbox/workspace.service.d.ts +13 -0
- package/dist/sandbox/workspace.service.js +67 -0
- package/dist/session/phase-completion.artifacts.d.ts +10 -0
- package/dist/session/phase-completion.artifacts.js +620 -0
- package/dist/session/phase-completion.d.ts +32 -44
- package/dist/session/phase-completion.helpers.d.ts +112 -0
- package/dist/session/phase-completion.helpers.js +665 -0
- package/dist/session/phase-completion.js +28 -3485
- package/dist/session/phase-completion.prompts.d.ts +14 -0
- package/dist/session/phase-completion.prompts.js +220 -0
- package/dist/session/phase-completion.runtime.d.ts +36 -0
- package/dist/session/phase-completion.runtime.js +611 -0
- package/dist/session/phase-completion.schemas.d.ts +3 -0
- package/dist/session/phase-completion.schemas.js +707 -0
- package/dist/session/phase-completion.validation.d.ts +34 -0
- package/dist/session/phase-completion.validation.js +724 -0
- package/dist/session/provider-cli/provider-cli.artifacts.d.ts +93 -0
- package/dist/session/provider-cli/provider-cli.artifacts.js +778 -0
- package/dist/session/provider-cli/provider-cli.base.d.ts +1 -0
- package/dist/session/provider-cli/provider-cli.deterministic-payloads.d.ts +12 -0
- package/dist/session/provider-cli/provider-cli.deterministic-payloads.js +279 -0
- package/dist/session/provider-cli/provider-cli.deterministic.d.ts +14 -0
- package/dist/session/provider-cli/provider-cli.deterministic.js +329 -0
- package/dist/session/provider-cli/provider-cli.process.d.ts +34 -0
- package/dist/session/provider-cli/provider-cli.process.js +316 -0
- package/dist/session/provider-cli/provider-cli.prompts.d.ts +47 -0
- package/dist/session/provider-cli/provider-cli.prompts.js +573 -0
- package/dist/session/provider-cli/provider-cli.qa-sidecars.d.ts +11 -0
- package/dist/session/provider-cli/provider-cli.qa-sidecars.js +65 -0
- package/dist/session/provider-cli/provider-cli.runtime.d.ts +9 -0
- package/dist/session/provider-cli/provider-cli.runtime.js +41 -0
- package/dist/session/provider-cli/provider-cli.service.d.ts +9 -53
- package/dist/session/provider-cli/provider-cli.service.js +182 -2536
- package/dist/session/provider-cli/provider-cli.types.d.ts +3 -0
- package/dist/session/provider-cli/provider-output-state.d.ts +20 -0
- package/dist/session/provider-cli/provider-output-state.js +445 -0
- package/dist/session/session-redaction.d.ts +3 -0
- package/dist/session/session-redaction.js +23 -0
- package/dist/session/session.base.d.ts +4 -0
- package/dist/session/session.service.d.ts +6 -1
- package/dist/session/session.service.js +202 -53
- package/dist/session/session.types.d.ts +123 -1
- package/dist/session/session.types.js +301 -3
- package/dist/shared/path/path.js +3 -1
- package/dist/shared/process/host-process-recovery.d.ts +12 -0
- package/dist/shared/process/host-process-recovery.js +78 -0
- package/dist/shared/process/host-process-supervisor.d.ts +40 -0
- package/dist/shared/process/host-process-supervisor.js +195 -0
- package/dist/shared/process/windows-launch.d.ts +8 -0
- package/dist/shared/process/windows-launch.js +41 -0
- package/dist/shared/provider-capabilities.d.ts +1 -0
- package/dist/shared/provider-capabilities.js +3 -0
- package/dist/shared/redaction/redaction.js +10 -3
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/worktree/artifact-injector.js +39 -6
- package/dist/worktree/dependency-warmer.d.ts +11 -0
- package/dist/worktree/dependency-warmer.js +45 -1
- package/dist/worktree/index.d.ts +1 -0
- package/dist/worktree/index.js +1 -0
- package/dist/worktree/prior-phase-artifact-hydrator.d.ts +2 -0
- package/dist/worktree/prior-phase-artifact-hydrator.js +136 -0
- package/dist/worktree/worktree-paths.d.ts +4 -0
- package/dist/worktree/worktree-paths.js +30 -0
- package/dist/worktree/worktree.base.d.ts +3 -0
- package/dist/worktree/worktree.service.d.ts +8 -3
- package/dist/worktree/worktree.service.js +89 -52
- package/dist/worktree/worktree.types.d.ts +3 -1
- package/package.json +6 -3
- package/runtime/windows-job-supervisor.ps1 +287 -0
- package/runtime/windows-listener-owner.ps1 +33 -0
|
@@ -0,0 +1,1241 @@
|
|
|
1
|
+
import fs from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { StreamingProviderProcessRunner, } from "../session/provider-cli/provider-cli.process.js";
|
|
4
|
+
import { createRuntimeIsolationLease, renewRuntimeIsolationLease, transitionRuntimeIsolationLease, validRuntimeIsolationLease, withRuntimeIsolationResources, } from "./runtime-isolation-lease.js";
|
|
5
|
+
import { hostPortEnvironment, loadHostRuntimeContract, } from "./host-runtime-contract.js";
|
|
6
|
+
import { HostPortBroker, } from "./host-port-broker.js";
|
|
7
|
+
import { NodeHostRuntimeAdapter, } from "./node-host-runtime-adapter.js";
|
|
8
|
+
import { AttemptProcessSupervisorRegistry } from "./attempt-process-supervisor-registry.js";
|
|
9
|
+
import { AttemptNodeRuntimeLauncher } from "./attempt-node-runtime-launcher.js";
|
|
10
|
+
import { HostListenerInspector } from "./host-listener-inspector.js";
|
|
11
|
+
import { PlaywrightHostRuntimeAdapter } from "./playwright-host-runtime-adapter.js";
|
|
12
|
+
import { HostParallelPromotionStore, configuredLaunchPortNames, hostRuntimeContractFingerprint, hostRuntimeParallelEligible, } from "./host-runtime-promotion.js";
|
|
13
|
+
import { ExactHostProcessRecovery, } from "../shared/process/host-process-recovery.js";
|
|
14
|
+
export class HostRuntimeLeaseManager {
|
|
15
|
+
stateRoot;
|
|
16
|
+
processSupervisors;
|
|
17
|
+
processRunner;
|
|
18
|
+
listenerInspector;
|
|
19
|
+
promotionStore;
|
|
20
|
+
playwrightAdapter;
|
|
21
|
+
processRecovery;
|
|
22
|
+
active = new Map();
|
|
23
|
+
pendingRenewals = new Map();
|
|
24
|
+
admissions = new Map();
|
|
25
|
+
singletonOwners = new Map();
|
|
26
|
+
singletonWaiters = [];
|
|
27
|
+
parallelProjectOwners = new Map();
|
|
28
|
+
parallelProjectWaiters = [];
|
|
29
|
+
degraded = new Map();
|
|
30
|
+
portBroker;
|
|
31
|
+
nodeAdapter;
|
|
32
|
+
constructor(stateRoot, portBroker = new HostPortBroker(), nodeAdapter, processSupervisors = new AttemptProcessSupervisorRegistry(), processRunner = new StreamingProviderProcessRunner(), listenerInspector = new HostListenerInspector(), promotionStore = new HostParallelPromotionStore(stateRoot), playwrightAdapter = new PlaywrightHostRuntimeAdapter(), processRecovery = new ExactHostProcessRecovery()) {
|
|
33
|
+
this.stateRoot = stateRoot;
|
|
34
|
+
this.processSupervisors = processSupervisors;
|
|
35
|
+
this.processRunner = processRunner;
|
|
36
|
+
this.listenerInspector = listenerInspector;
|
|
37
|
+
this.promotionStore = promotionStore;
|
|
38
|
+
this.playwrightAdapter = playwrightAdapter;
|
|
39
|
+
this.processRecovery = processRecovery;
|
|
40
|
+
this.portBroker = portBroker;
|
|
41
|
+
this.nodeAdapter = nodeAdapter ?? new NodeHostRuntimeAdapter(portBroker);
|
|
42
|
+
}
|
|
43
|
+
async reap(options) {
|
|
44
|
+
const report = {
|
|
45
|
+
examined: 0,
|
|
46
|
+
reaped: 0,
|
|
47
|
+
processesTerminated: 0,
|
|
48
|
+
reusedPidsProtected: 0,
|
|
49
|
+
degraded: 0,
|
|
50
|
+
};
|
|
51
|
+
for (const ledger of await this.readLedgers()) {
|
|
52
|
+
const lease = ledger.runtimeLease;
|
|
53
|
+
if (lease.runnerId !== options.runnerId || this.active.has(lease.dispatchAttemptId))
|
|
54
|
+
continue;
|
|
55
|
+
const staleInstance = lease.runnerInstanceId !== options.runnerInstanceId;
|
|
56
|
+
const expired = Date.parse(lease.expiresAt) <= (options.nowMs ?? Date.now());
|
|
57
|
+
if (!staleInstance && !expired && lease.state !== "released" && lease.state !== "degraded")
|
|
58
|
+
continue;
|
|
59
|
+
report.examined += 1;
|
|
60
|
+
const attemptRoot = this.attemptRoot(lease.dispatchAttemptId);
|
|
61
|
+
let outcomes;
|
|
62
|
+
try {
|
|
63
|
+
outcomes = await this.recoverPersistedProcesses(lease, attemptRoot);
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
await this.markLedgerDegraded(ledger);
|
|
67
|
+
report.degraded += 1;
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
report.processesTerminated += outcomes.filter(outcome => outcome === "terminated").length;
|
|
71
|
+
report.reusedPidsProtected += outcomes.filter(outcome => outcome === "pid_reused").length;
|
|
72
|
+
if (outcomes.includes("unverifiable")) {
|
|
73
|
+
await this.markLedgerDegraded(ledger);
|
|
74
|
+
report.degraded += 1;
|
|
75
|
+
continue;
|
|
76
|
+
}
|
|
77
|
+
try {
|
|
78
|
+
const releasing = transitionHostLeaseForCleanup(lease, "releasing");
|
|
79
|
+
await this.writeLedger(releasing, ledger.repoId);
|
|
80
|
+
await fs.rm(attemptRoot, { recursive: true, force: true });
|
|
81
|
+
await fs.rm(this.ledgerPath(lease.dispatchAttemptId), { force: true });
|
|
82
|
+
this.forgetDegraded(lease.dispatchAttemptId, lease.fencingToken);
|
|
83
|
+
report.reaped += 1;
|
|
84
|
+
}
|
|
85
|
+
catch {
|
|
86
|
+
await this.markLedgerDegraded(ledger);
|
|
87
|
+
report.degraded += 1;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return report;
|
|
91
|
+
}
|
|
92
|
+
snapshot() {
|
|
93
|
+
const projects = new Map();
|
|
94
|
+
for (const [dispatchAttemptId, admission] of this.admissions) {
|
|
95
|
+
const active = this.active.get(dispatchAttemptId);
|
|
96
|
+
const project = capacityProject(projects, admission.repoId);
|
|
97
|
+
if (active) {
|
|
98
|
+
if (active.lease.state === "degraded") {
|
|
99
|
+
project.degradedAttempts += 1;
|
|
100
|
+
project.safetyClass = conservativeSafetyClass(project.safetyClass, "host_serial_only");
|
|
101
|
+
}
|
|
102
|
+
else {
|
|
103
|
+
project.activeAttempts += 1;
|
|
104
|
+
const effectiveConcurrency = effectiveHostConcurrency(active.lease, active.runtimeContract);
|
|
105
|
+
project.safetyClass = conservativeSafetyClass(project.safetyClass, effectiveConcurrency === "host_parallel_safe"
|
|
106
|
+
? effectiveConcurrency
|
|
107
|
+
: admission.parallelProjectKey && admission.provisionalPromotionFingerprint
|
|
108
|
+
? "host_parallel_candidate"
|
|
109
|
+
: "host_serial_only");
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
else {
|
|
113
|
+
project.reservedAttempts += 1;
|
|
114
|
+
project.safetyClass = conservativeSafetyClass(project.safetyClass, admission.provisionalPromotionFingerprint ? "host_parallel_candidate" : "host_serial_only");
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
for (const [dispatchAttemptId, degraded] of this.degraded) {
|
|
118
|
+
if (this.active.has(dispatchAttemptId) || !degraded.repoId)
|
|
119
|
+
continue;
|
|
120
|
+
const project = capacityProject(projects, degraded.repoId);
|
|
121
|
+
project.degradedAttempts += 1;
|
|
122
|
+
project.safetyClass = conservativeSafetyClass(project.safetyClass, "host_serial_only");
|
|
123
|
+
}
|
|
124
|
+
for (const waiter of this.singletonWaiters) {
|
|
125
|
+
const repoId = repoIdFromProjectSingletonKeys(waiter.keys);
|
|
126
|
+
if (!repoId)
|
|
127
|
+
continue;
|
|
128
|
+
const project = capacityProject(projects, repoId);
|
|
129
|
+
project.waitingAttempts += 1;
|
|
130
|
+
project.safetyClass = conservativeSafetyClass(project.safetyClass, "host_serial_only");
|
|
131
|
+
}
|
|
132
|
+
for (const waiter of this.parallelProjectWaiters) {
|
|
133
|
+
const repoId = repoIdFromProjectSingletonKey(waiter.key);
|
|
134
|
+
if (!repoId)
|
|
135
|
+
continue;
|
|
136
|
+
const project = capacityProject(projects, repoId);
|
|
137
|
+
project.waitingAttempts += 1;
|
|
138
|
+
project.safetyClass = conservativeSafetyClass(project.safetyClass, "host_parallel_candidate");
|
|
139
|
+
}
|
|
140
|
+
const activeAttempts = [...this.active.values()]
|
|
141
|
+
.filter(active => active.lease.state !== "degraded").length;
|
|
142
|
+
const degradedAttempts = new Set([
|
|
143
|
+
...this.degraded.keys(),
|
|
144
|
+
...[...this.active.entries()]
|
|
145
|
+
.filter(([, active]) => active.lease.state === "degraded")
|
|
146
|
+
.map(([dispatchAttemptId]) => dispatchAttemptId),
|
|
147
|
+
]).size;
|
|
148
|
+
return {
|
|
149
|
+
activeAttempts,
|
|
150
|
+
reservedAttempts: Math.max(0, this.admissions.size - this.active.size),
|
|
151
|
+
waitingAttempts: this.singletonWaiters.length + this.parallelProjectWaiters.length,
|
|
152
|
+
degradedAttempts,
|
|
153
|
+
exclusiveReservations: this.singletonOwners.size,
|
|
154
|
+
sharedReservations: [...this.parallelProjectOwners.values()]
|
|
155
|
+
.reduce((count, owners) => count + owners.size, 0),
|
|
156
|
+
projects: [...projects.values()]
|
|
157
|
+
.map(project => ({
|
|
158
|
+
...project,
|
|
159
|
+
safetyClass: project.safetyClass ?? "host_serial_only",
|
|
160
|
+
}))
|
|
161
|
+
.sort((left, right) => left.repoId.localeCompare(right.repoId)),
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
async acquire(dispatch, signal) {
|
|
165
|
+
const sandbox = dispatch.sandbox;
|
|
166
|
+
if (!sandbox || sandbox.executionMode !== "host") {
|
|
167
|
+
throw new Error("host_runtime_lease_invalid: managed host sandbox context is required.");
|
|
168
|
+
}
|
|
169
|
+
uuidHex(sandbox.dispatchAttemptId, "dispatch attempt");
|
|
170
|
+
uuidHex(dispatch.lease.fencingToken, "fencing token");
|
|
171
|
+
const owner = {
|
|
172
|
+
dispatchAttemptId: sandbox.dispatchAttemptId,
|
|
173
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
174
|
+
};
|
|
175
|
+
const existing = this.admissions.get(sandbox.dispatchAttemptId);
|
|
176
|
+
if (existing) {
|
|
177
|
+
if (existing.fencingToken !== owner.fencingToken) {
|
|
178
|
+
throw new Error("host_runtime_singleton_conflict: dispatch attempt already holds admission under a different fence.");
|
|
179
|
+
}
|
|
180
|
+
return existing.keys;
|
|
181
|
+
}
|
|
182
|
+
const promotion = await this.promotionStore.load(dispatch.repo.id);
|
|
183
|
+
const projectKey = projectHostRuntimeSingletonKey(dispatch.repo.id);
|
|
184
|
+
const keys = promotion ? [] : [projectKey];
|
|
185
|
+
const expiresAt = this.pendingRenewals.get(sandbox.dispatchAttemptId)?.fencingToken === owner.fencingToken
|
|
186
|
+
? this.pendingRenewals.get(sandbox.dispatchAttemptId).expiresAt
|
|
187
|
+
: dispatch.lease.expiresAt;
|
|
188
|
+
try {
|
|
189
|
+
if (promotion) {
|
|
190
|
+
await this.reserveParallelProject(projectKey, owner, expiresAt, signal);
|
|
191
|
+
}
|
|
192
|
+
else {
|
|
193
|
+
await this.reserveSingletonKeys(keys, owner, expiresAt, signal);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
catch (error) {
|
|
197
|
+
this.clearPendingRenewal(owner);
|
|
198
|
+
throw error;
|
|
199
|
+
}
|
|
200
|
+
this.admissions.set(sandbox.dispatchAttemptId, {
|
|
201
|
+
repoId: dispatch.repo.id,
|
|
202
|
+
fencingToken: owner.fencingToken,
|
|
203
|
+
provisionalPromotionFingerprint: promotion?.runtimeContractFingerprint ?? null,
|
|
204
|
+
keys,
|
|
205
|
+
parallelProjectKey: promotion ? projectKey : null,
|
|
206
|
+
});
|
|
207
|
+
return keys;
|
|
208
|
+
}
|
|
209
|
+
async reserveSingletonKeys(keys, owner, expiresAt, signal) {
|
|
210
|
+
const owned = keys.every(key => sameSingletonOwner(this.singletonOwners.get(key), owner));
|
|
211
|
+
if (owned)
|
|
212
|
+
return;
|
|
213
|
+
const staleFence = [...this.singletonOwners.values()].find(existing => existing.dispatchAttemptId === owner.dispatchAttemptId
|
|
214
|
+
&& existing.fencingToken !== owner.fencingToken);
|
|
215
|
+
if (staleFence) {
|
|
216
|
+
throw new Error("host_runtime_singleton_conflict: dispatch attempt already holds the singleton under a different fence.");
|
|
217
|
+
}
|
|
218
|
+
if (signal?.aborted)
|
|
219
|
+
throw cancellationError(signal);
|
|
220
|
+
throwIfAdmissionExpired(expiresAt);
|
|
221
|
+
if (this.singletonKeysAvailable(keys)) {
|
|
222
|
+
this.assignSingletonKeys(keys, owner);
|
|
223
|
+
return;
|
|
224
|
+
}
|
|
225
|
+
await new Promise((resolve, reject) => {
|
|
226
|
+
const waiter = {
|
|
227
|
+
owner,
|
|
228
|
+
keys,
|
|
229
|
+
resolve: () => resolve(),
|
|
230
|
+
reject,
|
|
231
|
+
signal,
|
|
232
|
+
expiresAt,
|
|
233
|
+
deadlineTimer: null,
|
|
234
|
+
abort: () => {
|
|
235
|
+
const index = this.singletonWaiters.indexOf(waiter);
|
|
236
|
+
if (index >= 0)
|
|
237
|
+
this.singletonWaiters.splice(index, 1);
|
|
238
|
+
clearAdmissionDeadline(waiter);
|
|
239
|
+
this.clearPendingRenewal(owner);
|
|
240
|
+
reject(cancellationError(signal));
|
|
241
|
+
this.drainAdmissionWaiters();
|
|
242
|
+
},
|
|
243
|
+
expire: () => {
|
|
244
|
+
const index = this.singletonWaiters.indexOf(waiter);
|
|
245
|
+
if (index < 0)
|
|
246
|
+
return;
|
|
247
|
+
this.singletonWaiters.splice(index, 1);
|
|
248
|
+
clearAdmissionDeadline(waiter);
|
|
249
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
250
|
+
this.clearPendingRenewal(owner);
|
|
251
|
+
reject(new HostRuntimeAdmissionExpiredError());
|
|
252
|
+
this.drainAdmissionWaiters();
|
|
253
|
+
},
|
|
254
|
+
};
|
|
255
|
+
this.singletonWaiters.push(waiter);
|
|
256
|
+
signal?.addEventListener("abort", waiter.abort, { once: true });
|
|
257
|
+
waiter.deadlineTimer = scheduleAdmissionDeadline(waiter.expiresAt, waiter.expire);
|
|
258
|
+
this.drainAdmissionWaiters();
|
|
259
|
+
});
|
|
260
|
+
}
|
|
261
|
+
async reserveParallelProject(key, owner, expiresAt, signal) {
|
|
262
|
+
if (this.parallelProjectOwned(key, owner))
|
|
263
|
+
return;
|
|
264
|
+
const staleFence = [...(this.parallelProjectOwners.get(key)?.values() ?? [])]
|
|
265
|
+
.find(existing => existing.dispatchAttemptId === owner.dispatchAttemptId
|
|
266
|
+
&& existing.fencingToken !== owner.fencingToken);
|
|
267
|
+
if (staleFence) {
|
|
268
|
+
throw new Error("host_runtime_parallel_conflict: dispatch attempt already holds shared admission under a different fence.");
|
|
269
|
+
}
|
|
270
|
+
if (signal?.aborted)
|
|
271
|
+
throw cancellationError(signal);
|
|
272
|
+
throwIfAdmissionExpired(expiresAt);
|
|
273
|
+
if (this.parallelProjectAvailable(key)) {
|
|
274
|
+
this.assignParallelProject(key, owner);
|
|
275
|
+
return;
|
|
276
|
+
}
|
|
277
|
+
await new Promise((resolve, reject) => {
|
|
278
|
+
const waiter = {
|
|
279
|
+
owner,
|
|
280
|
+
key,
|
|
281
|
+
resolve,
|
|
282
|
+
reject,
|
|
283
|
+
signal,
|
|
284
|
+
expiresAt,
|
|
285
|
+
deadlineTimer: null,
|
|
286
|
+
abort: () => {
|
|
287
|
+
const index = this.parallelProjectWaiters.indexOf(waiter);
|
|
288
|
+
if (index >= 0)
|
|
289
|
+
this.parallelProjectWaiters.splice(index, 1);
|
|
290
|
+
clearAdmissionDeadline(waiter);
|
|
291
|
+
this.clearPendingRenewal(owner);
|
|
292
|
+
reject(cancellationError(signal));
|
|
293
|
+
this.drainAdmissionWaiters();
|
|
294
|
+
},
|
|
295
|
+
expire: () => {
|
|
296
|
+
const index = this.parallelProjectWaiters.indexOf(waiter);
|
|
297
|
+
if (index < 0)
|
|
298
|
+
return;
|
|
299
|
+
this.parallelProjectWaiters.splice(index, 1);
|
|
300
|
+
clearAdmissionDeadline(waiter);
|
|
301
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
302
|
+
this.clearPendingRenewal(owner);
|
|
303
|
+
reject(new HostRuntimeAdmissionExpiredError());
|
|
304
|
+
this.drainAdmissionWaiters();
|
|
305
|
+
},
|
|
306
|
+
};
|
|
307
|
+
this.parallelProjectWaiters.push(waiter);
|
|
308
|
+
signal?.addEventListener("abort", waiter.abort, { once: true });
|
|
309
|
+
waiter.deadlineTimer = scheduleAdmissionDeadline(waiter.expiresAt, waiter.expire);
|
|
310
|
+
this.drainAdmissionWaiters();
|
|
311
|
+
});
|
|
312
|
+
}
|
|
313
|
+
async start(dispatch, worktreePath, signal) {
|
|
314
|
+
const sandbox = dispatch.sandbox;
|
|
315
|
+
if (!sandbox || sandbox.executionMode !== "host") {
|
|
316
|
+
throw new Error("host_runtime_lease_invalid: managed host sandbox context is required.");
|
|
317
|
+
}
|
|
318
|
+
const existing = this.active.get(sandbox.dispatchAttemptId);
|
|
319
|
+
if (existing) {
|
|
320
|
+
if (existing.lease.fencingToken !== dispatch.lease.fencingToken) {
|
|
321
|
+
throw new Error("host_runtime_lease_conflict: an active host lease has a different fence.");
|
|
322
|
+
}
|
|
323
|
+
return {
|
|
324
|
+
lease: existing.lease,
|
|
325
|
+
environment: hostRuntimeEnvironment(dispatch.session.id, existing.lease, existing.runtimeContract),
|
|
326
|
+
runtimeContract: existing.runtimeContract,
|
|
327
|
+
};
|
|
328
|
+
}
|
|
329
|
+
const admission = this.admissions.get(sandbox.dispatchAttemptId);
|
|
330
|
+
const owner = { dispatchAttemptId: sandbox.dispatchAttemptId, fencingToken: dispatch.lease.fencingToken };
|
|
331
|
+
if (!admission || admission.fencingToken !== owner.fencingToken) {
|
|
332
|
+
throw new Error("host_runtime_singleton_unreserved: managed host admission is required before start.");
|
|
333
|
+
}
|
|
334
|
+
const attemptHex = uuidHex(sandbox.dispatchAttemptId, "dispatch attempt");
|
|
335
|
+
const normalizedWorktreePath = path.resolve(worktreePath);
|
|
336
|
+
const attemptRoot = path.join(this.stateRoot, "host-attempts", attemptHex);
|
|
337
|
+
const temporaryPath = path.join(attemptRoot, "tmp");
|
|
338
|
+
const logPath = path.join(attemptRoot, "logs");
|
|
339
|
+
const cachePath = path.join(attemptRoot, "cache");
|
|
340
|
+
const browserProfilePath = path.join(attemptRoot, "browser-profile");
|
|
341
|
+
const testOutputPath = path.join(attemptRoot, "test-output");
|
|
342
|
+
const processSupervisorMetadataPath = path.join(logPath, "attempt-process-supervisor.json");
|
|
343
|
+
const processSupervisorId = `host-${attemptHex}`;
|
|
344
|
+
const renewedExpiry = this.pendingRenewals.get(sandbox.dispatchAttemptId);
|
|
345
|
+
const expiresAt = renewedExpiry?.fencingToken === dispatch.lease.fencingToken
|
|
346
|
+
? renewedExpiry.expiresAt
|
|
347
|
+
: dispatch.lease.expiresAt;
|
|
348
|
+
let processSupervisor = null;
|
|
349
|
+
try {
|
|
350
|
+
let runtimeContract;
|
|
351
|
+
try {
|
|
352
|
+
runtimeContract = await loadHostRuntimeContract(normalizedWorktreePath);
|
|
353
|
+
}
|
|
354
|
+
catch (error) {
|
|
355
|
+
if (admission.provisionalPromotionFingerprint) {
|
|
356
|
+
await this.promotionStore.invalidate(dispatch.repo.id).catch(() => undefined);
|
|
357
|
+
}
|
|
358
|
+
throw error;
|
|
359
|
+
}
|
|
360
|
+
if (runtimeContract.contract.runtime.hostConcurrency === "disabled") {
|
|
361
|
+
throw new Error("host_runtime_disabled: project runtime configuration requires Docker execution.");
|
|
362
|
+
}
|
|
363
|
+
const runtimeContractFingerprint = hostRuntimeContractFingerprint(runtimeContract);
|
|
364
|
+
const promotion = await this.promotionStore.load(dispatch.repo.id);
|
|
365
|
+
const promotionMatches = hostRuntimeParallelEligible(runtimeContract)
|
|
366
|
+
&& promotion?.runtimeContractFingerprint === runtimeContractFingerprint;
|
|
367
|
+
if (admission.keys.length === 0 && !promotionMatches) {
|
|
368
|
+
await this.promotionStore.invalidate(dispatch.repo.id);
|
|
369
|
+
const projectKeys = hostRuntimeSingletonKeys(dispatch);
|
|
370
|
+
if (admission.parallelProjectKey) {
|
|
371
|
+
this.releaseParallelProjectNoDrain(admission.parallelProjectKey, owner);
|
|
372
|
+
admission.parallelProjectKey = null;
|
|
373
|
+
}
|
|
374
|
+
const expiresAt = this.pendingRenewals.get(sandbox.dispatchAttemptId)?.fencingToken === owner.fencingToken
|
|
375
|
+
? this.pendingRenewals.get(sandbox.dispatchAttemptId).expiresAt
|
|
376
|
+
: dispatch.lease.expiresAt;
|
|
377
|
+
await this.reserveSingletonKeys(projectKeys, owner, expiresAt, signal);
|
|
378
|
+
admission.keys = projectKeys;
|
|
379
|
+
}
|
|
380
|
+
const singletonKeys = admission.keys;
|
|
381
|
+
if (!singletonKeys.every(key => sameSingletonOwner(this.singletonOwners.get(key), owner))) {
|
|
382
|
+
throw new Error("host_runtime_singleton_unreserved: managed host singleton ownership is required before start.");
|
|
383
|
+
}
|
|
384
|
+
if (admission.parallelProjectKey && !this.parallelProjectOwned(admission.parallelProjectKey, owner)) {
|
|
385
|
+
throw new Error("host_runtime_parallel_unreserved: matching shared project admission is required before start.");
|
|
386
|
+
}
|
|
387
|
+
await Promise.all([
|
|
388
|
+
ensurePrivateDirectory(temporaryPath),
|
|
389
|
+
ensurePrivateDirectory(logPath),
|
|
390
|
+
ensurePrivateDirectory(cachePath),
|
|
391
|
+
ensurePrivateDirectory(browserProfilePath),
|
|
392
|
+
ensurePrivateDirectory(testOutputPath),
|
|
393
|
+
]);
|
|
394
|
+
let lease = createRuntimeIsolationLease({
|
|
395
|
+
leaseId: sandbox.executionLeaseId,
|
|
396
|
+
dispatchAttemptId: sandbox.dispatchAttemptId,
|
|
397
|
+
executionLeaseId: sandbox.executionLeaseId,
|
|
398
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
399
|
+
runnerId: dispatch.execution.runnerId ?? "",
|
|
400
|
+
runnerInstanceId: sandbox.runnerInstanceId,
|
|
401
|
+
worktreeKey: dispatch.lease.worktreeKey,
|
|
402
|
+
worktreePath: normalizedWorktreePath,
|
|
403
|
+
mode: "host",
|
|
404
|
+
expiresAt,
|
|
405
|
+
resources: {
|
|
406
|
+
memoryReservationBytes: 0,
|
|
407
|
+
cpuMillis: 0,
|
|
408
|
+
pids: 0,
|
|
409
|
+
ports: [],
|
|
410
|
+
paths: [
|
|
411
|
+
{ path: normalizedWorktreePath, role: "worktree" },
|
|
412
|
+
{ path: temporaryPath, role: "temporary" },
|
|
413
|
+
{ path: logPath, role: "logs" },
|
|
414
|
+
{ path: cachePath, role: "cache" },
|
|
415
|
+
{ path: browserProfilePath, role: "browser_profile" },
|
|
416
|
+
{ path: testOutputPath, role: "test_output" },
|
|
417
|
+
{ path: processSupervisorMetadataPath, role: "process_supervisor_metadata" },
|
|
418
|
+
],
|
|
419
|
+
processSupervisorId,
|
|
420
|
+
dockerResources: [],
|
|
421
|
+
singletonKeys,
|
|
422
|
+
},
|
|
423
|
+
});
|
|
424
|
+
await this.writeLedger(lease);
|
|
425
|
+
lease = transitionRuntimeIsolationLease(lease, "starting");
|
|
426
|
+
await this.writeLedger(lease);
|
|
427
|
+
lease = transitionRuntimeIsolationLease(lease, "active");
|
|
428
|
+
await this.writeLedger(lease);
|
|
429
|
+
processSupervisor = this.processSupervisors.register({
|
|
430
|
+
processSupervisorId,
|
|
431
|
+
dispatchAttemptId: lease.dispatchAttemptId,
|
|
432
|
+
executionLeaseId: lease.executionLeaseId,
|
|
433
|
+
fencingToken: lease.fencingToken,
|
|
434
|
+
logDirectory: logPath,
|
|
435
|
+
}, this.processRunner, { signal });
|
|
436
|
+
this.active.set(sandbox.dispatchAttemptId, {
|
|
437
|
+
lease,
|
|
438
|
+
repoId: dispatch.repo.id,
|
|
439
|
+
sessionId: dispatch.session.id,
|
|
440
|
+
attemptRoot,
|
|
441
|
+
runtimeContract,
|
|
442
|
+
runtimeContractFingerprint,
|
|
443
|
+
parallelPromotionEligible: hostRuntimeParallelEligible(runtimeContract),
|
|
444
|
+
processSupervisor,
|
|
445
|
+
adapterCleanups: [],
|
|
446
|
+
pendingAdapterLaunches: new Map(),
|
|
447
|
+
releasing: false,
|
|
448
|
+
});
|
|
449
|
+
this.pendingRenewals.delete(sandbox.dispatchAttemptId);
|
|
450
|
+
return {
|
|
451
|
+
lease,
|
|
452
|
+
environment: hostRuntimeEnvironment(dispatch.session.id, lease, runtimeContract),
|
|
453
|
+
runtimeContract,
|
|
454
|
+
};
|
|
455
|
+
}
|
|
456
|
+
catch (error) {
|
|
457
|
+
this.pendingRenewals.delete(sandbox.dispatchAttemptId);
|
|
458
|
+
if (processSupervisor) {
|
|
459
|
+
await processSupervisor.stopAll(dispatch.lease.fencingToken, "Host runtime start rolled back.").catch(() => undefined);
|
|
460
|
+
this.processSupervisors.unregister(sandbox.dispatchAttemptId, dispatch.lease.fencingToken);
|
|
461
|
+
}
|
|
462
|
+
await fs.rm(attemptRoot, { recursive: true, force: true }).catch(() => undefined);
|
|
463
|
+
await fs.rm(this.ledgerPath(sandbox.dispatchAttemptId), { force: true }).catch(() => undefined);
|
|
464
|
+
this.releaseSingletonKeys(sandbox.dispatchAttemptId, dispatch.lease.fencingToken);
|
|
465
|
+
this.releaseParallelProjects(sandbox.dispatchAttemptId, dispatch.lease.fencingToken);
|
|
466
|
+
this.deleteAdmission(sandbox.dispatchAttemptId, dispatch.lease.fencingToken);
|
|
467
|
+
throw error;
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
async updateLease(dispatchAttemptId, fencingToken, expiresAt) {
|
|
471
|
+
try {
|
|
472
|
+
uuidHex(dispatchAttemptId, "dispatch attempt");
|
|
473
|
+
uuidHex(fencingToken, "fencing token");
|
|
474
|
+
}
|
|
475
|
+
catch {
|
|
476
|
+
return;
|
|
477
|
+
}
|
|
478
|
+
const active = this.active.get(dispatchAttemptId);
|
|
479
|
+
if (!active) {
|
|
480
|
+
if (Number.isFinite(Date.parse(expiresAt))) {
|
|
481
|
+
const pending = this.pendingRenewals.get(dispatchAttemptId);
|
|
482
|
+
if (!pending
|
|
483
|
+
|| pending.fencingToken !== fencingToken
|
|
484
|
+
|| Date.parse(expiresAt) > Date.parse(pending.expiresAt)) {
|
|
485
|
+
this.pendingRenewals.set(dispatchAttemptId, { fencingToken, expiresAt });
|
|
486
|
+
}
|
|
487
|
+
this.renewAdmissionWaiterDeadline(dispatchAttemptId, fencingToken, expiresAt);
|
|
488
|
+
}
|
|
489
|
+
return;
|
|
490
|
+
}
|
|
491
|
+
const renewed = renewRuntimeIsolationLease(active.lease, fencingToken, expiresAt);
|
|
492
|
+
if (!renewed)
|
|
493
|
+
return;
|
|
494
|
+
active.lease = renewed;
|
|
495
|
+
await this.writeLedger(renewed);
|
|
496
|
+
}
|
|
497
|
+
async startConfiguredProject(dispatchAttemptId, fencingToken, signal) {
|
|
498
|
+
const active = this.active.get(dispatchAttemptId);
|
|
499
|
+
if (!active || active.lease.fencingToken !== fencingToken || active.lease.state !== "active" || active.releasing) {
|
|
500
|
+
throw new Error("host_runtime_project_unavailable: an active matching host runtime lease is required.");
|
|
501
|
+
}
|
|
502
|
+
const configuredPort = active.runtimeContract.contract.runtime.ports.find(port => port.launch !== null);
|
|
503
|
+
if (!configuredPort?.launch) {
|
|
504
|
+
return {
|
|
505
|
+
lease: active.lease,
|
|
506
|
+
environment: hostRuntimeEnvironment(active.sessionId, active.lease, active.runtimeContract),
|
|
507
|
+
runtimeContract: active.runtimeContract,
|
|
508
|
+
};
|
|
509
|
+
}
|
|
510
|
+
const workingDirectory = await containedWorkingDirectory(active.lease.worktreePath, configuredPort.launch.workingDirectory);
|
|
511
|
+
const owner = hostPortOwner(active.lease);
|
|
512
|
+
const logDirectory = active.lease.resources.paths.find(entry => entry.role === "logs")?.path;
|
|
513
|
+
if (!logDirectory)
|
|
514
|
+
throw new Error("host_runtime_project_invalid: attempt log directory is unavailable.");
|
|
515
|
+
const launcher = new AttemptNodeRuntimeLauncher(active.processSupervisor, owner, logDirectory, this.listenerInspector);
|
|
516
|
+
const launched = await this.launchNodeRuntime(dispatchAttemptId, fencingToken, configuredPort.name, {
|
|
517
|
+
command: configuredPort.launch.command,
|
|
518
|
+
args: configuredPort.launch.args,
|
|
519
|
+
cwd: workingDirectory,
|
|
520
|
+
environment: hostRuntimeEnvironment(active.sessionId, active.lease, active.runtimeContract),
|
|
521
|
+
}, launcher, signal);
|
|
522
|
+
const promotedLease = await this.promoteVerifiedRuntime(active, launched.runtimeLease);
|
|
523
|
+
const runtimeEnvironment = hostRuntimeEnvironment(active.sessionId, promotedLease, active.runtimeContract);
|
|
524
|
+
const playwright = active.runtimeContract.contract.runtime.playwright;
|
|
525
|
+
if (!playwright) {
|
|
526
|
+
return {
|
|
527
|
+
lease: promotedLease,
|
|
528
|
+
environment: runtimeEnvironment,
|
|
529
|
+
runtimeContract: active.runtimeContract,
|
|
530
|
+
};
|
|
531
|
+
}
|
|
532
|
+
const playwrightPort = promotedLease.resources.ports.find(port => port.name === playwright.baseUrlPort);
|
|
533
|
+
const playwrightPortConfig = active.runtimeContract.contract.runtime.ports
|
|
534
|
+
.find(port => port.name === playwright.baseUrlPort);
|
|
535
|
+
if (!playwrightPort || !playwrightPortConfig?.baseUrlEnv) {
|
|
536
|
+
throw new Error("host_runtime_playwright_unavailable: the configured verified base URL is unavailable.");
|
|
537
|
+
}
|
|
538
|
+
if (playwrightPort.hostIp !== "127.0.0.1") {
|
|
539
|
+
throw new Error("host_runtime_playwright_unavailable: Playwright requires an IPv4 loopback listener.");
|
|
540
|
+
}
|
|
541
|
+
const baseUrl = hostPortEnvironment(playwrightPortConfig, playwrightPort.hostIp, playwrightPort.hostPort)[playwrightPortConfig.baseUrlEnv];
|
|
542
|
+
if (!baseUrl)
|
|
543
|
+
throw new Error("host_runtime_playwright_unavailable: the configured base URL could not be rendered.");
|
|
544
|
+
const pathFor = (role) => promotedLease.resources.paths.find(entry => entry.role === role)?.path ?? "";
|
|
545
|
+
const preparedPlaywright = await this.playwrightAdapter.prepare(playwright, {
|
|
546
|
+
worktreePath: promotedLease.worktreePath,
|
|
547
|
+
temporaryDirectory: pathFor("temporary"),
|
|
548
|
+
testOutputDirectory: pathFor("test_output"),
|
|
549
|
+
browserProfileDirectory: pathFor("browser_profile"),
|
|
550
|
+
baseUrl,
|
|
551
|
+
});
|
|
552
|
+
return {
|
|
553
|
+
lease: promotedLease,
|
|
554
|
+
environment: { ...runtimeEnvironment, ...preparedPlaywright.environment },
|
|
555
|
+
runtimeContract: active.runtimeContract,
|
|
556
|
+
};
|
|
557
|
+
}
|
|
558
|
+
async promoteVerifiedRuntime(active, runtimeLease) {
|
|
559
|
+
if (!active.parallelPromotionEligible)
|
|
560
|
+
return runtimeLease;
|
|
561
|
+
const verifiedPortNames = runtimeLease.resources.ports.map(port => port.name).sort((left, right) => left.localeCompare(right));
|
|
562
|
+
const requiredPortNames = configuredLaunchPortNames(active.runtimeContract);
|
|
563
|
+
if (requiredPortNames.length !== verifiedPortNames.length
|
|
564
|
+
|| requiredPortNames.some((name, index) => name !== verifiedPortNames[index])) {
|
|
565
|
+
return runtimeLease;
|
|
566
|
+
}
|
|
567
|
+
const projectKey = projectHostRuntimeSingletonKey(active.repoId);
|
|
568
|
+
const promotedLease = withRuntimeIsolationResources(runtimeLease, {
|
|
569
|
+
...runtimeLease.resources,
|
|
570
|
+
singletonKeys: runtimeLease.resources.singletonKeys.filter(key => key !== projectKey),
|
|
571
|
+
});
|
|
572
|
+
await this.writeLedger(promotedLease);
|
|
573
|
+
active.lease = promotedLease;
|
|
574
|
+
try {
|
|
575
|
+
const promotion = await this.promotionStore.observe(active.repoId, active.runtimeContract, runtimeLease.dispatchAttemptId, verifiedPortNames);
|
|
576
|
+
this.convertSingletonToParallelProject(projectKey, {
|
|
577
|
+
dispatchAttemptId: runtimeLease.dispatchAttemptId,
|
|
578
|
+
fencingToken: runtimeLease.fencingToken,
|
|
579
|
+
});
|
|
580
|
+
const admission = this.admissions.get(runtimeLease.dispatchAttemptId);
|
|
581
|
+
if (admission?.fencingToken === runtimeLease.fencingToken) {
|
|
582
|
+
this.admissions.set(runtimeLease.dispatchAttemptId, {
|
|
583
|
+
...admission,
|
|
584
|
+
keys: promotedLease.resources.singletonKeys,
|
|
585
|
+
provisionalPromotionFingerprint: promotion.runtimeContractFingerprint,
|
|
586
|
+
parallelProjectKey: projectKey,
|
|
587
|
+
});
|
|
588
|
+
}
|
|
589
|
+
return promotedLease;
|
|
590
|
+
}
|
|
591
|
+
catch {
|
|
592
|
+
await this.promotionStore.invalidate(active.repoId).catch(() => undefined);
|
|
593
|
+
active.lease = runtimeLease;
|
|
594
|
+
await this.writeLedger(runtimeLease);
|
|
595
|
+
return runtimeLease;
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
async launchNodeRuntime(dispatchAttemptId, fencingToken, portName, command, launcher, signal) {
|
|
599
|
+
const active = this.active.get(dispatchAttemptId);
|
|
600
|
+
if (!active || active.lease.fencingToken !== fencingToken || active.lease.state !== "active" || active.releasing) {
|
|
601
|
+
throw new Error("host_runtime_adapter_unavailable: an active matching host runtime lease is required.");
|
|
602
|
+
}
|
|
603
|
+
const portConfig = active.runtimeContract.contract.runtime.ports.find(port => port.name === portName);
|
|
604
|
+
if (!portConfig)
|
|
605
|
+
throw new Error(`host_runtime_adapter_invalid: configured port '${portName}' was not found.`);
|
|
606
|
+
if (active.lease.resources.ports.some(port => port.name === portName) || active.pendingAdapterLaunches.has(portName)) {
|
|
607
|
+
throw new Error(`host_runtime_adapter_conflict: configured port '${portName}' is already active.`);
|
|
608
|
+
}
|
|
609
|
+
const controller = linkedAbortController(signal);
|
|
610
|
+
let retainedByActiveRuntime = false;
|
|
611
|
+
const operation = (async () => {
|
|
612
|
+
const owner = hostPortOwner(active.lease);
|
|
613
|
+
const launched = await this.nodeAdapter.launch(owner, portConfig, command, launcher, controller.signal);
|
|
614
|
+
let cleaned = false;
|
|
615
|
+
const cleanup = async () => {
|
|
616
|
+
if (cleaned)
|
|
617
|
+
return;
|
|
618
|
+
await launcher.stop(launched.value);
|
|
619
|
+
await this.portBroker.release(launched.reservation.reservationId, owner);
|
|
620
|
+
cleaned = true;
|
|
621
|
+
unlinkAbortController(controller, signal);
|
|
622
|
+
};
|
|
623
|
+
active.adapterCleanups.push(cleanup);
|
|
624
|
+
try {
|
|
625
|
+
if (active.releasing)
|
|
626
|
+
throw new Error("host_runtime_adapter_cancelled: host runtime release started during launch.");
|
|
627
|
+
const runtimeLease = withRuntimeIsolationResources(active.lease, {
|
|
628
|
+
...active.lease.resources,
|
|
629
|
+
ports: [...active.lease.resources.ports, {
|
|
630
|
+
name: launched.reservation.name,
|
|
631
|
+
protocol: launched.reservation.protocol,
|
|
632
|
+
hostIp: launched.reservation.hostIp,
|
|
633
|
+
hostPort: launched.reservation.hostPort,
|
|
634
|
+
targetPort: launched.reservation.targetPort,
|
|
635
|
+
}],
|
|
636
|
+
});
|
|
637
|
+
await this.writeLedger(runtimeLease);
|
|
638
|
+
active.lease = runtimeLease;
|
|
639
|
+
retainedByActiveRuntime = true;
|
|
640
|
+
return { ...launched, runtimeLease };
|
|
641
|
+
}
|
|
642
|
+
catch (error) {
|
|
643
|
+
try {
|
|
644
|
+
await cleanup();
|
|
645
|
+
removeCleanup(active.adapterCleanups, cleanup);
|
|
646
|
+
}
|
|
647
|
+
catch {
|
|
648
|
+
// Preserve the cleanup and broker claim for the fenced release retry.
|
|
649
|
+
}
|
|
650
|
+
throw error;
|
|
651
|
+
}
|
|
652
|
+
})();
|
|
653
|
+
active.pendingAdapterLaunches.set(portName, { controller, promise: operation });
|
|
654
|
+
try {
|
|
655
|
+
return await operation;
|
|
656
|
+
}
|
|
657
|
+
finally {
|
|
658
|
+
active.pendingAdapterLaunches.delete(portName);
|
|
659
|
+
if (!retainedByActiveRuntime)
|
|
660
|
+
unlinkAbortController(controller, signal);
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
async release(dispatchAttemptId, fencingToken) {
|
|
664
|
+
const active = this.active.get(dispatchAttemptId);
|
|
665
|
+
this.pendingRenewals.delete(dispatchAttemptId);
|
|
666
|
+
if (!active) {
|
|
667
|
+
this.cancelAdmissionWaiters(dispatchAttemptId, fencingToken, "Managed host admission was released before acquisition.");
|
|
668
|
+
this.releaseSingletonKeys(dispatchAttemptId, fencingToken);
|
|
669
|
+
this.releaseParallelProjects(dispatchAttemptId, fencingToken);
|
|
670
|
+
this.deleteAdmission(dispatchAttemptId, fencingToken);
|
|
671
|
+
return null;
|
|
672
|
+
}
|
|
673
|
+
if (active.lease.fencingToken !== fencingToken)
|
|
674
|
+
return null;
|
|
675
|
+
active.releasing = true;
|
|
676
|
+
for (const pending of active.pendingAdapterLaunches.values()) {
|
|
677
|
+
pending.controller.abort("Host runtime release cancelled the pending adapter launch.");
|
|
678
|
+
}
|
|
679
|
+
await Promise.allSettled([...active.pendingAdapterLaunches.values()].map(pending => pending.promise));
|
|
680
|
+
let releasing = transitionRuntimeIsolationLease(active.lease, "releasing");
|
|
681
|
+
active.lease = releasing;
|
|
682
|
+
await this.writeLedger(releasing);
|
|
683
|
+
try {
|
|
684
|
+
await active.processSupervisor.stopAll(fencingToken, "Host runtime lease released.");
|
|
685
|
+
await releaseAdapterResources(active.adapterCleanups);
|
|
686
|
+
await fs.rm(active.attemptRoot, { recursive: true, force: true });
|
|
687
|
+
releasing = transitionRuntimeIsolationLease(releasing, "released");
|
|
688
|
+
active.lease = releasing;
|
|
689
|
+
await this.writeLedger(releasing);
|
|
690
|
+
await fs.rm(this.ledgerPath(dispatchAttemptId), { force: true });
|
|
691
|
+
this.active.delete(dispatchAttemptId);
|
|
692
|
+
this.processSupervisors.unregister(dispatchAttemptId, fencingToken);
|
|
693
|
+
this.releaseSingletonKeys(dispatchAttemptId, fencingToken);
|
|
694
|
+
this.releaseParallelProjects(dispatchAttemptId, fencingToken);
|
|
695
|
+
this.deleteAdmission(dispatchAttemptId, fencingToken);
|
|
696
|
+
return releasing;
|
|
697
|
+
}
|
|
698
|
+
catch (error) {
|
|
699
|
+
const degraded = transitionRuntimeIsolationLease(releasing, "degraded");
|
|
700
|
+
active.lease = degraded;
|
|
701
|
+
await this.writeLedger(degraded).catch(() => undefined);
|
|
702
|
+
throw new HostRuntimeLeaseCleanupError(degraded, error);
|
|
703
|
+
}
|
|
704
|
+
}
|
|
705
|
+
async readLedgers() {
|
|
706
|
+
const root = path.join(this.stateRoot, "runtime-ledger", "host");
|
|
707
|
+
const entries = await fs.readdir(root, { withFileTypes: true }).catch(error => {
|
|
708
|
+
if (error.code === "ENOENT")
|
|
709
|
+
return [];
|
|
710
|
+
throw error;
|
|
711
|
+
});
|
|
712
|
+
const ledgers = [];
|
|
713
|
+
for (const entry of entries) {
|
|
714
|
+
if (!entry.isFile() || !/^[0-9a-f]{32}\.json$/u.test(entry.name))
|
|
715
|
+
continue;
|
|
716
|
+
try {
|
|
717
|
+
const value = JSON.parse(await fs.readFile(path.join(root, entry.name), "utf8"));
|
|
718
|
+
if (!validHostRuntimeLedger(value))
|
|
719
|
+
continue;
|
|
720
|
+
const expectedName = `${uuidHex(value.runtimeLease.dispatchAttemptId, "dispatch attempt")}.json`;
|
|
721
|
+
if (entry.name !== expectedName)
|
|
722
|
+
continue;
|
|
723
|
+
ledgers.push(value);
|
|
724
|
+
}
|
|
725
|
+
catch {
|
|
726
|
+
// An invalid ledger has no trustworthy ownership identity and is never reaped.
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
return ledgers;
|
|
730
|
+
}
|
|
731
|
+
async recoverPersistedProcesses(lease, attemptRoot) {
|
|
732
|
+
const logsRoot = path.join(attemptRoot, "logs");
|
|
733
|
+
const expectedAttemptRoot = path.resolve(this.attemptRoot(lease.dispatchAttemptId));
|
|
734
|
+
if (path.resolve(attemptRoot) !== expectedAttemptRoot) {
|
|
735
|
+
throw new Error("host_runtime_reaper_invalid: attempt path is outside the fenced attempt root.");
|
|
736
|
+
}
|
|
737
|
+
const entries = await fs.readdir(logsRoot, { withFileTypes: true }).catch(error => {
|
|
738
|
+
if (error.code === "ENOENT")
|
|
739
|
+
return [];
|
|
740
|
+
throw error;
|
|
741
|
+
});
|
|
742
|
+
const outcomes = [];
|
|
743
|
+
for (const entry of entries) {
|
|
744
|
+
if (!entry.isFile() || !/^process-supervisor\.[a-z][a-z0-9_-]{0,31}\.\d+\.json$/u.test(entry.name))
|
|
745
|
+
continue;
|
|
746
|
+
const metadataPath = path.join(logsRoot, entry.name);
|
|
747
|
+
const value = JSON.parse(await fs.readFile(metadataPath, "utf8"));
|
|
748
|
+
if (!validHostProcessSupervisorMetadata(value, lease)) {
|
|
749
|
+
throw new Error("host_runtime_reaper_invalid: process ownership metadata did not match the runtime lease.");
|
|
750
|
+
}
|
|
751
|
+
outcomes.push(await this.processRecovery.recover(value));
|
|
752
|
+
}
|
|
753
|
+
return outcomes;
|
|
754
|
+
}
|
|
755
|
+
async markLedgerDegraded(ledger) {
|
|
756
|
+
const lease = ledger.runtimeLease;
|
|
757
|
+
if (lease.state === "released")
|
|
758
|
+
return;
|
|
759
|
+
const degraded = transitionHostLeaseForCleanup(lease, "degraded");
|
|
760
|
+
const repoId = ledger.repoId ?? repoIdFromProjectSingletonKeys(lease.resources.singletonKeys);
|
|
761
|
+
this.rememberDegraded(degraded, repoId);
|
|
762
|
+
await this.writeLedger(degraded, repoId ?? undefined).catch(() => undefined);
|
|
763
|
+
}
|
|
764
|
+
rememberDegraded(lease, repoId) {
|
|
765
|
+
this.degraded.set(lease.dispatchAttemptId, { lease, repoId });
|
|
766
|
+
if (!repoId)
|
|
767
|
+
return;
|
|
768
|
+
const owner = { dispatchAttemptId: lease.dispatchAttemptId, fencingToken: lease.fencingToken };
|
|
769
|
+
const projectKey = projectHostRuntimeSingletonKey(repoId);
|
|
770
|
+
const existing = this.singletonOwners.get(projectKey);
|
|
771
|
+
if (!existing || sameSingletonOwner(existing, owner))
|
|
772
|
+
this.singletonOwners.set(projectKey, owner);
|
|
773
|
+
}
|
|
774
|
+
forgetDegraded(dispatchAttemptId, fencingToken) {
|
|
775
|
+
if (this.degraded.get(dispatchAttemptId)?.lease.fencingToken !== fencingToken)
|
|
776
|
+
return;
|
|
777
|
+
this.degraded.delete(dispatchAttemptId);
|
|
778
|
+
this.releaseSingletonKeys(dispatchAttemptId, fencingToken);
|
|
779
|
+
this.releaseParallelProjects(dispatchAttemptId, fencingToken);
|
|
780
|
+
}
|
|
781
|
+
attemptRoot(dispatchAttemptId) {
|
|
782
|
+
return path.join(this.stateRoot, "host-attempts", uuidHex(dispatchAttemptId, "dispatch attempt"));
|
|
783
|
+
}
|
|
784
|
+
async writeLedger(runtimeLease, explicitRepoId) {
|
|
785
|
+
const root = path.join(this.stateRoot, "runtime-ledger", "host");
|
|
786
|
+
await ensurePrivateDirectory(root);
|
|
787
|
+
const target = this.ledgerPath(runtimeLease.dispatchAttemptId);
|
|
788
|
+
const temporary = `${target}.${process.pid}.${Date.now()}.tmp`;
|
|
789
|
+
try {
|
|
790
|
+
const handle = await fs.open(temporary, "wx", 0o600);
|
|
791
|
+
try {
|
|
792
|
+
const repoId = explicitRepoId
|
|
793
|
+
?? this.active.get(runtimeLease.dispatchAttemptId)?.repoId
|
|
794
|
+
?? this.admissions.get(runtimeLease.dispatchAttemptId)?.repoId
|
|
795
|
+
?? this.degraded.get(runtimeLease.dispatchAttemptId)?.repoId
|
|
796
|
+
?? undefined;
|
|
797
|
+
const ledger = {
|
|
798
|
+
schemaVersion: 1,
|
|
799
|
+
...(repoId ? { repoId } : {}),
|
|
800
|
+
runtimeLease,
|
|
801
|
+
};
|
|
802
|
+
await handle.writeFile(`${JSON.stringify(ledger)}\n`);
|
|
803
|
+
await handle.sync();
|
|
804
|
+
}
|
|
805
|
+
finally {
|
|
806
|
+
await handle.close();
|
|
807
|
+
}
|
|
808
|
+
await fs.rename(temporary, target);
|
|
809
|
+
}
|
|
810
|
+
catch (error) {
|
|
811
|
+
await fs.rm(temporary, { force: true }).catch(() => undefined);
|
|
812
|
+
throw error;
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
ledgerPath(dispatchAttemptId) {
|
|
816
|
+
const attemptHex = uuidHex(dispatchAttemptId, "dispatch attempt");
|
|
817
|
+
return path.join(this.stateRoot, "runtime-ledger", "host", `${attemptHex}.json`);
|
|
818
|
+
}
|
|
819
|
+
singletonKeysAvailable(keys) {
|
|
820
|
+
return keys.every(key => !this.singletonOwners.has(key) && !this.parallelProjectOwners.has(key));
|
|
821
|
+
}
|
|
822
|
+
assignSingletonKeys(keys, owner) {
|
|
823
|
+
for (const key of keys)
|
|
824
|
+
this.singletonOwners.set(key, owner);
|
|
825
|
+
}
|
|
826
|
+
releaseSingletonKeys(dispatchAttemptId, fencingToken) {
|
|
827
|
+
for (const [key, owner] of this.singletonOwners) {
|
|
828
|
+
if (owner.dispatchAttemptId === dispatchAttemptId && owner.fencingToken === fencingToken) {
|
|
829
|
+
this.singletonOwners.delete(key);
|
|
830
|
+
}
|
|
831
|
+
}
|
|
832
|
+
this.drainAdmissionWaiters();
|
|
833
|
+
}
|
|
834
|
+
convertSingletonToParallelProject(key, owner) {
|
|
835
|
+
if (!sameSingletonOwner(this.singletonOwners.get(key), owner)) {
|
|
836
|
+
throw new Error("host_runtime_parallel_promotion_stale: matching exclusive project admission is required.");
|
|
837
|
+
}
|
|
838
|
+
this.assignParallelProject(key, owner);
|
|
839
|
+
this.singletonOwners.delete(key);
|
|
840
|
+
this.drainAdmissionWaiters();
|
|
841
|
+
}
|
|
842
|
+
parallelProjectOwned(key, owner) {
|
|
843
|
+
return sameSingletonOwner(this.parallelProjectOwners.get(key)?.get(owner.dispatchAttemptId), owner);
|
|
844
|
+
}
|
|
845
|
+
parallelProjectAvailable(key) {
|
|
846
|
+
return !this.singletonOwners.has(key)
|
|
847
|
+
&& !this.singletonWaiters.some(waiter => waiter.keys.includes(key));
|
|
848
|
+
}
|
|
849
|
+
assignParallelProject(key, owner) {
|
|
850
|
+
const owners = this.parallelProjectOwners.get(key) ?? new Map();
|
|
851
|
+
owners.set(owner.dispatchAttemptId, owner);
|
|
852
|
+
this.parallelProjectOwners.set(key, owners);
|
|
853
|
+
}
|
|
854
|
+
releaseParallelProjectNoDrain(key, owner) {
|
|
855
|
+
const owners = this.parallelProjectOwners.get(key);
|
|
856
|
+
if (!sameSingletonOwner(owners?.get(owner.dispatchAttemptId), owner))
|
|
857
|
+
return;
|
|
858
|
+
owners.delete(owner.dispatchAttemptId);
|
|
859
|
+
if (owners.size === 0)
|
|
860
|
+
this.parallelProjectOwners.delete(key);
|
|
861
|
+
}
|
|
862
|
+
releaseParallelProjects(dispatchAttemptId, fencingToken) {
|
|
863
|
+
const owner = { dispatchAttemptId, fencingToken };
|
|
864
|
+
for (const key of [...this.parallelProjectOwners.keys()]) {
|
|
865
|
+
this.releaseParallelProjectNoDrain(key, owner);
|
|
866
|
+
}
|
|
867
|
+
this.drainAdmissionWaiters();
|
|
868
|
+
}
|
|
869
|
+
deleteAdmission(dispatchAttemptId, fencingToken) {
|
|
870
|
+
if (this.admissions.get(dispatchAttemptId)?.fencingToken === fencingToken) {
|
|
871
|
+
this.admissions.delete(dispatchAttemptId);
|
|
872
|
+
}
|
|
873
|
+
}
|
|
874
|
+
renewAdmissionWaiterDeadline(dispatchAttemptId, fencingToken, expiresAt) {
|
|
875
|
+
for (const waiter of [...this.singletonWaiters, ...this.parallelProjectWaiters]) {
|
|
876
|
+
if (!sameSingletonOwner(waiter.owner, { dispatchAttemptId, fencingToken })
|
|
877
|
+
|| Date.parse(expiresAt) <= Date.parse(waiter.expiresAt))
|
|
878
|
+
continue;
|
|
879
|
+
waiter.expiresAt = expiresAt;
|
|
880
|
+
clearAdmissionDeadline(waiter);
|
|
881
|
+
waiter.deadlineTimer = scheduleAdmissionDeadline(waiter.expiresAt, waiter.expire);
|
|
882
|
+
}
|
|
883
|
+
}
|
|
884
|
+
clearPendingRenewal(owner) {
|
|
885
|
+
if (this.pendingRenewals.get(owner.dispatchAttemptId)?.fencingToken === owner.fencingToken) {
|
|
886
|
+
this.pendingRenewals.delete(owner.dispatchAttemptId);
|
|
887
|
+
}
|
|
888
|
+
}
|
|
889
|
+
cancelAdmissionWaiters(dispatchAttemptId, fencingToken, reason) {
|
|
890
|
+
const owner = { dispatchAttemptId, fencingToken };
|
|
891
|
+
for (const waiters of [this.singletonWaiters, this.parallelProjectWaiters]) {
|
|
892
|
+
for (let index = waiters.length - 1; index >= 0; index -= 1) {
|
|
893
|
+
const waiter = waiters[index];
|
|
894
|
+
if (!sameSingletonOwner(waiter.owner, owner))
|
|
895
|
+
continue;
|
|
896
|
+
waiters.splice(index, 1);
|
|
897
|
+
clearAdmissionDeadline(waiter);
|
|
898
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
899
|
+
waiter.reject(new Error(`host_runtime_admission_cancelled: ${reason}`));
|
|
900
|
+
}
|
|
901
|
+
}
|
|
902
|
+
this.drainAdmissionWaiters();
|
|
903
|
+
}
|
|
904
|
+
drainSingletonWaiters() {
|
|
905
|
+
for (let index = 0; index < this.singletonWaiters.length;) {
|
|
906
|
+
const waiter = this.singletonWaiters[index];
|
|
907
|
+
if (waiter.signal?.aborted) {
|
|
908
|
+
this.singletonWaiters.splice(index, 1);
|
|
909
|
+
clearAdmissionDeadline(waiter);
|
|
910
|
+
waiter.signal.removeEventListener("abort", waiter.abort);
|
|
911
|
+
this.clearPendingRenewal(waiter.owner);
|
|
912
|
+
waiter.reject(cancellationError(waiter.signal));
|
|
913
|
+
continue;
|
|
914
|
+
}
|
|
915
|
+
if (admissionExpired(waiter.expiresAt)) {
|
|
916
|
+
this.singletonWaiters.splice(index, 1);
|
|
917
|
+
clearAdmissionDeadline(waiter);
|
|
918
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
919
|
+
this.clearPendingRenewal(waiter.owner);
|
|
920
|
+
waiter.reject(new HostRuntimeAdmissionExpiredError());
|
|
921
|
+
continue;
|
|
922
|
+
}
|
|
923
|
+
if (!this.singletonKeysAvailable(waiter.keys)) {
|
|
924
|
+
index += 1;
|
|
925
|
+
continue;
|
|
926
|
+
}
|
|
927
|
+
this.singletonWaiters.splice(index, 1);
|
|
928
|
+
clearAdmissionDeadline(waiter);
|
|
929
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
930
|
+
this.assignSingletonKeys(waiter.keys, waiter.owner);
|
|
931
|
+
waiter.resolve(waiter.keys);
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
drainParallelProjectWaiters() {
|
|
935
|
+
for (let index = 0; index < this.parallelProjectWaiters.length;) {
|
|
936
|
+
const waiter = this.parallelProjectWaiters[index];
|
|
937
|
+
if (waiter.signal?.aborted) {
|
|
938
|
+
this.parallelProjectWaiters.splice(index, 1);
|
|
939
|
+
clearAdmissionDeadline(waiter);
|
|
940
|
+
waiter.signal.removeEventListener("abort", waiter.abort);
|
|
941
|
+
this.clearPendingRenewal(waiter.owner);
|
|
942
|
+
waiter.reject(cancellationError(waiter.signal));
|
|
943
|
+
continue;
|
|
944
|
+
}
|
|
945
|
+
if (admissionExpired(waiter.expiresAt)) {
|
|
946
|
+
this.parallelProjectWaiters.splice(index, 1);
|
|
947
|
+
clearAdmissionDeadline(waiter);
|
|
948
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
949
|
+
this.clearPendingRenewal(waiter.owner);
|
|
950
|
+
waiter.reject(new HostRuntimeAdmissionExpiredError());
|
|
951
|
+
continue;
|
|
952
|
+
}
|
|
953
|
+
if (!this.parallelProjectAvailable(waiter.key)) {
|
|
954
|
+
index += 1;
|
|
955
|
+
continue;
|
|
956
|
+
}
|
|
957
|
+
this.parallelProjectWaiters.splice(index, 1);
|
|
958
|
+
clearAdmissionDeadline(waiter);
|
|
959
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
960
|
+
this.assignParallelProject(waiter.key, waiter.owner);
|
|
961
|
+
waiter.resolve();
|
|
962
|
+
}
|
|
963
|
+
}
|
|
964
|
+
drainAdmissionWaiters() {
|
|
965
|
+
this.drainSingletonWaiters();
|
|
966
|
+
this.drainParallelProjectWaiters();
|
|
967
|
+
}
|
|
968
|
+
}
|
|
969
|
+
export class HostRuntimeLeaseCleanupError extends Error {
|
|
970
|
+
lease;
|
|
971
|
+
constructor(lease, cause) {
|
|
972
|
+
super(`host_runtime_cleanup_degraded: ${cause instanceof Error ? cause.message : String(cause)}`, { cause });
|
|
973
|
+
this.lease = lease;
|
|
974
|
+
this.name = "HostRuntimeLeaseCleanupError";
|
|
975
|
+
}
|
|
976
|
+
}
|
|
977
|
+
export class HostRuntimeAdmissionExpiredError extends Error {
|
|
978
|
+
constructor() {
|
|
979
|
+
super("host_runtime_admission_expired: dispatch lease expired while waiting for managed host admission.");
|
|
980
|
+
this.name = "HostRuntimeAdmissionExpiredError";
|
|
981
|
+
}
|
|
982
|
+
}
|
|
983
|
+
export function hostRuntimeEventResources(lease, runtimeContract) {
|
|
984
|
+
const requestedHostConcurrency = runtimeContract?.contract.runtime.hostConcurrency ?? "serial";
|
|
985
|
+
const configuredLaunches = runtimeContract?.contract.runtime.ports.filter(port => port.launch !== null) ?? [];
|
|
986
|
+
const verifiedLaunches = configuredLaunches.filter(configured => lease.resources.ports.some(port => port.name === configured.name));
|
|
987
|
+
const effectiveConcurrency = effectiveHostConcurrency(lease, runtimeContract);
|
|
988
|
+
return {
|
|
989
|
+
agentContainer: null,
|
|
990
|
+
sidecarContainers: [],
|
|
991
|
+
networks: [],
|
|
992
|
+
volumes: [],
|
|
993
|
+
publishedPorts: lease.resources.ports,
|
|
994
|
+
hostConcurrency: effectiveConcurrency,
|
|
995
|
+
requestedHostConcurrency,
|
|
996
|
+
hostParallelPromotion: effectiveConcurrency === "host_parallel_safe"
|
|
997
|
+
? "observed_probe_passed"
|
|
998
|
+
: requestedHostConcurrency !== "parallel"
|
|
999
|
+
? "not_requested"
|
|
1000
|
+
: configuredLaunches.length > 0 && verifiedLaunches.length === configuredLaunches.length
|
|
1001
|
+
? "pending_live_parallel_proof"
|
|
1002
|
+
: "pending_adapter_probe",
|
|
1003
|
+
verifiedRuntimePorts: verifiedLaunches.map(port => port.name),
|
|
1004
|
+
runtimeContractSource: runtimeContract?.source ?? "default",
|
|
1005
|
+
requestedPorts: runtimeContract?.contract.runtime.ports.map(port => port.name) ?? [],
|
|
1006
|
+
singletonKeys: lease.resources.singletonKeys,
|
|
1007
|
+
processSupervisorId: lease.resources.processSupervisorId,
|
|
1008
|
+
runtimeLease: lease,
|
|
1009
|
+
};
|
|
1010
|
+
}
|
|
1011
|
+
export function hostRuntimeSingletonKeys(dispatch) {
|
|
1012
|
+
return [projectHostRuntimeSingletonKey(dispatch.repo.id)];
|
|
1013
|
+
}
|
|
1014
|
+
function hostRuntimeEnvironment(sessionId, lease, runtimeContract) {
|
|
1015
|
+
const pathFor = (role) => lease.resources.paths.find(entry => entry.role === role)?.path ?? "";
|
|
1016
|
+
const environment = {
|
|
1017
|
+
FORGE_SESSION_ID: sessionId,
|
|
1018
|
+
FORGE_RUNTIME_MODE: "host",
|
|
1019
|
+
FORGE_RESOURCE_LEASE_ID: lease.leaseId,
|
|
1020
|
+
FORGE_DISPATCH_ATTEMPT_ID: lease.dispatchAttemptId,
|
|
1021
|
+
FORGE_EXECUTION_LEASE_ID: lease.executionLeaseId,
|
|
1022
|
+
FORGE_FENCING_TOKEN: lease.fencingToken,
|
|
1023
|
+
FORGE_WORKSPACE: lease.worktreePath,
|
|
1024
|
+
FORGE_WORKTREE_PATH: lease.worktreePath,
|
|
1025
|
+
FORGE_TMPDIR: pathFor("temporary"),
|
|
1026
|
+
FORGE_LOG_DIR: pathFor("logs"),
|
|
1027
|
+
FORGE_CACHE_DIR: pathFor("cache"),
|
|
1028
|
+
FORGE_BROWSER_PROFILE_DIR: pathFor("browser_profile"),
|
|
1029
|
+
FORGE_TEST_OUTPUT_DIR: pathFor("test_output"),
|
|
1030
|
+
FORGE_PROCESS_SUPERVISOR_ID: lease.resources.processSupervisorId ?? "",
|
|
1031
|
+
FORGE_PROCESS_SUPERVISOR_METADATA_PATH: pathFor("process_supervisor_metadata"),
|
|
1032
|
+
FORGE_HOST_CONCURRENCY: effectiveHostConcurrency(lease, runtimeContract),
|
|
1033
|
+
FORGE_HOST_CONCURRENCY_REQUESTED: runtimeContract.contract.runtime.hostConcurrency,
|
|
1034
|
+
FORGE_HOST_RUNTIME_CONTRACT_SOURCE: runtimeContract.source,
|
|
1035
|
+
FORGE_SINGLETON_KEYS: lease.resources.singletonKeys.join(","),
|
|
1036
|
+
TMPDIR: pathFor("temporary"),
|
|
1037
|
+
TMP: pathFor("temporary"),
|
|
1038
|
+
TEMP: pathFor("temporary"),
|
|
1039
|
+
XDG_CACHE_HOME: pathFor("cache"),
|
|
1040
|
+
NPM_CONFIG_CACHE: path.join(pathFor("cache"), "npm"),
|
|
1041
|
+
};
|
|
1042
|
+
if (lease.resources.singletonKeys.length === 0)
|
|
1043
|
+
delete environment.FORGE_SINGLETON_KEYS;
|
|
1044
|
+
const resolvedBaseUrls = [];
|
|
1045
|
+
for (const assigned of lease.resources.ports) {
|
|
1046
|
+
const configured = runtimeContract.contract.runtime.ports.find(port => port.name === assigned.name);
|
|
1047
|
+
if (!configured || assigned.hostIp !== "127.0.0.1")
|
|
1048
|
+
continue;
|
|
1049
|
+
Object.assign(environment, hostPortEnvironment(configured, assigned.hostIp, assigned.hostPort));
|
|
1050
|
+
const normalizedName = assigned.name.toUpperCase().replaceAll(/[^A-Z0-9]/gu, "_");
|
|
1051
|
+
environment[`FORGE_${normalizedName}_PORT`] = String(assigned.hostPort);
|
|
1052
|
+
if (configured.baseUrlEnv && configured.format) {
|
|
1053
|
+
const baseUrl = hostPortEnvironment(configured, assigned.hostIp, assigned.hostPort)[configured.baseUrlEnv];
|
|
1054
|
+
if (baseUrl) {
|
|
1055
|
+
environment[`FORGE_${normalizedName}_URL`] = baseUrl;
|
|
1056
|
+
resolvedBaseUrls.push(baseUrl);
|
|
1057
|
+
}
|
|
1058
|
+
}
|
|
1059
|
+
}
|
|
1060
|
+
if (resolvedBaseUrls.length === 1)
|
|
1061
|
+
environment.FORGE_BASE_URL = resolvedBaseUrls[0];
|
|
1062
|
+
return environment;
|
|
1063
|
+
}
|
|
1064
|
+
function effectiveHostConcurrency(lease, runtimeContract) {
|
|
1065
|
+
if (!runtimeContract || !hostRuntimeParallelEligible(runtimeContract))
|
|
1066
|
+
return "host_serial_only";
|
|
1067
|
+
const requiredPorts = configuredLaunchPortNames(runtimeContract);
|
|
1068
|
+
const everyPortVerified = requiredPorts.length > 0
|
|
1069
|
+
&& requiredPorts.every(name => lease.resources.ports.some(port => port.name === name));
|
|
1070
|
+
const projectSingletonHeld = lease.resources.singletonKeys.some(key => key.endsWith(":host-runtime"));
|
|
1071
|
+
return everyPortVerified && !projectSingletonHeld ? "host_parallel_safe" : "host_serial_only";
|
|
1072
|
+
}
|
|
1073
|
+
function projectHostRuntimeSingletonKey(repoId) {
|
|
1074
|
+
return `project:${repoId.toLowerCase()}:host-runtime`;
|
|
1075
|
+
}
|
|
1076
|
+
function capacityProject(projects, repoId) {
|
|
1077
|
+
const normalized = repoId.trim().toLowerCase();
|
|
1078
|
+
const existing = projects.get(normalized);
|
|
1079
|
+
if (existing)
|
|
1080
|
+
return existing;
|
|
1081
|
+
const created = {
|
|
1082
|
+
repoId: normalized,
|
|
1083
|
+
safetyClass: null,
|
|
1084
|
+
activeAttempts: 0,
|
|
1085
|
+
reservedAttempts: 0,
|
|
1086
|
+
waitingAttempts: 0,
|
|
1087
|
+
degradedAttempts: 0,
|
|
1088
|
+
};
|
|
1089
|
+
projects.set(normalized, created);
|
|
1090
|
+
return created;
|
|
1091
|
+
}
|
|
1092
|
+
function conservativeSafetyClass(left, right) {
|
|
1093
|
+
if (!left)
|
|
1094
|
+
return right;
|
|
1095
|
+
const rank = {
|
|
1096
|
+
host_serial_only: 0,
|
|
1097
|
+
host_parallel_candidate: 1,
|
|
1098
|
+
host_parallel_safe: 2,
|
|
1099
|
+
};
|
|
1100
|
+
return rank[right] < rank[left] ? right : left;
|
|
1101
|
+
}
|
|
1102
|
+
function repoIdFromProjectSingletonKeys(keys) {
|
|
1103
|
+
return keys.map(repoIdFromProjectSingletonKey).find((value) => value !== null) ?? null;
|
|
1104
|
+
}
|
|
1105
|
+
function repoIdFromProjectSingletonKey(key) {
|
|
1106
|
+
const prefix = "project:";
|
|
1107
|
+
const suffix = ":host-runtime";
|
|
1108
|
+
return key.startsWith(prefix) && key.endsWith(suffix)
|
|
1109
|
+
? key.slice(prefix.length, -suffix.length)
|
|
1110
|
+
: null;
|
|
1111
|
+
}
|
|
1112
|
+
function sameSingletonOwner(left, right) {
|
|
1113
|
+
return left?.dispatchAttemptId === right.dispatchAttemptId && left.fencingToken === right.fencingToken;
|
|
1114
|
+
}
|
|
1115
|
+
function hostPortOwner(lease) {
|
|
1116
|
+
const processSupervisorId = lease.resources.processSupervisorId;
|
|
1117
|
+
if (!processSupervisorId) {
|
|
1118
|
+
throw new Error("host_runtime_adapter_invalid: process supervisor identity is required.");
|
|
1119
|
+
}
|
|
1120
|
+
return {
|
|
1121
|
+
dispatchAttemptId: lease.dispatchAttemptId,
|
|
1122
|
+
executionLeaseId: lease.executionLeaseId,
|
|
1123
|
+
fencingToken: lease.fencingToken,
|
|
1124
|
+
processSupervisorId,
|
|
1125
|
+
};
|
|
1126
|
+
}
|
|
1127
|
+
function linkedAbortController(signal) {
|
|
1128
|
+
const controller = new AbortController();
|
|
1129
|
+
if (signal?.aborted) {
|
|
1130
|
+
controller.abort(signal.reason);
|
|
1131
|
+
}
|
|
1132
|
+
else {
|
|
1133
|
+
signal?.addEventListener("abort", () => controller.abort(signal.reason), { once: true, signal: controller.signal });
|
|
1134
|
+
}
|
|
1135
|
+
return controller;
|
|
1136
|
+
}
|
|
1137
|
+
function unlinkAbortController(controller, _signal) {
|
|
1138
|
+
if (!controller.signal.aborted)
|
|
1139
|
+
controller.abort("Host adapter launch settled.");
|
|
1140
|
+
}
|
|
1141
|
+
function removeCleanup(cleanups, cleanup) {
|
|
1142
|
+
const index = cleanups.indexOf(cleanup);
|
|
1143
|
+
if (index >= 0)
|
|
1144
|
+
cleanups.splice(index, 1);
|
|
1145
|
+
}
|
|
1146
|
+
async function releaseAdapterResources(cleanups) {
|
|
1147
|
+
const pending = cleanups.splice(0);
|
|
1148
|
+
const failed = [];
|
|
1149
|
+
let firstError;
|
|
1150
|
+
for (const cleanup of pending.reverse()) {
|
|
1151
|
+
try {
|
|
1152
|
+
await cleanup();
|
|
1153
|
+
}
|
|
1154
|
+
catch (error) {
|
|
1155
|
+
firstError ??= error;
|
|
1156
|
+
failed.unshift(cleanup);
|
|
1157
|
+
}
|
|
1158
|
+
}
|
|
1159
|
+
cleanups.push(...failed);
|
|
1160
|
+
if (firstError !== undefined)
|
|
1161
|
+
throw firstError;
|
|
1162
|
+
}
|
|
1163
|
+
function cancellationError(signal) {
|
|
1164
|
+
return new Error(String(signal?.reason ?? "Dispatch cancelled while waiting for host runtime singleton ownership."));
|
|
1165
|
+
}
|
|
1166
|
+
function validHostRuntimeLedger(value) {
|
|
1167
|
+
if (typeof value !== "object" || value === null)
|
|
1168
|
+
return false;
|
|
1169
|
+
const ledger = value;
|
|
1170
|
+
return ledger.schemaVersion === 1
|
|
1171
|
+
&& (ledger.repoId === undefined || /^[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}$/iu.test(ledger.repoId))
|
|
1172
|
+
&& validRuntimeIsolationLease(ledger.runtimeLease)
|
|
1173
|
+
&& ledger.runtimeLease.mode === "host";
|
|
1174
|
+
}
|
|
1175
|
+
function validHostProcessSupervisorMetadata(value, lease) {
|
|
1176
|
+
if (typeof value !== "object" || value === null)
|
|
1177
|
+
return false;
|
|
1178
|
+
const metadata = value;
|
|
1179
|
+
return metadata.schemaVersion === 1
|
|
1180
|
+
&& /^host-[0-9a-f]{32}$/u.test(metadata.supervisorId)
|
|
1181
|
+
&& (metadata.strategy === "windows_job_object" || metadata.strategy === "unix_process_group")
|
|
1182
|
+
&& Number.isSafeInteger(metadata.rootPid)
|
|
1183
|
+
&& metadata.rootPid > 0
|
|
1184
|
+
&& Number.isFinite(Date.parse(metadata.rootCreatedAt))
|
|
1185
|
+
&& (metadata.rootIdentity == null
|
|
1186
|
+
|| (typeof metadata.rootIdentity === "string" && metadata.rootIdentity.length <= 256))
|
|
1187
|
+
&& /^[0-9a-f]{64}$/u.test(metadata.commandDigest)
|
|
1188
|
+
&& path.resolve(metadata.worktreePath) === path.resolve(lease.worktreePath)
|
|
1189
|
+
&& metadata.runtimeLeaseId === lease.executionLeaseId
|
|
1190
|
+
&& metadata.dispatchAttemptId === lease.dispatchAttemptId
|
|
1191
|
+
&& metadata.fencingToken === lease.fencingToken
|
|
1192
|
+
&& Number.isSafeInteger(metadata.bridgePid)
|
|
1193
|
+
&& metadata.bridgePid > 0;
|
|
1194
|
+
}
|
|
1195
|
+
function transitionHostLeaseForCleanup(lease, state) {
|
|
1196
|
+
if (lease.state === state || lease.state === "released")
|
|
1197
|
+
return lease;
|
|
1198
|
+
return transitionRuntimeIsolationLease(lease, state);
|
|
1199
|
+
}
|
|
1200
|
+
function throwIfAdmissionExpired(expiresAt) {
|
|
1201
|
+
if (admissionExpired(expiresAt))
|
|
1202
|
+
throw new HostRuntimeAdmissionExpiredError();
|
|
1203
|
+
}
|
|
1204
|
+
function admissionExpired(expiresAt) {
|
|
1205
|
+
const deadline = Date.parse(expiresAt);
|
|
1206
|
+
return !Number.isFinite(deadline) || deadline <= Date.now();
|
|
1207
|
+
}
|
|
1208
|
+
function scheduleAdmissionDeadline(expiresAt, expire) {
|
|
1209
|
+
const delay = Math.max(0, Date.parse(expiresAt) - Date.now());
|
|
1210
|
+
const timer = setTimeout(expire, Math.min(delay, 2_147_483_647));
|
|
1211
|
+
timer.unref?.();
|
|
1212
|
+
return timer;
|
|
1213
|
+
}
|
|
1214
|
+
function clearAdmissionDeadline(waiter) {
|
|
1215
|
+
if (waiter.deadlineTimer)
|
|
1216
|
+
clearTimeout(waiter.deadlineTimer);
|
|
1217
|
+
waiter.deadlineTimer = null;
|
|
1218
|
+
}
|
|
1219
|
+
async function ensurePrivateDirectory(directory) {
|
|
1220
|
+
await fs.mkdir(directory, { recursive: true, mode: 0o700 });
|
|
1221
|
+
if (process.platform !== "win32")
|
|
1222
|
+
await fs.chmod(directory, 0o700);
|
|
1223
|
+
}
|
|
1224
|
+
async function containedWorkingDirectory(worktreePath, configuredDirectory) {
|
|
1225
|
+
const [realWorktree, realDirectory] = await Promise.all([
|
|
1226
|
+
fs.realpath(worktreePath),
|
|
1227
|
+
fs.realpath(path.resolve(worktreePath, configuredDirectory)),
|
|
1228
|
+
]);
|
|
1229
|
+
const relative = path.relative(realWorktree, realDirectory);
|
|
1230
|
+
if (relative.startsWith("..") || path.isAbsolute(relative)) {
|
|
1231
|
+
throw new Error("host_runtime_project_invalid: launch working directory must remain inside the prepared worktree.");
|
|
1232
|
+
}
|
|
1233
|
+
return realDirectory;
|
|
1234
|
+
}
|
|
1235
|
+
function uuidHex(value, label) {
|
|
1236
|
+
const normalized = value.replaceAll("-", "").toLowerCase();
|
|
1237
|
+
if (!/^[0-9a-f]{32}$/u.test(normalized)) {
|
|
1238
|
+
throw new Error(`host_runtime_lease_invalid: ${label} id must be a full UUID.`);
|
|
1239
|
+
}
|
|
1240
|
+
return normalized;
|
|
1241
|
+
}
|