@loucompanion/forge-bridge 0.1.3-dev.8687c5099dce → 0.1.3-dev.88e93f53db5b
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -1
- package/dist/bridge/bridge.base.d.ts +2 -0
- package/dist/bridge/bridge.service.d.ts +25 -1
- package/dist/bridge/bridge.service.js +341 -74
- package/dist/bridge/bridge.types.d.ts +4 -0
- package/dist/cleanup/cleanup.service.d.ts +1 -0
- package/dist/cleanup/cleanup.service.js +70 -9
- package/dist/cli/cli.service.js +9 -4
- package/dist/runtime/aspnet-runtime-prebuild.d.ts +36 -0
- package/dist/runtime/aspnet-runtime-prebuild.js +340 -0
- package/dist/runtime/attempt-node-runtime-launcher.js +5 -1
- package/dist/runtime/attempt-process-supervisor.d.ts +11 -0
- package/dist/runtime/attempt-process-supervisor.js +36 -2
- package/dist/runtime/host-listener-inspector.d.ts +2 -0
- package/dist/runtime/host-listener-inspector.js +25 -0
- package/dist/runtime/host-runtime-lease.d.ts +66 -5
- package/dist/runtime/host-runtime-lease.js +714 -38
- package/dist/runtime/memory-capacity.d.ts +68 -0
- package/dist/runtime/memory-capacity.js +121 -0
- package/dist/runtime/node-host-runtime-adapter.d.ts +3 -0
- package/dist/runtime/node-host-runtime-adapter.js +24 -8
- package/dist/runtime/playwright-host-runtime-adapter.d.ts +1 -0
- package/dist/runtime/playwright-host-runtime-adapter.js +5 -0
- package/dist/runtime/workspace-resource-estimator.d.ts +17 -0
- package/dist/runtime/workspace-resource-estimator.js +174 -0
- package/dist/sandbox/preflight.d.ts +31 -1
- package/dist/sandbox/preflight.js +237 -51
- package/dist/sandbox/repository-lock.js +4 -2
- package/dist/sandbox/sandbox.types.d.ts +7 -0
- package/dist/session/managed-attempt-tools.d.ts +7 -1
- package/dist/session/managed-attempt-tools.js +257 -13
- package/dist/session/managed-build-coordinator.d.ts +5 -0
- package/dist/session/managed-build-coordinator.js +105 -0
- package/dist/session/managed-build-receipt.d.ts +21 -0
- package/dist/session/managed-build-receipt.js +335 -0
- package/dist/session/phase-completion.artifacts.d.ts +17 -0
- package/dist/session/phase-completion.artifacts.js +224 -22
- package/dist/session/phase-completion.prompts.js +2 -2
- package/dist/session/phase-completion.runtime.js +10 -3
- package/dist/session/phase-completion.schemas.js +1 -1
- package/dist/session/phase-completion.validation.js +7 -3
- package/dist/session/provider-cli/ci-repair-readiness.d.ts +6 -0
- package/dist/session/provider-cli/ci-repair-readiness.js +12 -2
- package/dist/session/provider-cli/provider-cli.artifacts.d.ts +2 -1
- package/dist/session/provider-cli/provider-cli.artifacts.js +37 -3
- package/dist/session/provider-cli/provider-cli.process.d.ts +7 -0
- package/dist/session/provider-cli/provider-cli.process.js +218 -17
- package/dist/session/provider-cli/provider-cli.prompts.js +17 -4
- package/dist/session/provider-cli/provider-cli.runtime.js +3 -1
- package/dist/session/provider-cli/provider-cli.service.d.ts +5 -2
- package/dist/session/provider-cli/provider-cli.service.js +83 -34
- package/dist/session/provider-cli/provider-output-state.js +61 -14
- package/dist/session/provider-cli/provider-usage.d.ts +7 -0
- package/dist/session/provider-cli/provider-usage.js +28 -5
- package/dist/session/session-redaction.js +16 -1
- package/dist/session/session.service.js +81 -9
- package/dist/session/session.types.d.ts +15 -0
- package/dist/session/session.types.js +3 -1
- package/dist/shared/format/format-bytes.d.ts +1 -0
- package/dist/shared/format/format-bytes.js +9 -0
- package/dist/shared/path/path.d.ts +1 -0
- package/dist/shared/path/path.js +18 -0
- package/dist/shared/process/host-process-supervisor.d.ts +2 -0
- package/dist/shared/process/host-process-supervisor.js +57 -9
- package/dist/shared/redaction/redaction.js +6 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/worktree/dependency-warmer.d.ts +18 -0
- package/dist/worktree/dependency-warmer.js +218 -8
- package/dist/worktree/index.d.ts +1 -1
- package/dist/worktree/worktree.service.d.ts +2 -0
- package/dist/worktree/worktree.service.js +158 -17
- package/dist/worktree/worktree.types.d.ts +2 -0
- package/package.json +3 -2
- package/runtime/windows-job-supervisor.ps1 +89 -1
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import fs from "node:fs/promises";
|
|
2
|
+
import { statfsSync } from "node:fs";
|
|
3
|
+
import os from "node:os";
|
|
2
4
|
import path from "node:path";
|
|
5
|
+
import { resolveExistingDirectoryPath } from "../shared/path/path.js";
|
|
3
6
|
import { StreamingProviderProcessRunner, } from "../session/provider-cli/provider-cli.process.js";
|
|
4
7
|
import { createRuntimeIsolationLease, renewRuntimeIsolationLease, transitionRuntimeIsolationLease, validRuntimeIsolationLease, withRuntimeIsolationResources, } from "./runtime-isolation-lease.js";
|
|
5
8
|
import { hostPortEnvironment, loadHostRuntimeContract, } from "./host-runtime-contract.js";
|
|
@@ -8,10 +11,14 @@ import { NodeHostRuntimeAdapter, } from "./node-host-runtime-adapter.js";
|
|
|
8
11
|
import { AttemptProcessSupervisorRegistry } from "./attempt-process-supervisor-registry.js";
|
|
9
12
|
import { AttemptNodeRuntimeLauncher } from "./attempt-node-runtime-launcher.js";
|
|
10
13
|
import { HostListenerInspector } from "./host-listener-inspector.js";
|
|
14
|
+
import { RecoveryStabilizedMemory, buildMemoryAdmissionDiagnostics, resolveAdaptiveMemoryReserveBytes, } from "./memory-capacity.js";
|
|
11
15
|
import { PlaywrightHostRuntimeAdapter } from "./playwright-host-runtime-adapter.js";
|
|
12
16
|
import { HostParallelPromotionStore, configuredLaunchPortNames, hostRuntimeContractFingerprint, hostRuntimeParallelEligible, } from "./host-runtime-promotion.js";
|
|
13
17
|
import { ExactHostProcessRecovery, } from "../shared/process/host-process-recovery.js";
|
|
14
18
|
import { resetGeneratedOutputs } from "./generated-output-reset.js";
|
|
19
|
+
import { architectMinimumWorkspaceEstimateBytes } from "./workspace-resource-estimator.js";
|
|
20
|
+
import { createAspNetRuntimePrebuildPlan, ensureAspNetRuntimePrebuilt, resolveWorktreeCommitSha, } from "./aspnet-runtime-prebuild.js";
|
|
21
|
+
const hostCapacityWaiterScanLimit = 64;
|
|
15
22
|
export class HostRuntimeLeaseManager {
|
|
16
23
|
stateRoot;
|
|
17
24
|
processSupervisors;
|
|
@@ -20,6 +27,7 @@ export class HostRuntimeLeaseManager {
|
|
|
20
27
|
promotionStore;
|
|
21
28
|
playwrightAdapter;
|
|
22
29
|
processRecovery;
|
|
30
|
+
capacityResourceSample;
|
|
23
31
|
active = new Map();
|
|
24
32
|
pendingRenewals = new Map();
|
|
25
33
|
admissions = new Map();
|
|
@@ -27,11 +35,19 @@ export class HostRuntimeLeaseManager {
|
|
|
27
35
|
singletonWaiters = [];
|
|
28
36
|
parallelProjectOwners = new Map();
|
|
29
37
|
parallelProjectWaiters = [];
|
|
38
|
+
capacityReservations = new Map();
|
|
39
|
+
capacityWaiters = [];
|
|
40
|
+
capacityWaiterScanCursor = 0;
|
|
41
|
+
backgroundAdmissionWaiters = [];
|
|
30
42
|
degraded = new Map();
|
|
31
43
|
knownPromotions = new Map();
|
|
44
|
+
workspaceAvailableBytes = Number.MAX_SAFE_INTEGER;
|
|
45
|
+
ambientMemoryCommitmentBytes = null;
|
|
32
46
|
portBroker;
|
|
33
47
|
nodeAdapter;
|
|
34
|
-
|
|
48
|
+
capacityConfiguration;
|
|
49
|
+
memoryAvailability;
|
|
50
|
+
constructor(stateRoot, portBroker = new HostPortBroker(), nodeAdapter, processSupervisors = new AttemptProcessSupervisorRegistry(), processRunner = new StreamingProviderProcessRunner(), listenerInspector = new HostListenerInspector(), promotionStore = new HostParallelPromotionStore(stateRoot), playwrightAdapter = new PlaywrightHostRuntimeAdapter(), processRecovery = new ExactHostProcessRecovery(), capacityResourceSample) {
|
|
35
51
|
this.stateRoot = stateRoot;
|
|
36
52
|
this.processSupervisors = processSupervisors;
|
|
37
53
|
this.processRunner = processRunner;
|
|
@@ -39,8 +55,11 @@ export class HostRuntimeLeaseManager {
|
|
|
39
55
|
this.promotionStore = promotionStore;
|
|
40
56
|
this.playwrightAdapter = playwrightAdapter;
|
|
41
57
|
this.processRecovery = processRecovery;
|
|
58
|
+
this.capacityResourceSample = capacityResourceSample;
|
|
42
59
|
this.portBroker = portBroker;
|
|
43
60
|
this.nodeAdapter = nodeAdapter ?? new NodeHostRuntimeAdapter(portBroker);
|
|
61
|
+
this.capacityConfiguration = loadHostCapacityConfiguration(capacityResourceSample);
|
|
62
|
+
this.memoryAvailability = new RecoveryStabilizedMemory(this.capacityConfiguration.memoryRecoverySamples, this.capacityConfiguration.memoryRecoverySampleIntervalMs);
|
|
44
63
|
}
|
|
45
64
|
async reap(options) {
|
|
46
65
|
await this.refreshKnownPromotions();
|
|
@@ -168,6 +187,14 @@ export class HostRuntimeLeaseManager {
|
|
|
168
187
|
project.waitingAttempts += 1;
|
|
169
188
|
project.safetyClass = conservativeSafetyClass(project.safetyClass, "host_parallel_candidate");
|
|
170
189
|
}
|
|
190
|
+
for (const waiter of this.backgroundAdmissionWaiters) {
|
|
191
|
+
const repoId = repoIdFromProjectSingletonKeys(waiter.keys);
|
|
192
|
+
if (!repoId)
|
|
193
|
+
continue;
|
|
194
|
+
const project = capacityProject(projects, repoId);
|
|
195
|
+
project.waitingAttempts += 1;
|
|
196
|
+
project.safetyClass = conservativeSafetyClass(project.safetyClass, "host_serial_only");
|
|
197
|
+
}
|
|
171
198
|
const activeAttempts = [...this.active.values()]
|
|
172
199
|
.filter(active => active.lease.state !== "degraded").length;
|
|
173
200
|
const degradedAttempts = new Set([
|
|
@@ -176,14 +203,19 @@ export class HostRuntimeLeaseManager {
|
|
|
176
203
|
.filter(([, active]) => active.lease.state === "degraded")
|
|
177
204
|
.map(([dispatchAttemptId]) => dispatchAttemptId),
|
|
178
205
|
]).size;
|
|
206
|
+
const hostResources = this.hostResourceSnapshot();
|
|
179
207
|
return {
|
|
180
208
|
activeAttempts,
|
|
181
209
|
reservedAttempts: Math.max(0, this.admissions.size - this.active.size),
|
|
182
|
-
waitingAttempts: this.singletonWaiters.length
|
|
210
|
+
waitingAttempts: this.singletonWaiters.length
|
|
211
|
+
+ this.parallelProjectWaiters.length
|
|
212
|
+
+ this.capacityWaiters.length
|
|
213
|
+
+ this.backgroundAdmissionWaiters.length,
|
|
183
214
|
degradedAttempts,
|
|
184
215
|
exclusiveReservations: this.singletonOwners.size,
|
|
185
216
|
sharedReservations: [...this.parallelProjectOwners.values()]
|
|
186
217
|
.reduce((count, owners) => count + owners.size, 0),
|
|
218
|
+
...hostResources,
|
|
187
219
|
projects: [...projects.values()]
|
|
188
220
|
.map(project => ({
|
|
189
221
|
...project,
|
|
@@ -192,6 +224,12 @@ export class HostRuntimeLeaseManager {
|
|
|
192
224
|
.sort((left, right) => left.repoId.localeCompare(right.repoId)),
|
|
193
225
|
};
|
|
194
226
|
}
|
|
227
|
+
hasForegroundDemand() {
|
|
228
|
+
return [...this.admissions.values()].some(admission => admission.priority === "foreground")
|
|
229
|
+
|| this.singletonWaiters.some(waiter => waiter.priority === "foreground")
|
|
230
|
+
|| this.parallelProjectWaiters.some(waiter => waiter.priority === "foreground")
|
|
231
|
+
|| this.capacityWaiters.some(waiter => waiter.priority === "foreground");
|
|
232
|
+
}
|
|
195
233
|
async acquire(dispatch, signal) {
|
|
196
234
|
const sandbox = dispatch.sandbox;
|
|
197
235
|
if (!sandbox || sandbox.executionMode !== "host") {
|
|
@@ -211,6 +249,7 @@ export class HostRuntimeLeaseManager {
|
|
|
211
249
|
return existing.keys;
|
|
212
250
|
}
|
|
213
251
|
const executionClass = runtimeExecutionClass(dispatch);
|
|
252
|
+
const priority = runtimeAdmissionPriority(dispatch);
|
|
214
253
|
const promotion = executionClass === "managed_runtime"
|
|
215
254
|
? await this.promotionStore.load(dispatch.repo.id)
|
|
216
255
|
: null;
|
|
@@ -219,37 +258,345 @@ export class HostRuntimeLeaseManager {
|
|
|
219
258
|
const projectKey = executionClass === "analysis_only"
|
|
220
259
|
? projectHostAnalysisKey(dispatch.repo.id)
|
|
221
260
|
: projectHostRuntimeSingletonKey(dispatch.repo.id);
|
|
222
|
-
|
|
261
|
+
const useParallelAdmission = priority === "foreground"
|
|
262
|
+
&& (promotion !== null || executionClass === "analysis_only");
|
|
263
|
+
let keys = useParallelAdmission ? [] : [projectKey];
|
|
264
|
+
const resources = normalizedHostResourceRequest(sandbox.requestedResources);
|
|
223
265
|
const expiresAt = this.pendingRenewals.get(sandbox.dispatchAttemptId)?.fencingToken === owner.fencingToken
|
|
224
266
|
? this.pendingRenewals.get(sandbox.dispatchAttemptId).expiresAt
|
|
225
267
|
: dispatch.lease.expiresAt;
|
|
226
268
|
try {
|
|
227
|
-
if (
|
|
228
|
-
await this.
|
|
269
|
+
if (priority === "background") {
|
|
270
|
+
await this.refreshWorkspaceCapacity();
|
|
271
|
+
keys = await this.reserveBackgroundAdmission(keys, owner, resources, expiresAt, signal);
|
|
272
|
+
}
|
|
273
|
+
else if (useParallelAdmission) {
|
|
274
|
+
await this.reserveParallelProject(projectKey, owner, expiresAt, signal, priority);
|
|
229
275
|
}
|
|
230
276
|
else {
|
|
231
|
-
keys = await this.reserveSingletonKeys(keys, owner, expiresAt, signal);
|
|
277
|
+
keys = await this.reserveSingletonKeys(keys, owner, expiresAt, signal, priority);
|
|
278
|
+
}
|
|
279
|
+
if (priority === "foreground") {
|
|
280
|
+
await this.refreshWorkspaceCapacity();
|
|
281
|
+
const capacityExpiresAt = this.pendingRenewals.get(sandbox.dispatchAttemptId)?.fencingToken === owner.fencingToken
|
|
282
|
+
? this.pendingRenewals.get(sandbox.dispatchAttemptId).expiresAt
|
|
283
|
+
: expiresAt;
|
|
284
|
+
await this.reserveHostCapacity(owner, resources, capacityExpiresAt, signal, priority);
|
|
232
285
|
}
|
|
233
286
|
}
|
|
234
287
|
catch (error) {
|
|
288
|
+
this.releaseHostCapacity(owner.dispatchAttemptId, owner.fencingToken);
|
|
289
|
+
this.releaseSingletonKeys(owner.dispatchAttemptId, owner.fencingToken);
|
|
290
|
+
this.releaseParallelProjects(owner.dispatchAttemptId, owner.fencingToken);
|
|
235
291
|
this.clearPendingRenewal(owner);
|
|
236
292
|
throw error;
|
|
237
293
|
}
|
|
238
|
-
const admittedPromotion =
|
|
239
|
-
?? (executionClass === "managed_runtime" && keys.length === 0
|
|
294
|
+
const admittedPromotion = useParallelAdmission
|
|
295
|
+
? promotion ?? (executionClass === "managed_runtime" && keys.length === 0
|
|
240
296
|
? this.knownPromotions.get(dispatch.repo.id) ?? null
|
|
241
|
-
: null)
|
|
297
|
+
: null)
|
|
298
|
+
: null;
|
|
242
299
|
this.admissions.set(sandbox.dispatchAttemptId, {
|
|
243
300
|
repoId: dispatch.repo.id,
|
|
244
301
|
fencingToken: owner.fencingToken,
|
|
245
302
|
executionClass,
|
|
303
|
+
priority,
|
|
246
304
|
provisionalPromotionFingerprint: admittedPromotion?.runtimeContractFingerprint ?? null,
|
|
247
305
|
keys,
|
|
248
|
-
parallelProjectKey:
|
|
306
|
+
parallelProjectKey: useParallelAdmission ? projectKey : null,
|
|
249
307
|
});
|
|
250
308
|
return keys;
|
|
251
309
|
}
|
|
252
|
-
async
|
|
310
|
+
async reserveHostCapacity(owner, resources, expiresAt, signal, priority = "foreground") {
|
|
311
|
+
const existing = this.capacityReservations.get(owner.dispatchAttemptId);
|
|
312
|
+
if (existing) {
|
|
313
|
+
if (!sameSingletonOwner(existing.owner, owner)) {
|
|
314
|
+
throw new Error("host_runtime_capacity_conflict: dispatch attempt already holds capacity under a different fence.");
|
|
315
|
+
}
|
|
316
|
+
return;
|
|
317
|
+
}
|
|
318
|
+
if (signal?.aborted)
|
|
319
|
+
throw cancellationError(signal);
|
|
320
|
+
throwIfAdmissionExpired(expiresAt);
|
|
321
|
+
const impossibleReason = this.hostCapacityImpossibleReason(resources);
|
|
322
|
+
if (impossibleReason)
|
|
323
|
+
throw new HostRuntimeCapacityRequestImpossibleError(impossibleReason);
|
|
324
|
+
if (this.capacityWaiters.length === 0 && this.hostCapacityAvailable(resources)) {
|
|
325
|
+
this.capacityReservations.set(owner.dispatchAttemptId, { owner, resources, state: "pending" });
|
|
326
|
+
return;
|
|
327
|
+
}
|
|
328
|
+
return new Promise((resolve, reject) => {
|
|
329
|
+
const waiter = {
|
|
330
|
+
owner,
|
|
331
|
+
resources,
|
|
332
|
+
priority,
|
|
333
|
+
resolve,
|
|
334
|
+
reject,
|
|
335
|
+
signal,
|
|
336
|
+
expiresAt,
|
|
337
|
+
deadlineTimer: null,
|
|
338
|
+
capacityRecheckTimer: null,
|
|
339
|
+
abort: () => {
|
|
340
|
+
if (!this.detachCapacityWaiter(waiter))
|
|
341
|
+
return;
|
|
342
|
+
this.clearPendingRenewal(owner);
|
|
343
|
+
reject(cancellationError(signal));
|
|
344
|
+
this.drainAdmissionWaiters();
|
|
345
|
+
},
|
|
346
|
+
expire: () => {
|
|
347
|
+
if (!this.detachCapacityWaiter(waiter))
|
|
348
|
+
return;
|
|
349
|
+
this.clearPendingRenewal(owner);
|
|
350
|
+
reject(new HostRuntimeAdmissionExpiredError());
|
|
351
|
+
this.drainAdmissionWaiters();
|
|
352
|
+
},
|
|
353
|
+
};
|
|
354
|
+
waiter.deadlineTimer = scheduleAdmissionDeadline(expiresAt, waiter.expire);
|
|
355
|
+
// Memory, CPU and filesystem capacity can recover without one of our leases
|
|
356
|
+
// being released (for example when an unrelated host process exits). Recheck
|
|
357
|
+
// ambient capacity while queued so readiness probes and attempts do not remain
|
|
358
|
+
// stranded behind a stale low-resource sample.
|
|
359
|
+
waiter.capacityRecheckTimer = setInterval(() => this.drainCapacityWaiters(), 1_000);
|
|
360
|
+
waiter.capacityRecheckTimer.unref?.();
|
|
361
|
+
signal?.addEventListener("abort", waiter.abort, { once: true });
|
|
362
|
+
enqueueAdmissionWaiter(this.capacityWaiters, waiter);
|
|
363
|
+
this.drainCapacityWaiters();
|
|
364
|
+
});
|
|
365
|
+
}
|
|
366
|
+
async reserveBackgroundAdmission(keys, owner, resources, expiresAt, signal) {
|
|
367
|
+
if (signal?.aborted)
|
|
368
|
+
throw cancellationError(signal);
|
|
369
|
+
throwIfAdmissionExpired(expiresAt);
|
|
370
|
+
return await new Promise((resolve, reject) => {
|
|
371
|
+
const waiter = {
|
|
372
|
+
owner,
|
|
373
|
+
keys,
|
|
374
|
+
resources,
|
|
375
|
+
priority: "background",
|
|
376
|
+
resolve,
|
|
377
|
+
reject,
|
|
378
|
+
signal,
|
|
379
|
+
expiresAt,
|
|
380
|
+
deadlineTimer: null,
|
|
381
|
+
capacityRecheckTimer: null,
|
|
382
|
+
abort: () => {
|
|
383
|
+
removeWaiter(this.backgroundAdmissionWaiters, waiter);
|
|
384
|
+
clearAdmissionDeadline(waiter);
|
|
385
|
+
clearCapacityRecheck(waiter);
|
|
386
|
+
this.clearPendingRenewal(owner);
|
|
387
|
+
reject(cancellationError(signal));
|
|
388
|
+
this.drainAdmissionWaiters();
|
|
389
|
+
},
|
|
390
|
+
expire: () => {
|
|
391
|
+
removeWaiter(this.backgroundAdmissionWaiters, waiter);
|
|
392
|
+
clearCapacityRecheck(waiter);
|
|
393
|
+
signal?.removeEventListener("abort", waiter.abort);
|
|
394
|
+
this.clearPendingRenewal(owner);
|
|
395
|
+
reject(new HostRuntimeAdmissionExpiredError());
|
|
396
|
+
this.drainAdmissionWaiters();
|
|
397
|
+
},
|
|
398
|
+
};
|
|
399
|
+
waiter.deadlineTimer = scheduleAdmissionDeadline(expiresAt, waiter.expire);
|
|
400
|
+
waiter.capacityRecheckTimer = setInterval(() => this.drainAdmissionWaiters(), 1_000);
|
|
401
|
+
waiter.capacityRecheckTimer.unref?.();
|
|
402
|
+
signal?.addEventListener("abort", waiter.abort, { once: true });
|
|
403
|
+
this.backgroundAdmissionWaiters.push(waiter);
|
|
404
|
+
this.drainAdmissionWaiters();
|
|
405
|
+
});
|
|
406
|
+
}
|
|
407
|
+
hostCapacityAvailable(resources) {
|
|
408
|
+
const capacity = this.hostResourceSnapshot();
|
|
409
|
+
return capacity.availableAttempts >= resources.sandboxSlots
|
|
410
|
+
&& capacity.availableMemoryBytes >= resources.memoryBytes
|
|
411
|
+
&& capacity.availableCpuMillis >= resources.cpuMillis
|
|
412
|
+
&& capacity.availableWorkspaceBytes >= resources.workspaceBytes;
|
|
413
|
+
}
|
|
414
|
+
hostCapacityImpossibleReason(resources) {
|
|
415
|
+
const { hardMaxAttempts, hostReserveBytes, cpuReserveMillis } = this.capacityConfiguration;
|
|
416
|
+
if (resources.sandboxSlots > hardMaxAttempts) {
|
|
417
|
+
return `requested ${resources.sandboxSlots} sandbox slots exceed the host maximum of ${hardMaxAttempts}.`;
|
|
418
|
+
}
|
|
419
|
+
const totalMemoryBytes = this.capacityResourceSample?.totalMemoryBytes ?? os.totalmem();
|
|
420
|
+
const maximumMemoryBytes = Math.max(0, totalMemoryBytes - hostReserveBytes);
|
|
421
|
+
if (resources.memoryBytes > maximumMemoryBytes) {
|
|
422
|
+
return `requested memory ${resources.memoryBytes} exceeds the host maximum of ${maximumMemoryBytes} bytes after reserve.`;
|
|
423
|
+
}
|
|
424
|
+
const totalCpuMillis = this.capacityResourceSample?.totalCpuMillis ?? os.cpus().length * 1_000;
|
|
425
|
+
const maximumCpuMillis = Math.max(0, totalCpuMillis - cpuReserveMillis);
|
|
426
|
+
if (resources.cpuMillis > maximumCpuMillis) {
|
|
427
|
+
return `requested CPU ${resources.cpuMillis} millis exceeds the host maximum of ${maximumCpuMillis} millis after reserve.`;
|
|
428
|
+
}
|
|
429
|
+
return null;
|
|
430
|
+
}
|
|
431
|
+
hostResourceSnapshot() {
|
|
432
|
+
if (!this.capacityResourceSample) {
|
|
433
|
+
try {
|
|
434
|
+
const stats = statfsSync(this.stateRoot);
|
|
435
|
+
this.workspaceAvailableBytes = Math.max(0, stats.bavail * stats.bsize);
|
|
436
|
+
}
|
|
437
|
+
catch {
|
|
438
|
+
// State root may not exist during the first capability heartbeat. The first
|
|
439
|
+
// admission creates it and refreshes the sample before reserving resources.
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
const { hardMaxAttempts, hostReserveBytes, cpuReserveMillis, workspaceReserveBytes } = this.capacityConfiguration;
|
|
443
|
+
const activeReservationMemoryBytes = [...this.capacityReservations.values()]
|
|
444
|
+
.reduce((total, reservation) => total + reservation.resources.memoryBytes, 0);
|
|
445
|
+
const pendingReservationMemoryBytes = [...this.capacityReservations.values()]
|
|
446
|
+
.filter(reservation => reservation.state === "pending")
|
|
447
|
+
.reduce((total, reservation) => total + reservation.resources.memoryBytes, 0);
|
|
448
|
+
const activeReservationCpuMillis = [...this.capacityReservations.values()]
|
|
449
|
+
.reduce((total, reservation) => total + reservation.resources.cpuMillis, 0);
|
|
450
|
+
const activeReservationWorkspaceBytes = [...this.capacityReservations.values()]
|
|
451
|
+
.reduce((total, reservation) => total + reservation.resources.workspaceBytes, 0);
|
|
452
|
+
const sampledAt = new Date().toISOString();
|
|
453
|
+
const totalMemoryBytes = this.capacityResourceSample?.totalMemoryBytes ?? os.totalmem();
|
|
454
|
+
const rawAvailableMemoryBytes = this.capacityResourceSample?.availableMemoryBytes ?? os.freemem();
|
|
455
|
+
const sampledAvailableMemoryBytes = this.memoryAvailability.observe(rawAvailableMemoryBytes);
|
|
456
|
+
if (this.capacityReservations.size === 0) {
|
|
457
|
+
// Capture non-Forge resident use before the first reservation. Refreshing
|
|
458
|
+
// while idle lets the baseline recover when an unrelated process exits;
|
|
459
|
+
// freezing it while leases exist prevents pending/low-RSS workers from
|
|
460
|
+
// borrowing memory already occupied by the host and control plane.
|
|
461
|
+
this.ambientMemoryCommitmentBytes = Math.max(0, totalMemoryBytes - rawAvailableMemoryBytes);
|
|
462
|
+
}
|
|
463
|
+
const committedMemoryBytes = Math.min(totalMemoryBytes, (this.ambientMemoryCommitmentBytes ?? 0) + activeReservationMemoryBytes);
|
|
464
|
+
const totalCpuMillis = this.capacityResourceSample?.totalCpuMillis ?? os.cpus().length * 1000;
|
|
465
|
+
const sampledWorkspaceBytes = this.capacityResourceSample?.availableWorkspaceBytes ?? this.workspaceAvailableBytes;
|
|
466
|
+
const memoryAdmission = buildMemoryAdmissionDiagnostics({
|
|
467
|
+
sampledAt,
|
|
468
|
+
totalMemoryBytes,
|
|
469
|
+
rawAvailableMemoryBytes,
|
|
470
|
+
stabilizedAvailableMemoryBytes: sampledAvailableMemoryBytes,
|
|
471
|
+
controlPlaneMemoryBytes: process.memoryUsage().rss,
|
|
472
|
+
reserveMemoryBytes: hostReserveBytes,
|
|
473
|
+
committedMemoryBytes,
|
|
474
|
+
pendingCommitmentMemoryBytes: pendingReservationMemoryBytes,
|
|
475
|
+
});
|
|
476
|
+
const availableMemoryBytes = memoryAdmission.availableForAdmissionBytes;
|
|
477
|
+
const availableCpuMillis = Math.max(0, totalCpuMillis - cpuReserveMillis - activeReservationCpuMillis);
|
|
478
|
+
const availableWorkspaceBytes = Math.max(0, sampledWorkspaceBytes - workspaceReserveBytes - activeReservationWorkspaceBytes);
|
|
479
|
+
const slotAvailability = Math.max(0, hardMaxAttempts - this.capacityReservations.size);
|
|
480
|
+
// Capability snapshots advertise whether any supported attempt can fit.
|
|
481
|
+
// The queue/admission path still checks the exact project-derived request.
|
|
482
|
+
const minimumRequest = minimumSupportedHostResourceRequest();
|
|
483
|
+
const resourceAvailability = Math.min(resourceSlots(availableMemoryBytes, minimumRequest.memoryBytes), resourceSlots(availableCpuMillis, minimumRequest.cpuMillis), resourceSlots(availableWorkspaceBytes, minimumRequest.workspaceBytes));
|
|
484
|
+
const availableAttempts = Math.max(0, Math.min(slotAvailability, resourceAvailability));
|
|
485
|
+
const unavailableReason = slotAvailability <= 0
|
|
486
|
+
? `Host attempt capacity is full (${this.capacityReservations.size}/${hardMaxAttempts}).`
|
|
487
|
+
: availableMemoryBytes < minimumRequest.memoryBytes
|
|
488
|
+
? "Host memory is below the admission reserve."
|
|
489
|
+
: availableCpuMillis < minimumRequest.cpuMillis
|
|
490
|
+
? "Host CPU capacity is below the admission reserve."
|
|
491
|
+
: availableWorkspaceBytes < minimumRequest.workspaceBytes
|
|
492
|
+
? "Host workspace capacity is below the admission reserve."
|
|
493
|
+
: null;
|
|
494
|
+
return {
|
|
495
|
+
hardMaxAttempts,
|
|
496
|
+
effectiveMaxAttempts: this.capacityReservations.size + availableAttempts,
|
|
497
|
+
availableAttempts,
|
|
498
|
+
totalMemoryBytes,
|
|
499
|
+
availableMemoryBytes,
|
|
500
|
+
hostReserveBytes,
|
|
501
|
+
availableCpuMillis,
|
|
502
|
+
availableWorkspaceBytes,
|
|
503
|
+
activeReservationMemoryBytes,
|
|
504
|
+
memoryAdmission,
|
|
505
|
+
unavailableCode: unavailableReason ? "capacity_unavailable" : null,
|
|
506
|
+
unavailableReason,
|
|
507
|
+
};
|
|
508
|
+
}
|
|
509
|
+
async refreshWorkspaceCapacity() {
|
|
510
|
+
if (this.capacityResourceSample)
|
|
511
|
+
return;
|
|
512
|
+
try {
|
|
513
|
+
await fs.mkdir(this.stateRoot, { recursive: true });
|
|
514
|
+
const stats = await fs.statfs(this.stateRoot);
|
|
515
|
+
this.workspaceAvailableBytes = Math.max(0, stats.bavail * stats.bsize);
|
|
516
|
+
}
|
|
517
|
+
catch {
|
|
518
|
+
// Keep the last trustworthy sample. Memory/CPU and the hard ceiling still
|
|
519
|
+
// protect admission when the filesystem cannot be sampled temporarily.
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
releaseHostCapacity(dispatchAttemptId, fencingToken) {
|
|
523
|
+
const reservation = this.capacityReservations.get(dispatchAttemptId);
|
|
524
|
+
if (!reservation || reservation.owner.fencingToken !== fencingToken)
|
|
525
|
+
return;
|
|
526
|
+
this.capacityReservations.delete(dispatchAttemptId);
|
|
527
|
+
this.drainAdmissionWaiters();
|
|
528
|
+
}
|
|
529
|
+
drainCapacityWaiters() {
|
|
530
|
+
for (const waiter of this.capacityWaitersForScan()) {
|
|
531
|
+
if (!this.capacityWaiters.includes(waiter))
|
|
532
|
+
continue;
|
|
533
|
+
if (waiter.signal?.aborted) {
|
|
534
|
+
this.detachCapacityWaiter(waiter);
|
|
535
|
+
this.clearPendingRenewal(waiter.owner);
|
|
536
|
+
waiter.reject(cancellationError(waiter.signal));
|
|
537
|
+
continue;
|
|
538
|
+
}
|
|
539
|
+
if (admissionExpired(waiter.expiresAt)) {
|
|
540
|
+
this.detachCapacityWaiter(waiter);
|
|
541
|
+
this.clearPendingRenewal(waiter.owner);
|
|
542
|
+
waiter.reject(new HostRuntimeAdmissionExpiredError());
|
|
543
|
+
continue;
|
|
544
|
+
}
|
|
545
|
+
const impossibleReason = this.hostCapacityImpossibleReason(waiter.resources);
|
|
546
|
+
if (impossibleReason) {
|
|
547
|
+
this.detachCapacityWaiter(waiter);
|
|
548
|
+
this.clearPendingRenewal(waiter.owner);
|
|
549
|
+
waiter.reject(new HostRuntimeCapacityRequestImpossibleError(impossibleReason));
|
|
550
|
+
continue;
|
|
551
|
+
}
|
|
552
|
+
if (!this.hostCapacityAvailable(waiter.resources))
|
|
553
|
+
continue;
|
|
554
|
+
this.detachCapacityWaiter(waiter);
|
|
555
|
+
this.capacityReservations.set(waiter.owner.dispatchAttemptId, {
|
|
556
|
+
owner: waiter.owner,
|
|
557
|
+
resources: waiter.resources,
|
|
558
|
+
state: "pending",
|
|
559
|
+
});
|
|
560
|
+
waiter.resolve();
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
capacityWaitersForScan() {
|
|
564
|
+
const head = this.capacityWaiters[0];
|
|
565
|
+
if (!head) {
|
|
566
|
+
this.capacityWaiterScanCursor = 0;
|
|
567
|
+
return [];
|
|
568
|
+
}
|
|
569
|
+
const scanCount = Math.min(hostCapacityWaiterScanLimit, this.capacityWaiters.length);
|
|
570
|
+
if (scanCount === this.capacityWaiters.length) {
|
|
571
|
+
this.capacityWaiterScanCursor = 0;
|
|
572
|
+
return this.capacityWaiters.slice();
|
|
573
|
+
}
|
|
574
|
+
// The head always gets the first opportunity, while a rotating bounded
|
|
575
|
+
// window prevents an unfit renewed head (or a very deep queue) from
|
|
576
|
+
// indefinitely hiding a later request that fits current capacity.
|
|
577
|
+
const laterCount = this.capacityWaiters.length - 1;
|
|
578
|
+
const laterScanCount = scanCount - 1;
|
|
579
|
+
const start = this.capacityWaiterScanCursor % laterCount;
|
|
580
|
+
const result = [head];
|
|
581
|
+
for (let offset = 0; offset < laterScanCount; offset += 1) {
|
|
582
|
+
result.push(this.capacityWaiters[1 + ((start + offset) % laterCount)]);
|
|
583
|
+
}
|
|
584
|
+
this.capacityWaiterScanCursor = (start + laterScanCount) % laterCount;
|
|
585
|
+
return result;
|
|
586
|
+
}
|
|
587
|
+
detachCapacityWaiter(waiter) {
|
|
588
|
+
const index = this.capacityWaiters.indexOf(waiter);
|
|
589
|
+
if (index < 0)
|
|
590
|
+
return false;
|
|
591
|
+
this.capacityWaiters.splice(index, 1);
|
|
592
|
+
clearAdmissionDeadline(waiter);
|
|
593
|
+
clearCapacityRecheck(waiter);
|
|
594
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
595
|
+
if (this.capacityWaiters.length === 0)
|
|
596
|
+
this.capacityWaiterScanCursor = 0;
|
|
597
|
+
return true;
|
|
598
|
+
}
|
|
599
|
+
async reserveSingletonKeys(keys, owner, expiresAt, signal, priority = "foreground") {
|
|
253
600
|
const owned = keys.every(key => sameSingletonOwner(this.singletonOwners.get(key), owner));
|
|
254
601
|
if (owned)
|
|
255
602
|
return keys;
|
|
@@ -269,6 +616,7 @@ export class HostRuntimeLeaseManager {
|
|
|
269
616
|
const waiter = {
|
|
270
617
|
owner,
|
|
271
618
|
keys,
|
|
619
|
+
priority,
|
|
272
620
|
resolve,
|
|
273
621
|
reject,
|
|
274
622
|
signal,
|
|
@@ -295,13 +643,13 @@ export class HostRuntimeLeaseManager {
|
|
|
295
643
|
this.drainAdmissionWaiters();
|
|
296
644
|
},
|
|
297
645
|
};
|
|
298
|
-
this.singletonWaiters
|
|
646
|
+
enqueueAdmissionWaiter(this.singletonWaiters, waiter);
|
|
299
647
|
signal?.addEventListener("abort", waiter.abort, { once: true });
|
|
300
648
|
waiter.deadlineTimer = scheduleAdmissionDeadline(waiter.expiresAt, waiter.expire);
|
|
301
649
|
this.drainAdmissionWaiters();
|
|
302
650
|
});
|
|
303
651
|
}
|
|
304
|
-
async reserveParallelProject(key, owner, expiresAt, signal) {
|
|
652
|
+
async reserveParallelProject(key, owner, expiresAt, signal, priority = "foreground") {
|
|
305
653
|
if (this.parallelProjectOwned(key, owner))
|
|
306
654
|
return;
|
|
307
655
|
const staleFence = [...(this.parallelProjectOwners.get(key)?.values() ?? [])]
|
|
@@ -313,7 +661,7 @@ export class HostRuntimeLeaseManager {
|
|
|
313
661
|
if (signal?.aborted)
|
|
314
662
|
throw cancellationError(signal);
|
|
315
663
|
throwIfAdmissionExpired(expiresAt);
|
|
316
|
-
if (this.parallelProjectAvailable(key)) {
|
|
664
|
+
if (this.parallelProjectAvailable(key, priority)) {
|
|
317
665
|
this.assignParallelProject(key, owner);
|
|
318
666
|
return;
|
|
319
667
|
}
|
|
@@ -321,6 +669,7 @@ export class HostRuntimeLeaseManager {
|
|
|
321
669
|
const waiter = {
|
|
322
670
|
owner,
|
|
323
671
|
key,
|
|
672
|
+
priority,
|
|
324
673
|
resolve,
|
|
325
674
|
reject,
|
|
326
675
|
signal,
|
|
@@ -347,13 +696,61 @@ export class HostRuntimeLeaseManager {
|
|
|
347
696
|
this.drainAdmissionWaiters();
|
|
348
697
|
},
|
|
349
698
|
};
|
|
350
|
-
this.parallelProjectWaiters
|
|
699
|
+
enqueueAdmissionWaiter(this.parallelProjectWaiters, waiter);
|
|
700
|
+
signal?.addEventListener("abort", waiter.abort, { once: true });
|
|
701
|
+
waiter.deadlineTimer = scheduleAdmissionDeadline(waiter.expiresAt, waiter.expire);
|
|
702
|
+
this.drainAdmissionWaiters();
|
|
703
|
+
});
|
|
704
|
+
}
|
|
705
|
+
async upgradeParallelProjectToSingleton(keys, parallelProjectKey, owner, expiresAt, signal, priority = "foreground") {
|
|
706
|
+
if (!this.parallelProjectOwned(parallelProjectKey, owner)) {
|
|
707
|
+
throw new Error("host_runtime_parallel_unreserved: matching shared project admission is required before serialization.");
|
|
708
|
+
}
|
|
709
|
+
if (signal?.aborted)
|
|
710
|
+
throw cancellationError(signal);
|
|
711
|
+
throwIfAdmissionExpired(expiresAt);
|
|
712
|
+
return await new Promise((resolve, reject) => {
|
|
713
|
+
const waiter = {
|
|
714
|
+
owner,
|
|
715
|
+
keys,
|
|
716
|
+
priority,
|
|
717
|
+
resolve,
|
|
718
|
+
reject,
|
|
719
|
+
signal,
|
|
720
|
+
expiresAt,
|
|
721
|
+
deadlineTimer: null,
|
|
722
|
+
abort: () => {
|
|
723
|
+
const index = this.singletonWaiters.indexOf(waiter);
|
|
724
|
+
if (index >= 0)
|
|
725
|
+
this.singletonWaiters.splice(index, 1);
|
|
726
|
+
clearAdmissionDeadline(waiter);
|
|
727
|
+
this.clearPendingRenewal(owner);
|
|
728
|
+
reject(cancellationError(signal));
|
|
729
|
+
this.drainAdmissionWaiters();
|
|
730
|
+
},
|
|
731
|
+
expire: () => {
|
|
732
|
+
const index = this.singletonWaiters.indexOf(waiter);
|
|
733
|
+
if (index < 0)
|
|
734
|
+
return;
|
|
735
|
+
this.singletonWaiters.splice(index, 1);
|
|
736
|
+
clearAdmissionDeadline(waiter);
|
|
737
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
738
|
+
this.clearPendingRenewal(owner);
|
|
739
|
+
reject(new HostRuntimeAdmissionExpiredError());
|
|
740
|
+
this.drainAdmissionWaiters();
|
|
741
|
+
},
|
|
742
|
+
};
|
|
743
|
+
// Queue the exclusive claim before relinquishing the shared claim. This
|
|
744
|
+
// makes the promotion downgrade atomic from the broker's perspective and
|
|
745
|
+
// prevents a background readiness probe from jumping ahead in between.
|
|
746
|
+
enqueueAdmissionWaiter(this.singletonWaiters, waiter);
|
|
351
747
|
signal?.addEventListener("abort", waiter.abort, { once: true });
|
|
352
748
|
waiter.deadlineTimer = scheduleAdmissionDeadline(waiter.expiresAt, waiter.expire);
|
|
749
|
+
this.releaseParallelProjectNoDrain(parallelProjectKey, owner);
|
|
353
750
|
this.drainAdmissionWaiters();
|
|
354
751
|
});
|
|
355
752
|
}
|
|
356
|
-
async start(dispatch, worktreePath, signal) {
|
|
753
|
+
async start(dispatch, worktreePath, signal, onAdmissionCheckpoint) {
|
|
357
754
|
const sandbox = dispatch.sandbox;
|
|
358
755
|
if (!sandbox || sandbox.executionMode !== "host") {
|
|
359
756
|
throw new Error("host_runtime_lease_invalid: managed host sandbox context is required.");
|
|
@@ -376,7 +773,7 @@ export class HostRuntimeLeaseManager {
|
|
|
376
773
|
throw new Error("host_runtime_singleton_unreserved: managed host admission is required before start.");
|
|
377
774
|
}
|
|
378
775
|
const attemptHex = uuidHex(sandbox.dispatchAttemptId, "dispatch attempt");
|
|
379
|
-
const normalizedWorktreePath =
|
|
776
|
+
const normalizedWorktreePath = await resolveExistingDirectoryPath(worktreePath);
|
|
380
777
|
const attemptRoot = path.join(this.stateRoot, "host-attempts", attemptHex);
|
|
381
778
|
const temporaryPath = path.join(attemptRoot, "tmp");
|
|
382
779
|
const logPath = path.join(attemptRoot, "logs");
|
|
@@ -413,14 +810,19 @@ export class HostRuntimeLeaseManager {
|
|
|
413
810
|
if (admission.executionClass === "managed_runtime" && admission.keys.length === 0 && !promotionMatches) {
|
|
414
811
|
await this.invalidatePromotion(dispatch.repo.id);
|
|
415
812
|
const projectKeys = hostRuntimeSingletonKeys(dispatch);
|
|
416
|
-
if (admission.parallelProjectKey) {
|
|
417
|
-
this.releaseParallelProjectNoDrain(admission.parallelProjectKey, owner);
|
|
418
|
-
admission.parallelProjectKey = null;
|
|
419
|
-
}
|
|
420
813
|
const expiresAt = this.pendingRenewals.get(sandbox.dispatchAttemptId)?.fencingToken === owner.fencingToken
|
|
421
814
|
? this.pendingRenewals.get(sandbox.dispatchAttemptId).expiresAt
|
|
422
815
|
: dispatch.lease.expiresAt;
|
|
423
|
-
|
|
816
|
+
await onAdmissionCheckpoint?.({
|
|
817
|
+
code: "host_runtime_serial_admission_waiting",
|
|
818
|
+
message: "The prepared runtime contract no longer matches the parallel-readiness proof. Waiting for foreground-priority serialized admission before launch.",
|
|
819
|
+
expiresAt,
|
|
820
|
+
});
|
|
821
|
+
const parallelProjectKey = admission.parallelProjectKey;
|
|
822
|
+
admission.parallelProjectKey = null;
|
|
823
|
+
admission.keys = parallelProjectKey
|
|
824
|
+
? await this.upgradeParallelProjectToSingleton(projectKeys, parallelProjectKey, owner, expiresAt, signal, admission.priority)
|
|
825
|
+
: await this.reserveSingletonKeys(projectKeys, owner, expiresAt, signal, admission.priority);
|
|
424
826
|
while (admission.keys.length === 0) {
|
|
425
827
|
const promoted = this.knownPromotions.get(dispatch.repo.id);
|
|
426
828
|
if (!promoted) {
|
|
@@ -429,7 +831,7 @@ export class HostRuntimeLeaseManager {
|
|
|
429
831
|
if (promoted.runtimeContractFingerprint !== runtimeContractFingerprint) {
|
|
430
832
|
this.releaseParallelProjectNoDrain(projectKeys[0], owner);
|
|
431
833
|
await this.invalidatePromotion(dispatch.repo.id);
|
|
432
|
-
admission.keys = await this.reserveSingletonKeys(projectKeys, owner, expiresAt, signal);
|
|
834
|
+
admission.keys = await this.reserveSingletonKeys(projectKeys, owner, expiresAt, signal, admission.priority);
|
|
433
835
|
continue;
|
|
434
836
|
}
|
|
435
837
|
admission.parallelProjectKey = projectKeys[0] ?? null;
|
|
@@ -444,7 +846,7 @@ export class HostRuntimeLeaseManager {
|
|
|
444
846
|
const expiresAt = this.pendingRenewals.get(sandbox.dispatchAttemptId)?.fencingToken === owner.fencingToken
|
|
445
847
|
? this.pendingRenewals.get(sandbox.dispatchAttemptId).expiresAt
|
|
446
848
|
: dispatch.lease.expiresAt;
|
|
447
|
-
const reservedSingletonKeys = await this.reserveSingletonKeys(missingSingletonKeys, owner, expiresAt, signal);
|
|
849
|
+
const reservedSingletonKeys = await this.reserveSingletonKeys(missingSingletonKeys, owner, expiresAt, signal, admission.priority);
|
|
448
850
|
admission.keys = [...admission.keys, ...reservedSingletonKeys]
|
|
449
851
|
.sort((left, right) => left.localeCompare(right));
|
|
450
852
|
}
|
|
@@ -463,6 +865,7 @@ export class HostRuntimeLeaseManager {
|
|
|
463
865
|
ensurePrivateDirectory(testOutputPath),
|
|
464
866
|
]);
|
|
465
867
|
await resetGeneratedOutputs(normalizedWorktreePath, testOutputPath);
|
|
868
|
+
const inputCommitSha = await resolveWorktreeCommitSha(normalizedWorktreePath, sandbox.inputCommitSha);
|
|
466
869
|
let lease = createRuntimeIsolationLease({
|
|
467
870
|
leaseId: sandbox.executionLeaseId,
|
|
468
871
|
dispatchAttemptId: sandbox.dispatchAttemptId,
|
|
@@ -512,6 +915,7 @@ export class HostRuntimeLeaseManager {
|
|
|
512
915
|
attemptRoot,
|
|
513
916
|
runtimeContract,
|
|
514
917
|
runtimeContractFingerprint,
|
|
918
|
+
inputCommitSha,
|
|
515
919
|
executionClass: admission.executionClass,
|
|
516
920
|
parallelPromotionEligible: hostRuntimeParallelEligible(runtimeContract),
|
|
517
921
|
processSupervisor,
|
|
@@ -568,7 +972,7 @@ export class HostRuntimeLeaseManager {
|
|
|
568
972
|
active.lease = renewed;
|
|
569
973
|
await this.writeLedger(renewed);
|
|
570
974
|
}
|
|
571
|
-
async startConfiguredProject(dispatchAttemptId, fencingToken, signal) {
|
|
975
|
+
async startConfiguredProject(dispatchAttemptId, fencingToken, signal, onProjectCheckpoint) {
|
|
572
976
|
const active = this.active.get(dispatchAttemptId);
|
|
573
977
|
if (!active || active.lease.fencingToken !== fencingToken || active.lease.state !== "active" || active.releasing) {
|
|
574
978
|
throw new Error("host_runtime_project_unavailable: an active matching host runtime lease is required.");
|
|
@@ -597,14 +1001,43 @@ export class HostRuntimeLeaseManager {
|
|
|
597
1001
|
const launcher = new AttemptNodeRuntimeLauncher(active.processSupervisor, owner, logDirectory, this.listenerInspector);
|
|
598
1002
|
for (const configuredPort of configuredPorts) {
|
|
599
1003
|
const workingDirectory = await containedWorkingDirectory(active.lease.worktreePath, configuredPort.launch.workingDirectory);
|
|
1004
|
+
const launchEnvironment = {
|
|
1005
|
+
...configuredPort.launch.environment,
|
|
1006
|
+
...hostRuntimeEnvironment(active.sessionId, active.lease, active.runtimeContract, active.executionClass),
|
|
1007
|
+
};
|
|
1008
|
+
const prebuild = createAspNetRuntimePrebuildPlan({
|
|
1009
|
+
command: configuredPort.launch.command,
|
|
1010
|
+
args: configuredPort.launch.args,
|
|
1011
|
+
cwd: workingDirectory,
|
|
1012
|
+
worktreePath: active.lease.worktreePath,
|
|
1013
|
+
environment: launchEnvironment,
|
|
1014
|
+
inputCommitSha: active.inputCommitSha,
|
|
1015
|
+
runtimeContractFingerprint: active.runtimeContractFingerprint,
|
|
1016
|
+
});
|
|
1017
|
+
if (prebuild) {
|
|
1018
|
+
await onProjectCheckpoint?.({
|
|
1019
|
+
code: "host_runtime_build_started",
|
|
1020
|
+
message: `Preparing the ASP.NET runtime for commit ${active.inputCommitSha.slice(0, 12)} before binding its assigned port.`,
|
|
1021
|
+
data: { input_commit_sha: active.inputCommitSha, project_path: prebuild.projectPath },
|
|
1022
|
+
});
|
|
1023
|
+
const build = await ensureAspNetRuntimePrebuilt(prebuild, async (plan, buildSignal) => await executeAspNetRuntimeBuild(active.processSupervisor, logDirectory, plan, buildSignal), signal);
|
|
1024
|
+
await onProjectCheckpoint?.({
|
|
1025
|
+
code: build.cacheHit ? "host_runtime_build_cached" : "host_runtime_build_completed",
|
|
1026
|
+
message: build.cacheHit
|
|
1027
|
+
? `Reused the verified ASP.NET build for commit ${active.inputCommitSha.slice(0, 12)}.`
|
|
1028
|
+
: `ASP.NET runtime build completed for commit ${active.inputCommitSha.slice(0, 12)}; launching without another build.`,
|
|
1029
|
+
data: {
|
|
1030
|
+
input_commit_sha: active.inputCommitSha,
|
|
1031
|
+
build_fingerprint: build.fingerprint,
|
|
1032
|
+
receipt_path: build.receiptPath,
|
|
1033
|
+
},
|
|
1034
|
+
});
|
|
1035
|
+
}
|
|
600
1036
|
await this.launchNodeRuntime(dispatchAttemptId, fencingToken, configuredPort.name, {
|
|
601
1037
|
command: configuredPort.launch.command,
|
|
602
1038
|
args: configuredPort.launch.args,
|
|
603
1039
|
cwd: workingDirectory,
|
|
604
|
-
environment:
|
|
605
|
-
...configuredPort.launch.environment,
|
|
606
|
-
...hostRuntimeEnvironment(active.sessionId, active.lease, active.runtimeContract, active.executionClass),
|
|
607
|
-
},
|
|
1040
|
+
environment: launchEnvironment,
|
|
608
1041
|
}, launcher, signal);
|
|
609
1042
|
}
|
|
610
1043
|
const promotedLease = await this.promoteVerifiedRuntime(active, active.lease);
|
|
@@ -637,6 +1070,7 @@ export class HostRuntimeLeaseManager {
|
|
|
637
1070
|
testOutputDirectory: pathFor("test_output"),
|
|
638
1071
|
browserProfileDirectory: pathFor("browser_profile"),
|
|
639
1072
|
baseUrl,
|
|
1073
|
+
ownershipToken: uuidHex(promotedLease.dispatchAttemptId, "dispatch attempt"),
|
|
640
1074
|
});
|
|
641
1075
|
return {
|
|
642
1076
|
lease: promotedLease,
|
|
@@ -649,7 +1083,18 @@ export class HostRuntimeLeaseManager {
|
|
|
649
1083
|
const active = this.active.get(dispatchAttemptId);
|
|
650
1084
|
if (!active || active.lease.fencingToken !== fencingToken || active.releasing)
|
|
651
1085
|
return null;
|
|
652
|
-
return new AttemptSupervisedProviderProcessRunner(active.processSupervisor, hostPortOwner(active.lease), this.listenerInspector, active.executionClass === "analysis_only");
|
|
1086
|
+
return new AttemptSupervisedProviderProcessRunner(active.processSupervisor, hostPortOwner(active.lease), this.listenerInspector, active.executionClass === "analysis_only", () => this.markCapacityReservationRunning(dispatchAttemptId, fencingToken));
|
|
1087
|
+
}
|
|
1088
|
+
markCapacityReservationRunning(dispatchAttemptId, fencingToken) {
|
|
1089
|
+
const reservation = this.capacityReservations.get(dispatchAttemptId);
|
|
1090
|
+
if (reservation?.owner.fencingToken === fencingToken)
|
|
1091
|
+
reservation.state = "running";
|
|
1092
|
+
}
|
|
1093
|
+
providerMemoryTelemetry(dispatchAttemptId, fencingToken) {
|
|
1094
|
+
const active = this.active.get(dispatchAttemptId);
|
|
1095
|
+
if (!active || active.lease.fencingToken !== fencingToken || active.releasing)
|
|
1096
|
+
return null;
|
|
1097
|
+
return active.processSupervisor.providerMemoryTelemetry();
|
|
653
1098
|
}
|
|
654
1099
|
async promoteVerifiedRuntime(active, runtimeLease) {
|
|
655
1100
|
if (!active.parallelPromotionEligible)
|
|
@@ -936,7 +1381,7 @@ export class HostRuntimeLeaseManager {
|
|
|
936
1381
|
this.singletonOwners.delete(key);
|
|
937
1382
|
for (let index = 0; index < this.singletonWaiters.length;) {
|
|
938
1383
|
const waiter = this.singletonWaiters[index];
|
|
939
|
-
if (waiter.keys.length !== 1 || waiter.keys[0] !== key) {
|
|
1384
|
+
if (waiter.priority !== "foreground" || waiter.keys.length !== 1 || waiter.keys[0] !== key) {
|
|
940
1385
|
index += 1;
|
|
941
1386
|
continue;
|
|
942
1387
|
}
|
|
@@ -952,9 +1397,10 @@ export class HostRuntimeLeaseManager {
|
|
|
952
1397
|
parallelProjectOwned(key, owner) {
|
|
953
1398
|
return sameSingletonOwner(this.parallelProjectOwners.get(key)?.get(owner.dispatchAttemptId), owner);
|
|
954
1399
|
}
|
|
955
|
-
parallelProjectAvailable(key) {
|
|
1400
|
+
parallelProjectAvailable(key, priority = "foreground") {
|
|
956
1401
|
return !this.singletonOwners.has(key)
|
|
957
|
-
&& !this.singletonWaiters.some(waiter => waiter.keys.includes(key)
|
|
1402
|
+
&& !this.singletonWaiters.some(waiter => waiter.keys.includes(key)
|
|
1403
|
+
&& (priority === "background" || waiter.priority === "foreground"));
|
|
958
1404
|
}
|
|
959
1405
|
assignParallelProject(key, owner) {
|
|
960
1406
|
const owners = this.parallelProjectOwners.get(key) ?? new Map();
|
|
@@ -980,9 +1426,15 @@ export class HostRuntimeLeaseManager {
|
|
|
980
1426
|
if (this.admissions.get(dispatchAttemptId)?.fencingToken === fencingToken) {
|
|
981
1427
|
this.admissions.delete(dispatchAttemptId);
|
|
982
1428
|
}
|
|
1429
|
+
this.releaseHostCapacity(dispatchAttemptId, fencingToken);
|
|
983
1430
|
}
|
|
984
1431
|
renewAdmissionWaiterDeadline(dispatchAttemptId, fencingToken, expiresAt) {
|
|
985
|
-
for (const waiter of [
|
|
1432
|
+
for (const waiter of [
|
|
1433
|
+
...this.singletonWaiters,
|
|
1434
|
+
...this.parallelProjectWaiters,
|
|
1435
|
+
...this.capacityWaiters,
|
|
1436
|
+
...this.backgroundAdmissionWaiters,
|
|
1437
|
+
]) {
|
|
986
1438
|
if (!sameSingletonOwner(waiter.owner, { dispatchAttemptId, fencingToken })
|
|
987
1439
|
|| Date.parse(expiresAt) <= Date.parse(waiter.expiresAt))
|
|
988
1440
|
continue;
|
|
@@ -998,13 +1450,20 @@ export class HostRuntimeLeaseManager {
|
|
|
998
1450
|
}
|
|
999
1451
|
cancelAdmissionWaiters(dispatchAttemptId, fencingToken, reason) {
|
|
1000
1452
|
const owner = { dispatchAttemptId, fencingToken };
|
|
1001
|
-
for (const waiters of [
|
|
1453
|
+
for (const waiters of [
|
|
1454
|
+
this.singletonWaiters,
|
|
1455
|
+
this.parallelProjectWaiters,
|
|
1456
|
+
this.capacityWaiters,
|
|
1457
|
+
this.backgroundAdmissionWaiters,
|
|
1458
|
+
]) {
|
|
1002
1459
|
for (let index = waiters.length - 1; index >= 0; index -= 1) {
|
|
1003
1460
|
const waiter = waiters[index];
|
|
1004
1461
|
if (!sameSingletonOwner(waiter.owner, owner))
|
|
1005
1462
|
continue;
|
|
1006
1463
|
waiters.splice(index, 1);
|
|
1007
1464
|
clearAdmissionDeadline(waiter);
|
|
1465
|
+
if ("capacityRecheckTimer" in waiter)
|
|
1466
|
+
clearCapacityRecheck(waiter);
|
|
1008
1467
|
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
1009
1468
|
waiter.reject(new Error(`host_runtime_admission_cancelled: ${reason}`));
|
|
1010
1469
|
}
|
|
@@ -1060,7 +1519,7 @@ export class HostRuntimeLeaseManager {
|
|
|
1060
1519
|
waiter.reject(new HostRuntimeAdmissionExpiredError());
|
|
1061
1520
|
continue;
|
|
1062
1521
|
}
|
|
1063
|
-
if (!this.parallelProjectAvailable(waiter.key)) {
|
|
1522
|
+
if (!this.parallelProjectAvailable(waiter.key, waiter.priority)) {
|
|
1064
1523
|
index += 1;
|
|
1065
1524
|
continue;
|
|
1066
1525
|
}
|
|
@@ -1071,6 +1530,46 @@ export class HostRuntimeLeaseManager {
|
|
|
1071
1530
|
waiter.resolve();
|
|
1072
1531
|
}
|
|
1073
1532
|
}
|
|
1533
|
+
drainBackgroundAdmissionWaiters() {
|
|
1534
|
+
if (this.hasForegroundDemand())
|
|
1535
|
+
return;
|
|
1536
|
+
for (let index = 0; index < this.backgroundAdmissionWaiters.length;) {
|
|
1537
|
+
const waiter = this.backgroundAdmissionWaiters[index];
|
|
1538
|
+
if (waiter.signal?.aborted) {
|
|
1539
|
+
this.backgroundAdmissionWaiters.splice(index, 1);
|
|
1540
|
+
clearAdmissionDeadline(waiter);
|
|
1541
|
+
clearCapacityRecheck(waiter);
|
|
1542
|
+
waiter.signal.removeEventListener("abort", waiter.abort);
|
|
1543
|
+
this.clearPendingRenewal(waiter.owner);
|
|
1544
|
+
waiter.reject(cancellationError(waiter.signal));
|
|
1545
|
+
continue;
|
|
1546
|
+
}
|
|
1547
|
+
if (admissionExpired(waiter.expiresAt)) {
|
|
1548
|
+
this.backgroundAdmissionWaiters.splice(index, 1);
|
|
1549
|
+
clearAdmissionDeadline(waiter);
|
|
1550
|
+
clearCapacityRecheck(waiter);
|
|
1551
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
1552
|
+
this.clearPendingRenewal(waiter.owner);
|
|
1553
|
+
waiter.reject(new HostRuntimeAdmissionExpiredError());
|
|
1554
|
+
continue;
|
|
1555
|
+
}
|
|
1556
|
+
if (!this.singletonKeysAvailable(waiter.keys) || !this.hostCapacityAvailable(waiter.resources)) {
|
|
1557
|
+
index += 1;
|
|
1558
|
+
continue;
|
|
1559
|
+
}
|
|
1560
|
+
this.backgroundAdmissionWaiters.splice(index, 1);
|
|
1561
|
+
clearAdmissionDeadline(waiter);
|
|
1562
|
+
clearCapacityRecheck(waiter);
|
|
1563
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
1564
|
+
this.assignSingletonKeys(waiter.keys, waiter.owner);
|
|
1565
|
+
this.capacityReservations.set(waiter.owner.dispatchAttemptId, {
|
|
1566
|
+
owner: waiter.owner,
|
|
1567
|
+
resources: waiter.resources,
|
|
1568
|
+
state: "pending",
|
|
1569
|
+
});
|
|
1570
|
+
waiter.resolve(waiter.keys);
|
|
1571
|
+
}
|
|
1572
|
+
}
|
|
1074
1573
|
async refreshKnownPromotions() {
|
|
1075
1574
|
const promotions = await this.promotionStore.list();
|
|
1076
1575
|
this.knownPromotions.clear();
|
|
@@ -1084,6 +1583,8 @@ export class HostRuntimeLeaseManager {
|
|
|
1084
1583
|
drainAdmissionWaiters() {
|
|
1085
1584
|
this.drainSingletonWaiters();
|
|
1086
1585
|
this.drainParallelProjectWaiters();
|
|
1586
|
+
this.drainCapacityWaiters();
|
|
1587
|
+
this.drainBackgroundAdmissionWaiters();
|
|
1087
1588
|
}
|
|
1088
1589
|
}
|
|
1089
1590
|
class AttemptSupervisedProviderProcessRunner {
|
|
@@ -1091,14 +1592,20 @@ class AttemptSupervisedProviderProcessRunner {
|
|
|
1091
1592
|
owner;
|
|
1092
1593
|
listenerInspector;
|
|
1093
1594
|
enforceNoListeners;
|
|
1094
|
-
|
|
1595
|
+
onProcessTreeStarted;
|
|
1596
|
+
constructor(supervisor, owner, listenerInspector, enforceNoListeners, onProcessTreeStarted) {
|
|
1095
1597
|
this.supervisor = supervisor;
|
|
1096
1598
|
this.owner = owner;
|
|
1097
1599
|
this.listenerInspector = listenerInspector;
|
|
1098
1600
|
this.enforceNoListeners = enforceNoListeners;
|
|
1601
|
+
this.onProcessTreeStarted = onProcessTreeStarted;
|
|
1099
1602
|
}
|
|
1100
1603
|
async run(request, handlers = {}, signal) {
|
|
1604
|
+
const baselineListeners = this.enforceNoListeners
|
|
1605
|
+
? new Set(await this.listenerInspector.inspectLoopbackListeners())
|
|
1606
|
+
: new Set();
|
|
1101
1607
|
const child = await this.supervisor.startChildReady("provider", request, handlers, signal, { maxStartAttempts: request.stdin === undefined ? 1 : 3, retryDelayMs: 100 });
|
|
1608
|
+
this.onProcessTreeStarted();
|
|
1102
1609
|
if (!this.enforceNoListeners)
|
|
1103
1610
|
return await child.result;
|
|
1104
1611
|
let settled = false;
|
|
@@ -1106,7 +1613,8 @@ class AttemptSupervisedProviderProcessRunner {
|
|
|
1106
1613
|
const monitor = (async () => {
|
|
1107
1614
|
while (!settled) {
|
|
1108
1615
|
try {
|
|
1109
|
-
const ports = await this.listenerInspector.inspectOwnedLoopbackListeners(child, this.owner)
|
|
1616
|
+
const ports = (await this.listenerInspector.inspectOwnedLoopbackListeners(child, this.owner))
|
|
1617
|
+
.filter(port => !baselineListeners.has(port));
|
|
1110
1618
|
if (ports.length > 0) {
|
|
1111
1619
|
inspectionFailure = new Error(`host_analysis_listener_violation: analysis-only provider opened undeclared loopback listener(s): ${ports.join(", ")}.`);
|
|
1112
1620
|
await child.stop(this.owner.fencingToken).catch(() => undefined);
|
|
@@ -1170,6 +1678,12 @@ export class HostRuntimeAdmissionExpiredError extends Error {
|
|
|
1170
1678
|
this.name = "HostRuntimeAdmissionExpiredError";
|
|
1171
1679
|
}
|
|
1172
1680
|
}
|
|
1681
|
+
export class HostRuntimeCapacityRequestImpossibleError extends Error {
|
|
1682
|
+
constructor(reason) {
|
|
1683
|
+
super(`host_runtime_capacity_request_impossible: ${reason}`);
|
|
1684
|
+
this.name = "HostRuntimeCapacityRequestImpossibleError";
|
|
1685
|
+
}
|
|
1686
|
+
}
|
|
1173
1687
|
export function hostRuntimeEventResources(lease, runtimeContract, executionClass = "managed_runtime") {
|
|
1174
1688
|
const requestedHostConcurrency = runtimeContract?.contract.runtime.hostConcurrency ?? "serial";
|
|
1175
1689
|
const configuredLaunches = runtimeContract?.contract.runtime.ports.filter(port => port.launch !== null) ?? [];
|
|
@@ -1209,6 +1723,8 @@ function hostRuntimeEnvironment(sessionId, lease, runtimeContract, executionClas
|
|
|
1209
1723
|
FORGE_RUNTIME_MODE: "host",
|
|
1210
1724
|
FORGE_RESOURCE_LEASE_ID: lease.leaseId,
|
|
1211
1725
|
FORGE_DISPATCH_ATTEMPT_ID: lease.dispatchAttemptId,
|
|
1726
|
+
FORGE_E2E_OWNERSHIP_MODE: "attempt",
|
|
1727
|
+
FORGE_E2E_OWNERSHIP_TOKEN: uuidHex(lease.dispatchAttemptId, "dispatch attempt"),
|
|
1212
1728
|
FORGE_EXECUTION_LEASE_ID: lease.executionLeaseId,
|
|
1213
1729
|
FORGE_FENCING_TOKEN: lease.fencingToken,
|
|
1214
1730
|
FORGE_WORKSPACE: lease.worktreePath,
|
|
@@ -1259,6 +1775,45 @@ function hostRuntimeEnvironment(sessionId, lease, runtimeContract, executionClas
|
|
|
1259
1775
|
environment.FORGE_BASE_URL = resolvedBaseUrls[0];
|
|
1260
1776
|
return environment;
|
|
1261
1777
|
}
|
|
1778
|
+
async function executeAspNetRuntimeBuild(supervisor, logDirectory, plan, signal) {
|
|
1779
|
+
const childName = `runtime-build-${plan.fingerprint.slice(-12)}`;
|
|
1780
|
+
const stdoutLogPath = path.join(logDirectory, `${childName}.stdout.log`);
|
|
1781
|
+
const stderrLogPath = path.join(logDirectory, `${childName}.stderr.log`);
|
|
1782
|
+
let stdoutTail = "";
|
|
1783
|
+
let stderrTail = "";
|
|
1784
|
+
const append = async (target, line) => {
|
|
1785
|
+
await fs.appendFile(target, `${line}\n`, { encoding: "utf8", mode: 0o600 });
|
|
1786
|
+
};
|
|
1787
|
+
const child = await supervisor.startChildReady(childName, {
|
|
1788
|
+
command: plan.command,
|
|
1789
|
+
args: plan.args,
|
|
1790
|
+
cwd: plan.cwd,
|
|
1791
|
+
env: plan.environment,
|
|
1792
|
+
captureOutput: false,
|
|
1793
|
+
}, {
|
|
1794
|
+
onStdoutLine: async (line) => {
|
|
1795
|
+
stdoutTail = appendRuntimeBuildTail(stdoutTail, line);
|
|
1796
|
+
await append(stdoutLogPath, line);
|
|
1797
|
+
},
|
|
1798
|
+
onStderrLine: async (line) => {
|
|
1799
|
+
stderrTail = appendRuntimeBuildTail(stderrTail, line);
|
|
1800
|
+
await append(stderrLogPath, line);
|
|
1801
|
+
},
|
|
1802
|
+
}, signal, { maxStartAttempts: 2, retryDelayMs: 250 });
|
|
1803
|
+
const result = await child.result;
|
|
1804
|
+
return {
|
|
1805
|
+
exitCode: result.exitCode,
|
|
1806
|
+
stdoutTail: appendRuntimeBuildTail(stdoutTail, result.stdout),
|
|
1807
|
+
stderrTail: appendRuntimeBuildTail(stderrTail, result.stderr),
|
|
1808
|
+
stdoutLogPath,
|
|
1809
|
+
stderrLogPath,
|
|
1810
|
+
};
|
|
1811
|
+
}
|
|
1812
|
+
function appendRuntimeBuildTail(current, value) {
|
|
1813
|
+
if (!value)
|
|
1814
|
+
return current;
|
|
1815
|
+
return `${current}${current ? "\n" : ""}${value}`.slice(-4_000);
|
|
1816
|
+
}
|
|
1262
1817
|
function effectiveHostConcurrency(lease, runtimeContract, executionClass = "managed_runtime") {
|
|
1263
1818
|
if (executionClass === "analysis_only") {
|
|
1264
1819
|
const analysisSingletonHeld = lease.resources.singletonKeys.length > 0;
|
|
@@ -1287,6 +1842,122 @@ function runtimeExecutionClass(dispatch) {
|
|
|
1287
1842
|
? "analysis_only"
|
|
1288
1843
|
: "managed_runtime";
|
|
1289
1844
|
}
|
|
1845
|
+
function runtimeAdmissionPriority(dispatch) {
|
|
1846
|
+
return dispatch.execution.lane.trim().toLowerCase() === "runtime_readiness"
|
|
1847
|
+
|| dispatch.cycleRun.phase.trim().toLowerCase() === "readiness_probe"
|
|
1848
|
+
? "background"
|
|
1849
|
+
: "foreground";
|
|
1850
|
+
}
|
|
1851
|
+
function enqueueAdmissionWaiter(waiters, waiter) {
|
|
1852
|
+
if (waiter.priority === "background") {
|
|
1853
|
+
waiters.push(waiter);
|
|
1854
|
+
return;
|
|
1855
|
+
}
|
|
1856
|
+
const backgroundIndex = waiters.findIndex(candidate => candidate.priority === "background");
|
|
1857
|
+
if (backgroundIndex < 0)
|
|
1858
|
+
waiters.push(waiter);
|
|
1859
|
+
else
|
|
1860
|
+
waiters.splice(backgroundIndex, 0, waiter);
|
|
1861
|
+
}
|
|
1862
|
+
function defaultHostResourceRequest() {
|
|
1863
|
+
const workerMemoryLimitBytes = configuredInteger("FORGE_HOST_DEFAULT_WORKER_MEMORY_LIMIT_BYTES", 2 * 1024 ** 3, 128 * 1024 ** 2, 512 * 1024 ** 3);
|
|
1864
|
+
const runtimeOverheadBytes = 128 * 1024 ** 2;
|
|
1865
|
+
return {
|
|
1866
|
+
sandboxSlots: 1,
|
|
1867
|
+
cpuMillis: 2000,
|
|
1868
|
+
workerMemoryLimitBytes,
|
|
1869
|
+
serviceMemoryLimitBytes: 0,
|
|
1870
|
+
runtimeOverheadBytes,
|
|
1871
|
+
memoryBytes: workerMemoryLimitBytes + runtimeOverheadBytes,
|
|
1872
|
+
workspaceBytes: 10 * 1024 ** 3,
|
|
1873
|
+
pids: 512,
|
|
1874
|
+
buildSlots: 1,
|
|
1875
|
+
};
|
|
1876
|
+
}
|
|
1877
|
+
function minimumSupportedHostResourceRequest() {
|
|
1878
|
+
return {
|
|
1879
|
+
sandboxSlots: 1,
|
|
1880
|
+
cpuMillis: 1000,
|
|
1881
|
+
workerMemoryLimitBytes: 1024 ** 3,
|
|
1882
|
+
serviceMemoryLimitBytes: 0,
|
|
1883
|
+
runtimeOverheadBytes: 128 * 1024 ** 2,
|
|
1884
|
+
memoryBytes: 1152 * 1024 ** 2,
|
|
1885
|
+
workspaceBytes: architectMinimumWorkspaceEstimateBytes,
|
|
1886
|
+
pids: 256,
|
|
1887
|
+
buildSlots: 0,
|
|
1888
|
+
};
|
|
1889
|
+
}
|
|
1890
|
+
function normalizedHostResourceRequest(resources) {
|
|
1891
|
+
const defaults = defaultHostResourceRequest();
|
|
1892
|
+
const workerMemoryLimitBytes = positiveResourceValue(resources?.workerMemoryLimitBytes, defaults.workerMemoryLimitBytes);
|
|
1893
|
+
const serviceMemoryLimitBytes = nonNegativeResourceValue(resources?.serviceMemoryLimitBytes, defaults.serviceMemoryLimitBytes);
|
|
1894
|
+
const runtimeOverheadBytes = nonNegativeResourceValue(resources?.runtimeOverheadBytes, defaults.runtimeOverheadBytes);
|
|
1895
|
+
return {
|
|
1896
|
+
sandboxSlots: positiveResourceValue(resources?.sandboxSlots, defaults.sandboxSlots),
|
|
1897
|
+
cpuMillis: positiveResourceValue(resources?.cpuMillis, defaults.cpuMillis),
|
|
1898
|
+
workerMemoryLimitBytes,
|
|
1899
|
+
serviceMemoryLimitBytes,
|
|
1900
|
+
runtimeOverheadBytes,
|
|
1901
|
+
memoryBytes: workerMemoryLimitBytes + serviceMemoryLimitBytes + runtimeOverheadBytes,
|
|
1902
|
+
workspaceBytes: positiveResourceValue(resources?.workspaceBytes, defaults.workspaceBytes),
|
|
1903
|
+
pids: positiveResourceValue(resources?.pids, defaults.pids),
|
|
1904
|
+
buildSlots: nonNegativeResourceValue(resources?.buildSlots, defaults.buildSlots),
|
|
1905
|
+
};
|
|
1906
|
+
}
|
|
1907
|
+
function positiveResourceValue(value, fallback) {
|
|
1908
|
+
return Number.isFinite(value) && value > 0 ? Math.floor(value) : fallback;
|
|
1909
|
+
}
|
|
1910
|
+
function nonNegativeResourceValue(value, fallback) {
|
|
1911
|
+
return Number.isFinite(value) && value >= 0 ? Math.floor(value) : fallback;
|
|
1912
|
+
}
|
|
1913
|
+
function configuredInteger(name, fallback, minimum, maximum) {
|
|
1914
|
+
const parsed = Number.parseInt(process.env[name] ?? "", 10);
|
|
1915
|
+
return Number.isFinite(parsed) ? Math.max(minimum, Math.min(maximum, parsed)) : fallback;
|
|
1916
|
+
}
|
|
1917
|
+
function loadHostCapacityConfiguration(capacityResourceSample) {
|
|
1918
|
+
const sampledCpuCount = capacityResourceSample
|
|
1919
|
+
? Math.max(1, Math.floor(capacityResourceSample.totalCpuMillis / 1_000))
|
|
1920
|
+
: os.cpus().length;
|
|
1921
|
+
const totalMemoryBytes = capacityResourceSample?.totalMemoryBytes ?? os.totalmem();
|
|
1922
|
+
const reserveFloorBytes = configuredInteger("FORGE_HOST_MEMORY_RESERVE_FLOOR_BYTES", 1024 ** 3, 512 * 1024 ** 2, 128 * 1024 ** 3);
|
|
1923
|
+
const reserveCeilingBytes = configuredInteger("FORGE_HOST_MEMORY_RESERVE_CEILING_BYTES", 4 * 1024 ** 3, reserveFloorBytes, 128 * 1024 ** 3);
|
|
1924
|
+
return {
|
|
1925
|
+
hardMaxAttempts: configuredInteger("FORGE_HOST_MAX_CONCURRENT_ATTEMPTS", Math.max(1, Math.min(8, Math.floor(sampledCpuCount / 2))), 1, 32),
|
|
1926
|
+
hostReserveBytes: resolveAdaptiveMemoryReserveBytes(totalMemoryBytes, {
|
|
1927
|
+
explicitBytes: configuredOptionalInteger("FORGE_HOST_MEMORY_RESERVE_BYTES", 512 * 1024 ** 2, 128 * 1024 ** 3),
|
|
1928
|
+
floorBytes: reserveFloorBytes,
|
|
1929
|
+
ceilingBytes: reserveCeilingBytes,
|
|
1930
|
+
percent: configuredNumber("FORGE_HOST_MEMORY_RESERVE_PERCENT", 12.5, 1, 50),
|
|
1931
|
+
}),
|
|
1932
|
+
cpuReserveMillis: configuredInteger("FORGE_HOST_CPU_RESERVE_MILLIS", 1000, 0, 32_000),
|
|
1933
|
+
workspaceReserveBytes: configuredInteger("FORGE_HOST_WORKSPACE_RESERVE_BYTES", 10 * 1024 ** 3, 1024 ** 3, Number.MAX_SAFE_INTEGER),
|
|
1934
|
+
memoryRecoverySamples: configuredInteger("FORGE_HOST_MEMORY_RECOVERY_SAMPLES", 2, 1, 10),
|
|
1935
|
+
memoryRecoverySampleIntervalMs: configuredInteger("FORGE_HOST_MEMORY_RECOVERY_SAMPLE_INTERVAL_MS", 1_000, 100, 30_000),
|
|
1936
|
+
};
|
|
1937
|
+
}
|
|
1938
|
+
function configuredOptionalInteger(name, minimum, maximum) {
|
|
1939
|
+
const raw = process.env[name];
|
|
1940
|
+
if (raw === undefined || raw.trim().length === 0)
|
|
1941
|
+
return null;
|
|
1942
|
+
const parsed = Number.parseInt(raw, 10);
|
|
1943
|
+
return Number.isFinite(parsed) ? Math.max(minimum, Math.min(maximum, parsed)) : null;
|
|
1944
|
+
}
|
|
1945
|
+
function configuredNumber(name, fallback, minimum, maximum) {
|
|
1946
|
+
const parsed = Number(process.env[name] ?? "");
|
|
1947
|
+
return Number.isFinite(parsed) && parsed > 0
|
|
1948
|
+
? Math.max(minimum, Math.min(maximum, parsed))
|
|
1949
|
+
: fallback;
|
|
1950
|
+
}
|
|
1951
|
+
function resourceSlots(available, requested) {
|
|
1952
|
+
if (requested <= 0)
|
|
1953
|
+
return Number.MAX_SAFE_INTEGER;
|
|
1954
|
+
return Math.max(0, Math.floor(available / requested));
|
|
1955
|
+
}
|
|
1956
|
+
function removeWaiter(waiters, waiter) {
|
|
1957
|
+
const index = waiters.indexOf(waiter);
|
|
1958
|
+
if (index >= 0)
|
|
1959
|
+
waiters.splice(index, 1);
|
|
1960
|
+
}
|
|
1290
1961
|
function capacityProject(projects, repoId) {
|
|
1291
1962
|
const normalized = repoId.trim().toLowerCase();
|
|
1292
1963
|
const existing = projects.get(normalized);
|
|
@@ -1437,6 +2108,11 @@ function clearAdmissionDeadline(waiter) {
|
|
|
1437
2108
|
clearTimeout(waiter.deadlineTimer);
|
|
1438
2109
|
waiter.deadlineTimer = null;
|
|
1439
2110
|
}
|
|
2111
|
+
function clearCapacityRecheck(waiter) {
|
|
2112
|
+
if (waiter.capacityRecheckTimer)
|
|
2113
|
+
clearInterval(waiter.capacityRecheckTimer);
|
|
2114
|
+
waiter.capacityRecheckTimer = null;
|
|
2115
|
+
}
|
|
1440
2116
|
async function ensurePrivateDirectory(directory) {
|
|
1441
2117
|
await fs.mkdir(directory, { recursive: true, mode: 0o700 });
|
|
1442
2118
|
if (process.platform !== "win32")
|