@loucompanion/forge-bridge 0.1.3-dev.fbe15023cdfe → 0.1.4-dev.bfa11ebe98c6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +88 -2
- package/dist/bridge/bridge-chat.types.d.ts +111 -0
- package/dist/bridge/bridge-chat.types.js +401 -0
- package/dist/bridge/bridge.base.d.ts +4 -1
- package/dist/bridge/bridge.service.d.ts +72 -3
- package/dist/bridge/bridge.service.js +1562 -152
- package/dist/bridge/bridge.types.d.ts +46 -1
- package/dist/bridge/dispatch-result-outbox.d.ts +25 -0
- package/dist/bridge/dispatch-result-outbox.js +62 -0
- package/dist/bridge/dispatch-result-worker.d.ts +2 -0
- package/dist/bridge/dispatch-result-worker.js +122 -0
- package/dist/bridge/internals.d.ts +2 -0
- package/dist/bridge/internals.js +1 -0
- package/dist/bridge/readiness-result-outbox.d.ts +21 -0
- package/dist/bridge/readiness-result-outbox.js +54 -0
- package/dist/bridge/readiness-result-worker.d.ts +2 -0
- package/dist/bridge/readiness-result-worker.js +121 -0
- package/dist/bridge/runner-host-capabilities.d.ts +15 -0
- package/dist/bridge/runner-host-capabilities.js +93 -0
- package/dist/cleanup/cleanup.service.d.ts +1 -0
- package/dist/cleanup/cleanup.service.js +70 -9
- package/dist/cli/cli.help.js +2 -2
- package/dist/cli/cli.service.d.ts +14 -6
- package/dist/cli/cli.service.js +75 -16
- package/dist/cli/local-config.d.ts +1 -0
- package/dist/cli/local-config.js +4 -0
- package/dist/runtime/aspnet-runtime-prebuild.d.ts +36 -0
- package/dist/runtime/aspnet-runtime-prebuild.js +341 -0
- package/dist/runtime/attempt-node-runtime-launcher.d.ts +2 -1
- package/dist/runtime/attempt-node-runtime-launcher.js +57 -18
- package/dist/runtime/attempt-path-operation.d.ts +21 -0
- package/dist/runtime/attempt-path-operation.js +94 -0
- package/dist/runtime/attempt-private-directory.d.ts +6 -0
- package/dist/runtime/attempt-private-directory.js +54 -0
- package/dist/runtime/attempt-process-supervisor.d.ts +11 -0
- package/dist/runtime/attempt-process-supervisor.js +67 -20
- package/dist/runtime/canonical-source-file.d.ts +3 -0
- package/dist/runtime/canonical-source-file.js +54 -0
- package/dist/runtime/docker-engine-probe.d.ts +90 -0
- package/dist/runtime/docker-engine-probe.js +333 -0
- package/dist/runtime/generated-output-reset.d.ts +10 -0
- package/dist/runtime/generated-output-reset.js +62 -8
- package/dist/runtime/host-database-capability-boundary.d.ts +16 -0
- package/dist/runtime/host-database-capability-boundary.js +62 -0
- package/dist/runtime/host-diagnostics.d.ts +11 -0
- package/dist/runtime/host-diagnostics.js +191 -0
- package/dist/runtime/host-execution-root.d.ts +4 -0
- package/dist/runtime/host-execution-root.js +39 -0
- package/dist/runtime/host-listener-inspector.d.ts +8 -1
- package/dist/runtime/host-listener-inspector.js +121 -60
- package/dist/runtime/host-runtime-contract.d.ts +12 -1
- package/dist/runtime/host-runtime-contract.js +107 -53
- package/dist/runtime/host-runtime-lease.d.ts +90 -8
- package/dist/runtime/host-runtime-lease.js +1097 -108
- package/dist/runtime/host-runtime-promotion.d.ts +12 -3
- package/dist/runtime/host-runtime-promotion.js +59 -12
- package/dist/runtime/memory-capacity.d.ts +70 -0
- package/dist/runtime/memory-capacity.js +123 -0
- package/dist/runtime/native-test-postgres-fallback.d.ts +25 -0
- package/dist/runtime/native-test-postgres-fallback.js +91 -0
- package/dist/runtime/node-host-runtime-adapter.d.ts +3 -0
- package/dist/runtime/node-host-runtime-adapter.js +28 -11
- package/dist/runtime/pinned-directory-process.d.ts +9 -0
- package/dist/runtime/pinned-directory-process.js +61 -0
- package/dist/runtime/pinned-source-directory.d.ts +17 -0
- package/dist/runtime/pinned-source-directory.js +94 -0
- package/dist/runtime/playwright-host-runtime-adapter.d.ts +1 -0
- package/dist/runtime/playwright-host-runtime-adapter.js +53 -5
- package/dist/runtime/prepared-runtime-identity.d.ts +6 -0
- package/dist/runtime/prepared-runtime-identity.js +218 -0
- package/dist/runtime/prepared-runtime-readiness.d.ts +30 -0
- package/dist/runtime/prepared-runtime-readiness.js +285 -0
- package/dist/runtime/runtime-isolation-lease.js +4 -1
- package/dist/runtime/workspace-resource-estimator.d.ts +17 -0
- package/dist/runtime/workspace-resource-estimator.js +174 -0
- package/dist/sandbox/preflight.d.ts +31 -1
- package/dist/sandbox/preflight.js +237 -51
- package/dist/sandbox/repository-lock.d.ts +2 -1
- package/dist/sandbox/repository-lock.js +17 -8
- package/dist/sandbox/runtime.d.ts +6 -1
- package/dist/sandbox/runtime.js +132 -103
- package/dist/sandbox/sandbox.types.d.ts +7 -0
- package/dist/sandbox/workspace.service.js +2 -0
- package/dist/session/managed-attempt-tools.d.ts +17 -1
- package/dist/session/managed-attempt-tools.js +481 -31
- package/dist/session/managed-build-coordinator.d.ts +5 -0
- package/dist/session/managed-build-coordinator.js +105 -0
- package/dist/session/managed-build-receipt.d.ts +21 -0
- package/dist/session/managed-build-receipt.js +337 -0
- package/dist/session/managed-verification-timeouts.d.ts +8 -0
- package/dist/session/managed-verification-timeouts.js +17 -0
- package/dist/session/phase-completion.artifacts.d.ts +17 -0
- package/dist/session/phase-completion.artifacts.js +224 -22
- package/dist/session/phase-completion.d.ts +11 -1
- package/dist/session/phase-completion.helpers.d.ts +5 -0
- package/dist/session/phase-completion.helpers.js +30 -7
- package/dist/session/phase-completion.js +14 -3
- package/dist/session/phase-completion.prompts.js +17 -18
- package/dist/session/phase-completion.runtime.d.ts +3 -1
- package/dist/session/phase-completion.runtime.js +118 -16
- package/dist/session/phase-completion.schemas.js +49 -2
- package/dist/session/phase-completion.validation.d.ts +1 -0
- package/dist/session/phase-completion.validation.js +111 -21
- package/dist/session/pre-publication-validation.d.ts +75 -0
- package/dist/session/pre-publication-validation.js +657 -0
- package/dist/session/provider-cli/bridge-cli.conformance-fixtures.d.ts +72 -0
- package/dist/session/provider-cli/bridge-cli.conformance-fixtures.js +462 -0
- package/dist/session/provider-cli/ci-repair-readiness.d.ts +8 -0
- package/dist/session/provider-cli/ci-repair-readiness.js +23 -3
- package/dist/session/provider-cli/provider-cli.artifacts.d.ts +4 -2
- package/dist/session/provider-cli/provider-cli.artifacts.js +47 -13
- package/dist/session/provider-cli/provider-cli.deterministic.js +34 -2
- package/dist/session/provider-cli/provider-cli.process.d.ts +51 -0
- package/dist/session/provider-cli/provider-cli.process.js +422 -22
- package/dist/session/provider-cli/provider-cli.prompts.d.ts +5 -0
- package/dist/session/provider-cli/provider-cli.prompts.js +136 -16
- package/dist/session/provider-cli/provider-cli.repair-epoch.d.ts +6 -0
- package/dist/session/provider-cli/provider-cli.repair-epoch.js +11 -0
- package/dist/session/provider-cli/provider-cli.runtime.d.ts +8 -1
- package/dist/session/provider-cli/provider-cli.runtime.js +40 -3
- package/dist/session/provider-cli/provider-cli.service.d.ts +30 -5
- package/dist/session/provider-cli/provider-cli.service.js +771 -68
- package/dist/session/provider-cli/provider-cli.types.d.ts +2 -0
- package/dist/session/provider-cli/provider-output-state.d.ts +10 -0
- package/dist/session/provider-cli/provider-output-state.js +361 -17
- package/dist/session/provider-cli/provider-usage.d.ts +7 -0
- package/dist/session/provider-cli/provider-usage.js +28 -5
- package/dist/session/session-redaction.js +434 -9
- package/dist/session/session.service.js +145 -23
- package/dist/session/session.types.d.ts +147 -2
- package/dist/session/session.types.js +587 -7
- package/dist/session/unreferenced-exports.d.ts +12 -0
- package/dist/session/unreferenced-exports.js +59 -0
- package/dist/shared/format/format-bytes.d.ts +1 -0
- package/dist/shared/format/format-bytes.js +9 -0
- package/dist/shared/git/git.js +2 -1
- package/dist/shared/path/path.d.ts +1 -0
- package/dist/shared/path/path.js +18 -0
- package/dist/shared/process/dotnet-toolchain.d.ts +17 -0
- package/dist/shared/process/dotnet-toolchain.js +307 -0
- package/dist/shared/process/host-process-supervisor.d.ts +2 -0
- package/dist/shared/process/host-process-supervisor.js +57 -9
- package/dist/shared/process/process.d.ts +2 -0
- package/dist/shared/process/process.js +12 -7
- package/dist/shared/provider-capabilities.d.ts +3 -0
- package/dist/shared/provider-capabilities.js +85 -0
- package/dist/shared/redaction/redaction.d.ts +1 -0
- package/dist/shared/redaction/redaction.js +39 -3
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/worktree/dependency-warmer.d.ts +41 -9
- package/dist/worktree/dependency-warmer.js +497 -22
- package/dist/worktree/index.d.ts +1 -1
- package/dist/worktree/story-qa-reference-hydrator.d.ts +4 -0
- package/dist/worktree/story-qa-reference-hydrator.js +253 -0
- package/dist/worktree/worktree.service.d.ts +7 -0
- package/dist/worktree/worktree.service.js +253 -21
- package/dist/worktree/worktree.types.d.ts +7 -1
- package/package.json +7 -5
- package/runtime/windows-job-supervisor.ps1 +380 -14
|
@@ -1,5 +1,14 @@
|
|
|
1
1
|
import fs from "node:fs/promises";
|
|
2
|
+
import { statfsSync } from "node:fs";
|
|
3
|
+
import { randomUUID } from "node:crypto";
|
|
4
|
+
import os from "node:os";
|
|
2
5
|
import path from "node:path";
|
|
6
|
+
import { assertHostAttemptRootIdentity, retainHostDiagnostics } from "./host-diagnostics.js";
|
|
7
|
+
import { assertHostExecutionRoot, compactHostAttemptId } from "./host-execution-root.js";
|
|
8
|
+
import { PreparedRuntimeReadiness } from "./prepared-runtime-readiness.js";
|
|
9
|
+
import { preparedRuntimeIdentity } from "./prepared-runtime-identity.js";
|
|
10
|
+
import { resolveExistingDirectoryPath } from "../shared/path/path.js";
|
|
11
|
+
import { formatBytes } from "../shared/format/format-bytes.js";
|
|
3
12
|
import { StreamingProviderProcessRunner, } from "../session/provider-cli/provider-cli.process.js";
|
|
4
13
|
import { createRuntimeIsolationLease, renewRuntimeIsolationLease, transitionRuntimeIsolationLease, validRuntimeIsolationLease, withRuntimeIsolationResources, } from "./runtime-isolation-lease.js";
|
|
5
14
|
import { hostPortEnvironment, loadHostRuntimeContract, } from "./host-runtime-contract.js";
|
|
@@ -8,10 +17,18 @@ import { NodeHostRuntimeAdapter, } from "./node-host-runtime-adapter.js";
|
|
|
8
17
|
import { AttemptProcessSupervisorRegistry } from "./attempt-process-supervisor-registry.js";
|
|
9
18
|
import { AttemptNodeRuntimeLauncher } from "./attempt-node-runtime-launcher.js";
|
|
10
19
|
import { HostListenerInspector } from "./host-listener-inspector.js";
|
|
20
|
+
import { RecoveryStabilizedMemory, buildMemoryAdmissionDiagnostics, resolveAdaptiveMemoryReserveBytes, } from "./memory-capacity.js";
|
|
11
21
|
import { PlaywrightHostRuntimeAdapter } from "./playwright-host-runtime-adapter.js";
|
|
12
22
|
import { HostParallelPromotionStore, configuredLaunchPortNames, hostRuntimeContractFingerprint, hostRuntimeParallelEligible, } from "./host-runtime-promotion.js";
|
|
13
23
|
import { ExactHostProcessRecovery, } from "../shared/process/host-process-recovery.js";
|
|
14
24
|
import { resetGeneratedOutputs } from "./generated-output-reset.js";
|
|
25
|
+
import { assertCanonicalDirectory, createCanonicalDirectory } from "./attempt-private-directory.js";
|
|
26
|
+
import { readCanonicalSourceFile } from "./canonical-source-file.js";
|
|
27
|
+
import { readPinnedSourceDirectory } from "./pinned-source-directory.js";
|
|
28
|
+
import { architectMinimumWorkspaceEstimateBytes } from "./workspace-resource-estimator.js";
|
|
29
|
+
import { createAspNetRuntimePrebuildPlan, ensureAspNetRuntimePrebuilt, resolveWorktreeCommitSha, } from "./aspnet-runtime-prebuild.js";
|
|
30
|
+
const hostCapacityWaiterScanLimit = 64;
|
|
31
|
+
const maximumProcessRecoveryMetadataBytes = 1024 * 1024;
|
|
15
32
|
export class HostRuntimeLeaseManager {
|
|
16
33
|
stateRoot;
|
|
17
34
|
processSupervisors;
|
|
@@ -20,18 +37,30 @@ export class HostRuntimeLeaseManager {
|
|
|
20
37
|
promotionStore;
|
|
21
38
|
playwrightAdapter;
|
|
22
39
|
processRecovery;
|
|
40
|
+
capacityResourceSample;
|
|
41
|
+
preparedReadiness;
|
|
23
42
|
active = new Map();
|
|
24
43
|
pendingRenewals = new Map();
|
|
44
|
+
acquiringAdmissions = new Set();
|
|
25
45
|
admissions = new Map();
|
|
26
46
|
singletonOwners = new Map();
|
|
27
47
|
singletonWaiters = [];
|
|
28
48
|
parallelProjectOwners = new Map();
|
|
29
49
|
parallelProjectWaiters = [];
|
|
50
|
+
capacityReservations = new Map();
|
|
51
|
+
capacityWaiters = [];
|
|
52
|
+
capacityWaiterScanCursor = 0;
|
|
53
|
+
backgroundAdmissionWaiters = [];
|
|
30
54
|
degraded = new Map();
|
|
31
55
|
knownPromotions = new Map();
|
|
56
|
+
ledgerWriteTail = Promise.resolve();
|
|
57
|
+
workspaceAvailableBytes = Number.MAX_SAFE_INTEGER;
|
|
32
58
|
portBroker;
|
|
33
59
|
nodeAdapter;
|
|
34
|
-
|
|
60
|
+
capacityConfiguration;
|
|
61
|
+
memoryAvailability;
|
|
62
|
+
executionRoot;
|
|
63
|
+
constructor(stateRoot, portBroker = new HostPortBroker(), nodeAdapter, processSupervisors = new AttemptProcessSupervisorRegistry(), processRunner = new StreamingProviderProcessRunner(), listenerInspector = new HostListenerInspector(), promotionStore = new HostParallelPromotionStore(stateRoot), playwrightAdapter = new PlaywrightHostRuntimeAdapter(), processRecovery = new ExactHostProcessRecovery(), capacityResourceSample, executionRoot) {
|
|
35
64
|
this.stateRoot = stateRoot;
|
|
36
65
|
this.processSupervisors = processSupervisors;
|
|
37
66
|
this.processRunner = processRunner;
|
|
@@ -39,22 +68,46 @@ export class HostRuntimeLeaseManager {
|
|
|
39
68
|
this.promotionStore = promotionStore;
|
|
40
69
|
this.playwrightAdapter = playwrightAdapter;
|
|
41
70
|
this.processRecovery = processRecovery;
|
|
71
|
+
this.capacityResourceSample = capacityResourceSample;
|
|
72
|
+
this.executionRoot = path.resolve(executionRoot ?? stateRoot);
|
|
73
|
+
if (executionRoot !== undefined)
|
|
74
|
+
assertHostExecutionRoot(this.executionRoot);
|
|
75
|
+
this.preparedReadiness = new PreparedRuntimeReadiness(this, promotionStore, (id, fence) => this.release(id, fence, true), () => new HostRuntimeAdmissionExpiredError());
|
|
42
76
|
this.portBroker = portBroker;
|
|
43
77
|
this.nodeAdapter = nodeAdapter ?? new NodeHostRuntimeAdapter(portBroker);
|
|
78
|
+
this.capacityConfiguration = loadHostCapacityConfiguration(capacityResourceSample);
|
|
79
|
+
this.memoryAvailability = new RecoveryStabilizedMemory(this.capacityConfiguration.memoryRecoverySamples, this.capacityConfiguration.memoryRecoverySampleIntervalMs);
|
|
44
80
|
}
|
|
45
81
|
async reap(options) {
|
|
46
82
|
await this.refreshKnownPromotions();
|
|
83
|
+
const recoveredAttempts = [];
|
|
47
84
|
const report = {
|
|
48
85
|
examined: 0,
|
|
49
86
|
reaped: 0,
|
|
50
87
|
processesTerminated: 0,
|
|
51
88
|
reusedPidsProtected: 0,
|
|
52
89
|
degraded: 0,
|
|
90
|
+
recoveredAttempts,
|
|
53
91
|
};
|
|
54
92
|
for (const ledger of await this.readLedgers()) {
|
|
55
93
|
const lease = ledger.runtimeLease;
|
|
56
94
|
if (lease.runnerId !== options.runnerId)
|
|
57
95
|
continue;
|
|
96
|
+
// A released ledger with its original attempt root is intentionally retained
|
|
97
|
+
// evidence that exceeded the diagnostics archive budget. It owns no runtime
|
|
98
|
+
// resources, but still requires process-identity recovery below on restart.
|
|
99
|
+
if (lease.state === "released") {
|
|
100
|
+
const attemptRoot = this.attemptRoot(lease.dispatchAttemptId);
|
|
101
|
+
const rootExists = await fs.stat(attemptRoot).then(() => true, error => {
|
|
102
|
+
if (error.code === "ENOENT")
|
|
103
|
+
return false;
|
|
104
|
+
throw error;
|
|
105
|
+
});
|
|
106
|
+
if (!rootExists) {
|
|
107
|
+
await fs.rm(this.ledgerPath(lease.dispatchAttemptId), { force: true });
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
58
111
|
const active = this.active.get(lease.dispatchAttemptId);
|
|
59
112
|
if (active) {
|
|
60
113
|
if (active.lease.state !== "degraded")
|
|
@@ -78,9 +131,13 @@ export class HostRuntimeLeaseManager {
|
|
|
78
131
|
continue;
|
|
79
132
|
report.examined += 1;
|
|
80
133
|
const attemptRoot = this.attemptRoot(lease.dispatchAttemptId);
|
|
134
|
+
const attemptRootIdentity = deserializeHostAttemptRootIdentity(ledger.attemptRootIdentity);
|
|
81
135
|
let outcomes;
|
|
82
136
|
try {
|
|
83
|
-
|
|
137
|
+
if (!attemptRootIdentity) {
|
|
138
|
+
throw new Error("host_runtime_reaper_invalid: original attempt-root identity is unavailable.");
|
|
139
|
+
}
|
|
140
|
+
outcomes = await this.recoverPersistedProcesses(lease, attemptRoot, attemptRootIdentity);
|
|
84
141
|
}
|
|
85
142
|
catch {
|
|
86
143
|
await this.markLedgerDegraded(ledger);
|
|
@@ -94,10 +151,45 @@ export class HostRuntimeLeaseManager {
|
|
|
94
151
|
report.degraded += 1;
|
|
95
152
|
continue;
|
|
96
153
|
}
|
|
154
|
+
// The runtime has already been released. Keep the over-budget original
|
|
155
|
+
// evidence without re-copying it or reacquiring a capacity claim.
|
|
156
|
+
if (lease.state === "released")
|
|
157
|
+
continue;
|
|
158
|
+
// Every supervised tree of this previous-instance attempt is proved gone,
|
|
159
|
+
// so no provider can still act for it. Diagnostics handling below never
|
|
160
|
+
// revives a process, so the proof stands even if cleanup degrades. An
|
|
161
|
+
// attempt with no ownership record at all may have been mid-launch, so
|
|
162
|
+
// it is not reported and keeps its unknown outcome.
|
|
163
|
+
if (staleInstance && outcomes.length > 0) {
|
|
164
|
+
recoveredAttempts.push({
|
|
165
|
+
dispatchAttemptId: lease.dispatchAttemptId,
|
|
166
|
+
fencingToken: lease.fencingToken,
|
|
167
|
+
runnerInstanceId: lease.runnerInstanceId,
|
|
168
|
+
});
|
|
169
|
+
}
|
|
97
170
|
try {
|
|
98
171
|
const releasing = transitionHostLeaseForCleanup(lease, "releasing");
|
|
99
|
-
await this.writeLedger(releasing, ledger.repoId);
|
|
100
|
-
|
|
172
|
+
await this.writeLedger(releasing, ledger.repoId, attemptRootIdentity);
|
|
173
|
+
let evidenceRetainedAtAttemptRoot = false;
|
|
174
|
+
try {
|
|
175
|
+
await retainHostDiagnostics(this.stateRoot, attemptRoot, lease.dispatchAttemptId, attemptRootIdentity, this.executionRoot);
|
|
176
|
+
}
|
|
177
|
+
catch {
|
|
178
|
+
// Exact process recovery above already proved that the runtime is gone.
|
|
179
|
+
// Diagnostics retention can independently refuse a changed inventory,
|
|
180
|
+
// an unsafe archive path, or an over-budget source. Preserve the
|
|
181
|
+
// original attempt root and a released ledger for reconciliation, but
|
|
182
|
+
// never turn evidence-only work back into a capacity-owning runtime.
|
|
183
|
+
evidenceRetainedAtAttemptRoot = true;
|
|
184
|
+
}
|
|
185
|
+
if (evidenceRetainedAtAttemptRoot) {
|
|
186
|
+
const released = transitionRuntimeIsolationLease(releasing, "released");
|
|
187
|
+
await this.writeLedger(released, ledger.repoId, attemptRootIdentity);
|
|
188
|
+
this.forgetDegraded(lease.dispatchAttemptId, lease.fencingToken);
|
|
189
|
+
report.reaped += 1;
|
|
190
|
+
continue;
|
|
191
|
+
}
|
|
192
|
+
await removeHostAttemptRoot(attemptRoot);
|
|
101
193
|
await fs.rm(this.ledgerPath(lease.dispatchAttemptId), { force: true });
|
|
102
194
|
this.forgetDegraded(lease.dispatchAttemptId, lease.fencingToken);
|
|
103
195
|
report.reaped += 1;
|
|
@@ -110,6 +202,7 @@ export class HostRuntimeLeaseManager {
|
|
|
110
202
|
return report;
|
|
111
203
|
}
|
|
112
204
|
snapshot() {
|
|
205
|
+
this.reconcileReservationOwners();
|
|
113
206
|
const projects = new Map();
|
|
114
207
|
for (const promotion of this.knownPromotions.values()) {
|
|
115
208
|
const project = capacityProject(projects, promotion.repoId);
|
|
@@ -168,6 +261,26 @@ export class HostRuntimeLeaseManager {
|
|
|
168
261
|
project.waitingAttempts += 1;
|
|
169
262
|
project.safetyClass = conservativeSafetyClass(project.safetyClass, "host_parallel_candidate");
|
|
170
263
|
}
|
|
264
|
+
for (const waiter of this.backgroundAdmissionWaiters) {
|
|
265
|
+
const repoId = repoIdFromProjectSingletonKeys(waiter.keys);
|
|
266
|
+
if (!repoId)
|
|
267
|
+
continue;
|
|
268
|
+
const project = capacityProject(projects, repoId);
|
|
269
|
+
project.waitingAttempts += 1;
|
|
270
|
+
project.safetyClass = conservativeSafetyClass(project.safetyClass, "host_serial_only");
|
|
271
|
+
}
|
|
272
|
+
// A prepared-runtime verification remains foreground demand after its
|
|
273
|
+
// preparation admission is released and before the internal probe has
|
|
274
|
+
// acquired a normal admission. Report that barrier as waiting capacity so
|
|
275
|
+
// the server cannot mistake the transient gap for an idle host and dispatch
|
|
276
|
+
// readiness probes in a tight loop.
|
|
277
|
+
const unrepresentedPrepared = this.preparedReadiness.pendingSnapshots()
|
|
278
|
+
.filter(pending => !this.hasAttemptRepresentation(pending.probeDispatchAttemptId ?? pending.dispatchAttemptId, pending.probeFencingToken ?? pending.fencingToken));
|
|
279
|
+
for (const pending of unrepresentedPrepared) {
|
|
280
|
+
const project = capacityProject(projects, pending.repoId);
|
|
281
|
+
project.waitingAttempts += 1;
|
|
282
|
+
project.safetyClass = conservativeSafetyClass(project.safetyClass, "host_parallel_candidate");
|
|
283
|
+
}
|
|
171
284
|
const activeAttempts = [...this.active.values()]
|
|
172
285
|
.filter(active => active.lease.state !== "degraded").length;
|
|
173
286
|
const degradedAttempts = new Set([
|
|
@@ -176,14 +289,21 @@ export class HostRuntimeLeaseManager {
|
|
|
176
289
|
.filter(([, active]) => active.lease.state === "degraded")
|
|
177
290
|
.map(([dispatchAttemptId]) => dispatchAttemptId),
|
|
178
291
|
]).size;
|
|
292
|
+
const hostResources = this.hostResourceSnapshot();
|
|
179
293
|
return {
|
|
180
294
|
activeAttempts,
|
|
295
|
+
preparedWorktreeReadiness: true,
|
|
181
296
|
reservedAttempts: Math.max(0, this.admissions.size - this.active.size),
|
|
182
|
-
waitingAttempts: this.singletonWaiters.length
|
|
297
|
+
waitingAttempts: this.singletonWaiters.length
|
|
298
|
+
+ this.parallelProjectWaiters.length
|
|
299
|
+
+ this.capacityWaiters.length
|
|
300
|
+
+ this.backgroundAdmissionWaiters.length
|
|
301
|
+
+ unrepresentedPrepared.length,
|
|
183
302
|
degradedAttempts,
|
|
184
303
|
exclusiveReservations: this.singletonOwners.size,
|
|
185
304
|
sharedReservations: [...this.parallelProjectOwners.values()]
|
|
186
305
|
.reduce((count, owners) => count + owners.size, 0),
|
|
306
|
+
...hostResources,
|
|
187
307
|
projects: [...projects.values()]
|
|
188
308
|
.map(project => ({
|
|
189
309
|
...project,
|
|
@@ -192,6 +312,21 @@ export class HostRuntimeLeaseManager {
|
|
|
192
312
|
.sort((left, right) => left.repoId.localeCompare(right.repoId)),
|
|
193
313
|
};
|
|
194
314
|
}
|
|
315
|
+
hasForegroundDemand() {
|
|
316
|
+
return this.preparedReadiness.hasPending()
|
|
317
|
+
|| [...this.admissions.values()].some(admission => admission.priority === "foreground")
|
|
318
|
+
|| this.singletonWaiters.some(waiter => waiter.priority === "foreground")
|
|
319
|
+
|| this.parallelProjectWaiters.some(waiter => waiter.priority === "foreground")
|
|
320
|
+
|| this.capacityWaiters.some(waiter => waiter.priority === "foreground");
|
|
321
|
+
}
|
|
322
|
+
async acquirePreparation(dispatch, signal) {
|
|
323
|
+
// Discovery probes already own their verification lifecycle. Analysis-only
|
|
324
|
+
// work never launches the application and must not wait for a runtime wave.
|
|
325
|
+
if (runtimeExecutionClass(dispatch) === "analysis_only" || runtimeAdmissionPriority(dispatch) === "background") {
|
|
326
|
+
return await this.acquire(dispatch, signal);
|
|
327
|
+
}
|
|
328
|
+
return await this.preparedReadiness.acquire(dispatch, signal);
|
|
329
|
+
}
|
|
195
330
|
async acquire(dispatch, signal) {
|
|
196
331
|
const sandbox = dispatch.sandbox;
|
|
197
332
|
if (!sandbox || sandbox.executionMode !== "host") {
|
|
@@ -211,6 +346,7 @@ export class HostRuntimeLeaseManager {
|
|
|
211
346
|
return existing.keys;
|
|
212
347
|
}
|
|
213
348
|
const executionClass = runtimeExecutionClass(dispatch);
|
|
349
|
+
const priority = runtimeAdmissionPriority(dispatch);
|
|
214
350
|
const promotion = executionClass === "managed_runtime"
|
|
215
351
|
? await this.promotionStore.load(dispatch.repo.id)
|
|
216
352
|
: null;
|
|
@@ -219,37 +355,353 @@ export class HostRuntimeLeaseManager {
|
|
|
219
355
|
const projectKey = executionClass === "analysis_only"
|
|
220
356
|
? projectHostAnalysisKey(dispatch.repo.id)
|
|
221
357
|
: projectHostRuntimeSingletonKey(dispatch.repo.id);
|
|
222
|
-
|
|
358
|
+
const useParallelAdmission = priority === "foreground"
|
|
359
|
+
&& dispatch.cycleRun.phase !== "prepared_readiness"
|
|
360
|
+
&& (promotion !== null || executionClass === "analysis_only");
|
|
361
|
+
let keys = useParallelAdmission ? [] : [projectKey];
|
|
362
|
+
const resources = normalizedHostResourceRequest(sandbox.requestedResources);
|
|
223
363
|
const expiresAt = this.pendingRenewals.get(sandbox.dispatchAttemptId)?.fencingToken === owner.fencingToken
|
|
224
364
|
? this.pendingRenewals.get(sandbox.dispatchAttemptId).expiresAt
|
|
225
365
|
: dispatch.lease.expiresAt;
|
|
366
|
+
const acquisitionKey = hostRuntimeOwnerKey(owner);
|
|
367
|
+
this.acquiringAdmissions.add(acquisitionKey);
|
|
226
368
|
try {
|
|
227
|
-
|
|
228
|
-
|
|
369
|
+
try {
|
|
370
|
+
if (priority === "background") {
|
|
371
|
+
await this.refreshWorkspaceCapacity();
|
|
372
|
+
keys = await this.reserveBackgroundAdmission(keys, owner, resources, expiresAt, signal);
|
|
373
|
+
}
|
|
374
|
+
else if (useParallelAdmission) {
|
|
375
|
+
await this.reserveParallelProject(projectKey, owner, expiresAt, signal, priority);
|
|
376
|
+
}
|
|
377
|
+
else {
|
|
378
|
+
keys = await this.reserveSingletonKeys(keys, owner, expiresAt, signal, priority);
|
|
379
|
+
}
|
|
380
|
+
if (priority === "foreground") {
|
|
381
|
+
await this.refreshWorkspaceCapacity();
|
|
382
|
+
const capacityExpiresAt = this.pendingRenewals.get(sandbox.dispatchAttemptId)?.fencingToken === owner.fencingToken
|
|
383
|
+
? this.pendingRenewals.get(sandbox.dispatchAttemptId).expiresAt
|
|
384
|
+
: expiresAt;
|
|
385
|
+
await this.reserveHostCapacity(owner, resources, capacityExpiresAt, signal, priority);
|
|
386
|
+
}
|
|
229
387
|
}
|
|
230
|
-
|
|
231
|
-
|
|
388
|
+
catch (error) {
|
|
389
|
+
this.releaseHostCapacity(owner.dispatchAttemptId, owner.fencingToken);
|
|
390
|
+
this.releaseSingletonKeys(owner.dispatchAttemptId, owner.fencingToken);
|
|
391
|
+
this.releaseParallelProjects(owner.dispatchAttemptId, owner.fencingToken);
|
|
392
|
+
this.clearPendingRenewal(owner);
|
|
393
|
+
throw error;
|
|
232
394
|
}
|
|
395
|
+
const admittedPromotion = useParallelAdmission
|
|
396
|
+
? promotion ?? (executionClass === "managed_runtime" && keys.length === 0
|
|
397
|
+
? this.knownPromotions.get(dispatch.repo.id) ?? null
|
|
398
|
+
: null)
|
|
399
|
+
: null;
|
|
400
|
+
this.admissions.set(sandbox.dispatchAttemptId, {
|
|
401
|
+
repoId: dispatch.repo.id,
|
|
402
|
+
fencingToken: owner.fencingToken,
|
|
403
|
+
executionClass,
|
|
404
|
+
priority,
|
|
405
|
+
provisionalPromotionFingerprint: admittedPromotion?.runtimeContractFingerprint ?? null,
|
|
406
|
+
keys,
|
|
407
|
+
parallelProjectKey: useParallelAdmission ? projectKey : null,
|
|
408
|
+
});
|
|
409
|
+
return keys;
|
|
233
410
|
}
|
|
234
|
-
|
|
235
|
-
this.
|
|
236
|
-
|
|
411
|
+
finally {
|
|
412
|
+
this.acquiringAdmissions.delete(acquisitionKey);
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
async reserveHostCapacity(owner, resources, expiresAt, signal, priority = "foreground") {
|
|
416
|
+
const existing = this.capacityReservations.get(owner.dispatchAttemptId);
|
|
417
|
+
if (existing) {
|
|
418
|
+
if (!sameSingletonOwner(existing.owner, owner)) {
|
|
419
|
+
throw new Error("host_runtime_capacity_conflict: dispatch attempt already holds capacity under a different fence.");
|
|
420
|
+
}
|
|
421
|
+
return;
|
|
422
|
+
}
|
|
423
|
+
if (signal?.aborted)
|
|
424
|
+
throw cancellationError(signal);
|
|
425
|
+
throwIfAdmissionExpired(expiresAt);
|
|
426
|
+
const impossibleReason = this.hostCapacityImpossibleReason(resources);
|
|
427
|
+
if (impossibleReason)
|
|
428
|
+
throw new HostRuntimeCapacityRequestImpossibleError(impossibleReason);
|
|
429
|
+
if (this.capacityWaiters.length === 0 && this.hostCapacityAvailable(resources)) {
|
|
430
|
+
this.capacityReservations.set(owner.dispatchAttemptId, { owner, resources, state: "pending" });
|
|
431
|
+
return;
|
|
237
432
|
}
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
433
|
+
return new Promise((resolve, reject) => {
|
|
434
|
+
const waiter = {
|
|
435
|
+
owner,
|
|
436
|
+
resources,
|
|
437
|
+
priority,
|
|
438
|
+
resolve,
|
|
439
|
+
reject,
|
|
440
|
+
signal,
|
|
441
|
+
expiresAt,
|
|
442
|
+
deadlineTimer: null,
|
|
443
|
+
capacityRecheckTimer: null,
|
|
444
|
+
abort: () => {
|
|
445
|
+
if (!this.detachCapacityWaiter(waiter))
|
|
446
|
+
return;
|
|
447
|
+
this.clearPendingRenewal(owner);
|
|
448
|
+
reject(cancellationError(signal));
|
|
449
|
+
this.drainAdmissionWaiters();
|
|
450
|
+
},
|
|
451
|
+
expire: () => {
|
|
452
|
+
if (!this.detachCapacityWaiter(waiter))
|
|
453
|
+
return;
|
|
454
|
+
this.clearPendingRenewal(owner);
|
|
455
|
+
reject(new HostRuntimeAdmissionExpiredError());
|
|
456
|
+
this.drainAdmissionWaiters();
|
|
457
|
+
},
|
|
458
|
+
};
|
|
459
|
+
waiter.deadlineTimer = scheduleAdmissionDeadline(expiresAt, waiter.expire);
|
|
460
|
+
// Memory, CPU and filesystem capacity can recover without one of our leases
|
|
461
|
+
// being released (for example when an unrelated host process exits). Recheck
|
|
462
|
+
// ambient capacity while queued so readiness probes and attempts do not remain
|
|
463
|
+
// stranded behind a stale low-resource sample.
|
|
464
|
+
waiter.capacityRecheckTimer = setInterval(() => this.drainCapacityWaiters(), 1_000);
|
|
465
|
+
waiter.capacityRecheckTimer.unref?.();
|
|
466
|
+
signal?.addEventListener("abort", waiter.abort, { once: true });
|
|
467
|
+
enqueueAdmissionWaiter(this.capacityWaiters, waiter);
|
|
468
|
+
this.drainCapacityWaiters();
|
|
249
469
|
});
|
|
250
|
-
return keys;
|
|
251
470
|
}
|
|
252
|
-
async
|
|
471
|
+
async reserveBackgroundAdmission(keys, owner, resources, expiresAt, signal) {
|
|
472
|
+
if (signal?.aborted)
|
|
473
|
+
throw cancellationError(signal);
|
|
474
|
+
throwIfAdmissionExpired(expiresAt);
|
|
475
|
+
return await new Promise((resolve, reject) => {
|
|
476
|
+
const waiter = {
|
|
477
|
+
owner,
|
|
478
|
+
keys,
|
|
479
|
+
resources,
|
|
480
|
+
priority: "background",
|
|
481
|
+
resolve,
|
|
482
|
+
reject,
|
|
483
|
+
signal,
|
|
484
|
+
expiresAt,
|
|
485
|
+
deadlineTimer: null,
|
|
486
|
+
capacityRecheckTimer: null,
|
|
487
|
+
abort: () => {
|
|
488
|
+
removeWaiter(this.backgroundAdmissionWaiters, waiter);
|
|
489
|
+
clearAdmissionDeadline(waiter);
|
|
490
|
+
clearCapacityRecheck(waiter);
|
|
491
|
+
this.clearPendingRenewal(owner);
|
|
492
|
+
reject(cancellationError(signal));
|
|
493
|
+
this.drainAdmissionWaiters();
|
|
494
|
+
},
|
|
495
|
+
expire: () => {
|
|
496
|
+
removeWaiter(this.backgroundAdmissionWaiters, waiter);
|
|
497
|
+
clearCapacityRecheck(waiter);
|
|
498
|
+
signal?.removeEventListener("abort", waiter.abort);
|
|
499
|
+
this.clearPendingRenewal(owner);
|
|
500
|
+
reject(new HostRuntimeAdmissionExpiredError());
|
|
501
|
+
this.drainAdmissionWaiters();
|
|
502
|
+
},
|
|
503
|
+
};
|
|
504
|
+
waiter.deadlineTimer = scheduleAdmissionDeadline(expiresAt, waiter.expire);
|
|
505
|
+
waiter.capacityRecheckTimer = setInterval(() => this.drainAdmissionWaiters(), 1_000);
|
|
506
|
+
waiter.capacityRecheckTimer.unref?.();
|
|
507
|
+
signal?.addEventListener("abort", waiter.abort, { once: true });
|
|
508
|
+
this.backgroundAdmissionWaiters.push(waiter);
|
|
509
|
+
this.drainAdmissionWaiters();
|
|
510
|
+
});
|
|
511
|
+
}
|
|
512
|
+
hostCapacityAvailable(resources) {
|
|
513
|
+
const capacity = this.hostResourceSnapshot();
|
|
514
|
+
return capacity.availableAttempts >= resources.sandboxSlots
|
|
515
|
+
&& capacity.availableMemoryBytes >= resources.memoryBytes
|
|
516
|
+
&& capacity.availableCpuMillis >= resources.cpuMillis
|
|
517
|
+
&& capacity.availableWorkspaceBytes >= resources.workspaceBytes;
|
|
518
|
+
}
|
|
519
|
+
hostCapacityImpossibleReason(resources) {
|
|
520
|
+
const { hardMaxAttempts, idleHostReserveBytes, cpuReserveMillis } = this.capacityConfiguration;
|
|
521
|
+
if (resources.sandboxSlots > hardMaxAttempts) {
|
|
522
|
+
return `requested ${resources.sandboxSlots} sandbox slots exceed the host maximum of ${hardMaxAttempts}.`;
|
|
523
|
+
}
|
|
524
|
+
const totalMemoryBytes = this.capacityResourceSample?.totalMemoryBytes ?? os.totalmem();
|
|
525
|
+
const maximumMemoryBytes = Math.max(0, totalMemoryBytes - idleHostReserveBytes);
|
|
526
|
+
if (resources.memoryBytes > maximumMemoryBytes) {
|
|
527
|
+
return `requested memory ${resources.memoryBytes} exceeds the host maximum of ${maximumMemoryBytes} bytes after reserve.`;
|
|
528
|
+
}
|
|
529
|
+
const totalCpuMillis = this.capacityResourceSample?.totalCpuMillis ?? os.cpus().length * 1_000;
|
|
530
|
+
const maximumCpuMillis = Math.max(0, totalCpuMillis - cpuReserveMillis);
|
|
531
|
+
if (resources.cpuMillis > maximumCpuMillis) {
|
|
532
|
+
return `requested CPU ${resources.cpuMillis} millis exceeds the host maximum of ${maximumCpuMillis} millis after reserve.`;
|
|
533
|
+
}
|
|
534
|
+
return null;
|
|
535
|
+
}
|
|
536
|
+
hostResourceSnapshot() {
|
|
537
|
+
if (!this.capacityResourceSample) {
|
|
538
|
+
try {
|
|
539
|
+
const stats = statfsSync(this.stateRoot);
|
|
540
|
+
this.workspaceAvailableBytes = Math.max(0, stats.bavail * stats.bsize);
|
|
541
|
+
}
|
|
542
|
+
catch {
|
|
543
|
+
// State root may not exist during the first capability heartbeat. The first
|
|
544
|
+
// admission creates it and refreshes the sample before reserving resources.
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
const { hardMaxAttempts, cpuReserveMillis, workspaceReserveBytes } = this.capacityConfiguration;
|
|
548
|
+
const hostReserveBytes = this.capacityReservations.size === 0
|
|
549
|
+
? this.capacityConfiguration.idleHostReserveBytes
|
|
550
|
+
: this.capacityConfiguration.hostReserveBytes;
|
|
551
|
+
const activeReservationMemoryBytes = [...this.capacityReservations.values()]
|
|
552
|
+
.reduce((total, reservation) => total + reservation.resources.memoryBytes, 0);
|
|
553
|
+
const pendingReservationMemoryBytes = [...this.capacityReservations.values()]
|
|
554
|
+
.filter(reservation => reservation.state === "pending")
|
|
555
|
+
.reduce((total, reservation) => total + reservation.resources.memoryBytes, 0);
|
|
556
|
+
const activeReservationCpuMillis = [...this.capacityReservations.values()]
|
|
557
|
+
.reduce((total, reservation) => total + reservation.resources.cpuMillis, 0);
|
|
558
|
+
const activeReservationWorkspaceBytes = [...this.capacityReservations.values()]
|
|
559
|
+
.reduce((total, reservation) => total + reservation.resources.workspaceBytes, 0);
|
|
560
|
+
const sampledAt = new Date().toISOString();
|
|
561
|
+
const totalMemoryBytes = this.capacityResourceSample?.totalMemoryBytes ?? os.totalmem();
|
|
562
|
+
const rawAvailableMemoryBytes = this.capacityResourceSample?.availableMemoryBytes ?? os.freemem();
|
|
563
|
+
const sampledAvailableMemoryBytes = this.memoryAvailability.observe(rawAvailableMemoryBytes);
|
|
564
|
+
// raw/stabilized available memory already includes every resident host and
|
|
565
|
+
// Forge process. Keep the independent total ceiling scoped to Forge's own
|
|
566
|
+
// reservations; adding ambient use here charges it a second time and can
|
|
567
|
+
// strand work even while more than the requested memory is physically free.
|
|
568
|
+
const committedMemoryBytes = Math.min(totalMemoryBytes, activeReservationMemoryBytes);
|
|
569
|
+
const totalCpuMillis = this.capacityResourceSample?.totalCpuMillis ?? os.cpus().length * 1000;
|
|
570
|
+
const sampledWorkspaceBytes = this.capacityResourceSample?.availableWorkspaceBytes ?? this.workspaceAvailableBytes;
|
|
571
|
+
const memoryAdmission = buildMemoryAdmissionDiagnostics({
|
|
572
|
+
sampledAt,
|
|
573
|
+
totalMemoryBytes,
|
|
574
|
+
rawAvailableMemoryBytes,
|
|
575
|
+
stabilizedAvailableMemoryBytes: sampledAvailableMemoryBytes,
|
|
576
|
+
controlPlaneMemoryBytes: process.memoryUsage().rss,
|
|
577
|
+
reserveMemoryBytes: hostReserveBytes,
|
|
578
|
+
committedMemoryBytes,
|
|
579
|
+
pendingCommitmentMemoryBytes: pendingReservationMemoryBytes,
|
|
580
|
+
});
|
|
581
|
+
const availableMemoryBytes = memoryAdmission.availableForAdmissionBytes;
|
|
582
|
+
const availableCpuMillis = Math.max(0, totalCpuMillis - cpuReserveMillis - activeReservationCpuMillis);
|
|
583
|
+
const availableWorkspaceBytes = Math.max(0, sampledWorkspaceBytes - workspaceReserveBytes - activeReservationWorkspaceBytes);
|
|
584
|
+
const slotAvailability = Math.max(0, hardMaxAttempts - this.capacityReservations.size);
|
|
585
|
+
// Capability snapshots advertise whether any supported attempt can fit.
|
|
586
|
+
// The queue/admission path still checks the exact project-derived request.
|
|
587
|
+
const minimumRequest = minimumSupportedHostResourceRequest();
|
|
588
|
+
const resourceAvailability = Math.min(resourceSlots(availableMemoryBytes, minimumRequest.memoryBytes), resourceSlots(availableCpuMillis, minimumRequest.cpuMillis), resourceSlots(availableWorkspaceBytes, minimumRequest.workspaceBytes));
|
|
589
|
+
const availableAttempts = Math.max(0, Math.min(slotAvailability, resourceAvailability));
|
|
590
|
+
const unavailableReason = slotAvailability <= 0
|
|
591
|
+
? `Host attempt capacity is full (${this.capacityReservations.size}/${hardMaxAttempts}).`
|
|
592
|
+
: availableMemoryBytes < minimumRequest.memoryBytes
|
|
593
|
+
? "Host memory is below the admission reserve."
|
|
594
|
+
: availableCpuMillis < minimumRequest.cpuMillis
|
|
595
|
+
? "Host CPU capacity is below the admission reserve."
|
|
596
|
+
: availableWorkspaceBytes < minimumRequest.workspaceBytes
|
|
597
|
+
? "Host workspace capacity is below the admission reserve."
|
|
598
|
+
: null;
|
|
599
|
+
return {
|
|
600
|
+
hardMaxAttempts,
|
|
601
|
+
effectiveMaxAttempts: this.capacityReservations.size + availableAttempts,
|
|
602
|
+
availableAttempts,
|
|
603
|
+
totalMemoryBytes,
|
|
604
|
+
availableMemoryBytes,
|
|
605
|
+
hostReserveBytes,
|
|
606
|
+
availableCpuMillis,
|
|
607
|
+
availableWorkspaceBytes,
|
|
608
|
+
activeReservationMemoryBytes,
|
|
609
|
+
memoryAdmission,
|
|
610
|
+
unavailableCode: unavailableReason ? "capacity_unavailable" : null,
|
|
611
|
+
unavailableReason,
|
|
612
|
+
};
|
|
613
|
+
}
|
|
614
|
+
async refreshWorkspaceCapacity() {
|
|
615
|
+
if (this.capacityResourceSample)
|
|
616
|
+
return;
|
|
617
|
+
try {
|
|
618
|
+
await fs.mkdir(this.stateRoot, { recursive: true });
|
|
619
|
+
const stats = await fs.statfs(this.stateRoot);
|
|
620
|
+
this.workspaceAvailableBytes = Math.max(0, stats.bavail * stats.bsize);
|
|
621
|
+
}
|
|
622
|
+
catch {
|
|
623
|
+
// Keep the last trustworthy sample. Memory/CPU and the hard ceiling still
|
|
624
|
+
// protect admission when the filesystem cannot be sampled temporarily.
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
releaseHostCapacity(dispatchAttemptId, fencingToken) {
|
|
628
|
+
const reservation = this.capacityReservations.get(dispatchAttemptId);
|
|
629
|
+
if (!reservation || reservation.owner.fencingToken !== fencingToken)
|
|
630
|
+
return;
|
|
631
|
+
this.capacityReservations.delete(dispatchAttemptId);
|
|
632
|
+
this.drainAdmissionWaiters();
|
|
633
|
+
}
|
|
634
|
+
drainCapacityWaiters() {
|
|
635
|
+
for (const waiter of this.capacityWaitersForScan()) {
|
|
636
|
+
if (!this.capacityWaiters.includes(waiter))
|
|
637
|
+
continue;
|
|
638
|
+
if (waiter.signal?.aborted) {
|
|
639
|
+
this.detachCapacityWaiter(waiter);
|
|
640
|
+
this.clearPendingRenewal(waiter.owner);
|
|
641
|
+
waiter.reject(cancellationError(waiter.signal));
|
|
642
|
+
continue;
|
|
643
|
+
}
|
|
644
|
+
if (admissionExpired(waiter.expiresAt)) {
|
|
645
|
+
this.detachCapacityWaiter(waiter);
|
|
646
|
+
this.clearPendingRenewal(waiter.owner);
|
|
647
|
+
waiter.reject(new HostRuntimeAdmissionExpiredError());
|
|
648
|
+
continue;
|
|
649
|
+
}
|
|
650
|
+
const impossibleReason = this.hostCapacityImpossibleReason(waiter.resources);
|
|
651
|
+
if (impossibleReason) {
|
|
652
|
+
this.detachCapacityWaiter(waiter);
|
|
653
|
+
this.clearPendingRenewal(waiter.owner);
|
|
654
|
+
waiter.reject(new HostRuntimeCapacityRequestImpossibleError(impossibleReason));
|
|
655
|
+
continue;
|
|
656
|
+
}
|
|
657
|
+
if (!this.hostCapacityAvailable(waiter.resources))
|
|
658
|
+
continue;
|
|
659
|
+
this.detachCapacityWaiter(waiter);
|
|
660
|
+
this.capacityReservations.set(waiter.owner.dispatchAttemptId, {
|
|
661
|
+
owner: waiter.owner,
|
|
662
|
+
resources: waiter.resources,
|
|
663
|
+
state: "pending",
|
|
664
|
+
});
|
|
665
|
+
waiter.resolve();
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
capacityWaitersForScan() {
|
|
669
|
+
const head = this.capacityWaiters[0];
|
|
670
|
+
if (!head) {
|
|
671
|
+
this.capacityWaiterScanCursor = 0;
|
|
672
|
+
return [];
|
|
673
|
+
}
|
|
674
|
+
const scanCount = Math.min(hostCapacityWaiterScanLimit, this.capacityWaiters.length);
|
|
675
|
+
if (scanCount === this.capacityWaiters.length) {
|
|
676
|
+
this.capacityWaiterScanCursor = 0;
|
|
677
|
+
return this.capacityWaiters.slice();
|
|
678
|
+
}
|
|
679
|
+
// The head always gets the first opportunity, while a rotating bounded
|
|
680
|
+
// window prevents an unfit renewed head (or a very deep queue) from
|
|
681
|
+
// indefinitely hiding a later request that fits current capacity.
|
|
682
|
+
const laterCount = this.capacityWaiters.length - 1;
|
|
683
|
+
const laterScanCount = scanCount - 1;
|
|
684
|
+
const start = this.capacityWaiterScanCursor % laterCount;
|
|
685
|
+
const result = [head];
|
|
686
|
+
for (let offset = 0; offset < laterScanCount; offset += 1) {
|
|
687
|
+
result.push(this.capacityWaiters[1 + ((start + offset) % laterCount)]);
|
|
688
|
+
}
|
|
689
|
+
this.capacityWaiterScanCursor = (start + laterScanCount) % laterCount;
|
|
690
|
+
return result;
|
|
691
|
+
}
|
|
692
|
+
detachCapacityWaiter(waiter) {
|
|
693
|
+
const index = this.capacityWaiters.indexOf(waiter);
|
|
694
|
+
if (index < 0)
|
|
695
|
+
return false;
|
|
696
|
+
this.capacityWaiters.splice(index, 1);
|
|
697
|
+
clearAdmissionDeadline(waiter);
|
|
698
|
+
clearCapacityRecheck(waiter);
|
|
699
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
700
|
+
if (this.capacityWaiters.length === 0)
|
|
701
|
+
this.capacityWaiterScanCursor = 0;
|
|
702
|
+
return true;
|
|
703
|
+
}
|
|
704
|
+
async reserveSingletonKeys(keys, owner, expiresAt, signal, priority = "foreground") {
|
|
253
705
|
const owned = keys.every(key => sameSingletonOwner(this.singletonOwners.get(key), owner));
|
|
254
706
|
if (owned)
|
|
255
707
|
return keys;
|
|
@@ -269,6 +721,7 @@ export class HostRuntimeLeaseManager {
|
|
|
269
721
|
const waiter = {
|
|
270
722
|
owner,
|
|
271
723
|
keys,
|
|
724
|
+
priority,
|
|
272
725
|
resolve,
|
|
273
726
|
reject,
|
|
274
727
|
signal,
|
|
@@ -295,13 +748,13 @@ export class HostRuntimeLeaseManager {
|
|
|
295
748
|
this.drainAdmissionWaiters();
|
|
296
749
|
},
|
|
297
750
|
};
|
|
298
|
-
this.singletonWaiters
|
|
751
|
+
enqueueAdmissionWaiter(this.singletonWaiters, waiter);
|
|
299
752
|
signal?.addEventListener("abort", waiter.abort, { once: true });
|
|
300
753
|
waiter.deadlineTimer = scheduleAdmissionDeadline(waiter.expiresAt, waiter.expire);
|
|
301
754
|
this.drainAdmissionWaiters();
|
|
302
755
|
});
|
|
303
756
|
}
|
|
304
|
-
async reserveParallelProject(key, owner, expiresAt, signal) {
|
|
757
|
+
async reserveParallelProject(key, owner, expiresAt, signal, priority = "foreground") {
|
|
305
758
|
if (this.parallelProjectOwned(key, owner))
|
|
306
759
|
return;
|
|
307
760
|
const staleFence = [...(this.parallelProjectOwners.get(key)?.values() ?? [])]
|
|
@@ -313,7 +766,7 @@ export class HostRuntimeLeaseManager {
|
|
|
313
766
|
if (signal?.aborted)
|
|
314
767
|
throw cancellationError(signal);
|
|
315
768
|
throwIfAdmissionExpired(expiresAt);
|
|
316
|
-
if (this.parallelProjectAvailable(key)) {
|
|
769
|
+
if (this.parallelProjectAvailable(key, priority)) {
|
|
317
770
|
this.assignParallelProject(key, owner);
|
|
318
771
|
return;
|
|
319
772
|
}
|
|
@@ -321,6 +774,7 @@ export class HostRuntimeLeaseManager {
|
|
|
321
774
|
const waiter = {
|
|
322
775
|
owner,
|
|
323
776
|
key,
|
|
777
|
+
priority,
|
|
324
778
|
resolve,
|
|
325
779
|
reject,
|
|
326
780
|
signal,
|
|
@@ -347,17 +801,85 @@ export class HostRuntimeLeaseManager {
|
|
|
347
801
|
this.drainAdmissionWaiters();
|
|
348
802
|
},
|
|
349
803
|
};
|
|
350
|
-
this.parallelProjectWaiters
|
|
804
|
+
enqueueAdmissionWaiter(this.parallelProjectWaiters, waiter);
|
|
351
805
|
signal?.addEventListener("abort", waiter.abort, { once: true });
|
|
352
806
|
waiter.deadlineTimer = scheduleAdmissionDeadline(waiter.expiresAt, waiter.expire);
|
|
353
807
|
this.drainAdmissionWaiters();
|
|
354
808
|
});
|
|
355
809
|
}
|
|
356
|
-
async
|
|
810
|
+
async upgradeParallelProjectToSingleton(keys, parallelProjectKey, owner, expiresAt, signal, priority = "foreground") {
|
|
811
|
+
if (!this.parallelProjectOwned(parallelProjectKey, owner)) {
|
|
812
|
+
throw new Error("host_runtime_parallel_unreserved: matching shared project admission is required before serialization.");
|
|
813
|
+
}
|
|
814
|
+
if (signal?.aborted)
|
|
815
|
+
throw cancellationError(signal);
|
|
816
|
+
throwIfAdmissionExpired(expiresAt);
|
|
817
|
+
return await new Promise((resolve, reject) => {
|
|
818
|
+
const waiter = {
|
|
819
|
+
owner,
|
|
820
|
+
keys,
|
|
821
|
+
priority,
|
|
822
|
+
resolve,
|
|
823
|
+
reject,
|
|
824
|
+
signal,
|
|
825
|
+
expiresAt,
|
|
826
|
+
deadlineTimer: null,
|
|
827
|
+
abort: () => {
|
|
828
|
+
const index = this.singletonWaiters.indexOf(waiter);
|
|
829
|
+
if (index >= 0)
|
|
830
|
+
this.singletonWaiters.splice(index, 1);
|
|
831
|
+
clearAdmissionDeadline(waiter);
|
|
832
|
+
this.clearPendingRenewal(owner);
|
|
833
|
+
reject(cancellationError(signal));
|
|
834
|
+
this.drainAdmissionWaiters();
|
|
835
|
+
},
|
|
836
|
+
expire: () => {
|
|
837
|
+
const index = this.singletonWaiters.indexOf(waiter);
|
|
838
|
+
if (index < 0)
|
|
839
|
+
return;
|
|
840
|
+
this.singletonWaiters.splice(index, 1);
|
|
841
|
+
clearAdmissionDeadline(waiter);
|
|
842
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
843
|
+
this.clearPendingRenewal(owner);
|
|
844
|
+
reject(new HostRuntimeAdmissionExpiredError());
|
|
845
|
+
this.drainAdmissionWaiters();
|
|
846
|
+
},
|
|
847
|
+
};
|
|
848
|
+
// Queue the exclusive claim before relinquishing the shared claim. This
|
|
849
|
+
// makes the promotion downgrade atomic from the broker's perspective and
|
|
850
|
+
// prevents a background readiness probe from jumping ahead in between.
|
|
851
|
+
enqueueAdmissionWaiter(this.singletonWaiters, waiter);
|
|
852
|
+
signal?.addEventListener("abort", waiter.abort, { once: true });
|
|
853
|
+
waiter.deadlineTimer = scheduleAdmissionDeadline(waiter.expiresAt, waiter.expire);
|
|
854
|
+
this.releaseParallelProjectNoDrain(parallelProjectKey, owner);
|
|
855
|
+
this.drainAdmissionWaiters();
|
|
856
|
+
});
|
|
857
|
+
}
|
|
858
|
+
async start(dispatch, worktreePath, signal, onAdmissionCheckpoint) {
|
|
357
859
|
const sandbox = dispatch.sandbox;
|
|
358
860
|
if (!sandbox || sandbox.executionMode !== "host") {
|
|
359
861
|
throw new Error("host_runtime_lease_invalid: managed host sandbox context is required.");
|
|
360
862
|
}
|
|
863
|
+
const wasPrepared = this.preparedReadiness.has(sandbox.dispatchAttemptId);
|
|
864
|
+
const preparedProof = wasPrepared
|
|
865
|
+
? await this.preparedReadiness.verify(dispatch, worktreePath, onAdmissionCheckpoint)
|
|
866
|
+
: undefined;
|
|
867
|
+
if (wasPrepared) {
|
|
868
|
+
signal?.throwIfAborted();
|
|
869
|
+
const admission = this.hostResourceSnapshot().memoryAdmission;
|
|
870
|
+
const requestedMemoryBytes = normalizedHostResourceRequest(sandbox.requestedResources).memoryBytes;
|
|
871
|
+
await onAdmissionCheckpoint?.({
|
|
872
|
+
code: "host_runtime_prepared_admission_waiting",
|
|
873
|
+
message: `Prepared runtime verification passed. Waiting for host admission before agent startup: ${formatBytes(requestedMemoryBytes)} requested, ${formatBytes(admission.availableForAdmissionBytes)} currently available under the host reserve.`,
|
|
874
|
+
expiresAt: dispatch.lease.expiresAt,
|
|
875
|
+
});
|
|
876
|
+
await this.acquire(dispatch, signal);
|
|
877
|
+
await onAdmissionCheckpoint?.({
|
|
878
|
+
code: "host_runtime_prepared_admission_acquired",
|
|
879
|
+
message: "Prepared runtime acquired host admission; agent startup is continuing.",
|
|
880
|
+
expiresAt: dispatch.lease.expiresAt,
|
|
881
|
+
});
|
|
882
|
+
}
|
|
361
883
|
const existing = this.active.get(sandbox.dispatchAttemptId);
|
|
362
884
|
if (existing) {
|
|
363
885
|
if (existing.lease.fencingToken !== dispatch.lease.fencingToken) {
|
|
@@ -376,12 +898,13 @@ export class HostRuntimeLeaseManager {
|
|
|
376
898
|
throw new Error("host_runtime_singleton_unreserved: managed host admission is required before start.");
|
|
377
899
|
}
|
|
378
900
|
const attemptHex = uuidHex(sandbox.dispatchAttemptId, "dispatch attempt");
|
|
379
|
-
const
|
|
380
|
-
const
|
|
381
|
-
const
|
|
901
|
+
const compactAttemptHex = compactHostAttemptId(sandbox.dispatchAttemptId);
|
|
902
|
+
const normalizedWorktreePath = await resolveExistingDirectoryPath(worktreePath);
|
|
903
|
+
const attemptRoot = path.join(this.executionRoot, "a", compactAttemptHex);
|
|
904
|
+
const temporaryPath = path.join(attemptRoot, "t");
|
|
382
905
|
const logPath = path.join(attemptRoot, "logs");
|
|
383
|
-
const cachePath = path.join(attemptRoot, "
|
|
384
|
-
const browserProfilePath = path.join(attemptRoot, "
|
|
906
|
+
const cachePath = path.join(attemptRoot, "c");
|
|
907
|
+
const browserProfilePath = path.join(attemptRoot, "b");
|
|
385
908
|
const testOutputPath = path.join(attemptRoot, "test-output");
|
|
386
909
|
const processSupervisorMetadataPath = path.join(logPath, "attempt-process-supervisor.json");
|
|
387
910
|
const processSupervisorId = `host-${attemptHex}`;
|
|
@@ -390,50 +913,86 @@ export class HostRuntimeLeaseManager {
|
|
|
390
913
|
? renewedExpiry.expiresAt
|
|
391
914
|
: dispatch.lease.expiresAt;
|
|
392
915
|
let processSupervisor = null;
|
|
916
|
+
let attemptRootIdentity = null;
|
|
393
917
|
try {
|
|
394
|
-
|
|
395
|
-
try {
|
|
396
|
-
runtimeContract = await loadHostRuntimeContract(normalizedWorktreePath);
|
|
397
|
-
}
|
|
398
|
-
catch (error) {
|
|
399
|
-
if (admission.executionClass === "managed_runtime" && admission.provisionalPromotionFingerprint) {
|
|
400
|
-
await this.invalidatePromotion(dispatch.repo.id).catch(() => undefined);
|
|
401
|
-
}
|
|
402
|
-
throw error;
|
|
403
|
-
}
|
|
918
|
+
const runtimeContract = await loadHostRuntimeContract(normalizedWorktreePath);
|
|
404
919
|
if (runtimeContract.contract.runtime.hostConcurrency === "disabled") {
|
|
405
920
|
throw new Error("host_runtime_disabled: project runtime configuration requires Docker execution.");
|
|
406
921
|
}
|
|
922
|
+
if (wasPrepared && !preparedProof && admission.executionClass === "managed_runtime"
|
|
923
|
+
&& hostRuntimeParallelEligible(runtimeContract)) {
|
|
924
|
+
throw new Error("host_runtime_prepared_source_changed: runtime became parallel-eligible after preparation; a new foreground proof is required.");
|
|
925
|
+
}
|
|
407
926
|
const runtimeContractFingerprint = hostRuntimeContractFingerprint(runtimeContract);
|
|
927
|
+
const inputCommitSha = preparedProof?.inputCommitSha
|
|
928
|
+
?? await resolveWorktreeCommitSha(normalizedWorktreePath, sandbox.inputCommitSha);
|
|
929
|
+
if (preparedProof && (runtimeContractFingerprint !== preparedProof.runtimeContractFingerprint
|
|
930
|
+
|| JSON.stringify(preparedProof) !== JSON.stringify(await preparedRuntimeIdentity(normalizedWorktreePath, signal)))) {
|
|
931
|
+
throw new Error("host_runtime_prepared_source_changed: source changed after its foreground proof; agent was not started.");
|
|
932
|
+
}
|
|
408
933
|
const promotion = admission.executionClass === "managed_runtime"
|
|
409
|
-
? await this.promotionStore.load(dispatch.repo.id)
|
|
934
|
+
? await this.promotionStore.load(dispatch.repo.id, preparedProof ?? { runtimeContractFingerprint, inputCommitSha })
|
|
410
935
|
: null;
|
|
411
936
|
const promotionMatches = hostRuntimeParallelEligible(runtimeContract)
|
|
412
|
-
&& promotion?.runtimeContractFingerprint === runtimeContractFingerprint
|
|
937
|
+
&& promotion?.runtimeContractFingerprint === runtimeContractFingerprint
|
|
938
|
+
&& promotion.inputCommitSha === inputCommitSha;
|
|
413
939
|
if (admission.executionClass === "managed_runtime" && admission.keys.length === 0 && !promotionMatches) {
|
|
414
|
-
await this.invalidatePromotion(dispatch.repo.id);
|
|
415
940
|
const projectKeys = hostRuntimeSingletonKeys(dispatch);
|
|
416
|
-
if (admission.parallelProjectKey) {
|
|
417
|
-
this.releaseParallelProjectNoDrain(admission.parallelProjectKey, owner);
|
|
418
|
-
admission.parallelProjectKey = null;
|
|
419
|
-
}
|
|
420
941
|
const expiresAt = this.pendingRenewals.get(sandbox.dispatchAttemptId)?.fencingToken === owner.fencingToken
|
|
421
942
|
? this.pendingRenewals.get(sandbox.dispatchAttemptId).expiresAt
|
|
422
943
|
: dispatch.lease.expiresAt;
|
|
423
|
-
|
|
944
|
+
const parallelProjectKey = admission.parallelProjectKey;
|
|
945
|
+
admission.parallelProjectKey = null;
|
|
946
|
+
const capacityReservation = this.capacityReservations.get(owner.dispatchAttemptId);
|
|
947
|
+
if (capacityReservation && !sameSingletonOwner(capacityReservation.owner, owner)) {
|
|
948
|
+
throw new Error("host_runtime_capacity_conflict: dispatch attempt holds capacity under a different fence.");
|
|
949
|
+
}
|
|
950
|
+
const serializedAdmission = parallelProjectKey
|
|
951
|
+
? this.upgradeParallelProjectToSingleton(projectKeys, parallelProjectKey, owner, expiresAt, signal, admission.priority)
|
|
952
|
+
: this.reserveSingletonKeys(projectKeys, owner, expiresAt, signal, admission.priority);
|
|
953
|
+
// A sibling can already hold the same provisional shared project claim
|
|
954
|
+
// while waiting for this attempt's host-capacity slot. Retaining capacity
|
|
955
|
+
// while waiting for that sibling to release its shared claim creates a
|
|
956
|
+
// cycle: exclusive waits on shared, shared waits on capacity. Relinquish
|
|
957
|
+
// the capacity slot only after the exclusive waiter is queued, then
|
|
958
|
+
// reacquire it after serialization is granted.
|
|
959
|
+
if (capacityReservation) {
|
|
960
|
+
this.releaseHostCapacity(owner.dispatchAttemptId, owner.fencingToken);
|
|
961
|
+
}
|
|
962
|
+
// The checkpoint is allowed to do I/O. Observe an early admission
|
|
963
|
+
// failure now so a slow acknowledgement cannot create an unhandled
|
|
964
|
+
// rejection before the operational promise is awaited below.
|
|
965
|
+
void serializedAdmission.catch(() => undefined);
|
|
966
|
+
// Begin the operational downgrade before publishing telemetry. A lost
|
|
967
|
+
// callback acknowledgement must not leave the attempt holding its old
|
|
968
|
+
// shared admission while every recovery probe waits behind it.
|
|
969
|
+
await onAdmissionCheckpoint?.({
|
|
970
|
+
code: "host_runtime_serial_admission_waiting",
|
|
971
|
+
message: "The prepared runtime contract no longer matches the parallel-readiness proof. Waiting for foreground-priority serialized admission before launch.",
|
|
972
|
+
expiresAt,
|
|
973
|
+
});
|
|
974
|
+
admission.keys = await serializedAdmission;
|
|
424
975
|
while (admission.keys.length === 0) {
|
|
425
976
|
const promoted = this.knownPromotions.get(dispatch.repo.id);
|
|
426
977
|
if (!promoted) {
|
|
427
978
|
throw new Error("host_runtime_parallel_promotion_missing: shared admission requires matching promotion evidence.");
|
|
428
979
|
}
|
|
429
|
-
|
|
980
|
+
const matching = await this.promotionStore.load(dispatch.repo.id, preparedProof ?? { runtimeContractFingerprint, inputCommitSha });
|
|
981
|
+
if (!matching) {
|
|
982
|
+
// Drop obsolete proof before asking for exclusive admission; otherwise
|
|
983
|
+
// the queue can grant the same stale shared promotion indefinitely.
|
|
430
984
|
this.releaseParallelProjectNoDrain(projectKeys[0], owner);
|
|
431
|
-
await this.
|
|
432
|
-
admission.keys = await this.reserveSingletonKeys(projectKeys, owner, expiresAt, signal);
|
|
985
|
+
admission.keys = await this.reserveSingletonKeys(projectKeys, owner, expiresAt, signal, admission.priority);
|
|
433
986
|
continue;
|
|
434
987
|
}
|
|
435
988
|
admission.parallelProjectKey = projectKeys[0] ?? null;
|
|
436
989
|
admission.provisionalPromotionFingerprint = promoted.runtimeContractFingerprint;
|
|
990
|
+
break;
|
|
991
|
+
}
|
|
992
|
+
// Do not hold a capacity slot while a stale shared re-grant is still
|
|
993
|
+
// being converted to exclusive admission.
|
|
994
|
+
if (capacityReservation) {
|
|
995
|
+
await this.reserveHostCapacity(owner, capacityReservation.resources, expiresAt, signal, admission.priority);
|
|
437
996
|
}
|
|
438
997
|
}
|
|
439
998
|
const declaredSingletonKeys = runtimeContract.contract.runtime.singletonResources
|
|
@@ -444,7 +1003,7 @@ export class HostRuntimeLeaseManager {
|
|
|
444
1003
|
const expiresAt = this.pendingRenewals.get(sandbox.dispatchAttemptId)?.fencingToken === owner.fencingToken
|
|
445
1004
|
? this.pendingRenewals.get(sandbox.dispatchAttemptId).expiresAt
|
|
446
1005
|
: dispatch.lease.expiresAt;
|
|
447
|
-
const reservedSingletonKeys = await this.reserveSingletonKeys(missingSingletonKeys, owner, expiresAt, signal);
|
|
1006
|
+
const reservedSingletonKeys = await this.reserveSingletonKeys(missingSingletonKeys, owner, expiresAt, signal, admission.priority);
|
|
448
1007
|
admission.keys = [...admission.keys, ...reservedSingletonKeys]
|
|
449
1008
|
.sort((left, right) => left.localeCompare(right));
|
|
450
1009
|
}
|
|
@@ -455,13 +1014,16 @@ export class HostRuntimeLeaseManager {
|
|
|
455
1014
|
if (admission.parallelProjectKey && !this.parallelProjectOwned(admission.parallelProjectKey, owner)) {
|
|
456
1015
|
throw new Error("host_runtime_parallel_unreserved: matching shared project admission is required before start.");
|
|
457
1016
|
}
|
|
1017
|
+
await createCanonicalDirectory(attemptRoot);
|
|
1018
|
+
attemptRootIdentity = (await assertCanonicalDirectory(attemptRoot));
|
|
458
1019
|
await Promise.all([
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
1020
|
+
createCanonicalDirectory(temporaryPath),
|
|
1021
|
+
createCanonicalDirectory(logPath),
|
|
1022
|
+
createCanonicalDirectory(cachePath),
|
|
1023
|
+
createCanonicalDirectory(browserProfilePath),
|
|
1024
|
+
createCanonicalDirectory(testOutputPath),
|
|
464
1025
|
]);
|
|
1026
|
+
await assertHostAttemptRootIdentity(attemptRoot, attemptRootIdentity);
|
|
465
1027
|
await resetGeneratedOutputs(normalizedWorktreePath, testOutputPath);
|
|
466
1028
|
let lease = createRuntimeIsolationLease({
|
|
467
1029
|
leaseId: sandbox.executionLeaseId,
|
|
@@ -493,11 +1055,11 @@ export class HostRuntimeLeaseManager {
|
|
|
493
1055
|
singletonKeys,
|
|
494
1056
|
},
|
|
495
1057
|
});
|
|
496
|
-
await this.writeLedger(lease);
|
|
1058
|
+
await this.writeLedger(lease, undefined, attemptRootIdentity);
|
|
497
1059
|
lease = transitionRuntimeIsolationLease(lease, "starting");
|
|
498
|
-
await this.writeLedger(lease);
|
|
1060
|
+
await this.writeLedger(lease, undefined, attemptRootIdentity);
|
|
499
1061
|
lease = transitionRuntimeIsolationLease(lease, "active");
|
|
500
|
-
await this.writeLedger(lease);
|
|
1062
|
+
await this.writeLedger(lease, undefined, attemptRootIdentity);
|
|
501
1063
|
processSupervisor = this.processSupervisors.register({
|
|
502
1064
|
processSupervisorId,
|
|
503
1065
|
dispatchAttemptId: lease.dispatchAttemptId,
|
|
@@ -510,13 +1072,16 @@ export class HostRuntimeLeaseManager {
|
|
|
510
1072
|
repoId: dispatch.repo.id,
|
|
511
1073
|
sessionId: dispatch.session.id,
|
|
512
1074
|
attemptRoot,
|
|
1075
|
+
attemptRootIdentity,
|
|
513
1076
|
runtimeContract,
|
|
514
1077
|
runtimeContractFingerprint,
|
|
1078
|
+
inputCommitSha,
|
|
515
1079
|
executionClass: admission.executionClass,
|
|
516
|
-
parallelPromotionEligible: hostRuntimeParallelEligible(runtimeContract),
|
|
1080
|
+
parallelPromotionEligible: hostRuntimeParallelEligible(runtimeContract) && dispatch.cycleRun.phase !== "prepared_readiness",
|
|
517
1081
|
processSupervisor,
|
|
518
1082
|
adapterCleanups: [],
|
|
519
1083
|
pendingAdapterLaunches: new Map(),
|
|
1084
|
+
runtimeResourcesReleased: false,
|
|
520
1085
|
releasing: false,
|
|
521
1086
|
});
|
|
522
1087
|
this.pendingRenewals.delete(sandbox.dispatchAttemptId);
|
|
@@ -533,8 +1098,13 @@ export class HostRuntimeLeaseManager {
|
|
|
533
1098
|
await processSupervisor.stopAll(dispatch.lease.fencingToken, "Host runtime start rolled back.").catch(() => undefined);
|
|
534
1099
|
this.processSupervisors.unregister(sandbox.dispatchAttemptId, dispatch.lease.fencingToken);
|
|
535
1100
|
}
|
|
536
|
-
await
|
|
537
|
-
|
|
1101
|
+
const attemptRemoved = await retainHostDiagnostics(this.stateRoot, attemptRoot, sandbox.dispatchAttemptId, attemptRootIdentity, this.executionRoot).then(async () => {
|
|
1102
|
+
await removeHostAttemptRoot(attemptRoot);
|
|
1103
|
+
return true;
|
|
1104
|
+
}).catch(() => false); // A retention failure preserves the original evidence and ownership ledger.
|
|
1105
|
+
if (attemptRemoved) {
|
|
1106
|
+
await fs.rm(this.ledgerPath(sandbox.dispatchAttemptId), { force: true }).catch(() => undefined);
|
|
1107
|
+
}
|
|
538
1108
|
this.releaseSingletonKeys(sandbox.dispatchAttemptId, dispatch.lease.fencingToken);
|
|
539
1109
|
this.releaseParallelProjects(sandbox.dispatchAttemptId, dispatch.lease.fencingToken);
|
|
540
1110
|
this.deleteAdmission(sandbox.dispatchAttemptId, dispatch.lease.fencingToken);
|
|
@@ -549,6 +1119,7 @@ export class HostRuntimeLeaseManager {
|
|
|
549
1119
|
catch {
|
|
550
1120
|
return;
|
|
551
1121
|
}
|
|
1122
|
+
await this.preparedReadiness.renew(dispatchAttemptId, fencingToken, expiresAt);
|
|
552
1123
|
const active = this.active.get(dispatchAttemptId);
|
|
553
1124
|
if (!active) {
|
|
554
1125
|
if (Number.isFinite(Date.parse(expiresAt))) {
|
|
@@ -562,13 +1133,15 @@ export class HostRuntimeLeaseManager {
|
|
|
562
1133
|
}
|
|
563
1134
|
return;
|
|
564
1135
|
}
|
|
1136
|
+
if (active.releasing || active.lease.state === "releasing" || active.lease.state === "released")
|
|
1137
|
+
return;
|
|
565
1138
|
const renewed = renewRuntimeIsolationLease(active.lease, fencingToken, expiresAt);
|
|
566
1139
|
if (!renewed)
|
|
567
1140
|
return;
|
|
568
1141
|
active.lease = renewed;
|
|
569
1142
|
await this.writeLedger(renewed);
|
|
570
1143
|
}
|
|
571
|
-
async startConfiguredProject(dispatchAttemptId, fencingToken, signal) {
|
|
1144
|
+
async startConfiguredProject(dispatchAttemptId, fencingToken, signal, onProjectCheckpoint) {
|
|
572
1145
|
const active = this.active.get(dispatchAttemptId);
|
|
573
1146
|
if (!active || active.lease.fencingToken !== fencingToken || active.lease.state !== "active" || active.releasing) {
|
|
574
1147
|
throw new Error("host_runtime_project_unavailable: an active matching host runtime lease is required.");
|
|
@@ -597,13 +1170,44 @@ export class HostRuntimeLeaseManager {
|
|
|
597
1170
|
const launcher = new AttemptNodeRuntimeLauncher(active.processSupervisor, owner, logDirectory, this.listenerInspector);
|
|
598
1171
|
for (const configuredPort of configuredPorts) {
|
|
599
1172
|
const workingDirectory = await containedWorkingDirectory(active.lease.worktreePath, configuredPort.launch.workingDirectory);
|
|
1173
|
+
const buildEnvironment = {
|
|
1174
|
+
...configuredPort.launch.environment,
|
|
1175
|
+
...hostRuntimeEnvironment(active.sessionId, active.lease, active.runtimeContract, active.executionClass),
|
|
1176
|
+
};
|
|
1177
|
+
const prebuild = createAspNetRuntimePrebuildPlan({
|
|
1178
|
+
command: configuredPort.launch.command,
|
|
1179
|
+
args: configuredPort.launch.args,
|
|
1180
|
+
cwd: workingDirectory,
|
|
1181
|
+
worktreePath: active.lease.worktreePath,
|
|
1182
|
+
environment: buildEnvironment,
|
|
1183
|
+
inputCommitSha: active.inputCommitSha,
|
|
1184
|
+
runtimeContractFingerprint: active.runtimeContractFingerprint,
|
|
1185
|
+
});
|
|
1186
|
+
if (prebuild) {
|
|
1187
|
+
await onProjectCheckpoint?.({
|
|
1188
|
+
code: "host_runtime_build_started",
|
|
1189
|
+
message: `Preparing the ASP.NET runtime for commit ${active.inputCommitSha.slice(0, 12)} before binding its assigned port.`,
|
|
1190
|
+
data: { input_commit_sha: active.inputCommitSha, project_path: prebuild.projectPath },
|
|
1191
|
+
});
|
|
1192
|
+
const build = await ensureAspNetRuntimePrebuilt(prebuild, async (plan, buildSignal) => await executeAspNetRuntimeBuild(active.processSupervisor, logDirectory, plan, buildSignal), signal);
|
|
1193
|
+
await onProjectCheckpoint?.({
|
|
1194
|
+
code: build.cacheHit ? "host_runtime_build_cached" : "host_runtime_build_completed",
|
|
1195
|
+
message: build.cacheHit
|
|
1196
|
+
? `Reused the verified ASP.NET build for commit ${active.inputCommitSha.slice(0, 12)}.`
|
|
1197
|
+
: `ASP.NET runtime build completed for commit ${active.inputCommitSha.slice(0, 12)}; launching without another build.`,
|
|
1198
|
+
data: {
|
|
1199
|
+
input_commit_sha: active.inputCommitSha,
|
|
1200
|
+
build_fingerprint: build.fingerprint,
|
|
1201
|
+
receipt_path: build.receiptPath,
|
|
1202
|
+
},
|
|
1203
|
+
});
|
|
1204
|
+
}
|
|
600
1205
|
await this.launchNodeRuntime(dispatchAttemptId, fencingToken, configuredPort.name, {
|
|
601
1206
|
command: configuredPort.launch.command,
|
|
602
1207
|
args: configuredPort.launch.args,
|
|
603
1208
|
cwd: workingDirectory,
|
|
604
1209
|
environment: {
|
|
605
|
-
...
|
|
606
|
-
...hostRuntimeEnvironment(active.sessionId, active.lease, active.runtimeContract, active.executionClass),
|
|
1210
|
+
...buildEnvironment,
|
|
607
1211
|
},
|
|
608
1212
|
}, launcher, signal);
|
|
609
1213
|
}
|
|
@@ -637,6 +1241,7 @@ export class HostRuntimeLeaseManager {
|
|
|
637
1241
|
testOutputDirectory: pathFor("test_output"),
|
|
638
1242
|
browserProfileDirectory: pathFor("browser_profile"),
|
|
639
1243
|
baseUrl,
|
|
1244
|
+
ownershipToken: uuidHex(promotedLease.dispatchAttemptId, "dispatch attempt"),
|
|
640
1245
|
});
|
|
641
1246
|
return {
|
|
642
1247
|
lease: promotedLease,
|
|
@@ -649,7 +1254,18 @@ export class HostRuntimeLeaseManager {
|
|
|
649
1254
|
const active = this.active.get(dispatchAttemptId);
|
|
650
1255
|
if (!active || active.lease.fencingToken !== fencingToken || active.releasing)
|
|
651
1256
|
return null;
|
|
652
|
-
return new AttemptSupervisedProviderProcessRunner(active.processSupervisor, hostPortOwner(active.lease), this.listenerInspector, active.executionClass === "analysis_only");
|
|
1257
|
+
return new AttemptSupervisedProviderProcessRunner(active.processSupervisor, hostPortOwner(active.lease), this.listenerInspector, active.executionClass === "analysis_only", () => this.markCapacityReservationRunning(dispatchAttemptId, fencingToken));
|
|
1258
|
+
}
|
|
1259
|
+
markCapacityReservationRunning(dispatchAttemptId, fencingToken) {
|
|
1260
|
+
const reservation = this.capacityReservations.get(dispatchAttemptId);
|
|
1261
|
+
if (reservation?.owner.fencingToken === fencingToken)
|
|
1262
|
+
reservation.state = "running";
|
|
1263
|
+
}
|
|
1264
|
+
providerMemoryTelemetry(dispatchAttemptId, fencingToken) {
|
|
1265
|
+
const active = this.active.get(dispatchAttemptId);
|
|
1266
|
+
if (!active || active.lease.fencingToken !== fencingToken || active.releasing)
|
|
1267
|
+
return null;
|
|
1268
|
+
return active.processSupervisor.providerMemoryTelemetry();
|
|
653
1269
|
}
|
|
654
1270
|
async promoteVerifiedRuntime(active, runtimeLease) {
|
|
655
1271
|
if (!active.parallelPromotionEligible)
|
|
@@ -668,12 +1284,15 @@ export class HostRuntimeLeaseManager {
|
|
|
668
1284
|
await this.writeLedger(promotedLease);
|
|
669
1285
|
active.lease = promotedLease;
|
|
670
1286
|
try {
|
|
671
|
-
const promotion = await this.promotionStore.observe(active.repoId, active.runtimeContract, runtimeLease.dispatchAttemptId, verifiedPortNames);
|
|
1287
|
+
const promotion = await this.promotionStore.observe(active.repoId, active.runtimeContract, runtimeLease.dispatchAttemptId, verifiedPortNames, active.inputCommitSha);
|
|
672
1288
|
this.knownPromotions.set(promotion.repoId, promotion);
|
|
673
|
-
|
|
1289
|
+
const owner = {
|
|
674
1290
|
dispatchAttemptId: runtimeLease.dispatchAttemptId,
|
|
675
1291
|
fencingToken: runtimeLease.fencingToken,
|
|
676
|
-
}
|
|
1292
|
+
};
|
|
1293
|
+
if (!this.parallelProjectOwned(projectKey, owner)) {
|
|
1294
|
+
this.convertSingletonToParallelProject(projectKey, owner, promotion);
|
|
1295
|
+
}
|
|
677
1296
|
const admission = this.admissions.get(runtimeLease.dispatchAttemptId);
|
|
678
1297
|
if (admission?.fencingToken === runtimeLease.fencingToken) {
|
|
679
1298
|
this.admissions.set(runtimeLease.dispatchAttemptId, {
|
|
@@ -757,9 +1376,12 @@ export class HostRuntimeLeaseManager {
|
|
|
757
1376
|
unlinkAbortController(controller, signal);
|
|
758
1377
|
}
|
|
759
1378
|
}
|
|
760
|
-
async release(dispatchAttemptId, fencingToken) {
|
|
1379
|
+
async release(dispatchAttemptId, fencingToken, preservePreparation = false) {
|
|
1380
|
+
if (!preservePreparation)
|
|
1381
|
+
this.preparedReadiness.cancel(dispatchAttemptId, fencingToken);
|
|
761
1382
|
const active = this.active.get(dispatchAttemptId);
|
|
762
|
-
|
|
1383
|
+
if (!preservePreparation)
|
|
1384
|
+
this.pendingRenewals.delete(dispatchAttemptId);
|
|
763
1385
|
if (!active) {
|
|
764
1386
|
this.cancelAdmissionWaiters(dispatchAttemptId, fencingToken, "Managed host admission was released before acquisition.");
|
|
765
1387
|
this.releaseSingletonKeys(dispatchAttemptId, fencingToken);
|
|
@@ -778,13 +1400,48 @@ export class HostRuntimeLeaseManager {
|
|
|
778
1400
|
active.lease = releasing;
|
|
779
1401
|
await this.writeLedger(releasing);
|
|
780
1402
|
try {
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
1403
|
+
if (!active.runtimeResourcesReleased) {
|
|
1404
|
+
await active.processSupervisor.stopAll(fencingToken, "Host runtime lease released.");
|
|
1405
|
+
await releaseAdapterResources(active.adapterCleanups);
|
|
1406
|
+
active.runtimeResourcesReleased = true;
|
|
1407
|
+
// Runtime admission protects live processes, listeners, ports, and their
|
|
1408
|
+
// memory commitment. Once exact process and adapter cleanup succeeds,
|
|
1409
|
+
// diagnostics retention remains retryable evidence work and must not keep
|
|
1410
|
+
// consuming host capacity or blocking the next project dispatch.
|
|
1411
|
+
this.processSupervisors.unregister(dispatchAttemptId, fencingToken);
|
|
1412
|
+
this.releaseSingletonKeys(dispatchAttemptId, fencingToken);
|
|
1413
|
+
this.releaseParallelProjects(dispatchAttemptId, fencingToken);
|
|
1414
|
+
this.deleteAdmission(dispatchAttemptId, fencingToken);
|
|
1415
|
+
}
|
|
1416
|
+
let evidenceRetainedAtAttemptRoot = false;
|
|
1417
|
+
try {
|
|
1418
|
+
const diagnosticsPath = await retainHostDiagnostics(this.stateRoot, active.attemptRoot, dispatchAttemptId, active.attemptRootIdentity, this.executionRoot);
|
|
1419
|
+
releasing = withRuntimeIsolationResources(releasing, {
|
|
1420
|
+
...releasing.resources,
|
|
1421
|
+
paths: [...releasing.resources.paths.filter(entry => entry.role !== "diagnostics_archive"),
|
|
1422
|
+
{ role: "diagnostics_archive", path: diagnosticsPath }],
|
|
1423
|
+
});
|
|
1424
|
+
}
|
|
1425
|
+
catch {
|
|
1426
|
+
// Runtime teardown is already proven above. Diagnostics archival is
|
|
1427
|
+
// deliberately non-destructive and may refuse a changed or unsupported
|
|
1428
|
+
// attempt-root inventory. Preserve that root and its released ledger for
|
|
1429
|
+
// operator inspection/reconciliation without converting completed
|
|
1430
|
+
// teardown into a capacity-owning degraded runtime.
|
|
1431
|
+
evidenceRetainedAtAttemptRoot = true;
|
|
1432
|
+
}
|
|
1433
|
+
if (!evidenceRetainedAtAttemptRoot)
|
|
1434
|
+
await removeHostAttemptRoot(active.attemptRoot);
|
|
1435
|
+
const released = transitionRuntimeIsolationLease(releasing, "released");
|
|
1436
|
+
// Publish the terminal in-memory state before its serialized write. Any writer
|
|
1437
|
+
// that already captured this active owner must observe the terminal fence and
|
|
1438
|
+
// cannot recreate a nonterminal ledger after release completes.
|
|
1439
|
+
active.lease = released;
|
|
1440
|
+
await this.writeLedger(released);
|
|
1441
|
+
if (!evidenceRetainedAtAttemptRoot) {
|
|
1442
|
+
await fs.rm(this.ledgerPath(dispatchAttemptId), { force: true });
|
|
1443
|
+
}
|
|
1444
|
+
releasing = released;
|
|
788
1445
|
this.active.delete(dispatchAttemptId);
|
|
789
1446
|
this.processSupervisors.unregister(dispatchAttemptId, fencingToken);
|
|
790
1447
|
this.releaseSingletonKeys(dispatchAttemptId, fencingToken);
|
|
@@ -825,23 +1482,27 @@ export class HostRuntimeLeaseManager {
|
|
|
825
1482
|
}
|
|
826
1483
|
return ledgers;
|
|
827
1484
|
}
|
|
828
|
-
async recoverPersistedProcesses(lease, attemptRoot) {
|
|
1485
|
+
async recoverPersistedProcesses(lease, attemptRoot, attemptRootIdentity) {
|
|
829
1486
|
const logsRoot = path.join(attemptRoot, "logs");
|
|
830
1487
|
const expectedAttemptRoot = path.resolve(this.attemptRoot(lease.dispatchAttemptId));
|
|
831
1488
|
if (path.resolve(attemptRoot) !== expectedAttemptRoot) {
|
|
832
1489
|
throw new Error("host_runtime_reaper_invalid: attempt path is outside the fenced attempt root.");
|
|
833
1490
|
}
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
1491
|
+
await assertHostAttemptRootIdentity(attemptRoot, attemptRootIdentity);
|
|
1492
|
+
const rootEntries = await readPinnedSourceDirectory(attemptRoot, attemptRootIdentity);
|
|
1493
|
+
const logsEntry = rootEntries.find(entry => entry.name === "logs");
|
|
1494
|
+
if (!logsEntry)
|
|
1495
|
+
return [];
|
|
1496
|
+
if (logsEntry.kind !== "directory") {
|
|
1497
|
+
throw new Error("host_runtime_reaper_invalid: process ownership log path is not an ordinary directory.");
|
|
1498
|
+
}
|
|
1499
|
+
const entries = await readPinnedSourceDirectory(logsRoot, logsEntry.identity);
|
|
839
1500
|
const outcomes = [];
|
|
840
1501
|
for (const entry of entries) {
|
|
841
|
-
if (
|
|
1502
|
+
if (entry.kind !== "file" || !/^process-supervisor\.[a-z][a-z0-9_-]{0,31}\.\d+\.json$/u.test(entry.name))
|
|
842
1503
|
continue;
|
|
843
1504
|
const metadataPath = path.join(logsRoot, entry.name);
|
|
844
|
-
const value = JSON.parse(await
|
|
1505
|
+
const value = JSON.parse((await readCanonicalSourceFile(metadataPath, maximumProcessRecoveryMetadataBytes, entry.identity)).toString("utf8"));
|
|
845
1506
|
if (!validHostProcessSupervisorMetadata(value, lease)) {
|
|
846
1507
|
throw new Error("host_runtime_reaper_invalid: process ownership metadata did not match the runtime lease.");
|
|
847
1508
|
}
|
|
@@ -856,7 +1517,7 @@ export class HostRuntimeLeaseManager {
|
|
|
856
1517
|
const degraded = transitionHostLeaseForCleanup(lease, "degraded");
|
|
857
1518
|
const repoId = ledger.repoId ?? repoIdFromProjectSingletonKeys(lease.resources.singletonKeys);
|
|
858
1519
|
this.rememberDegraded(degraded, repoId);
|
|
859
|
-
await this.writeLedger(degraded, repoId ?? undefined).catch(() => undefined);
|
|
1520
|
+
await this.writeLedger(degraded, repoId ?? undefined, deserializeHostAttemptRootIdentity(ledger.attemptRootIdentity)).catch(() => undefined);
|
|
860
1521
|
}
|
|
861
1522
|
rememberDegraded(lease, repoId) {
|
|
862
1523
|
this.degraded.set(lease.dispatchAttemptId, { lease, repoId });
|
|
@@ -876,13 +1537,26 @@ export class HostRuntimeLeaseManager {
|
|
|
876
1537
|
this.releaseParallelProjects(dispatchAttemptId, fencingToken);
|
|
877
1538
|
}
|
|
878
1539
|
attemptRoot(dispatchAttemptId) {
|
|
879
|
-
return path.join(this.
|
|
1540
|
+
return path.join(this.executionRoot, "a", compactHostAttemptId(dispatchAttemptId));
|
|
1541
|
+
}
|
|
1542
|
+
writeLedger(runtimeLease, explicitRepoId, explicitAttemptRootIdentity) {
|
|
1543
|
+
const activeOwner = this.active.get(runtimeLease.dispatchAttemptId);
|
|
1544
|
+
const write = this.ledgerWriteTail.then(() => {
|
|
1545
|
+
if (activeOwner?.lease.fencingToken === runtimeLease.fencingToken
|
|
1546
|
+
&& activeOwner.lease.state === "released"
|
|
1547
|
+
&& runtimeLease.state !== "released") {
|
|
1548
|
+
return;
|
|
1549
|
+
}
|
|
1550
|
+
return this.writeLedgerAtomic(runtimeLease, explicitRepoId, explicitAttemptRootIdentity);
|
|
1551
|
+
});
|
|
1552
|
+
this.ledgerWriteTail = write.catch(() => undefined);
|
|
1553
|
+
return write;
|
|
880
1554
|
}
|
|
881
|
-
async
|
|
1555
|
+
async writeLedgerAtomic(runtimeLease, explicitRepoId, explicitAttemptRootIdentity) {
|
|
882
1556
|
const root = path.join(this.stateRoot, "runtime-ledger", "host");
|
|
883
1557
|
await ensurePrivateDirectory(root);
|
|
884
1558
|
const target = this.ledgerPath(runtimeLease.dispatchAttemptId);
|
|
885
|
-
const temporary = `${target}.${process.pid}.${Date.now()}.tmp`;
|
|
1559
|
+
const temporary = `${target}.${process.pid}.${Date.now()}.${randomUUID()}.tmp`;
|
|
886
1560
|
try {
|
|
887
1561
|
const handle = await fs.open(temporary, "wx", 0o600);
|
|
888
1562
|
try {
|
|
@@ -891,9 +1565,15 @@ export class HostRuntimeLeaseManager {
|
|
|
891
1565
|
?? this.admissions.get(runtimeLease.dispatchAttemptId)?.repoId
|
|
892
1566
|
?? this.degraded.get(runtimeLease.dispatchAttemptId)?.repoId
|
|
893
1567
|
?? undefined;
|
|
1568
|
+
const attemptRootIdentity = explicitAttemptRootIdentity
|
|
1569
|
+
?? this.active.get(runtimeLease.dispatchAttemptId)?.attemptRootIdentity
|
|
1570
|
+
?? undefined;
|
|
894
1571
|
const ledger = {
|
|
895
1572
|
schemaVersion: 1,
|
|
896
1573
|
...(repoId ? { repoId } : {}),
|
|
1574
|
+
...(attemptRootIdentity
|
|
1575
|
+
? { attemptRootIdentity: serializeHostAttemptRootIdentity(attemptRootIdentity) }
|
|
1576
|
+
: {}),
|
|
897
1577
|
runtimeLease,
|
|
898
1578
|
};
|
|
899
1579
|
await handle.writeFile(`${JSON.stringify(ledger)}\n`);
|
|
@@ -906,7 +1586,7 @@ export class HostRuntimeLeaseManager {
|
|
|
906
1586
|
}
|
|
907
1587
|
catch (error) {
|
|
908
1588
|
await fs.rm(temporary, { force: true }).catch(() => undefined);
|
|
909
|
-
throw error;
|
|
1589
|
+
throw new Error(`host_runtime_ledger_write_failed: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
910
1590
|
}
|
|
911
1591
|
}
|
|
912
1592
|
ledgerPath(dispatchAttemptId) {
|
|
@@ -936,7 +1616,7 @@ export class HostRuntimeLeaseManager {
|
|
|
936
1616
|
this.singletonOwners.delete(key);
|
|
937
1617
|
for (let index = 0; index < this.singletonWaiters.length;) {
|
|
938
1618
|
const waiter = this.singletonWaiters[index];
|
|
939
|
-
if (waiter.keys.length !== 1 || waiter.keys[0] !== key) {
|
|
1619
|
+
if (waiter.priority !== "foreground" || waiter.keys.length !== 1 || waiter.keys[0] !== key) {
|
|
940
1620
|
index += 1;
|
|
941
1621
|
continue;
|
|
942
1622
|
}
|
|
@@ -952,9 +1632,10 @@ export class HostRuntimeLeaseManager {
|
|
|
952
1632
|
parallelProjectOwned(key, owner) {
|
|
953
1633
|
return sameSingletonOwner(this.parallelProjectOwners.get(key)?.get(owner.dispatchAttemptId), owner);
|
|
954
1634
|
}
|
|
955
|
-
parallelProjectAvailable(key) {
|
|
1635
|
+
parallelProjectAvailable(key, priority = "foreground") {
|
|
956
1636
|
return !this.singletonOwners.has(key)
|
|
957
|
-
&& !this.singletonWaiters.some(waiter => waiter.keys.includes(key)
|
|
1637
|
+
&& !this.singletonWaiters.some(waiter => waiter.keys.includes(key)
|
|
1638
|
+
&& (priority === "background" || waiter.priority === "foreground"));
|
|
958
1639
|
}
|
|
959
1640
|
assignParallelProject(key, owner) {
|
|
960
1641
|
const owners = this.parallelProjectOwners.get(key) ?? new Map();
|
|
@@ -980,9 +1661,15 @@ export class HostRuntimeLeaseManager {
|
|
|
980
1661
|
if (this.admissions.get(dispatchAttemptId)?.fencingToken === fencingToken) {
|
|
981
1662
|
this.admissions.delete(dispatchAttemptId);
|
|
982
1663
|
}
|
|
1664
|
+
this.releaseHostCapacity(dispatchAttemptId, fencingToken);
|
|
983
1665
|
}
|
|
984
1666
|
renewAdmissionWaiterDeadline(dispatchAttemptId, fencingToken, expiresAt) {
|
|
985
|
-
for (const waiter of [
|
|
1667
|
+
for (const waiter of [
|
|
1668
|
+
...this.singletonWaiters,
|
|
1669
|
+
...this.parallelProjectWaiters,
|
|
1670
|
+
...this.capacityWaiters,
|
|
1671
|
+
...this.backgroundAdmissionWaiters,
|
|
1672
|
+
]) {
|
|
986
1673
|
if (!sameSingletonOwner(waiter.owner, { dispatchAttemptId, fencingToken })
|
|
987
1674
|
|| Date.parse(expiresAt) <= Date.parse(waiter.expiresAt))
|
|
988
1675
|
continue;
|
|
@@ -998,13 +1685,20 @@ export class HostRuntimeLeaseManager {
|
|
|
998
1685
|
}
|
|
999
1686
|
cancelAdmissionWaiters(dispatchAttemptId, fencingToken, reason) {
|
|
1000
1687
|
const owner = { dispatchAttemptId, fencingToken };
|
|
1001
|
-
for (const waiters of [
|
|
1688
|
+
for (const waiters of [
|
|
1689
|
+
this.singletonWaiters,
|
|
1690
|
+
this.parallelProjectWaiters,
|
|
1691
|
+
this.capacityWaiters,
|
|
1692
|
+
this.backgroundAdmissionWaiters,
|
|
1693
|
+
]) {
|
|
1002
1694
|
for (let index = waiters.length - 1; index >= 0; index -= 1) {
|
|
1003
1695
|
const waiter = waiters[index];
|
|
1004
1696
|
if (!sameSingletonOwner(waiter.owner, owner))
|
|
1005
1697
|
continue;
|
|
1006
1698
|
waiters.splice(index, 1);
|
|
1007
1699
|
clearAdmissionDeadline(waiter);
|
|
1700
|
+
if ("capacityRecheckTimer" in waiter)
|
|
1701
|
+
clearCapacityRecheck(waiter);
|
|
1008
1702
|
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
1009
1703
|
waiter.reject(new Error(`host_runtime_admission_cancelled: ${reason}`));
|
|
1010
1704
|
}
|
|
@@ -1060,7 +1754,7 @@ export class HostRuntimeLeaseManager {
|
|
|
1060
1754
|
waiter.reject(new HostRuntimeAdmissionExpiredError());
|
|
1061
1755
|
continue;
|
|
1062
1756
|
}
|
|
1063
|
-
if (!this.parallelProjectAvailable(waiter.key)) {
|
|
1757
|
+
if (!this.parallelProjectAvailable(waiter.key, waiter.priority)) {
|
|
1064
1758
|
index += 1;
|
|
1065
1759
|
continue;
|
|
1066
1760
|
}
|
|
@@ -1071,19 +1765,103 @@ export class HostRuntimeLeaseManager {
|
|
|
1071
1765
|
waiter.resolve();
|
|
1072
1766
|
}
|
|
1073
1767
|
}
|
|
1768
|
+
drainBackgroundAdmissionWaiters() {
|
|
1769
|
+
if (this.hasForegroundDemand())
|
|
1770
|
+
return;
|
|
1771
|
+
for (let index = 0; index < this.backgroundAdmissionWaiters.length;) {
|
|
1772
|
+
const waiter = this.backgroundAdmissionWaiters[index];
|
|
1773
|
+
if (waiter.signal?.aborted) {
|
|
1774
|
+
this.backgroundAdmissionWaiters.splice(index, 1);
|
|
1775
|
+
clearAdmissionDeadline(waiter);
|
|
1776
|
+
clearCapacityRecheck(waiter);
|
|
1777
|
+
waiter.signal.removeEventListener("abort", waiter.abort);
|
|
1778
|
+
this.clearPendingRenewal(waiter.owner);
|
|
1779
|
+
waiter.reject(cancellationError(waiter.signal));
|
|
1780
|
+
continue;
|
|
1781
|
+
}
|
|
1782
|
+
if (admissionExpired(waiter.expiresAt)) {
|
|
1783
|
+
this.backgroundAdmissionWaiters.splice(index, 1);
|
|
1784
|
+
clearAdmissionDeadline(waiter);
|
|
1785
|
+
clearCapacityRecheck(waiter);
|
|
1786
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
1787
|
+
this.clearPendingRenewal(waiter.owner);
|
|
1788
|
+
waiter.reject(new HostRuntimeAdmissionExpiredError());
|
|
1789
|
+
continue;
|
|
1790
|
+
}
|
|
1791
|
+
if (!this.singletonKeysAvailable(waiter.keys) || !this.hostCapacityAvailable(waiter.resources)) {
|
|
1792
|
+
index += 1;
|
|
1793
|
+
continue;
|
|
1794
|
+
}
|
|
1795
|
+
this.backgroundAdmissionWaiters.splice(index, 1);
|
|
1796
|
+
clearAdmissionDeadline(waiter);
|
|
1797
|
+
clearCapacityRecheck(waiter);
|
|
1798
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
1799
|
+
this.assignSingletonKeys(waiter.keys, waiter.owner);
|
|
1800
|
+
this.capacityReservations.set(waiter.owner.dispatchAttemptId, {
|
|
1801
|
+
owner: waiter.owner,
|
|
1802
|
+
resources: waiter.resources,
|
|
1803
|
+
state: "pending",
|
|
1804
|
+
});
|
|
1805
|
+
waiter.resolve(waiter.keys);
|
|
1806
|
+
}
|
|
1807
|
+
}
|
|
1074
1808
|
async refreshKnownPromotions() {
|
|
1075
1809
|
const promotions = await this.promotionStore.list();
|
|
1076
1810
|
this.knownPromotions.clear();
|
|
1077
1811
|
for (const promotion of promotions)
|
|
1078
1812
|
this.knownPromotions.set(promotion.repoId, promotion);
|
|
1079
1813
|
}
|
|
1080
|
-
async invalidatePromotion(repoId) {
|
|
1081
|
-
await this.promotionStore.invalidate(repoId);
|
|
1814
|
+
async invalidatePromotion(repoId, scope = "all") {
|
|
1815
|
+
await this.promotionStore.invalidate(repoId, scope);
|
|
1082
1816
|
this.knownPromotions.delete(repoId.trim().toLowerCase());
|
|
1083
1817
|
}
|
|
1084
1818
|
drainAdmissionWaiters() {
|
|
1819
|
+
this.reconcileReservationOwners();
|
|
1085
1820
|
this.drainSingletonWaiters();
|
|
1086
1821
|
this.drainParallelProjectWaiters();
|
|
1822
|
+
this.drainCapacityWaiters();
|
|
1823
|
+
this.drainBackgroundAdmissionWaiters();
|
|
1824
|
+
}
|
|
1825
|
+
reconcileReservationOwners() {
|
|
1826
|
+
for (const [key, owner] of this.singletonOwners) {
|
|
1827
|
+
if (!this.reservationOwnerIsLive(owner))
|
|
1828
|
+
this.singletonOwners.delete(key);
|
|
1829
|
+
}
|
|
1830
|
+
for (const [key, owners] of this.parallelProjectOwners) {
|
|
1831
|
+
for (const [dispatchAttemptId, owner] of owners) {
|
|
1832
|
+
const awaitingExclusiveUpgrade = this.singletonWaiters.some(waiter => waiter.keys.includes(key) && sameSingletonOwner(waiter.owner, owner));
|
|
1833
|
+
// A parallel owner cannot legitimately keep the shared claim after the
|
|
1834
|
+
// same fenced attempt has queued its exclusive downgrade. Reconcile the
|
|
1835
|
+
// half-upgraded state before testing singleton availability so one missed
|
|
1836
|
+
// release cannot make an attempt wait behind itself until lease expiry.
|
|
1837
|
+
if (awaitingExclusiveUpgrade || !this.reservationOwnerIsLive(owner)) {
|
|
1838
|
+
owners.delete(dispatchAttemptId);
|
|
1839
|
+
}
|
|
1840
|
+
}
|
|
1841
|
+
if (owners.size === 0)
|
|
1842
|
+
this.parallelProjectOwners.delete(key);
|
|
1843
|
+
}
|
|
1844
|
+
}
|
|
1845
|
+
reservationOwnerIsLive(owner) {
|
|
1846
|
+
if (this.acquiringAdmissions.has(hostRuntimeOwnerKey(owner)))
|
|
1847
|
+
return true;
|
|
1848
|
+
if (this.admissions.get(owner.dispatchAttemptId)?.fencingToken === owner.fencingToken)
|
|
1849
|
+
return true;
|
|
1850
|
+
if (this.active.get(owner.dispatchAttemptId)?.lease.fencingToken === owner.fencingToken)
|
|
1851
|
+
return true;
|
|
1852
|
+
if (this.degraded.get(owner.dispatchAttemptId)?.lease.fencingToken === owner.fencingToken)
|
|
1853
|
+
return true;
|
|
1854
|
+
if (sameSingletonOwner(this.capacityReservations.get(owner.dispatchAttemptId)?.owner, owner))
|
|
1855
|
+
return true;
|
|
1856
|
+
return [
|
|
1857
|
+
...this.singletonWaiters,
|
|
1858
|
+
...this.parallelProjectWaiters,
|
|
1859
|
+
...this.capacityWaiters,
|
|
1860
|
+
...this.backgroundAdmissionWaiters,
|
|
1861
|
+
].some(waiter => sameSingletonOwner(waiter.owner, owner));
|
|
1862
|
+
}
|
|
1863
|
+
hasAttemptRepresentation(dispatchAttemptId, fencingToken) {
|
|
1864
|
+
return this.reservationOwnerIsLive({ dispatchAttemptId, fencingToken });
|
|
1087
1865
|
}
|
|
1088
1866
|
}
|
|
1089
1867
|
class AttemptSupervisedProviderProcessRunner {
|
|
@@ -1091,14 +1869,20 @@ class AttemptSupervisedProviderProcessRunner {
|
|
|
1091
1869
|
owner;
|
|
1092
1870
|
listenerInspector;
|
|
1093
1871
|
enforceNoListeners;
|
|
1094
|
-
|
|
1872
|
+
onProcessTreeStarted;
|
|
1873
|
+
constructor(supervisor, owner, listenerInspector, enforceNoListeners, onProcessTreeStarted) {
|
|
1095
1874
|
this.supervisor = supervisor;
|
|
1096
1875
|
this.owner = owner;
|
|
1097
1876
|
this.listenerInspector = listenerInspector;
|
|
1098
1877
|
this.enforceNoListeners = enforceNoListeners;
|
|
1878
|
+
this.onProcessTreeStarted = onProcessTreeStarted;
|
|
1099
1879
|
}
|
|
1100
1880
|
async run(request, handlers = {}, signal) {
|
|
1881
|
+
const baselineListeners = this.enforceNoListeners
|
|
1882
|
+
? new Set(await this.listenerInspector.inspectLoopbackListeners())
|
|
1883
|
+
: new Set();
|
|
1101
1884
|
const child = await this.supervisor.startChildReady("provider", request, handlers, signal, { maxStartAttempts: request.stdin === undefined ? 1 : 3, retryDelayMs: 100 });
|
|
1885
|
+
this.onProcessTreeStarted();
|
|
1102
1886
|
if (!this.enforceNoListeners)
|
|
1103
1887
|
return await child.result;
|
|
1104
1888
|
let settled = false;
|
|
@@ -1106,7 +1890,8 @@ class AttemptSupervisedProviderProcessRunner {
|
|
|
1106
1890
|
const monitor = (async () => {
|
|
1107
1891
|
while (!settled) {
|
|
1108
1892
|
try {
|
|
1109
|
-
const ports = await this.listenerInspector.inspectOwnedLoopbackListeners(child, this.owner)
|
|
1893
|
+
const ports = (await this.listenerInspector.inspectOwnedLoopbackListeners(child, this.owner))
|
|
1894
|
+
.filter(port => !baselineListeners.has(port));
|
|
1110
1895
|
if (ports.length > 0) {
|
|
1111
1896
|
inspectionFailure = new Error(`host_analysis_listener_violation: analysis-only provider opened undeclared loopback listener(s): ${ports.join(", ")}.`);
|
|
1112
1897
|
await child.stop(this.owner.fencingToken).catch(() => undefined);
|
|
@@ -1170,6 +1955,12 @@ export class HostRuntimeAdmissionExpiredError extends Error {
|
|
|
1170
1955
|
this.name = "HostRuntimeAdmissionExpiredError";
|
|
1171
1956
|
}
|
|
1172
1957
|
}
|
|
1958
|
+
export class HostRuntimeCapacityRequestImpossibleError extends Error {
|
|
1959
|
+
constructor(reason) {
|
|
1960
|
+
super(`host_runtime_capacity_request_impossible: ${reason}`);
|
|
1961
|
+
this.name = "HostRuntimeCapacityRequestImpossibleError";
|
|
1962
|
+
}
|
|
1963
|
+
}
|
|
1173
1964
|
export function hostRuntimeEventResources(lease, runtimeContract, executionClass = "managed_runtime") {
|
|
1174
1965
|
const requestedHostConcurrency = runtimeContract?.contract.runtime.hostConcurrency ?? "serial";
|
|
1175
1966
|
const configuredLaunches = runtimeContract?.contract.runtime.ports.filter(port => port.launch !== null) ?? [];
|
|
@@ -1209,6 +2000,8 @@ function hostRuntimeEnvironment(sessionId, lease, runtimeContract, executionClas
|
|
|
1209
2000
|
FORGE_RUNTIME_MODE: "host",
|
|
1210
2001
|
FORGE_RESOURCE_LEASE_ID: lease.leaseId,
|
|
1211
2002
|
FORGE_DISPATCH_ATTEMPT_ID: lease.dispatchAttemptId,
|
|
2003
|
+
FORGE_E2E_OWNERSHIP_MODE: "attempt",
|
|
2004
|
+
FORGE_E2E_OWNERSHIP_TOKEN: uuidHex(lease.dispatchAttemptId, "dispatch attempt"),
|
|
1212
2005
|
FORGE_EXECUTION_LEASE_ID: lease.executionLeaseId,
|
|
1213
2006
|
FORGE_FENCING_TOKEN: lease.fencingToken,
|
|
1214
2007
|
FORGE_WORKSPACE: lease.worktreePath,
|
|
@@ -1218,6 +2011,12 @@ function hostRuntimeEnvironment(sessionId, lease, runtimeContract, executionClas
|
|
|
1218
2011
|
FORGE_CACHE_DIR: pathFor("cache"),
|
|
1219
2012
|
FORGE_BROWSER_PROFILE_DIR: pathFor("browser_profile"),
|
|
1220
2013
|
FORGE_TEST_OUTPUT_DIR: pathFor("test_output"),
|
|
2014
|
+
// Project-authored ASP.NET code must not fall back to the durable Forge
|
|
2015
|
+
// database when no brokered attempt database exists. The API policy treats
|
|
2016
|
+
// this marker as a fail-closed isolation requirement.
|
|
2017
|
+
FORGE_ATTEMPT_DATABASE_POLICY: "isolated_only",
|
|
2018
|
+
FORGE_ATTEMPT_DATABASE_SCHEMA: `forge_attempt_${uuidHex(lease.dispatchAttemptId, "dispatch attempt")}`,
|
|
2019
|
+
FORGE_ATTEMPT_DATABASE_READINESS_PATH: path.join(pathFor("logs"), "attempt-database-readiness.json"),
|
|
1221
2020
|
FORGE_PROCESS_SUPERVISOR_ID: lease.resources.processSupervisorId ?? "",
|
|
1222
2021
|
FORGE_PROCESS_SUPERVISOR_METADATA_PATH: pathFor("process_supervisor_metadata"),
|
|
1223
2022
|
FORGE_HOST_CONCURRENCY: effectiveHostConcurrency(lease, runtimeContract, executionClass),
|
|
@@ -1257,6 +2056,45 @@ function hostRuntimeEnvironment(sessionId, lease, runtimeContract, executionClas
|
|
|
1257
2056
|
environment.FORGE_BASE_URL = resolvedBaseUrls[0];
|
|
1258
2057
|
return environment;
|
|
1259
2058
|
}
|
|
2059
|
+
async function executeAspNetRuntimeBuild(supervisor, logDirectory, plan, signal) {
|
|
2060
|
+
const childName = `runtime-build-${plan.fingerprint.slice(-12)}`;
|
|
2061
|
+
const stdoutLogPath = path.join(logDirectory, `${childName}.stdout.log`);
|
|
2062
|
+
const stderrLogPath = path.join(logDirectory, `${childName}.stderr.log`);
|
|
2063
|
+
let stdoutTail = "";
|
|
2064
|
+
let stderrTail = "";
|
|
2065
|
+
const append = async (target, line) => {
|
|
2066
|
+
await fs.appendFile(target, `${line}\n`, { encoding: "utf8", mode: 0o600 });
|
|
2067
|
+
};
|
|
2068
|
+
const child = await supervisor.startChildReady(childName, {
|
|
2069
|
+
command: plan.command,
|
|
2070
|
+
args: plan.args,
|
|
2071
|
+
cwd: plan.cwd,
|
|
2072
|
+
env: plan.environment,
|
|
2073
|
+
captureOutput: false,
|
|
2074
|
+
}, {
|
|
2075
|
+
onStdoutLine: async (line) => {
|
|
2076
|
+
stdoutTail = appendRuntimeBuildTail(stdoutTail, line);
|
|
2077
|
+
await append(stdoutLogPath, line);
|
|
2078
|
+
},
|
|
2079
|
+
onStderrLine: async (line) => {
|
|
2080
|
+
stderrTail = appendRuntimeBuildTail(stderrTail, line);
|
|
2081
|
+
await append(stderrLogPath, line);
|
|
2082
|
+
},
|
|
2083
|
+
}, signal, { maxStartAttempts: 2, retryDelayMs: 250 });
|
|
2084
|
+
const result = await child.result;
|
|
2085
|
+
return {
|
|
2086
|
+
exitCode: result.exitCode,
|
|
2087
|
+
stdoutTail: appendRuntimeBuildTail(stdoutTail, result.stdout),
|
|
2088
|
+
stderrTail: appendRuntimeBuildTail(stderrTail, result.stderr),
|
|
2089
|
+
stdoutLogPath,
|
|
2090
|
+
stderrLogPath,
|
|
2091
|
+
};
|
|
2092
|
+
}
|
|
2093
|
+
function appendRuntimeBuildTail(current, value) {
|
|
2094
|
+
if (!value)
|
|
2095
|
+
return current;
|
|
2096
|
+
return `${current}${current ? "\n" : ""}${value}`.slice(-4_000);
|
|
2097
|
+
}
|
|
1260
2098
|
function effectiveHostConcurrency(lease, runtimeContract, executionClass = "managed_runtime") {
|
|
1261
2099
|
if (executionClass === "analysis_only") {
|
|
1262
2100
|
const analysisSingletonHeld = lease.resources.singletonKeys.length > 0;
|
|
@@ -1285,6 +2123,125 @@ function runtimeExecutionClass(dispatch) {
|
|
|
1285
2123
|
? "analysis_only"
|
|
1286
2124
|
: "managed_runtime";
|
|
1287
2125
|
}
|
|
2126
|
+
function runtimeAdmissionPriority(dispatch) {
|
|
2127
|
+
return dispatch.execution.lane.trim().toLowerCase() === "runtime_readiness"
|
|
2128
|
+
|| dispatch.cycleRun.phase.trim().toLowerCase() === "readiness_probe"
|
|
2129
|
+
? "background"
|
|
2130
|
+
: "foreground";
|
|
2131
|
+
}
|
|
2132
|
+
function enqueueAdmissionWaiter(waiters, waiter) {
|
|
2133
|
+
if (waiter.priority === "background") {
|
|
2134
|
+
waiters.push(waiter);
|
|
2135
|
+
return;
|
|
2136
|
+
}
|
|
2137
|
+
const backgroundIndex = waiters.findIndex(candidate => candidate.priority === "background");
|
|
2138
|
+
if (backgroundIndex < 0)
|
|
2139
|
+
waiters.push(waiter);
|
|
2140
|
+
else
|
|
2141
|
+
waiters.splice(backgroundIndex, 0, waiter);
|
|
2142
|
+
}
|
|
2143
|
+
function defaultHostResourceRequest() {
|
|
2144
|
+
const workerMemoryLimitBytes = configuredInteger("FORGE_HOST_DEFAULT_WORKER_MEMORY_LIMIT_BYTES", 2 * 1024 ** 3, 128 * 1024 ** 2, 512 * 1024 ** 3);
|
|
2145
|
+
const runtimeOverheadBytes = 128 * 1024 ** 2;
|
|
2146
|
+
return {
|
|
2147
|
+
sandboxSlots: 1,
|
|
2148
|
+
cpuMillis: 2000,
|
|
2149
|
+
workerMemoryLimitBytes,
|
|
2150
|
+
serviceMemoryLimitBytes: 0,
|
|
2151
|
+
runtimeOverheadBytes,
|
|
2152
|
+
memoryBytes: workerMemoryLimitBytes + runtimeOverheadBytes,
|
|
2153
|
+
workspaceBytes: 10 * 1024 ** 3,
|
|
2154
|
+
pids: 512,
|
|
2155
|
+
buildSlots: 1,
|
|
2156
|
+
};
|
|
2157
|
+
}
|
|
2158
|
+
function minimumSupportedHostResourceRequest() {
|
|
2159
|
+
return {
|
|
2160
|
+
sandboxSlots: 1,
|
|
2161
|
+
cpuMillis: 1000,
|
|
2162
|
+
workerMemoryLimitBytes: 1024 ** 3,
|
|
2163
|
+
serviceMemoryLimitBytes: 0,
|
|
2164
|
+
runtimeOverheadBytes: 128 * 1024 ** 2,
|
|
2165
|
+
memoryBytes: 1152 * 1024 ** 2,
|
|
2166
|
+
workspaceBytes: architectMinimumWorkspaceEstimateBytes,
|
|
2167
|
+
pids: 256,
|
|
2168
|
+
buildSlots: 0,
|
|
2169
|
+
};
|
|
2170
|
+
}
|
|
2171
|
+
function normalizedHostResourceRequest(resources) {
|
|
2172
|
+
const defaults = defaultHostResourceRequest();
|
|
2173
|
+
const workerMemoryLimitBytes = positiveResourceValue(resources?.workerMemoryLimitBytes, defaults.workerMemoryLimitBytes);
|
|
2174
|
+
const serviceMemoryLimitBytes = nonNegativeResourceValue(resources?.serviceMemoryLimitBytes, defaults.serviceMemoryLimitBytes);
|
|
2175
|
+
const runtimeOverheadBytes = nonNegativeResourceValue(resources?.runtimeOverheadBytes, defaults.runtimeOverheadBytes);
|
|
2176
|
+
return {
|
|
2177
|
+
sandboxSlots: positiveResourceValue(resources?.sandboxSlots, defaults.sandboxSlots),
|
|
2178
|
+
cpuMillis: positiveResourceValue(resources?.cpuMillis, defaults.cpuMillis),
|
|
2179
|
+
workerMemoryLimitBytes,
|
|
2180
|
+
serviceMemoryLimitBytes,
|
|
2181
|
+
runtimeOverheadBytes,
|
|
2182
|
+
memoryBytes: workerMemoryLimitBytes + serviceMemoryLimitBytes + runtimeOverheadBytes,
|
|
2183
|
+
workspaceBytes: positiveResourceValue(resources?.workspaceBytes, defaults.workspaceBytes),
|
|
2184
|
+
pids: positiveResourceValue(resources?.pids, defaults.pids),
|
|
2185
|
+
buildSlots: nonNegativeResourceValue(resources?.buildSlots, defaults.buildSlots),
|
|
2186
|
+
};
|
|
2187
|
+
}
|
|
2188
|
+
function positiveResourceValue(value, fallback) {
|
|
2189
|
+
return Number.isFinite(value) && value > 0 ? Math.floor(value) : fallback;
|
|
2190
|
+
}
|
|
2191
|
+
function nonNegativeResourceValue(value, fallback) {
|
|
2192
|
+
return Number.isFinite(value) && value >= 0 ? Math.floor(value) : fallback;
|
|
2193
|
+
}
|
|
2194
|
+
function configuredInteger(name, fallback, minimum, maximum) {
|
|
2195
|
+
const parsed = Number.parseInt(process.env[name] ?? "", 10);
|
|
2196
|
+
return Number.isFinite(parsed) ? Math.max(minimum, Math.min(maximum, parsed)) : fallback;
|
|
2197
|
+
}
|
|
2198
|
+
function loadHostCapacityConfiguration(capacityResourceSample) {
|
|
2199
|
+
const sampledCpuCount = capacityResourceSample
|
|
2200
|
+
? Math.max(1, Math.floor(capacityResourceSample.totalCpuMillis / 1_000))
|
|
2201
|
+
: os.cpus().length;
|
|
2202
|
+
const totalMemoryBytes = capacityResourceSample?.totalMemoryBytes ?? os.totalmem();
|
|
2203
|
+
const reserveFloorBytes = configuredInteger("FORGE_HOST_MEMORY_RESERVE_FLOOR_BYTES", 1024 ** 3, 512 * 1024 ** 2, 128 * 1024 ** 3);
|
|
2204
|
+
const reserveCeilingBytes = configuredInteger("FORGE_HOST_MEMORY_RESERVE_CEILING_BYTES", 4 * 1024 ** 3, reserveFloorBytes, 128 * 1024 ** 3);
|
|
2205
|
+
const hostReserveBytes = resolveAdaptiveMemoryReserveBytes(totalMemoryBytes, {
|
|
2206
|
+
explicitBytes: configuredOptionalInteger("FORGE_HOST_MEMORY_RESERVE_BYTES", 512 * 1024 ** 2, 128 * 1024 ** 3),
|
|
2207
|
+
floorBytes: reserveFloorBytes,
|
|
2208
|
+
ceilingBytes: reserveCeilingBytes,
|
|
2209
|
+
percent: configuredNumber("FORGE_HOST_MEMORY_RESERVE_PERCENT", 12.5, 1, 50),
|
|
2210
|
+
});
|
|
2211
|
+
const minimumIdleReserveBytes = Math.min(hostReserveBytes, reserveFloorBytes);
|
|
2212
|
+
return {
|
|
2213
|
+
hardMaxAttempts: configuredInteger("FORGE_HOST_MAX_CONCURRENT_ATTEMPTS", Math.max(1, Math.min(8, Math.floor(sampledCpuCount / 2))), 1, 32),
|
|
2214
|
+
hostReserveBytes,
|
|
2215
|
+
idleHostReserveBytes: configuredInteger("FORGE_HOST_IDLE_MEMORY_RESERVE_BYTES", minimumIdleReserveBytes, 512 * 1024 ** 2, hostReserveBytes),
|
|
2216
|
+
cpuReserveMillis: configuredInteger("FORGE_HOST_CPU_RESERVE_MILLIS", 1000, 0, 32_000),
|
|
2217
|
+
workspaceReserveBytes: configuredInteger("FORGE_HOST_WORKSPACE_RESERVE_BYTES", 10 * 1024 ** 3, 1024 ** 3, Number.MAX_SAFE_INTEGER),
|
|
2218
|
+
memoryRecoverySamples: configuredInteger("FORGE_HOST_MEMORY_RECOVERY_SAMPLES", 2, 1, 10),
|
|
2219
|
+
memoryRecoverySampleIntervalMs: configuredInteger("FORGE_HOST_MEMORY_RECOVERY_SAMPLE_INTERVAL_MS", 1_000, 100, 30_000),
|
|
2220
|
+
};
|
|
2221
|
+
}
|
|
2222
|
+
function configuredOptionalInteger(name, minimum, maximum) {
|
|
2223
|
+
const raw = process.env[name];
|
|
2224
|
+
if (raw === undefined || raw.trim().length === 0)
|
|
2225
|
+
return null;
|
|
2226
|
+
const parsed = Number.parseInt(raw, 10);
|
|
2227
|
+
return Number.isFinite(parsed) ? Math.max(minimum, Math.min(maximum, parsed)) : null;
|
|
2228
|
+
}
|
|
2229
|
+
function configuredNumber(name, fallback, minimum, maximum) {
|
|
2230
|
+
const parsed = Number(process.env[name] ?? "");
|
|
2231
|
+
return Number.isFinite(parsed) && parsed > 0
|
|
2232
|
+
? Math.max(minimum, Math.min(maximum, parsed))
|
|
2233
|
+
: fallback;
|
|
2234
|
+
}
|
|
2235
|
+
function resourceSlots(available, requested) {
|
|
2236
|
+
if (requested <= 0)
|
|
2237
|
+
return Number.MAX_SAFE_INTEGER;
|
|
2238
|
+
return Math.max(0, Math.floor(available / requested));
|
|
2239
|
+
}
|
|
2240
|
+
function removeWaiter(waiters, waiter) {
|
|
2241
|
+
const index = waiters.indexOf(waiter);
|
|
2242
|
+
if (index >= 0)
|
|
2243
|
+
waiters.splice(index, 1);
|
|
2244
|
+
}
|
|
1288
2245
|
function capacityProject(projects, repoId) {
|
|
1289
2246
|
const normalized = repoId.trim().toLowerCase();
|
|
1290
2247
|
const existing = projects.get(normalized);
|
|
@@ -1327,6 +2284,9 @@ function repoIdFromProjectSingletonKey(key) {
|
|
|
1327
2284
|
function sameSingletonOwner(left, right) {
|
|
1328
2285
|
return left?.dispatchAttemptId === right.dispatchAttemptId && left.fencingToken === right.fencingToken;
|
|
1329
2286
|
}
|
|
2287
|
+
function hostRuntimeOwnerKey(owner) {
|
|
2288
|
+
return `${owner.dispatchAttemptId}:${owner.fencingToken}`;
|
|
2289
|
+
}
|
|
1330
2290
|
function hostPortOwner(lease) {
|
|
1331
2291
|
const processSupervisorId = lease.resources.processSupervisorId;
|
|
1332
2292
|
if (!processSupervisorId) {
|
|
@@ -1375,6 +2335,14 @@ async function releaseAdapterResources(cleanups) {
|
|
|
1375
2335
|
if (firstError !== undefined)
|
|
1376
2336
|
throw firstError;
|
|
1377
2337
|
}
|
|
2338
|
+
async function removeHostAttemptRoot(attemptRoot) {
|
|
2339
|
+
await fs.rm(attemptRoot, {
|
|
2340
|
+
recursive: true,
|
|
2341
|
+
force: true,
|
|
2342
|
+
maxRetries: 10,
|
|
2343
|
+
retryDelay: 100,
|
|
2344
|
+
});
|
|
2345
|
+
}
|
|
1378
2346
|
function cancellationError(signal) {
|
|
1379
2347
|
return new Error(String(signal?.reason ?? "Dispatch cancelled while waiting for host runtime singleton ownership."));
|
|
1380
2348
|
}
|
|
@@ -1384,9 +2352,25 @@ function validHostRuntimeLedger(value) {
|
|
|
1384
2352
|
const ledger = value;
|
|
1385
2353
|
return ledger.schemaVersion === 1
|
|
1386
2354
|
&& (ledger.repoId === undefined || /^[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}$/iu.test(ledger.repoId))
|
|
2355
|
+
&& (ledger.attemptRootIdentity === undefined
|
|
2356
|
+
|| validSerializedHostAttemptRootIdentity(ledger.attemptRootIdentity))
|
|
1387
2357
|
&& validRuntimeIsolationLease(ledger.runtimeLease)
|
|
1388
2358
|
&& ledger.runtimeLease.mode === "host";
|
|
1389
2359
|
}
|
|
2360
|
+
function validSerializedHostAttemptRootIdentity(value) {
|
|
2361
|
+
if (typeof value !== "object" || value === null)
|
|
2362
|
+
return false;
|
|
2363
|
+
const identity = value;
|
|
2364
|
+
return /^-?\d{1,64}$/u.test(identity.dev) && /^-?\d{1,64}$/u.test(identity.ino);
|
|
2365
|
+
}
|
|
2366
|
+
function serializeHostAttemptRootIdentity(identity) {
|
|
2367
|
+
return { dev: identity.dev.toString(), ino: identity.ino.toString() };
|
|
2368
|
+
}
|
|
2369
|
+
function deserializeHostAttemptRootIdentity(identity) {
|
|
2370
|
+
return identity && validSerializedHostAttemptRootIdentity(identity)
|
|
2371
|
+
? { dev: BigInt(identity.dev), ino: BigInt(identity.ino) }
|
|
2372
|
+
: null;
|
|
2373
|
+
}
|
|
1390
2374
|
function validHostProcessSupervisorMetadata(value, lease) {
|
|
1391
2375
|
if (typeof value !== "object" || value === null)
|
|
1392
2376
|
return false;
|
|
@@ -1435,6 +2419,11 @@ function clearAdmissionDeadline(waiter) {
|
|
|
1435
2419
|
clearTimeout(waiter.deadlineTimer);
|
|
1436
2420
|
waiter.deadlineTimer = null;
|
|
1437
2421
|
}
|
|
2422
|
+
function clearCapacityRecheck(waiter) {
|
|
2423
|
+
if (waiter.capacityRecheckTimer)
|
|
2424
|
+
clearInterval(waiter.capacityRecheckTimer);
|
|
2425
|
+
waiter.capacityRecheckTimer = null;
|
|
2426
|
+
}
|
|
1438
2427
|
async function ensurePrivateDirectory(directory) {
|
|
1439
2428
|
await fs.mkdir(directory, { recursive: true, mode: 0o700 });
|
|
1440
2429
|
if (process.platform !== "win32")
|