@loucompanion/forge-bridge 0.1.3-dev.fbe15023cdfe → 0.1.4-dev.04a1e5b9e484
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +88 -2
- package/dist/bridge/bridge-chat.types.d.ts +111 -0
- package/dist/bridge/bridge-chat.types.js +401 -0
- package/dist/bridge/bridge.base.d.ts +4 -1
- package/dist/bridge/bridge.service.d.ts +72 -3
- package/dist/bridge/bridge.service.js +1563 -152
- package/dist/bridge/bridge.types.d.ts +46 -1
- package/dist/bridge/dispatch-result-outbox.d.ts +25 -0
- package/dist/bridge/dispatch-result-outbox.js +62 -0
- package/dist/bridge/dispatch-result-worker.d.ts +2 -0
- package/dist/bridge/dispatch-result-worker.js +122 -0
- package/dist/bridge/internals.d.ts +2 -0
- package/dist/bridge/internals.js +1 -0
- package/dist/bridge/readiness-result-outbox.d.ts +21 -0
- package/dist/bridge/readiness-result-outbox.js +54 -0
- package/dist/bridge/readiness-result-worker.d.ts +2 -0
- package/dist/bridge/readiness-result-worker.js +121 -0
- package/dist/bridge/runner-host-capabilities.d.ts +15 -0
- package/dist/bridge/runner-host-capabilities.js +93 -0
- package/dist/cleanup/cleanup.service.d.ts +1 -0
- package/dist/cleanup/cleanup.service.js +70 -9
- package/dist/cli/cli.help.js +2 -2
- package/dist/cli/cli.service.d.ts +14 -6
- package/dist/cli/cli.service.js +75 -16
- package/dist/cli/local-config.d.ts +1 -0
- package/dist/cli/local-config.js +4 -0
- package/dist/runtime/aspnet-runtime-prebuild.d.ts +36 -0
- package/dist/runtime/aspnet-runtime-prebuild.js +341 -0
- package/dist/runtime/attempt-database-broker.d.ts +89 -0
- package/dist/runtime/attempt-database-broker.js +390 -0
- package/dist/runtime/attempt-node-runtime-launcher.d.ts +2 -1
- package/dist/runtime/attempt-node-runtime-launcher.js +57 -18
- package/dist/runtime/attempt-path-operation.d.ts +21 -0
- package/dist/runtime/attempt-path-operation.js +94 -0
- package/dist/runtime/attempt-private-directory.d.ts +6 -0
- package/dist/runtime/attempt-private-directory.js +54 -0
- package/dist/runtime/attempt-process-supervisor.d.ts +16 -0
- package/dist/runtime/attempt-process-supervisor.js +94 -23
- package/dist/runtime/canonical-source-file.d.ts +3 -0
- package/dist/runtime/canonical-source-file.js +54 -0
- package/dist/runtime/docker-engine-probe.d.ts +90 -0
- package/dist/runtime/docker-engine-probe.js +333 -0
- package/dist/runtime/generated-output-reset.d.ts +10 -0
- package/dist/runtime/generated-output-reset.js +62 -8
- package/dist/runtime/host-database-capability-boundary.d.ts +16 -0
- package/dist/runtime/host-database-capability-boundary.js +62 -0
- package/dist/runtime/host-diagnostics.d.ts +11 -0
- package/dist/runtime/host-diagnostics.js +192 -0
- package/dist/runtime/host-execution-root.d.ts +4 -0
- package/dist/runtime/host-execution-root.js +39 -0
- package/dist/runtime/host-listener-inspector.d.ts +8 -1
- package/dist/runtime/host-listener-inspector.js +121 -60
- package/dist/runtime/host-runtime-contract.d.ts +12 -1
- package/dist/runtime/host-runtime-contract.js +168 -55
- package/dist/runtime/host-runtime-lease.d.ts +92 -8
- package/dist/runtime/host-runtime-lease.js +1141 -114
- package/dist/runtime/host-runtime-promotion.d.ts +12 -3
- package/dist/runtime/host-runtime-promotion.js +59 -12
- package/dist/runtime/memory-capacity.d.ts +70 -0
- package/dist/runtime/memory-capacity.js +123 -0
- package/dist/runtime/native-test-postgres-fallback.d.ts +25 -0
- package/dist/runtime/native-test-postgres-fallback.js +91 -0
- package/dist/runtime/node-host-runtime-adapter.d.ts +3 -0
- package/dist/runtime/node-host-runtime-adapter.js +29 -11
- package/dist/runtime/pinned-directory-process.d.ts +9 -0
- package/dist/runtime/pinned-directory-process.js +61 -0
- package/dist/runtime/pinned-source-directory.d.ts +17 -0
- package/dist/runtime/pinned-source-directory.js +94 -0
- package/dist/runtime/playwright-host-runtime-adapter.d.ts +1 -0
- package/dist/runtime/playwright-host-runtime-adapter.js +53 -5
- package/dist/runtime/prepared-runtime-identity.d.ts +6 -0
- package/dist/runtime/prepared-runtime-identity.js +218 -0
- package/dist/runtime/prepared-runtime-readiness.d.ts +30 -0
- package/dist/runtime/prepared-runtime-readiness.js +285 -0
- package/dist/runtime/runtime-isolation-lease.js +4 -1
- package/dist/runtime/workspace-resource-estimator.d.ts +17 -0
- package/dist/runtime/workspace-resource-estimator.js +174 -0
- package/dist/sandbox/preflight.d.ts +31 -1
- package/dist/sandbox/preflight.js +237 -51
- package/dist/sandbox/repository-lock.d.ts +2 -1
- package/dist/sandbox/repository-lock.js +19 -9
- package/dist/sandbox/runtime.d.ts +6 -1
- package/dist/sandbox/runtime.js +137 -107
- package/dist/sandbox/sandbox.types.d.ts +7 -0
- package/dist/sandbox/workspace.service.js +2 -0
- package/dist/session/managed-attempt-tools.d.ts +17 -1
- package/dist/session/managed-attempt-tools.js +489 -34
- package/dist/session/managed-build-coordinator.d.ts +5 -0
- package/dist/session/managed-build-coordinator.js +106 -0
- package/dist/session/managed-build-receipt.d.ts +21 -0
- package/dist/session/managed-build-receipt.js +337 -0
- package/dist/session/managed-verification-timeouts.d.ts +8 -0
- package/dist/session/managed-verification-timeouts.js +17 -0
- package/dist/session/phase-completion.artifacts.d.ts +17 -0
- package/dist/session/phase-completion.artifacts.js +224 -22
- package/dist/session/phase-completion.d.ts +11 -1
- package/dist/session/phase-completion.helpers.d.ts +5 -0
- package/dist/session/phase-completion.helpers.js +30 -7
- package/dist/session/phase-completion.js +14 -3
- package/dist/session/phase-completion.prompts.js +17 -18
- package/dist/session/phase-completion.runtime.d.ts +3 -1
- package/dist/session/phase-completion.runtime.js +126 -17
- package/dist/session/phase-completion.schemas.js +49 -2
- package/dist/session/phase-completion.validation.d.ts +1 -0
- package/dist/session/phase-completion.validation.js +111 -21
- package/dist/session/pre-publication-validation.d.ts +75 -0
- package/dist/session/pre-publication-validation.js +661 -0
- package/dist/session/provider-cli/bridge-cli.conformance-fixtures.d.ts +72 -0
- package/dist/session/provider-cli/bridge-cli.conformance-fixtures.js +462 -0
- package/dist/session/provider-cli/ci-repair-readiness.d.ts +8 -0
- package/dist/session/provider-cli/ci-repair-readiness.js +23 -3
- package/dist/session/provider-cli/provider-cli.artifacts.d.ts +4 -2
- package/dist/session/provider-cli/provider-cli.artifacts.js +47 -13
- package/dist/session/provider-cli/provider-cli.deterministic.js +34 -2
- package/dist/session/provider-cli/provider-cli.process.d.ts +51 -0
- package/dist/session/provider-cli/provider-cli.process.js +422 -22
- package/dist/session/provider-cli/provider-cli.prompts.d.ts +5 -0
- package/dist/session/provider-cli/provider-cli.prompts.js +136 -16
- package/dist/session/provider-cli/provider-cli.repair-epoch.d.ts +6 -0
- package/dist/session/provider-cli/provider-cli.repair-epoch.js +11 -0
- package/dist/session/provider-cli/provider-cli.runtime.d.ts +14 -1
- package/dist/session/provider-cli/provider-cli.runtime.js +54 -3
- package/dist/session/provider-cli/provider-cli.service.d.ts +30 -5
- package/dist/session/provider-cli/provider-cli.service.js +773 -69
- package/dist/session/provider-cli/provider-cli.types.d.ts +2 -0
- package/dist/session/provider-cli/provider-output-state.d.ts +10 -0
- package/dist/session/provider-cli/provider-output-state.js +361 -17
- package/dist/session/provider-cli/provider-usage.d.ts +7 -0
- package/dist/session/provider-cli/provider-usage.js +28 -5
- package/dist/session/session-redaction.js +434 -9
- package/dist/session/session.service.js +145 -23
- package/dist/session/session.types.d.ts +147 -2
- package/dist/session/session.types.js +587 -7
- package/dist/session/unreferenced-exports.d.ts +12 -0
- package/dist/session/unreferenced-exports.js +59 -0
- package/dist/shared/format/format-bytes.d.ts +1 -0
- package/dist/shared/format/format-bytes.js +9 -0
- package/dist/shared/git/git.js +2 -1
- package/dist/shared/path/path.d.ts +1 -0
- package/dist/shared/path/path.js +18 -0
- package/dist/shared/process/dotnet-toolchain.d.ts +17 -0
- package/dist/shared/process/dotnet-toolchain.js +307 -0
- package/dist/shared/process/host-process-recovery.js +3 -2
- package/dist/shared/process/host-process-supervisor.d.ts +2 -0
- package/dist/shared/process/host-process-supervisor.js +60 -11
- package/dist/shared/process/process.d.ts +2 -0
- package/dist/shared/process/process.js +12 -7
- package/dist/shared/provider-capabilities.d.ts +3 -0
- package/dist/shared/provider-capabilities.js +85 -0
- package/dist/shared/redaction/redaction.d.ts +1 -0
- package/dist/shared/redaction/redaction.js +39 -3
- package/dist/shared/time/monotonic.d.ts +1 -0
- package/dist/shared/time/monotonic.js +8 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/worktree/dependency-warmer.d.ts +41 -9
- package/dist/worktree/dependency-warmer.js +497 -22
- package/dist/worktree/index.d.ts +1 -1
- package/dist/worktree/story-qa-reference-hydrator.d.ts +4 -0
- package/dist/worktree/story-qa-reference-hydrator.js +253 -0
- package/dist/worktree/worktree.service.d.ts +7 -0
- package/dist/worktree/worktree.service.js +253 -21
- package/dist/worktree/worktree.types.d.ts +7 -1
- package/package.json +7 -5
- package/runtime/windows-job-supervisor.ps1 +380 -14
|
@@ -1,17 +1,36 @@
|
|
|
1
1
|
import fs from "node:fs/promises";
|
|
2
|
+
import { statfsSync } from "node:fs";
|
|
3
|
+
import { randomUUID } from "node:crypto";
|
|
4
|
+
import os from "node:os";
|
|
2
5
|
import path from "node:path";
|
|
6
|
+
import { assertHostAttemptRootIdentity, retainHostDiagnostics } from "./host-diagnostics.js";
|
|
7
|
+
import { assertHostExecutionRoot, compactHostAttemptId } from "./host-execution-root.js";
|
|
8
|
+
import { PreparedRuntimeReadiness } from "./prepared-runtime-readiness.js";
|
|
9
|
+
import { preparedRuntimeIdentity } from "./prepared-runtime-identity.js";
|
|
10
|
+
import { resolveExistingDirectoryPath } from "../shared/path/path.js";
|
|
11
|
+
import { formatBytes } from "../shared/format/format-bytes.js";
|
|
12
|
+
import { redactText } from "../shared/redaction/redaction.js";
|
|
3
13
|
import { StreamingProviderProcessRunner, } from "../session/provider-cli/provider-cli.process.js";
|
|
4
14
|
import { createRuntimeIsolationLease, renewRuntimeIsolationLease, transitionRuntimeIsolationLease, validRuntimeIsolationLease, withRuntimeIsolationResources, } from "./runtime-isolation-lease.js";
|
|
5
15
|
import { hostPortEnvironment, loadHostRuntimeContract, } from "./host-runtime-contract.js";
|
|
6
16
|
import { HostPortBroker, } from "./host-port-broker.js";
|
|
7
17
|
import { NodeHostRuntimeAdapter, } from "./node-host-runtime-adapter.js";
|
|
8
18
|
import { AttemptProcessSupervisorRegistry } from "./attempt-process-supervisor-registry.js";
|
|
19
|
+
import { attemptDatabaseCapabilityEnvironment, attemptDatabaseCapabilitySecrets, } from "./attempt-database-broker.js";
|
|
9
20
|
import { AttemptNodeRuntimeLauncher } from "./attempt-node-runtime-launcher.js";
|
|
10
21
|
import { HostListenerInspector } from "./host-listener-inspector.js";
|
|
22
|
+
import { RecoveryStabilizedMemory, buildMemoryAdmissionDiagnostics, resolveAdaptiveMemoryReserveBytes, } from "./memory-capacity.js";
|
|
11
23
|
import { PlaywrightHostRuntimeAdapter } from "./playwright-host-runtime-adapter.js";
|
|
12
24
|
import { HostParallelPromotionStore, configuredLaunchPortNames, hostRuntimeContractFingerprint, hostRuntimeParallelEligible, } from "./host-runtime-promotion.js";
|
|
13
25
|
import { ExactHostProcessRecovery, } from "../shared/process/host-process-recovery.js";
|
|
14
26
|
import { resetGeneratedOutputs } from "./generated-output-reset.js";
|
|
27
|
+
import { assertCanonicalDirectory, createCanonicalDirectory } from "./attempt-private-directory.js";
|
|
28
|
+
import { readCanonicalSourceFile } from "./canonical-source-file.js";
|
|
29
|
+
import { readPinnedSourceDirectory } from "./pinned-source-directory.js";
|
|
30
|
+
import { architectMinimumWorkspaceEstimateBytes } from "./workspace-resource-estimator.js";
|
|
31
|
+
import { createAspNetRuntimePrebuildPlan, ensureAspNetRuntimePrebuilt, resolveWorktreeCommitSha, } from "./aspnet-runtime-prebuild.js";
|
|
32
|
+
const hostCapacityWaiterScanLimit = 64;
|
|
33
|
+
const maximumProcessRecoveryMetadataBytes = 1024 * 1024;
|
|
15
34
|
export class HostRuntimeLeaseManager {
|
|
16
35
|
stateRoot;
|
|
17
36
|
processSupervisors;
|
|
@@ -20,18 +39,31 @@ export class HostRuntimeLeaseManager {
|
|
|
20
39
|
promotionStore;
|
|
21
40
|
playwrightAdapter;
|
|
22
41
|
processRecovery;
|
|
42
|
+
capacityResourceSample;
|
|
43
|
+
databaseBroker;
|
|
44
|
+
preparedReadiness;
|
|
23
45
|
active = new Map();
|
|
24
46
|
pendingRenewals = new Map();
|
|
47
|
+
acquiringAdmissions = new Set();
|
|
25
48
|
admissions = new Map();
|
|
26
49
|
singletonOwners = new Map();
|
|
27
50
|
singletonWaiters = [];
|
|
28
51
|
parallelProjectOwners = new Map();
|
|
29
52
|
parallelProjectWaiters = [];
|
|
53
|
+
capacityReservations = new Map();
|
|
54
|
+
capacityWaiters = [];
|
|
55
|
+
capacityWaiterScanCursor = 0;
|
|
56
|
+
backgroundAdmissionWaiters = [];
|
|
30
57
|
degraded = new Map();
|
|
31
58
|
knownPromotions = new Map();
|
|
59
|
+
ledgerWriteTail = Promise.resolve();
|
|
60
|
+
workspaceAvailableBytes = Number.MAX_SAFE_INTEGER;
|
|
32
61
|
portBroker;
|
|
33
62
|
nodeAdapter;
|
|
34
|
-
|
|
63
|
+
capacityConfiguration;
|
|
64
|
+
memoryAvailability;
|
|
65
|
+
executionRoot;
|
|
66
|
+
constructor(stateRoot, portBroker = new HostPortBroker(), nodeAdapter, processSupervisors = new AttemptProcessSupervisorRegistry(), processRunner = new StreamingProviderProcessRunner(), listenerInspector = new HostListenerInspector(), promotionStore = new HostParallelPromotionStore(stateRoot), playwrightAdapter = new PlaywrightHostRuntimeAdapter(), processRecovery = new ExactHostProcessRecovery(), capacityResourceSample, executionRoot, databaseBroker = null) {
|
|
35
67
|
this.stateRoot = stateRoot;
|
|
36
68
|
this.processSupervisors = processSupervisors;
|
|
37
69
|
this.processRunner = processRunner;
|
|
@@ -39,22 +71,47 @@ export class HostRuntimeLeaseManager {
|
|
|
39
71
|
this.promotionStore = promotionStore;
|
|
40
72
|
this.playwrightAdapter = playwrightAdapter;
|
|
41
73
|
this.processRecovery = processRecovery;
|
|
74
|
+
this.capacityResourceSample = capacityResourceSample;
|
|
75
|
+
this.databaseBroker = databaseBroker;
|
|
76
|
+
this.executionRoot = path.resolve(executionRoot ?? stateRoot);
|
|
77
|
+
if (executionRoot !== undefined)
|
|
78
|
+
assertHostExecutionRoot(this.executionRoot);
|
|
79
|
+
this.preparedReadiness = new PreparedRuntimeReadiness(this, promotionStore, (id, fence) => this.release(id, fence, true), () => new HostRuntimeAdmissionExpiredError());
|
|
42
80
|
this.portBroker = portBroker;
|
|
43
81
|
this.nodeAdapter = nodeAdapter ?? new NodeHostRuntimeAdapter(portBroker);
|
|
82
|
+
this.capacityConfiguration = loadHostCapacityConfiguration(capacityResourceSample);
|
|
83
|
+
this.memoryAvailability = new RecoveryStabilizedMemory(this.capacityConfiguration.memoryRecoverySamples, this.capacityConfiguration.memoryRecoverySampleIntervalMs);
|
|
44
84
|
}
|
|
45
85
|
async reap(options) {
|
|
46
86
|
await this.refreshKnownPromotions();
|
|
87
|
+
const recoveredAttempts = [];
|
|
47
88
|
const report = {
|
|
48
89
|
examined: 0,
|
|
49
90
|
reaped: 0,
|
|
50
91
|
processesTerminated: 0,
|
|
51
92
|
reusedPidsProtected: 0,
|
|
52
93
|
degraded: 0,
|
|
94
|
+
recoveredAttempts,
|
|
53
95
|
};
|
|
54
96
|
for (const ledger of await this.readLedgers()) {
|
|
55
97
|
const lease = ledger.runtimeLease;
|
|
56
98
|
if (lease.runnerId !== options.runnerId)
|
|
57
99
|
continue;
|
|
100
|
+
// A released ledger with its original attempt root is intentionally retained
|
|
101
|
+
// evidence that exceeded the diagnostics archive budget. It owns no runtime
|
|
102
|
+
// resources, but still requires process-identity recovery below on restart.
|
|
103
|
+
if (lease.state === "released") {
|
|
104
|
+
const attemptRoot = this.attemptRoot(lease.dispatchAttemptId);
|
|
105
|
+
const rootExists = await fs.stat(attemptRoot).then(() => true, error => {
|
|
106
|
+
if (error.code === "ENOENT")
|
|
107
|
+
return false;
|
|
108
|
+
throw error;
|
|
109
|
+
});
|
|
110
|
+
if (!rootExists) {
|
|
111
|
+
await fs.rm(this.ledgerPath(lease.dispatchAttemptId), { force: true });
|
|
112
|
+
continue;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
58
115
|
const active = this.active.get(lease.dispatchAttemptId);
|
|
59
116
|
if (active) {
|
|
60
117
|
if (active.lease.state !== "degraded")
|
|
@@ -78,9 +135,13 @@ export class HostRuntimeLeaseManager {
|
|
|
78
135
|
continue;
|
|
79
136
|
report.examined += 1;
|
|
80
137
|
const attemptRoot = this.attemptRoot(lease.dispatchAttemptId);
|
|
138
|
+
const attemptRootIdentity = deserializeHostAttemptRootIdentity(ledger.attemptRootIdentity);
|
|
81
139
|
let outcomes;
|
|
82
140
|
try {
|
|
83
|
-
|
|
141
|
+
if (!attemptRootIdentity) {
|
|
142
|
+
throw new Error("host_runtime_reaper_invalid: original attempt-root identity is unavailable.");
|
|
143
|
+
}
|
|
144
|
+
outcomes = await this.recoverPersistedProcesses(lease, attemptRoot, attemptRootIdentity);
|
|
84
145
|
}
|
|
85
146
|
catch {
|
|
86
147
|
await this.markLedgerDegraded(ledger);
|
|
@@ -94,10 +155,45 @@ export class HostRuntimeLeaseManager {
|
|
|
94
155
|
report.degraded += 1;
|
|
95
156
|
continue;
|
|
96
157
|
}
|
|
158
|
+
// The runtime has already been released. Keep the over-budget original
|
|
159
|
+
// evidence without re-copying it or reacquiring a capacity claim.
|
|
160
|
+
if (lease.state === "released")
|
|
161
|
+
continue;
|
|
162
|
+
// Every supervised tree of this previous-instance attempt is proved gone,
|
|
163
|
+
// so no provider can still act for it. Diagnostics handling below never
|
|
164
|
+
// revives a process, so the proof stands even if cleanup degrades. An
|
|
165
|
+
// attempt with no ownership record at all may have been mid-launch, so
|
|
166
|
+
// it is not reported and keeps its unknown outcome.
|
|
167
|
+
if (staleInstance && outcomes.length > 0) {
|
|
168
|
+
recoveredAttempts.push({
|
|
169
|
+
dispatchAttemptId: lease.dispatchAttemptId,
|
|
170
|
+
fencingToken: lease.fencingToken,
|
|
171
|
+
runnerInstanceId: lease.runnerInstanceId,
|
|
172
|
+
});
|
|
173
|
+
}
|
|
97
174
|
try {
|
|
98
175
|
const releasing = transitionHostLeaseForCleanup(lease, "releasing");
|
|
99
|
-
await this.writeLedger(releasing, ledger.repoId);
|
|
100
|
-
|
|
176
|
+
await this.writeLedger(releasing, ledger.repoId, attemptRootIdentity);
|
|
177
|
+
let evidenceRetainedAtAttemptRoot = false;
|
|
178
|
+
try {
|
|
179
|
+
await retainHostDiagnostics(this.stateRoot, attemptRoot, lease.dispatchAttemptId, attemptRootIdentity, this.executionRoot);
|
|
180
|
+
}
|
|
181
|
+
catch {
|
|
182
|
+
// Exact process recovery above already proved that the runtime is gone.
|
|
183
|
+
// Diagnostics retention can independently refuse a changed inventory,
|
|
184
|
+
// an unsafe archive path, or an over-budget source. Preserve the
|
|
185
|
+
// original attempt root and a released ledger for reconciliation, but
|
|
186
|
+
// never turn evidence-only work back into a capacity-owning runtime.
|
|
187
|
+
evidenceRetainedAtAttemptRoot = true;
|
|
188
|
+
}
|
|
189
|
+
if (evidenceRetainedAtAttemptRoot) {
|
|
190
|
+
const released = transitionRuntimeIsolationLease(releasing, "released");
|
|
191
|
+
await this.writeLedger(released, ledger.repoId, attemptRootIdentity);
|
|
192
|
+
this.forgetDegraded(lease.dispatchAttemptId, lease.fencingToken);
|
|
193
|
+
report.reaped += 1;
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
await removeHostAttemptRoot(attemptRoot);
|
|
101
197
|
await fs.rm(this.ledgerPath(lease.dispatchAttemptId), { force: true });
|
|
102
198
|
this.forgetDegraded(lease.dispatchAttemptId, lease.fencingToken);
|
|
103
199
|
report.reaped += 1;
|
|
@@ -107,9 +203,14 @@ export class HostRuntimeLeaseManager {
|
|
|
107
203
|
report.degraded += 1;
|
|
108
204
|
}
|
|
109
205
|
}
|
|
206
|
+
// Databases issued to attempts that ended without a clean release (a Bridge crash, or a
|
|
207
|
+
// drop that met a lingering session) are removed once no live lease holds them.
|
|
208
|
+
// A degraded lease may still own running processes after a restart, so its database is kept.
|
|
209
|
+
await this.databaseBroker?.reapOrphans(id => this.active.has(id) || this.degraded.has(id)).catch(() => undefined);
|
|
110
210
|
return report;
|
|
111
211
|
}
|
|
112
212
|
snapshot() {
|
|
213
|
+
this.reconcileReservationOwners();
|
|
113
214
|
const projects = new Map();
|
|
114
215
|
for (const promotion of this.knownPromotions.values()) {
|
|
115
216
|
const project = capacityProject(projects, promotion.repoId);
|
|
@@ -168,6 +269,26 @@ export class HostRuntimeLeaseManager {
|
|
|
168
269
|
project.waitingAttempts += 1;
|
|
169
270
|
project.safetyClass = conservativeSafetyClass(project.safetyClass, "host_parallel_candidate");
|
|
170
271
|
}
|
|
272
|
+
for (const waiter of this.backgroundAdmissionWaiters) {
|
|
273
|
+
const repoId = repoIdFromProjectSingletonKeys(waiter.keys);
|
|
274
|
+
if (!repoId)
|
|
275
|
+
continue;
|
|
276
|
+
const project = capacityProject(projects, repoId);
|
|
277
|
+
project.waitingAttempts += 1;
|
|
278
|
+
project.safetyClass = conservativeSafetyClass(project.safetyClass, "host_serial_only");
|
|
279
|
+
}
|
|
280
|
+
// A prepared-runtime verification remains foreground demand after its
|
|
281
|
+
// preparation admission is released and before the internal probe has
|
|
282
|
+
// acquired a normal admission. Report that barrier as waiting capacity so
|
|
283
|
+
// the server cannot mistake the transient gap for an idle host and dispatch
|
|
284
|
+
// readiness probes in a tight loop.
|
|
285
|
+
const unrepresentedPrepared = this.preparedReadiness.pendingSnapshots()
|
|
286
|
+
.filter(pending => !this.hasAttemptRepresentation(pending.probeDispatchAttemptId ?? pending.dispatchAttemptId, pending.probeFencingToken ?? pending.fencingToken));
|
|
287
|
+
for (const pending of unrepresentedPrepared) {
|
|
288
|
+
const project = capacityProject(projects, pending.repoId);
|
|
289
|
+
project.waitingAttempts += 1;
|
|
290
|
+
project.safetyClass = conservativeSafetyClass(project.safetyClass, "host_parallel_candidate");
|
|
291
|
+
}
|
|
171
292
|
const activeAttempts = [...this.active.values()]
|
|
172
293
|
.filter(active => active.lease.state !== "degraded").length;
|
|
173
294
|
const degradedAttempts = new Set([
|
|
@@ -176,14 +297,21 @@ export class HostRuntimeLeaseManager {
|
|
|
176
297
|
.filter(([, active]) => active.lease.state === "degraded")
|
|
177
298
|
.map(([dispatchAttemptId]) => dispatchAttemptId),
|
|
178
299
|
]).size;
|
|
300
|
+
const hostResources = this.hostResourceSnapshot();
|
|
179
301
|
return {
|
|
180
302
|
activeAttempts,
|
|
303
|
+
preparedWorktreeReadiness: true,
|
|
181
304
|
reservedAttempts: Math.max(0, this.admissions.size - this.active.size),
|
|
182
|
-
waitingAttempts: this.singletonWaiters.length
|
|
305
|
+
waitingAttempts: this.singletonWaiters.length
|
|
306
|
+
+ this.parallelProjectWaiters.length
|
|
307
|
+
+ this.capacityWaiters.length
|
|
308
|
+
+ this.backgroundAdmissionWaiters.length
|
|
309
|
+
+ unrepresentedPrepared.length,
|
|
183
310
|
degradedAttempts,
|
|
184
311
|
exclusiveReservations: this.singletonOwners.size,
|
|
185
312
|
sharedReservations: [...this.parallelProjectOwners.values()]
|
|
186
313
|
.reduce((count, owners) => count + owners.size, 0),
|
|
314
|
+
...hostResources,
|
|
187
315
|
projects: [...projects.values()]
|
|
188
316
|
.map(project => ({
|
|
189
317
|
...project,
|
|
@@ -192,6 +320,21 @@ export class HostRuntimeLeaseManager {
|
|
|
192
320
|
.sort((left, right) => left.repoId.localeCompare(right.repoId)),
|
|
193
321
|
};
|
|
194
322
|
}
|
|
323
|
+
hasForegroundDemand() {
|
|
324
|
+
return this.preparedReadiness.hasPending()
|
|
325
|
+
|| [...this.admissions.values()].some(admission => admission.priority === "foreground")
|
|
326
|
+
|| this.singletonWaiters.some(waiter => waiter.priority === "foreground")
|
|
327
|
+
|| this.parallelProjectWaiters.some(waiter => waiter.priority === "foreground")
|
|
328
|
+
|| this.capacityWaiters.some(waiter => waiter.priority === "foreground");
|
|
329
|
+
}
|
|
330
|
+
async acquirePreparation(dispatch, signal) {
|
|
331
|
+
// Discovery probes already own their verification lifecycle. Analysis-only
|
|
332
|
+
// work never launches the application and must not wait for a runtime wave.
|
|
333
|
+
if (runtimeExecutionClass(dispatch) === "analysis_only" || runtimeAdmissionPriority(dispatch) === "background") {
|
|
334
|
+
return await this.acquire(dispatch, signal);
|
|
335
|
+
}
|
|
336
|
+
return await this.preparedReadiness.acquire(dispatch, signal);
|
|
337
|
+
}
|
|
195
338
|
async acquire(dispatch, signal) {
|
|
196
339
|
const sandbox = dispatch.sandbox;
|
|
197
340
|
if (!sandbox || sandbox.executionMode !== "host") {
|
|
@@ -211,6 +354,7 @@ export class HostRuntimeLeaseManager {
|
|
|
211
354
|
return existing.keys;
|
|
212
355
|
}
|
|
213
356
|
const executionClass = runtimeExecutionClass(dispatch);
|
|
357
|
+
const priority = runtimeAdmissionPriority(dispatch);
|
|
214
358
|
const promotion = executionClass === "managed_runtime"
|
|
215
359
|
? await this.promotionStore.load(dispatch.repo.id)
|
|
216
360
|
: null;
|
|
@@ -219,37 +363,353 @@ export class HostRuntimeLeaseManager {
|
|
|
219
363
|
const projectKey = executionClass === "analysis_only"
|
|
220
364
|
? projectHostAnalysisKey(dispatch.repo.id)
|
|
221
365
|
: projectHostRuntimeSingletonKey(dispatch.repo.id);
|
|
222
|
-
|
|
366
|
+
const useParallelAdmission = priority === "foreground"
|
|
367
|
+
&& dispatch.cycleRun.phase !== "prepared_readiness"
|
|
368
|
+
&& (promotion !== null || executionClass === "analysis_only");
|
|
369
|
+
let keys = useParallelAdmission ? [] : [projectKey];
|
|
370
|
+
const resources = normalizedHostResourceRequest(sandbox.requestedResources);
|
|
223
371
|
const expiresAt = this.pendingRenewals.get(sandbox.dispatchAttemptId)?.fencingToken === owner.fencingToken
|
|
224
372
|
? this.pendingRenewals.get(sandbox.dispatchAttemptId).expiresAt
|
|
225
373
|
: dispatch.lease.expiresAt;
|
|
374
|
+
const acquisitionKey = hostRuntimeOwnerKey(owner);
|
|
375
|
+
this.acquiringAdmissions.add(acquisitionKey);
|
|
226
376
|
try {
|
|
227
|
-
|
|
228
|
-
|
|
377
|
+
try {
|
|
378
|
+
if (priority === "background") {
|
|
379
|
+
await this.refreshWorkspaceCapacity();
|
|
380
|
+
keys = await this.reserveBackgroundAdmission(keys, owner, resources, expiresAt, signal);
|
|
381
|
+
}
|
|
382
|
+
else if (useParallelAdmission) {
|
|
383
|
+
await this.reserveParallelProject(projectKey, owner, expiresAt, signal, priority);
|
|
384
|
+
}
|
|
385
|
+
else {
|
|
386
|
+
keys = await this.reserveSingletonKeys(keys, owner, expiresAt, signal, priority);
|
|
387
|
+
}
|
|
388
|
+
if (priority === "foreground") {
|
|
389
|
+
await this.refreshWorkspaceCapacity();
|
|
390
|
+
const capacityExpiresAt = this.pendingRenewals.get(sandbox.dispatchAttemptId)?.fencingToken === owner.fencingToken
|
|
391
|
+
? this.pendingRenewals.get(sandbox.dispatchAttemptId).expiresAt
|
|
392
|
+
: expiresAt;
|
|
393
|
+
await this.reserveHostCapacity(owner, resources, capacityExpiresAt, signal, priority);
|
|
394
|
+
}
|
|
229
395
|
}
|
|
230
|
-
|
|
231
|
-
|
|
396
|
+
catch (error) {
|
|
397
|
+
this.releaseHostCapacity(owner.dispatchAttemptId, owner.fencingToken);
|
|
398
|
+
this.releaseSingletonKeys(owner.dispatchAttemptId, owner.fencingToken);
|
|
399
|
+
this.releaseParallelProjects(owner.dispatchAttemptId, owner.fencingToken);
|
|
400
|
+
this.clearPendingRenewal(owner);
|
|
401
|
+
throw error;
|
|
402
|
+
}
|
|
403
|
+
const admittedPromotion = useParallelAdmission
|
|
404
|
+
? promotion ?? (executionClass === "managed_runtime" && keys.length === 0
|
|
405
|
+
? this.knownPromotions.get(dispatch.repo.id) ?? null
|
|
406
|
+
: null)
|
|
407
|
+
: null;
|
|
408
|
+
this.admissions.set(sandbox.dispatchAttemptId, {
|
|
409
|
+
repoId: dispatch.repo.id,
|
|
410
|
+
fencingToken: owner.fencingToken,
|
|
411
|
+
executionClass,
|
|
412
|
+
priority,
|
|
413
|
+
provisionalPromotionFingerprint: admittedPromotion?.runtimeContractFingerprint ?? null,
|
|
414
|
+
keys,
|
|
415
|
+
parallelProjectKey: useParallelAdmission ? projectKey : null,
|
|
416
|
+
});
|
|
417
|
+
return keys;
|
|
418
|
+
}
|
|
419
|
+
finally {
|
|
420
|
+
this.acquiringAdmissions.delete(acquisitionKey);
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
async reserveHostCapacity(owner, resources, expiresAt, signal, priority = "foreground") {
|
|
424
|
+
const existing = this.capacityReservations.get(owner.dispatchAttemptId);
|
|
425
|
+
if (existing) {
|
|
426
|
+
if (!sameSingletonOwner(existing.owner, owner)) {
|
|
427
|
+
throw new Error("host_runtime_capacity_conflict: dispatch attempt already holds capacity under a different fence.");
|
|
232
428
|
}
|
|
429
|
+
return;
|
|
233
430
|
}
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
431
|
+
if (signal?.aborted)
|
|
432
|
+
throw cancellationError(signal);
|
|
433
|
+
throwIfAdmissionExpired(expiresAt);
|
|
434
|
+
const impossibleReason = this.hostCapacityImpossibleReason(resources);
|
|
435
|
+
if (impossibleReason)
|
|
436
|
+
throw new HostRuntimeCapacityRequestImpossibleError(impossibleReason);
|
|
437
|
+
if (this.capacityWaiters.length === 0 && this.hostCapacityAvailable(resources)) {
|
|
438
|
+
this.capacityReservations.set(owner.dispatchAttemptId, { owner, resources, state: "pending" });
|
|
439
|
+
return;
|
|
237
440
|
}
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
441
|
+
return new Promise((resolve, reject) => {
|
|
442
|
+
const waiter = {
|
|
443
|
+
owner,
|
|
444
|
+
resources,
|
|
445
|
+
priority,
|
|
446
|
+
resolve,
|
|
447
|
+
reject,
|
|
448
|
+
signal,
|
|
449
|
+
expiresAt,
|
|
450
|
+
deadlineTimer: null,
|
|
451
|
+
capacityRecheckTimer: null,
|
|
452
|
+
abort: () => {
|
|
453
|
+
if (!this.detachCapacityWaiter(waiter))
|
|
454
|
+
return;
|
|
455
|
+
this.clearPendingRenewal(owner);
|
|
456
|
+
reject(cancellationError(signal));
|
|
457
|
+
this.drainAdmissionWaiters();
|
|
458
|
+
},
|
|
459
|
+
expire: () => {
|
|
460
|
+
if (!this.detachCapacityWaiter(waiter))
|
|
461
|
+
return;
|
|
462
|
+
this.clearPendingRenewal(owner);
|
|
463
|
+
reject(new HostRuntimeAdmissionExpiredError());
|
|
464
|
+
this.drainAdmissionWaiters();
|
|
465
|
+
},
|
|
466
|
+
};
|
|
467
|
+
waiter.deadlineTimer = scheduleAdmissionDeadline(expiresAt, waiter.expire);
|
|
468
|
+
// Memory, CPU and filesystem capacity can recover without one of our leases
|
|
469
|
+
// being released (for example when an unrelated host process exits). Recheck
|
|
470
|
+
// ambient capacity while queued so readiness probes and attempts do not remain
|
|
471
|
+
// stranded behind a stale low-resource sample.
|
|
472
|
+
waiter.capacityRecheckTimer = setInterval(() => this.drainCapacityWaiters(), 1_000);
|
|
473
|
+
waiter.capacityRecheckTimer.unref?.();
|
|
474
|
+
signal?.addEventListener("abort", waiter.abort, { once: true });
|
|
475
|
+
enqueueAdmissionWaiter(this.capacityWaiters, waiter);
|
|
476
|
+
this.drainCapacityWaiters();
|
|
249
477
|
});
|
|
250
|
-
return keys;
|
|
251
478
|
}
|
|
252
|
-
async
|
|
479
|
+
async reserveBackgroundAdmission(keys, owner, resources, expiresAt, signal) {
|
|
480
|
+
if (signal?.aborted)
|
|
481
|
+
throw cancellationError(signal);
|
|
482
|
+
throwIfAdmissionExpired(expiresAt);
|
|
483
|
+
return await new Promise((resolve, reject) => {
|
|
484
|
+
const waiter = {
|
|
485
|
+
owner,
|
|
486
|
+
keys,
|
|
487
|
+
resources,
|
|
488
|
+
priority: "background",
|
|
489
|
+
resolve,
|
|
490
|
+
reject,
|
|
491
|
+
signal,
|
|
492
|
+
expiresAt,
|
|
493
|
+
deadlineTimer: null,
|
|
494
|
+
capacityRecheckTimer: null,
|
|
495
|
+
abort: () => {
|
|
496
|
+
removeWaiter(this.backgroundAdmissionWaiters, waiter);
|
|
497
|
+
clearAdmissionDeadline(waiter);
|
|
498
|
+
clearCapacityRecheck(waiter);
|
|
499
|
+
this.clearPendingRenewal(owner);
|
|
500
|
+
reject(cancellationError(signal));
|
|
501
|
+
this.drainAdmissionWaiters();
|
|
502
|
+
},
|
|
503
|
+
expire: () => {
|
|
504
|
+
removeWaiter(this.backgroundAdmissionWaiters, waiter);
|
|
505
|
+
clearCapacityRecheck(waiter);
|
|
506
|
+
signal?.removeEventListener("abort", waiter.abort);
|
|
507
|
+
this.clearPendingRenewal(owner);
|
|
508
|
+
reject(new HostRuntimeAdmissionExpiredError());
|
|
509
|
+
this.drainAdmissionWaiters();
|
|
510
|
+
},
|
|
511
|
+
};
|
|
512
|
+
waiter.deadlineTimer = scheduleAdmissionDeadline(expiresAt, waiter.expire);
|
|
513
|
+
waiter.capacityRecheckTimer = setInterval(() => this.drainAdmissionWaiters(), 1_000);
|
|
514
|
+
waiter.capacityRecheckTimer.unref?.();
|
|
515
|
+
signal?.addEventListener("abort", waiter.abort, { once: true });
|
|
516
|
+
this.backgroundAdmissionWaiters.push(waiter);
|
|
517
|
+
this.drainAdmissionWaiters();
|
|
518
|
+
});
|
|
519
|
+
}
|
|
520
|
+
hostCapacityAvailable(resources) {
|
|
521
|
+
const capacity = this.hostResourceSnapshot();
|
|
522
|
+
return capacity.availableAttempts >= resources.sandboxSlots
|
|
523
|
+
&& capacity.availableMemoryBytes >= resources.memoryBytes
|
|
524
|
+
&& capacity.availableCpuMillis >= resources.cpuMillis
|
|
525
|
+
&& capacity.availableWorkspaceBytes >= resources.workspaceBytes;
|
|
526
|
+
}
|
|
527
|
+
hostCapacityImpossibleReason(resources) {
|
|
528
|
+
const { hardMaxAttempts, idleHostReserveBytes, cpuReserveMillis } = this.capacityConfiguration;
|
|
529
|
+
if (resources.sandboxSlots > hardMaxAttempts) {
|
|
530
|
+
return `requested ${resources.sandboxSlots} sandbox slots exceed the host maximum of ${hardMaxAttempts}.`;
|
|
531
|
+
}
|
|
532
|
+
const totalMemoryBytes = this.capacityResourceSample?.totalMemoryBytes ?? os.totalmem();
|
|
533
|
+
const maximumMemoryBytes = Math.max(0, totalMemoryBytes - idleHostReserveBytes);
|
|
534
|
+
if (resources.memoryBytes > maximumMemoryBytes) {
|
|
535
|
+
return `requested memory ${resources.memoryBytes} exceeds the host maximum of ${maximumMemoryBytes} bytes after reserve.`;
|
|
536
|
+
}
|
|
537
|
+
const totalCpuMillis = this.capacityResourceSample?.totalCpuMillis ?? os.cpus().length * 1_000;
|
|
538
|
+
const maximumCpuMillis = Math.max(0, totalCpuMillis - cpuReserveMillis);
|
|
539
|
+
if (resources.cpuMillis > maximumCpuMillis) {
|
|
540
|
+
return `requested CPU ${resources.cpuMillis} millis exceeds the host maximum of ${maximumCpuMillis} millis after reserve.`;
|
|
541
|
+
}
|
|
542
|
+
return null;
|
|
543
|
+
}
|
|
544
|
+
hostResourceSnapshot() {
|
|
545
|
+
if (!this.capacityResourceSample) {
|
|
546
|
+
try {
|
|
547
|
+
const stats = statfsSync(this.stateRoot);
|
|
548
|
+
this.workspaceAvailableBytes = Math.max(0, stats.bavail * stats.bsize);
|
|
549
|
+
}
|
|
550
|
+
catch {
|
|
551
|
+
// State root may not exist during the first capability heartbeat. The first
|
|
552
|
+
// admission creates it and refreshes the sample before reserving resources.
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
const { hardMaxAttempts, cpuReserveMillis, workspaceReserveBytes } = this.capacityConfiguration;
|
|
556
|
+
const hostReserveBytes = this.capacityReservations.size === 0
|
|
557
|
+
? this.capacityConfiguration.idleHostReserveBytes
|
|
558
|
+
: this.capacityConfiguration.hostReserveBytes;
|
|
559
|
+
const activeReservationMemoryBytes = [...this.capacityReservations.values()]
|
|
560
|
+
.reduce((total, reservation) => total + reservation.resources.memoryBytes, 0);
|
|
561
|
+
const pendingReservationMemoryBytes = [...this.capacityReservations.values()]
|
|
562
|
+
.filter(reservation => reservation.state === "pending")
|
|
563
|
+
.reduce((total, reservation) => total + reservation.resources.memoryBytes, 0);
|
|
564
|
+
const activeReservationCpuMillis = [...this.capacityReservations.values()]
|
|
565
|
+
.reduce((total, reservation) => total + reservation.resources.cpuMillis, 0);
|
|
566
|
+
const activeReservationWorkspaceBytes = [...this.capacityReservations.values()]
|
|
567
|
+
.reduce((total, reservation) => total + reservation.resources.workspaceBytes, 0);
|
|
568
|
+
const sampledAt = new Date().toISOString();
|
|
569
|
+
const totalMemoryBytes = this.capacityResourceSample?.totalMemoryBytes ?? os.totalmem();
|
|
570
|
+
const rawAvailableMemoryBytes = this.capacityResourceSample?.availableMemoryBytes ?? os.freemem();
|
|
571
|
+
const sampledAvailableMemoryBytes = this.memoryAvailability.observe(rawAvailableMemoryBytes);
|
|
572
|
+
// raw/stabilized available memory already includes every resident host and
|
|
573
|
+
// Forge process. Keep the independent total ceiling scoped to Forge's own
|
|
574
|
+
// reservations; adding ambient use here charges it a second time and can
|
|
575
|
+
// strand work even while more than the requested memory is physically free.
|
|
576
|
+
const committedMemoryBytes = Math.min(totalMemoryBytes, activeReservationMemoryBytes);
|
|
577
|
+
const totalCpuMillis = this.capacityResourceSample?.totalCpuMillis ?? os.cpus().length * 1000;
|
|
578
|
+
const sampledWorkspaceBytes = this.capacityResourceSample?.availableWorkspaceBytes ?? this.workspaceAvailableBytes;
|
|
579
|
+
const memoryAdmission = buildMemoryAdmissionDiagnostics({
|
|
580
|
+
sampledAt,
|
|
581
|
+
totalMemoryBytes,
|
|
582
|
+
rawAvailableMemoryBytes,
|
|
583
|
+
stabilizedAvailableMemoryBytes: sampledAvailableMemoryBytes,
|
|
584
|
+
controlPlaneMemoryBytes: process.memoryUsage().rss,
|
|
585
|
+
reserveMemoryBytes: hostReserveBytes,
|
|
586
|
+
committedMemoryBytes,
|
|
587
|
+
pendingCommitmentMemoryBytes: pendingReservationMemoryBytes,
|
|
588
|
+
});
|
|
589
|
+
const availableMemoryBytes = memoryAdmission.availableForAdmissionBytes;
|
|
590
|
+
const availableCpuMillis = Math.max(0, totalCpuMillis - cpuReserveMillis - activeReservationCpuMillis);
|
|
591
|
+
const availableWorkspaceBytes = Math.max(0, sampledWorkspaceBytes - workspaceReserveBytes - activeReservationWorkspaceBytes);
|
|
592
|
+
const slotAvailability = Math.max(0, hardMaxAttempts - this.capacityReservations.size);
|
|
593
|
+
// Capability snapshots advertise whether any supported attempt can fit.
|
|
594
|
+
// The queue/admission path still checks the exact project-derived request.
|
|
595
|
+
const minimumRequest = minimumSupportedHostResourceRequest();
|
|
596
|
+
const resourceAvailability = Math.min(resourceSlots(availableMemoryBytes, minimumRequest.memoryBytes), resourceSlots(availableCpuMillis, minimumRequest.cpuMillis), resourceSlots(availableWorkspaceBytes, minimumRequest.workspaceBytes));
|
|
597
|
+
const availableAttempts = Math.max(0, Math.min(slotAvailability, resourceAvailability));
|
|
598
|
+
const unavailableReason = slotAvailability <= 0
|
|
599
|
+
? `Host attempt capacity is full (${this.capacityReservations.size}/${hardMaxAttempts}).`
|
|
600
|
+
: availableMemoryBytes < minimumRequest.memoryBytes
|
|
601
|
+
? "Host memory is below the admission reserve."
|
|
602
|
+
: availableCpuMillis < minimumRequest.cpuMillis
|
|
603
|
+
? "Host CPU capacity is below the admission reserve."
|
|
604
|
+
: availableWorkspaceBytes < minimumRequest.workspaceBytes
|
|
605
|
+
? "Host workspace capacity is below the admission reserve."
|
|
606
|
+
: null;
|
|
607
|
+
return {
|
|
608
|
+
hardMaxAttempts,
|
|
609
|
+
effectiveMaxAttempts: this.capacityReservations.size + availableAttempts,
|
|
610
|
+
availableAttempts,
|
|
611
|
+
totalMemoryBytes,
|
|
612
|
+
availableMemoryBytes,
|
|
613
|
+
hostReserveBytes,
|
|
614
|
+
availableCpuMillis,
|
|
615
|
+
availableWorkspaceBytes,
|
|
616
|
+
activeReservationMemoryBytes,
|
|
617
|
+
memoryAdmission,
|
|
618
|
+
unavailableCode: unavailableReason ? "capacity_unavailable" : null,
|
|
619
|
+
unavailableReason,
|
|
620
|
+
};
|
|
621
|
+
}
|
|
622
|
+
async refreshWorkspaceCapacity() {
|
|
623
|
+
if (this.capacityResourceSample)
|
|
624
|
+
return;
|
|
625
|
+
try {
|
|
626
|
+
await fs.mkdir(this.stateRoot, { recursive: true });
|
|
627
|
+
const stats = await fs.statfs(this.stateRoot);
|
|
628
|
+
this.workspaceAvailableBytes = Math.max(0, stats.bavail * stats.bsize);
|
|
629
|
+
}
|
|
630
|
+
catch {
|
|
631
|
+
// Keep the last trustworthy sample. Memory/CPU and the hard ceiling still
|
|
632
|
+
// protect admission when the filesystem cannot be sampled temporarily.
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
releaseHostCapacity(dispatchAttemptId, fencingToken) {
|
|
636
|
+
const reservation = this.capacityReservations.get(dispatchAttemptId);
|
|
637
|
+
if (!reservation || reservation.owner.fencingToken !== fencingToken)
|
|
638
|
+
return;
|
|
639
|
+
this.capacityReservations.delete(dispatchAttemptId);
|
|
640
|
+
this.drainAdmissionWaiters();
|
|
641
|
+
}
|
|
642
|
+
drainCapacityWaiters() {
|
|
643
|
+
for (const waiter of this.capacityWaitersForScan()) {
|
|
644
|
+
if (!this.capacityWaiters.includes(waiter))
|
|
645
|
+
continue;
|
|
646
|
+
if (waiter.signal?.aborted) {
|
|
647
|
+
this.detachCapacityWaiter(waiter);
|
|
648
|
+
this.clearPendingRenewal(waiter.owner);
|
|
649
|
+
waiter.reject(cancellationError(waiter.signal));
|
|
650
|
+
continue;
|
|
651
|
+
}
|
|
652
|
+
if (admissionExpired(waiter.expiresAt)) {
|
|
653
|
+
this.detachCapacityWaiter(waiter);
|
|
654
|
+
this.clearPendingRenewal(waiter.owner);
|
|
655
|
+
waiter.reject(new HostRuntimeAdmissionExpiredError());
|
|
656
|
+
continue;
|
|
657
|
+
}
|
|
658
|
+
const impossibleReason = this.hostCapacityImpossibleReason(waiter.resources);
|
|
659
|
+
if (impossibleReason) {
|
|
660
|
+
this.detachCapacityWaiter(waiter);
|
|
661
|
+
this.clearPendingRenewal(waiter.owner);
|
|
662
|
+
waiter.reject(new HostRuntimeCapacityRequestImpossibleError(impossibleReason));
|
|
663
|
+
continue;
|
|
664
|
+
}
|
|
665
|
+
if (!this.hostCapacityAvailable(waiter.resources))
|
|
666
|
+
continue;
|
|
667
|
+
this.detachCapacityWaiter(waiter);
|
|
668
|
+
this.capacityReservations.set(waiter.owner.dispatchAttemptId, {
|
|
669
|
+
owner: waiter.owner,
|
|
670
|
+
resources: waiter.resources,
|
|
671
|
+
state: "pending",
|
|
672
|
+
});
|
|
673
|
+
waiter.resolve();
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
capacityWaitersForScan() {
|
|
677
|
+
const head = this.capacityWaiters[0];
|
|
678
|
+
if (!head) {
|
|
679
|
+
this.capacityWaiterScanCursor = 0;
|
|
680
|
+
return [];
|
|
681
|
+
}
|
|
682
|
+
const scanCount = Math.min(hostCapacityWaiterScanLimit, this.capacityWaiters.length);
|
|
683
|
+
if (scanCount === this.capacityWaiters.length) {
|
|
684
|
+
this.capacityWaiterScanCursor = 0;
|
|
685
|
+
return this.capacityWaiters.slice();
|
|
686
|
+
}
|
|
687
|
+
// The head always gets the first opportunity, while a rotating bounded
|
|
688
|
+
// window prevents an unfit renewed head (or a very deep queue) from
|
|
689
|
+
// indefinitely hiding a later request that fits current capacity.
|
|
690
|
+
const laterCount = this.capacityWaiters.length - 1;
|
|
691
|
+
const laterScanCount = scanCount - 1;
|
|
692
|
+
const start = this.capacityWaiterScanCursor % laterCount;
|
|
693
|
+
const result = [head];
|
|
694
|
+
for (let offset = 0; offset < laterScanCount; offset += 1) {
|
|
695
|
+
result.push(this.capacityWaiters[1 + ((start + offset) % laterCount)]);
|
|
696
|
+
}
|
|
697
|
+
this.capacityWaiterScanCursor = (start + laterScanCount) % laterCount;
|
|
698
|
+
return result;
|
|
699
|
+
}
|
|
700
|
+
detachCapacityWaiter(waiter) {
|
|
701
|
+
const index = this.capacityWaiters.indexOf(waiter);
|
|
702
|
+
if (index < 0)
|
|
703
|
+
return false;
|
|
704
|
+
this.capacityWaiters.splice(index, 1);
|
|
705
|
+
clearAdmissionDeadline(waiter);
|
|
706
|
+
clearCapacityRecheck(waiter);
|
|
707
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
708
|
+
if (this.capacityWaiters.length === 0)
|
|
709
|
+
this.capacityWaiterScanCursor = 0;
|
|
710
|
+
return true;
|
|
711
|
+
}
|
|
712
|
+
async reserveSingletonKeys(keys, owner, expiresAt, signal, priority = "foreground") {
|
|
253
713
|
const owned = keys.every(key => sameSingletonOwner(this.singletonOwners.get(key), owner));
|
|
254
714
|
if (owned)
|
|
255
715
|
return keys;
|
|
@@ -269,6 +729,7 @@ export class HostRuntimeLeaseManager {
|
|
|
269
729
|
const waiter = {
|
|
270
730
|
owner,
|
|
271
731
|
keys,
|
|
732
|
+
priority,
|
|
272
733
|
resolve,
|
|
273
734
|
reject,
|
|
274
735
|
signal,
|
|
@@ -295,13 +756,13 @@ export class HostRuntimeLeaseManager {
|
|
|
295
756
|
this.drainAdmissionWaiters();
|
|
296
757
|
},
|
|
297
758
|
};
|
|
298
|
-
this.singletonWaiters
|
|
759
|
+
enqueueAdmissionWaiter(this.singletonWaiters, waiter);
|
|
299
760
|
signal?.addEventListener("abort", waiter.abort, { once: true });
|
|
300
761
|
waiter.deadlineTimer = scheduleAdmissionDeadline(waiter.expiresAt, waiter.expire);
|
|
301
762
|
this.drainAdmissionWaiters();
|
|
302
763
|
});
|
|
303
764
|
}
|
|
304
|
-
async reserveParallelProject(key, owner, expiresAt, signal) {
|
|
765
|
+
async reserveParallelProject(key, owner, expiresAt, signal, priority = "foreground") {
|
|
305
766
|
if (this.parallelProjectOwned(key, owner))
|
|
306
767
|
return;
|
|
307
768
|
const staleFence = [...(this.parallelProjectOwners.get(key)?.values() ?? [])]
|
|
@@ -313,7 +774,7 @@ export class HostRuntimeLeaseManager {
|
|
|
313
774
|
if (signal?.aborted)
|
|
314
775
|
throw cancellationError(signal);
|
|
315
776
|
throwIfAdmissionExpired(expiresAt);
|
|
316
|
-
if (this.parallelProjectAvailable(key)) {
|
|
777
|
+
if (this.parallelProjectAvailable(key, priority)) {
|
|
317
778
|
this.assignParallelProject(key, owner);
|
|
318
779
|
return;
|
|
319
780
|
}
|
|
@@ -321,6 +782,7 @@ export class HostRuntimeLeaseManager {
|
|
|
321
782
|
const waiter = {
|
|
322
783
|
owner,
|
|
323
784
|
key,
|
|
785
|
+
priority,
|
|
324
786
|
resolve,
|
|
325
787
|
reject,
|
|
326
788
|
signal,
|
|
@@ -347,17 +809,85 @@ export class HostRuntimeLeaseManager {
|
|
|
347
809
|
this.drainAdmissionWaiters();
|
|
348
810
|
},
|
|
349
811
|
};
|
|
350
|
-
this.parallelProjectWaiters
|
|
812
|
+
enqueueAdmissionWaiter(this.parallelProjectWaiters, waiter);
|
|
813
|
+
signal?.addEventListener("abort", waiter.abort, { once: true });
|
|
814
|
+
waiter.deadlineTimer = scheduleAdmissionDeadline(waiter.expiresAt, waiter.expire);
|
|
815
|
+
this.drainAdmissionWaiters();
|
|
816
|
+
});
|
|
817
|
+
}
|
|
818
|
+
async upgradeParallelProjectToSingleton(keys, parallelProjectKey, owner, expiresAt, signal, priority = "foreground") {
|
|
819
|
+
if (!this.parallelProjectOwned(parallelProjectKey, owner)) {
|
|
820
|
+
throw new Error("host_runtime_parallel_unreserved: matching shared project admission is required before serialization.");
|
|
821
|
+
}
|
|
822
|
+
if (signal?.aborted)
|
|
823
|
+
throw cancellationError(signal);
|
|
824
|
+
throwIfAdmissionExpired(expiresAt);
|
|
825
|
+
return await new Promise((resolve, reject) => {
|
|
826
|
+
const waiter = {
|
|
827
|
+
owner,
|
|
828
|
+
keys,
|
|
829
|
+
priority,
|
|
830
|
+
resolve,
|
|
831
|
+
reject,
|
|
832
|
+
signal,
|
|
833
|
+
expiresAt,
|
|
834
|
+
deadlineTimer: null,
|
|
835
|
+
abort: () => {
|
|
836
|
+
const index = this.singletonWaiters.indexOf(waiter);
|
|
837
|
+
if (index >= 0)
|
|
838
|
+
this.singletonWaiters.splice(index, 1);
|
|
839
|
+
clearAdmissionDeadline(waiter);
|
|
840
|
+
this.clearPendingRenewal(owner);
|
|
841
|
+
reject(cancellationError(signal));
|
|
842
|
+
this.drainAdmissionWaiters();
|
|
843
|
+
},
|
|
844
|
+
expire: () => {
|
|
845
|
+
const index = this.singletonWaiters.indexOf(waiter);
|
|
846
|
+
if (index < 0)
|
|
847
|
+
return;
|
|
848
|
+
this.singletonWaiters.splice(index, 1);
|
|
849
|
+
clearAdmissionDeadline(waiter);
|
|
850
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
851
|
+
this.clearPendingRenewal(owner);
|
|
852
|
+
reject(new HostRuntimeAdmissionExpiredError());
|
|
853
|
+
this.drainAdmissionWaiters();
|
|
854
|
+
},
|
|
855
|
+
};
|
|
856
|
+
// Queue the exclusive claim before relinquishing the shared claim. This
|
|
857
|
+
// makes the promotion downgrade atomic from the broker's perspective and
|
|
858
|
+
// prevents a background readiness probe from jumping ahead in between.
|
|
859
|
+
enqueueAdmissionWaiter(this.singletonWaiters, waiter);
|
|
351
860
|
signal?.addEventListener("abort", waiter.abort, { once: true });
|
|
352
861
|
waiter.deadlineTimer = scheduleAdmissionDeadline(waiter.expiresAt, waiter.expire);
|
|
862
|
+
this.releaseParallelProjectNoDrain(parallelProjectKey, owner);
|
|
353
863
|
this.drainAdmissionWaiters();
|
|
354
864
|
});
|
|
355
865
|
}
|
|
356
|
-
async start(dispatch, worktreePath, signal) {
|
|
866
|
+
async start(dispatch, worktreePath, signal, onAdmissionCheckpoint) {
|
|
357
867
|
const sandbox = dispatch.sandbox;
|
|
358
868
|
if (!sandbox || sandbox.executionMode !== "host") {
|
|
359
869
|
throw new Error("host_runtime_lease_invalid: managed host sandbox context is required.");
|
|
360
870
|
}
|
|
871
|
+
const wasPrepared = this.preparedReadiness.has(sandbox.dispatchAttemptId);
|
|
872
|
+
const preparedProof = wasPrepared
|
|
873
|
+
? await this.preparedReadiness.verify(dispatch, worktreePath, onAdmissionCheckpoint)
|
|
874
|
+
: undefined;
|
|
875
|
+
if (wasPrepared) {
|
|
876
|
+
signal?.throwIfAborted();
|
|
877
|
+
const admission = this.hostResourceSnapshot().memoryAdmission;
|
|
878
|
+
const requestedMemoryBytes = normalizedHostResourceRequest(sandbox.requestedResources).memoryBytes;
|
|
879
|
+
await onAdmissionCheckpoint?.({
|
|
880
|
+
code: "host_runtime_prepared_admission_waiting",
|
|
881
|
+
message: `Prepared runtime verification passed. Waiting for host admission before agent startup: ${formatBytes(requestedMemoryBytes)} requested, ${formatBytes(admission.availableForAdmissionBytes)} currently available under the host reserve.`,
|
|
882
|
+
expiresAt: dispatch.lease.expiresAt,
|
|
883
|
+
});
|
|
884
|
+
await this.acquire(dispatch, signal);
|
|
885
|
+
await onAdmissionCheckpoint?.({
|
|
886
|
+
code: "host_runtime_prepared_admission_acquired",
|
|
887
|
+
message: "Prepared runtime acquired host admission; agent startup is continuing.",
|
|
888
|
+
expiresAt: dispatch.lease.expiresAt,
|
|
889
|
+
});
|
|
890
|
+
}
|
|
361
891
|
const existing = this.active.get(sandbox.dispatchAttemptId);
|
|
362
892
|
if (existing) {
|
|
363
893
|
if (existing.lease.fencingToken !== dispatch.lease.fencingToken) {
|
|
@@ -365,7 +895,7 @@ export class HostRuntimeLeaseManager {
|
|
|
365
895
|
}
|
|
366
896
|
return {
|
|
367
897
|
lease: existing.lease,
|
|
368
|
-
environment: hostRuntimeEnvironment(dispatch.session.id, existing.lease, existing.runtimeContract, existing.executionClass),
|
|
898
|
+
environment: hostRuntimeEnvironment(dispatch.session.id, existing.lease, existing.runtimeContract, existing.executionClass, existing.databaseCapability),
|
|
369
899
|
runtimeContract: existing.runtimeContract,
|
|
370
900
|
executionClass: existing.executionClass,
|
|
371
901
|
};
|
|
@@ -376,12 +906,13 @@ export class HostRuntimeLeaseManager {
|
|
|
376
906
|
throw new Error("host_runtime_singleton_unreserved: managed host admission is required before start.");
|
|
377
907
|
}
|
|
378
908
|
const attemptHex = uuidHex(sandbox.dispatchAttemptId, "dispatch attempt");
|
|
379
|
-
const
|
|
380
|
-
const
|
|
381
|
-
const
|
|
909
|
+
const compactAttemptHex = compactHostAttemptId(sandbox.dispatchAttemptId);
|
|
910
|
+
const normalizedWorktreePath = await resolveExistingDirectoryPath(worktreePath);
|
|
911
|
+
const attemptRoot = path.join(this.executionRoot, "a", compactAttemptHex);
|
|
912
|
+
const temporaryPath = path.join(attemptRoot, "t");
|
|
382
913
|
const logPath = path.join(attemptRoot, "logs");
|
|
383
|
-
const cachePath = path.join(attemptRoot, "
|
|
384
|
-
const browserProfilePath = path.join(attemptRoot, "
|
|
914
|
+
const cachePath = path.join(attemptRoot, "c");
|
|
915
|
+
const browserProfilePath = path.join(attemptRoot, "b");
|
|
385
916
|
const testOutputPath = path.join(attemptRoot, "test-output");
|
|
386
917
|
const processSupervisorMetadataPath = path.join(logPath, "attempt-process-supervisor.json");
|
|
387
918
|
const processSupervisorId = `host-${attemptHex}`;
|
|
@@ -390,50 +921,86 @@ export class HostRuntimeLeaseManager {
|
|
|
390
921
|
? renewedExpiry.expiresAt
|
|
391
922
|
: dispatch.lease.expiresAt;
|
|
392
923
|
let processSupervisor = null;
|
|
924
|
+
let attemptRootIdentity = null;
|
|
393
925
|
try {
|
|
394
|
-
|
|
395
|
-
try {
|
|
396
|
-
runtimeContract = await loadHostRuntimeContract(normalizedWorktreePath);
|
|
397
|
-
}
|
|
398
|
-
catch (error) {
|
|
399
|
-
if (admission.executionClass === "managed_runtime" && admission.provisionalPromotionFingerprint) {
|
|
400
|
-
await this.invalidatePromotion(dispatch.repo.id).catch(() => undefined);
|
|
401
|
-
}
|
|
402
|
-
throw error;
|
|
403
|
-
}
|
|
926
|
+
const runtimeContract = await loadHostRuntimeContract(normalizedWorktreePath);
|
|
404
927
|
if (runtimeContract.contract.runtime.hostConcurrency === "disabled") {
|
|
405
928
|
throw new Error("host_runtime_disabled: project runtime configuration requires Docker execution.");
|
|
406
929
|
}
|
|
930
|
+
if (wasPrepared && !preparedProof && admission.executionClass === "managed_runtime"
|
|
931
|
+
&& hostRuntimeParallelEligible(runtimeContract)) {
|
|
932
|
+
throw new Error("host_runtime_prepared_source_changed: runtime became parallel-eligible after preparation; a new foreground proof is required.");
|
|
933
|
+
}
|
|
407
934
|
const runtimeContractFingerprint = hostRuntimeContractFingerprint(runtimeContract);
|
|
935
|
+
const inputCommitSha = preparedProof?.inputCommitSha
|
|
936
|
+
?? await resolveWorktreeCommitSha(normalizedWorktreePath, sandbox.inputCommitSha);
|
|
937
|
+
if (preparedProof && (runtimeContractFingerprint !== preparedProof.runtimeContractFingerprint
|
|
938
|
+
|| JSON.stringify(preparedProof) !== JSON.stringify(await preparedRuntimeIdentity(normalizedWorktreePath, signal)))) {
|
|
939
|
+
throw new Error("host_runtime_prepared_source_changed: source changed after its foreground proof; agent was not started.");
|
|
940
|
+
}
|
|
408
941
|
const promotion = admission.executionClass === "managed_runtime"
|
|
409
|
-
? await this.promotionStore.load(dispatch.repo.id)
|
|
942
|
+
? await this.promotionStore.load(dispatch.repo.id, preparedProof ?? { runtimeContractFingerprint, inputCommitSha })
|
|
410
943
|
: null;
|
|
411
944
|
const promotionMatches = hostRuntimeParallelEligible(runtimeContract)
|
|
412
|
-
&& promotion?.runtimeContractFingerprint === runtimeContractFingerprint
|
|
945
|
+
&& promotion?.runtimeContractFingerprint === runtimeContractFingerprint
|
|
946
|
+
&& promotion.inputCommitSha === inputCommitSha;
|
|
413
947
|
if (admission.executionClass === "managed_runtime" && admission.keys.length === 0 && !promotionMatches) {
|
|
414
|
-
await this.invalidatePromotion(dispatch.repo.id);
|
|
415
948
|
const projectKeys = hostRuntimeSingletonKeys(dispatch);
|
|
416
|
-
if (admission.parallelProjectKey) {
|
|
417
|
-
this.releaseParallelProjectNoDrain(admission.parallelProjectKey, owner);
|
|
418
|
-
admission.parallelProjectKey = null;
|
|
419
|
-
}
|
|
420
949
|
const expiresAt = this.pendingRenewals.get(sandbox.dispatchAttemptId)?.fencingToken === owner.fencingToken
|
|
421
950
|
? this.pendingRenewals.get(sandbox.dispatchAttemptId).expiresAt
|
|
422
951
|
: dispatch.lease.expiresAt;
|
|
423
|
-
|
|
952
|
+
const parallelProjectKey = admission.parallelProjectKey;
|
|
953
|
+
admission.parallelProjectKey = null;
|
|
954
|
+
const capacityReservation = this.capacityReservations.get(owner.dispatchAttemptId);
|
|
955
|
+
if (capacityReservation && !sameSingletonOwner(capacityReservation.owner, owner)) {
|
|
956
|
+
throw new Error("host_runtime_capacity_conflict: dispatch attempt holds capacity under a different fence.");
|
|
957
|
+
}
|
|
958
|
+
const serializedAdmission = parallelProjectKey
|
|
959
|
+
? this.upgradeParallelProjectToSingleton(projectKeys, parallelProjectKey, owner, expiresAt, signal, admission.priority)
|
|
960
|
+
: this.reserveSingletonKeys(projectKeys, owner, expiresAt, signal, admission.priority);
|
|
961
|
+
// A sibling can already hold the same provisional shared project claim
|
|
962
|
+
// while waiting for this attempt's host-capacity slot. Retaining capacity
|
|
963
|
+
// while waiting for that sibling to release its shared claim creates a
|
|
964
|
+
// cycle: exclusive waits on shared, shared waits on capacity. Relinquish
|
|
965
|
+
// the capacity slot only after the exclusive waiter is queued, then
|
|
966
|
+
// reacquire it after serialization is granted.
|
|
967
|
+
if (capacityReservation) {
|
|
968
|
+
this.releaseHostCapacity(owner.dispatchAttemptId, owner.fencingToken);
|
|
969
|
+
}
|
|
970
|
+
// The checkpoint is allowed to do I/O. Observe an early admission
|
|
971
|
+
// failure now so a slow acknowledgement cannot create an unhandled
|
|
972
|
+
// rejection before the operational promise is awaited below.
|
|
973
|
+
void serializedAdmission.catch(() => undefined);
|
|
974
|
+
// Begin the operational downgrade before publishing telemetry. A lost
|
|
975
|
+
// callback acknowledgement must not leave the attempt holding its old
|
|
976
|
+
// shared admission while every recovery probe waits behind it.
|
|
977
|
+
await onAdmissionCheckpoint?.({
|
|
978
|
+
code: "host_runtime_serial_admission_waiting",
|
|
979
|
+
message: "The prepared runtime contract no longer matches the parallel-readiness proof. Waiting for foreground-priority serialized admission before launch.",
|
|
980
|
+
expiresAt,
|
|
981
|
+
});
|
|
982
|
+
admission.keys = await serializedAdmission;
|
|
424
983
|
while (admission.keys.length === 0) {
|
|
425
984
|
const promoted = this.knownPromotions.get(dispatch.repo.id);
|
|
426
985
|
if (!promoted) {
|
|
427
986
|
throw new Error("host_runtime_parallel_promotion_missing: shared admission requires matching promotion evidence.");
|
|
428
987
|
}
|
|
429
|
-
|
|
988
|
+
const matching = await this.promotionStore.load(dispatch.repo.id, preparedProof ?? { runtimeContractFingerprint, inputCommitSha });
|
|
989
|
+
if (!matching) {
|
|
990
|
+
// Drop obsolete proof before asking for exclusive admission; otherwise
|
|
991
|
+
// the queue can grant the same stale shared promotion indefinitely.
|
|
430
992
|
this.releaseParallelProjectNoDrain(projectKeys[0], owner);
|
|
431
|
-
await this.
|
|
432
|
-
admission.keys = await this.reserveSingletonKeys(projectKeys, owner, expiresAt, signal);
|
|
993
|
+
admission.keys = await this.reserveSingletonKeys(projectKeys, owner, expiresAt, signal, admission.priority);
|
|
433
994
|
continue;
|
|
434
995
|
}
|
|
435
996
|
admission.parallelProjectKey = projectKeys[0] ?? null;
|
|
436
997
|
admission.provisionalPromotionFingerprint = promoted.runtimeContractFingerprint;
|
|
998
|
+
break;
|
|
999
|
+
}
|
|
1000
|
+
// Do not hold a capacity slot while a stale shared re-grant is still
|
|
1001
|
+
// being converted to exclusive admission.
|
|
1002
|
+
if (capacityReservation) {
|
|
1003
|
+
await this.reserveHostCapacity(owner, capacityReservation.resources, expiresAt, signal, admission.priority);
|
|
437
1004
|
}
|
|
438
1005
|
}
|
|
439
1006
|
const declaredSingletonKeys = runtimeContract.contract.runtime.singletonResources
|
|
@@ -444,7 +1011,7 @@ export class HostRuntimeLeaseManager {
|
|
|
444
1011
|
const expiresAt = this.pendingRenewals.get(sandbox.dispatchAttemptId)?.fencingToken === owner.fencingToken
|
|
445
1012
|
? this.pendingRenewals.get(sandbox.dispatchAttemptId).expiresAt
|
|
446
1013
|
: dispatch.lease.expiresAt;
|
|
447
|
-
const reservedSingletonKeys = await this.reserveSingletonKeys(missingSingletonKeys, owner, expiresAt, signal);
|
|
1014
|
+
const reservedSingletonKeys = await this.reserveSingletonKeys(missingSingletonKeys, owner, expiresAt, signal, admission.priority);
|
|
448
1015
|
admission.keys = [...admission.keys, ...reservedSingletonKeys]
|
|
449
1016
|
.sort((left, right) => left.localeCompare(right));
|
|
450
1017
|
}
|
|
@@ -455,13 +1022,16 @@ export class HostRuntimeLeaseManager {
|
|
|
455
1022
|
if (admission.parallelProjectKey && !this.parallelProjectOwned(admission.parallelProjectKey, owner)) {
|
|
456
1023
|
throw new Error("host_runtime_parallel_unreserved: matching shared project admission is required before start.");
|
|
457
1024
|
}
|
|
1025
|
+
await createCanonicalDirectory(attemptRoot);
|
|
1026
|
+
attemptRootIdentity = (await assertCanonicalDirectory(attemptRoot));
|
|
458
1027
|
await Promise.all([
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
1028
|
+
createCanonicalDirectory(temporaryPath),
|
|
1029
|
+
createCanonicalDirectory(logPath),
|
|
1030
|
+
createCanonicalDirectory(cachePath),
|
|
1031
|
+
createCanonicalDirectory(browserProfilePath),
|
|
1032
|
+
createCanonicalDirectory(testOutputPath),
|
|
464
1033
|
]);
|
|
1034
|
+
await assertHostAttemptRootIdentity(attemptRoot, attemptRootIdentity);
|
|
465
1035
|
await resetGeneratedOutputs(normalizedWorktreePath, testOutputPath);
|
|
466
1036
|
let lease = createRuntimeIsolationLease({
|
|
467
1037
|
leaseId: sandbox.executionLeaseId,
|
|
@@ -493,11 +1063,11 @@ export class HostRuntimeLeaseManager {
|
|
|
493
1063
|
singletonKeys,
|
|
494
1064
|
},
|
|
495
1065
|
});
|
|
496
|
-
await this.writeLedger(lease);
|
|
1066
|
+
await this.writeLedger(lease, undefined, attemptRootIdentity);
|
|
497
1067
|
lease = transitionRuntimeIsolationLease(lease, "starting");
|
|
498
|
-
await this.writeLedger(lease);
|
|
1068
|
+
await this.writeLedger(lease, undefined, attemptRootIdentity);
|
|
499
1069
|
lease = transitionRuntimeIsolationLease(lease, "active");
|
|
500
|
-
await this.writeLedger(lease);
|
|
1070
|
+
await this.writeLedger(lease, undefined, attemptRootIdentity);
|
|
501
1071
|
processSupervisor = this.processSupervisors.register({
|
|
502
1072
|
processSupervisorId,
|
|
503
1073
|
dispatchAttemptId: lease.dispatchAttemptId,
|
|
@@ -510,13 +1080,16 @@ export class HostRuntimeLeaseManager {
|
|
|
510
1080
|
repoId: dispatch.repo.id,
|
|
511
1081
|
sessionId: dispatch.session.id,
|
|
512
1082
|
attemptRoot,
|
|
1083
|
+
attemptRootIdentity,
|
|
513
1084
|
runtimeContract,
|
|
514
1085
|
runtimeContractFingerprint,
|
|
1086
|
+
inputCommitSha,
|
|
515
1087
|
executionClass: admission.executionClass,
|
|
516
|
-
parallelPromotionEligible: hostRuntimeParallelEligible(runtimeContract),
|
|
1088
|
+
parallelPromotionEligible: hostRuntimeParallelEligible(runtimeContract) && dispatch.cycleRun.phase !== "prepared_readiness",
|
|
517
1089
|
processSupervisor,
|
|
518
1090
|
adapterCleanups: [],
|
|
519
1091
|
pendingAdapterLaunches: new Map(),
|
|
1092
|
+
runtimeResourcesReleased: false,
|
|
520
1093
|
releasing: false,
|
|
521
1094
|
});
|
|
522
1095
|
this.pendingRenewals.delete(sandbox.dispatchAttemptId);
|
|
@@ -533,8 +1106,13 @@ export class HostRuntimeLeaseManager {
|
|
|
533
1106
|
await processSupervisor.stopAll(dispatch.lease.fencingToken, "Host runtime start rolled back.").catch(() => undefined);
|
|
534
1107
|
this.processSupervisors.unregister(sandbox.dispatchAttemptId, dispatch.lease.fencingToken);
|
|
535
1108
|
}
|
|
536
|
-
await
|
|
537
|
-
|
|
1109
|
+
const attemptRemoved = await retainHostDiagnostics(this.stateRoot, attemptRoot, sandbox.dispatchAttemptId, attemptRootIdentity, this.executionRoot).then(async () => {
|
|
1110
|
+
await removeHostAttemptRoot(attemptRoot);
|
|
1111
|
+
return true;
|
|
1112
|
+
}).catch(() => false); // A retention failure preserves the original evidence and ownership ledger.
|
|
1113
|
+
if (attemptRemoved) {
|
|
1114
|
+
await fs.rm(this.ledgerPath(sandbox.dispatchAttemptId), { force: true }).catch(() => undefined);
|
|
1115
|
+
}
|
|
538
1116
|
this.releaseSingletonKeys(sandbox.dispatchAttemptId, dispatch.lease.fencingToken);
|
|
539
1117
|
this.releaseParallelProjects(sandbox.dispatchAttemptId, dispatch.lease.fencingToken);
|
|
540
1118
|
this.deleteAdmission(sandbox.dispatchAttemptId, dispatch.lease.fencingToken);
|
|
@@ -549,6 +1127,7 @@ export class HostRuntimeLeaseManager {
|
|
|
549
1127
|
catch {
|
|
550
1128
|
return;
|
|
551
1129
|
}
|
|
1130
|
+
await this.preparedReadiness.renew(dispatchAttemptId, fencingToken, expiresAt);
|
|
552
1131
|
const active = this.active.get(dispatchAttemptId);
|
|
553
1132
|
if (!active) {
|
|
554
1133
|
if (Number.isFinite(Date.parse(expiresAt))) {
|
|
@@ -562,13 +1141,15 @@ export class HostRuntimeLeaseManager {
|
|
|
562
1141
|
}
|
|
563
1142
|
return;
|
|
564
1143
|
}
|
|
1144
|
+
if (active.releasing || active.lease.state === "releasing" || active.lease.state === "released")
|
|
1145
|
+
return;
|
|
565
1146
|
const renewed = renewRuntimeIsolationLease(active.lease, fencingToken, expiresAt);
|
|
566
1147
|
if (!renewed)
|
|
567
1148
|
return;
|
|
568
1149
|
active.lease = renewed;
|
|
569
1150
|
await this.writeLedger(renewed);
|
|
570
1151
|
}
|
|
571
|
-
async startConfiguredProject(dispatchAttemptId, fencingToken, signal) {
|
|
1152
|
+
async startConfiguredProject(dispatchAttemptId, fencingToken, signal, onProjectCheckpoint) {
|
|
572
1153
|
const active = this.active.get(dispatchAttemptId);
|
|
573
1154
|
if (!active || active.lease.fencingToken !== fencingToken || active.lease.state !== "active" || active.releasing) {
|
|
574
1155
|
throw new Error("host_runtime_project_unavailable: an active matching host runtime lease is required.");
|
|
@@ -576,7 +1157,7 @@ export class HostRuntimeLeaseManager {
|
|
|
576
1157
|
if (active.executionClass === "analysis_only") {
|
|
577
1158
|
return {
|
|
578
1159
|
lease: active.lease,
|
|
579
|
-
environment: hostRuntimeEnvironment(active.sessionId, active.lease, active.runtimeContract, active.executionClass),
|
|
1160
|
+
environment: hostRuntimeEnvironment(active.sessionId, active.lease, active.runtimeContract, active.executionClass, active.databaseCapability),
|
|
580
1161
|
runtimeContract: active.runtimeContract,
|
|
581
1162
|
executionClass: active.executionClass,
|
|
582
1163
|
};
|
|
@@ -585,7 +1166,7 @@ export class HostRuntimeLeaseManager {
|
|
|
585
1166
|
if (configuredPorts.length === 0) {
|
|
586
1167
|
return {
|
|
587
1168
|
lease: active.lease,
|
|
588
|
-
environment: hostRuntimeEnvironment(active.sessionId, active.lease, active.runtimeContract, active.executionClass),
|
|
1169
|
+
environment: hostRuntimeEnvironment(active.sessionId, active.lease, active.runtimeContract, active.executionClass, active.databaseCapability),
|
|
589
1170
|
runtimeContract: active.runtimeContract,
|
|
590
1171
|
executionClass: active.executionClass,
|
|
591
1172
|
};
|
|
@@ -594,21 +1175,75 @@ export class HostRuntimeLeaseManager {
|
|
|
594
1175
|
const logDirectory = active.lease.resources.paths.find(entry => entry.role === "logs")?.path;
|
|
595
1176
|
if (!logDirectory)
|
|
596
1177
|
throw new Error("host_runtime_project_invalid: attempt log directory is unavailable.");
|
|
597
|
-
|
|
1178
|
+
if (this.databaseBroker && !active.databaseCapability
|
|
1179
|
+
&& configuredPorts.some(port => /^dotnet(?:\.exe)?$/iu.test(path.basename(port.launch.command)))) {
|
|
1180
|
+
// An ASP.NET runtime may require an isolated database (Forge.Api refuses to start in a
|
|
1181
|
+
// managed host attempt without one). Issue it before the first launch. Release waits for
|
|
1182
|
+
// an in-flight provision, so a database issued during release is still dropped.
|
|
1183
|
+
const provision = this.databaseBroker.provision(dispatchAttemptId, fencingToken, signal)
|
|
1184
|
+
.then(capability => {
|
|
1185
|
+
active.databaseCapability = capability;
|
|
1186
|
+
return capability;
|
|
1187
|
+
});
|
|
1188
|
+
active.databaseProvision = provision;
|
|
1189
|
+
await provision;
|
|
1190
|
+
if (active.releasing) {
|
|
1191
|
+
throw new Error("host_runtime_project_unavailable: the host runtime lease was released during database provisioning.");
|
|
1192
|
+
}
|
|
1193
|
+
await onProjectCheckpoint?.({
|
|
1194
|
+
code: "host_runtime_database_provisioned",
|
|
1195
|
+
message: `Provisioned isolated attempt database ${active.databaseCapability.databaseName} on the native test cluster.`,
|
|
1196
|
+
data: { database: active.databaseCapability.databaseName },
|
|
1197
|
+
});
|
|
1198
|
+
}
|
|
1199
|
+
// Runtime output is persisted to attempt logs; the database login it may print is redacted.
|
|
1200
|
+
const capabilitySecrets = attemptDatabaseCapabilitySecrets(active.databaseCapability?.connectionString);
|
|
1201
|
+
const launcher = new AttemptNodeRuntimeLauncher(active.processSupervisor, owner, logDirectory, this.listenerInspector, capabilitySecrets);
|
|
598
1202
|
for (const configuredPort of configuredPorts) {
|
|
599
1203
|
const workingDirectory = await containedWorkingDirectory(active.lease.worktreePath, configuredPort.launch.workingDirectory);
|
|
1204
|
+
const buildEnvironment = {
|
|
1205
|
+
...configuredPort.launch.environment,
|
|
1206
|
+
...hostRuntimeEnvironment(active.sessionId, active.lease, active.runtimeContract, active.executionClass, active.databaseCapability),
|
|
1207
|
+
};
|
|
1208
|
+
const prebuild = createAspNetRuntimePrebuildPlan({
|
|
1209
|
+
command: configuredPort.launch.command,
|
|
1210
|
+
args: configuredPort.launch.args,
|
|
1211
|
+
cwd: workingDirectory,
|
|
1212
|
+
worktreePath: active.lease.worktreePath,
|
|
1213
|
+
environment: buildEnvironment,
|
|
1214
|
+
inputCommitSha: active.inputCommitSha,
|
|
1215
|
+
runtimeContractFingerprint: active.runtimeContractFingerprint,
|
|
1216
|
+
});
|
|
1217
|
+
if (prebuild) {
|
|
1218
|
+
await onProjectCheckpoint?.({
|
|
1219
|
+
code: "host_runtime_build_started",
|
|
1220
|
+
message: `Preparing the ASP.NET runtime for commit ${active.inputCommitSha.slice(0, 12)} before binding its assigned port.`,
|
|
1221
|
+
data: { input_commit_sha: active.inputCommitSha, project_path: prebuild.projectPath },
|
|
1222
|
+
});
|
|
1223
|
+
const build = await ensureAspNetRuntimePrebuilt(prebuild, async (plan, buildSignal) => await executeAspNetRuntimeBuild(active.processSupervisor, logDirectory, plan, buildSignal, capabilitySecrets), signal);
|
|
1224
|
+
await onProjectCheckpoint?.({
|
|
1225
|
+
code: build.cacheHit ? "host_runtime_build_cached" : "host_runtime_build_completed",
|
|
1226
|
+
message: build.cacheHit
|
|
1227
|
+
? `Reused the verified ASP.NET build for commit ${active.inputCommitSha.slice(0, 12)}.`
|
|
1228
|
+
: `ASP.NET runtime build completed for commit ${active.inputCommitSha.slice(0, 12)}; launching without another build.`,
|
|
1229
|
+
data: {
|
|
1230
|
+
input_commit_sha: active.inputCommitSha,
|
|
1231
|
+
build_fingerprint: build.fingerprint,
|
|
1232
|
+
receipt_path: build.receiptPath,
|
|
1233
|
+
},
|
|
1234
|
+
});
|
|
1235
|
+
}
|
|
600
1236
|
await this.launchNodeRuntime(dispatchAttemptId, fencingToken, configuredPort.name, {
|
|
601
1237
|
command: configuredPort.launch.command,
|
|
602
1238
|
args: configuredPort.launch.args,
|
|
603
1239
|
cwd: workingDirectory,
|
|
604
1240
|
environment: {
|
|
605
|
-
...
|
|
606
|
-
...hostRuntimeEnvironment(active.sessionId, active.lease, active.runtimeContract, active.executionClass),
|
|
1241
|
+
...buildEnvironment,
|
|
607
1242
|
},
|
|
608
1243
|
}, launcher, signal);
|
|
609
1244
|
}
|
|
610
1245
|
const promotedLease = await this.promoteVerifiedRuntime(active, active.lease);
|
|
611
|
-
const runtimeEnvironment = hostRuntimeEnvironment(active.sessionId, promotedLease, active.runtimeContract, active.executionClass);
|
|
1246
|
+
const runtimeEnvironment = hostRuntimeEnvironment(active.sessionId, promotedLease, active.runtimeContract, active.executionClass, active.databaseCapability);
|
|
612
1247
|
const playwright = active.runtimeContract.contract.runtime.playwright;
|
|
613
1248
|
if (!playwright) {
|
|
614
1249
|
return {
|
|
@@ -637,6 +1272,7 @@ export class HostRuntimeLeaseManager {
|
|
|
637
1272
|
testOutputDirectory: pathFor("test_output"),
|
|
638
1273
|
browserProfileDirectory: pathFor("browser_profile"),
|
|
639
1274
|
baseUrl,
|
|
1275
|
+
ownershipToken: uuidHex(promotedLease.dispatchAttemptId, "dispatch attempt"),
|
|
640
1276
|
});
|
|
641
1277
|
return {
|
|
642
1278
|
lease: promotedLease,
|
|
@@ -649,7 +1285,18 @@ export class HostRuntimeLeaseManager {
|
|
|
649
1285
|
const active = this.active.get(dispatchAttemptId);
|
|
650
1286
|
if (!active || active.lease.fencingToken !== fencingToken || active.releasing)
|
|
651
1287
|
return null;
|
|
652
|
-
return new AttemptSupervisedProviderProcessRunner(active.processSupervisor, hostPortOwner(active.lease), this.listenerInspector, active.executionClass === "analysis_only");
|
|
1288
|
+
return new AttemptSupervisedProviderProcessRunner(active.processSupervisor, hostPortOwner(active.lease), this.listenerInspector, active.executionClass === "analysis_only", () => this.markCapacityReservationRunning(dispatchAttemptId, fencingToken));
|
|
1289
|
+
}
|
|
1290
|
+
markCapacityReservationRunning(dispatchAttemptId, fencingToken) {
|
|
1291
|
+
const reservation = this.capacityReservations.get(dispatchAttemptId);
|
|
1292
|
+
if (reservation?.owner.fencingToken === fencingToken)
|
|
1293
|
+
reservation.state = "running";
|
|
1294
|
+
}
|
|
1295
|
+
providerMemoryTelemetry(dispatchAttemptId, fencingToken) {
|
|
1296
|
+
const active = this.active.get(dispatchAttemptId);
|
|
1297
|
+
if (!active || active.lease.fencingToken !== fencingToken || active.releasing)
|
|
1298
|
+
return null;
|
|
1299
|
+
return active.processSupervisor.providerMemoryTelemetry();
|
|
653
1300
|
}
|
|
654
1301
|
async promoteVerifiedRuntime(active, runtimeLease) {
|
|
655
1302
|
if (!active.parallelPromotionEligible)
|
|
@@ -668,12 +1315,15 @@ export class HostRuntimeLeaseManager {
|
|
|
668
1315
|
await this.writeLedger(promotedLease);
|
|
669
1316
|
active.lease = promotedLease;
|
|
670
1317
|
try {
|
|
671
|
-
const promotion = await this.promotionStore.observe(active.repoId, active.runtimeContract, runtimeLease.dispatchAttemptId, verifiedPortNames);
|
|
1318
|
+
const promotion = await this.promotionStore.observe(active.repoId, active.runtimeContract, runtimeLease.dispatchAttemptId, verifiedPortNames, active.inputCommitSha);
|
|
672
1319
|
this.knownPromotions.set(promotion.repoId, promotion);
|
|
673
|
-
|
|
1320
|
+
const owner = {
|
|
674
1321
|
dispatchAttemptId: runtimeLease.dispatchAttemptId,
|
|
675
1322
|
fencingToken: runtimeLease.fencingToken,
|
|
676
|
-
}
|
|
1323
|
+
};
|
|
1324
|
+
if (!this.parallelProjectOwned(projectKey, owner)) {
|
|
1325
|
+
this.convertSingletonToParallelProject(projectKey, owner, promotion);
|
|
1326
|
+
}
|
|
677
1327
|
const admission = this.admissions.get(runtimeLease.dispatchAttemptId);
|
|
678
1328
|
if (admission?.fencingToken === runtimeLease.fencingToken) {
|
|
679
1329
|
this.admissions.set(runtimeLease.dispatchAttemptId, {
|
|
@@ -757,9 +1407,12 @@ export class HostRuntimeLeaseManager {
|
|
|
757
1407
|
unlinkAbortController(controller, signal);
|
|
758
1408
|
}
|
|
759
1409
|
}
|
|
760
|
-
async release(dispatchAttemptId, fencingToken) {
|
|
1410
|
+
async release(dispatchAttemptId, fencingToken, preservePreparation = false) {
|
|
1411
|
+
if (!preservePreparation)
|
|
1412
|
+
this.preparedReadiness.cancel(dispatchAttemptId, fencingToken);
|
|
761
1413
|
const active = this.active.get(dispatchAttemptId);
|
|
762
|
-
|
|
1414
|
+
if (!preservePreparation)
|
|
1415
|
+
this.pendingRenewals.delete(dispatchAttemptId);
|
|
763
1416
|
if (!active) {
|
|
764
1417
|
this.cancelAdmissionWaiters(dispatchAttemptId, fencingToken, "Managed host admission was released before acquisition.");
|
|
765
1418
|
this.releaseSingletonKeys(dispatchAttemptId, fencingToken);
|
|
@@ -774,17 +1427,57 @@ export class HostRuntimeLeaseManager {
|
|
|
774
1427
|
pending.controller.abort("Host runtime release cancelled the pending adapter launch.");
|
|
775
1428
|
}
|
|
776
1429
|
await Promise.allSettled([...active.pendingAdapterLaunches.values()].map(pending => pending.promise));
|
|
1430
|
+
await active.databaseProvision?.catch(() => undefined);
|
|
777
1431
|
let releasing = transitionRuntimeIsolationLease(active.lease, "releasing");
|
|
778
1432
|
active.lease = releasing;
|
|
779
1433
|
await this.writeLedger(releasing);
|
|
780
1434
|
try {
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
1435
|
+
if (!active.runtimeResourcesReleased) {
|
|
1436
|
+
await active.processSupervisor.stopAll(fencingToken, "Host runtime lease released.");
|
|
1437
|
+
await releaseAdapterResources(active.adapterCleanups);
|
|
1438
|
+
if (active.databaseCapability && this.databaseBroker) {
|
|
1439
|
+
// A database the broker cannot drop now is reaped later; it must not hold host capacity.
|
|
1440
|
+
await this.databaseBroker.release(dispatchAttemptId, fencingToken).catch(() => "retained");
|
|
1441
|
+
}
|
|
1442
|
+
active.runtimeResourcesReleased = true;
|
|
1443
|
+
// Runtime admission protects live processes, listeners, ports, and their
|
|
1444
|
+
// memory commitment. Once exact process and adapter cleanup succeeds,
|
|
1445
|
+
// diagnostics retention remains retryable evidence work and must not keep
|
|
1446
|
+
// consuming host capacity or blocking the next project dispatch.
|
|
1447
|
+
this.processSupervisors.unregister(dispatchAttemptId, fencingToken);
|
|
1448
|
+
this.releaseSingletonKeys(dispatchAttemptId, fencingToken);
|
|
1449
|
+
this.releaseParallelProjects(dispatchAttemptId, fencingToken);
|
|
1450
|
+
this.deleteAdmission(dispatchAttemptId, fencingToken);
|
|
1451
|
+
}
|
|
1452
|
+
let evidenceRetainedAtAttemptRoot = false;
|
|
1453
|
+
try {
|
|
1454
|
+
const diagnosticsPath = await retainHostDiagnostics(this.stateRoot, active.attemptRoot, dispatchAttemptId, active.attemptRootIdentity, this.executionRoot);
|
|
1455
|
+
releasing = withRuntimeIsolationResources(releasing, {
|
|
1456
|
+
...releasing.resources,
|
|
1457
|
+
paths: [...releasing.resources.paths.filter(entry => entry.role !== "diagnostics_archive"),
|
|
1458
|
+
{ role: "diagnostics_archive", path: diagnosticsPath }],
|
|
1459
|
+
});
|
|
1460
|
+
}
|
|
1461
|
+
catch {
|
|
1462
|
+
// Runtime teardown is already proven above. Diagnostics archival is
|
|
1463
|
+
// deliberately non-destructive and may refuse a changed or unsupported
|
|
1464
|
+
// attempt-root inventory. Preserve that root and its released ledger for
|
|
1465
|
+
// operator inspection/reconciliation without converting completed
|
|
1466
|
+
// teardown into a capacity-owning degraded runtime.
|
|
1467
|
+
evidenceRetainedAtAttemptRoot = true;
|
|
1468
|
+
}
|
|
1469
|
+
if (!evidenceRetainedAtAttemptRoot)
|
|
1470
|
+
await removeHostAttemptRoot(active.attemptRoot);
|
|
1471
|
+
const released = transitionRuntimeIsolationLease(releasing, "released");
|
|
1472
|
+
// Publish the terminal in-memory state before its serialized write. Any writer
|
|
1473
|
+
// that already captured this active owner must observe the terminal fence and
|
|
1474
|
+
// cannot recreate a nonterminal ledger after release completes.
|
|
1475
|
+
active.lease = released;
|
|
1476
|
+
await this.writeLedger(released);
|
|
1477
|
+
if (!evidenceRetainedAtAttemptRoot) {
|
|
1478
|
+
await fs.rm(this.ledgerPath(dispatchAttemptId), { force: true });
|
|
1479
|
+
}
|
|
1480
|
+
releasing = released;
|
|
788
1481
|
this.active.delete(dispatchAttemptId);
|
|
789
1482
|
this.processSupervisors.unregister(dispatchAttemptId, fencingToken);
|
|
790
1483
|
this.releaseSingletonKeys(dispatchAttemptId, fencingToken);
|
|
@@ -825,23 +1518,27 @@ export class HostRuntimeLeaseManager {
|
|
|
825
1518
|
}
|
|
826
1519
|
return ledgers;
|
|
827
1520
|
}
|
|
828
|
-
async recoverPersistedProcesses(lease, attemptRoot) {
|
|
1521
|
+
async recoverPersistedProcesses(lease, attemptRoot, attemptRootIdentity) {
|
|
829
1522
|
const logsRoot = path.join(attemptRoot, "logs");
|
|
830
1523
|
const expectedAttemptRoot = path.resolve(this.attemptRoot(lease.dispatchAttemptId));
|
|
831
1524
|
if (path.resolve(attemptRoot) !== expectedAttemptRoot) {
|
|
832
1525
|
throw new Error("host_runtime_reaper_invalid: attempt path is outside the fenced attempt root.");
|
|
833
1526
|
}
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
1527
|
+
await assertHostAttemptRootIdentity(attemptRoot, attemptRootIdentity);
|
|
1528
|
+
const rootEntries = await readPinnedSourceDirectory(attemptRoot, attemptRootIdentity);
|
|
1529
|
+
const logsEntry = rootEntries.find(entry => entry.name === "logs");
|
|
1530
|
+
if (!logsEntry)
|
|
1531
|
+
return [];
|
|
1532
|
+
if (logsEntry.kind !== "directory") {
|
|
1533
|
+
throw new Error("host_runtime_reaper_invalid: process ownership log path is not an ordinary directory.");
|
|
1534
|
+
}
|
|
1535
|
+
const entries = await readPinnedSourceDirectory(logsRoot, logsEntry.identity);
|
|
839
1536
|
const outcomes = [];
|
|
840
1537
|
for (const entry of entries) {
|
|
841
|
-
if (
|
|
1538
|
+
if (entry.kind !== "file" || !/^process-supervisor\.[a-z][a-z0-9_-]{0,31}\.\d+\.json$/u.test(entry.name))
|
|
842
1539
|
continue;
|
|
843
1540
|
const metadataPath = path.join(logsRoot, entry.name);
|
|
844
|
-
const value = JSON.parse(await
|
|
1541
|
+
const value = JSON.parse((await readCanonicalSourceFile(metadataPath, maximumProcessRecoveryMetadataBytes, entry.identity)).toString("utf8"));
|
|
845
1542
|
if (!validHostProcessSupervisorMetadata(value, lease)) {
|
|
846
1543
|
throw new Error("host_runtime_reaper_invalid: process ownership metadata did not match the runtime lease.");
|
|
847
1544
|
}
|
|
@@ -856,7 +1553,7 @@ export class HostRuntimeLeaseManager {
|
|
|
856
1553
|
const degraded = transitionHostLeaseForCleanup(lease, "degraded");
|
|
857
1554
|
const repoId = ledger.repoId ?? repoIdFromProjectSingletonKeys(lease.resources.singletonKeys);
|
|
858
1555
|
this.rememberDegraded(degraded, repoId);
|
|
859
|
-
await this.writeLedger(degraded, repoId ?? undefined).catch(() => undefined);
|
|
1556
|
+
await this.writeLedger(degraded, repoId ?? undefined, deserializeHostAttemptRootIdentity(ledger.attemptRootIdentity)).catch(() => undefined);
|
|
860
1557
|
}
|
|
861
1558
|
rememberDegraded(lease, repoId) {
|
|
862
1559
|
this.degraded.set(lease.dispatchAttemptId, { lease, repoId });
|
|
@@ -876,13 +1573,26 @@ export class HostRuntimeLeaseManager {
|
|
|
876
1573
|
this.releaseParallelProjects(dispatchAttemptId, fencingToken);
|
|
877
1574
|
}
|
|
878
1575
|
attemptRoot(dispatchAttemptId) {
|
|
879
|
-
return path.join(this.
|
|
1576
|
+
return path.join(this.executionRoot, "a", compactHostAttemptId(dispatchAttemptId));
|
|
880
1577
|
}
|
|
881
|
-
|
|
1578
|
+
writeLedger(runtimeLease, explicitRepoId, explicitAttemptRootIdentity) {
|
|
1579
|
+
const activeOwner = this.active.get(runtimeLease.dispatchAttemptId);
|
|
1580
|
+
const write = this.ledgerWriteTail.then(() => {
|
|
1581
|
+
if (activeOwner?.lease.fencingToken === runtimeLease.fencingToken
|
|
1582
|
+
&& activeOwner.lease.state === "released"
|
|
1583
|
+
&& runtimeLease.state !== "released") {
|
|
1584
|
+
return;
|
|
1585
|
+
}
|
|
1586
|
+
return this.writeLedgerAtomic(runtimeLease, explicitRepoId, explicitAttemptRootIdentity);
|
|
1587
|
+
});
|
|
1588
|
+
this.ledgerWriteTail = write.catch(() => undefined);
|
|
1589
|
+
return write;
|
|
1590
|
+
}
|
|
1591
|
+
async writeLedgerAtomic(runtimeLease, explicitRepoId, explicitAttemptRootIdentity) {
|
|
882
1592
|
const root = path.join(this.stateRoot, "runtime-ledger", "host");
|
|
883
1593
|
await ensurePrivateDirectory(root);
|
|
884
1594
|
const target = this.ledgerPath(runtimeLease.dispatchAttemptId);
|
|
885
|
-
const temporary = `${target}.${process.pid}.${Date.now()}.tmp`;
|
|
1595
|
+
const temporary = `${target}.${process.pid}.${Date.now()}.${randomUUID()}.tmp`;
|
|
886
1596
|
try {
|
|
887
1597
|
const handle = await fs.open(temporary, "wx", 0o600);
|
|
888
1598
|
try {
|
|
@@ -891,9 +1601,15 @@ export class HostRuntimeLeaseManager {
|
|
|
891
1601
|
?? this.admissions.get(runtimeLease.dispatchAttemptId)?.repoId
|
|
892
1602
|
?? this.degraded.get(runtimeLease.dispatchAttemptId)?.repoId
|
|
893
1603
|
?? undefined;
|
|
1604
|
+
const attemptRootIdentity = explicitAttemptRootIdentity
|
|
1605
|
+
?? this.active.get(runtimeLease.dispatchAttemptId)?.attemptRootIdentity
|
|
1606
|
+
?? undefined;
|
|
894
1607
|
const ledger = {
|
|
895
1608
|
schemaVersion: 1,
|
|
896
1609
|
...(repoId ? { repoId } : {}),
|
|
1610
|
+
...(attemptRootIdentity
|
|
1611
|
+
? { attemptRootIdentity: serializeHostAttemptRootIdentity(attemptRootIdentity) }
|
|
1612
|
+
: {}),
|
|
897
1613
|
runtimeLease,
|
|
898
1614
|
};
|
|
899
1615
|
await handle.writeFile(`${JSON.stringify(ledger)}\n`);
|
|
@@ -906,7 +1622,7 @@ export class HostRuntimeLeaseManager {
|
|
|
906
1622
|
}
|
|
907
1623
|
catch (error) {
|
|
908
1624
|
await fs.rm(temporary, { force: true }).catch(() => undefined);
|
|
909
|
-
throw error;
|
|
1625
|
+
throw new Error(`host_runtime_ledger_write_failed: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
910
1626
|
}
|
|
911
1627
|
}
|
|
912
1628
|
ledgerPath(dispatchAttemptId) {
|
|
@@ -936,7 +1652,7 @@ export class HostRuntimeLeaseManager {
|
|
|
936
1652
|
this.singletonOwners.delete(key);
|
|
937
1653
|
for (let index = 0; index < this.singletonWaiters.length;) {
|
|
938
1654
|
const waiter = this.singletonWaiters[index];
|
|
939
|
-
if (waiter.keys.length !== 1 || waiter.keys[0] !== key) {
|
|
1655
|
+
if (waiter.priority !== "foreground" || waiter.keys.length !== 1 || waiter.keys[0] !== key) {
|
|
940
1656
|
index += 1;
|
|
941
1657
|
continue;
|
|
942
1658
|
}
|
|
@@ -952,9 +1668,10 @@ export class HostRuntimeLeaseManager {
|
|
|
952
1668
|
parallelProjectOwned(key, owner) {
|
|
953
1669
|
return sameSingletonOwner(this.parallelProjectOwners.get(key)?.get(owner.dispatchAttemptId), owner);
|
|
954
1670
|
}
|
|
955
|
-
parallelProjectAvailable(key) {
|
|
1671
|
+
parallelProjectAvailable(key, priority = "foreground") {
|
|
956
1672
|
return !this.singletonOwners.has(key)
|
|
957
|
-
&& !this.singletonWaiters.some(waiter => waiter.keys.includes(key)
|
|
1673
|
+
&& !this.singletonWaiters.some(waiter => waiter.keys.includes(key)
|
|
1674
|
+
&& (priority === "background" || waiter.priority === "foreground"));
|
|
958
1675
|
}
|
|
959
1676
|
assignParallelProject(key, owner) {
|
|
960
1677
|
const owners = this.parallelProjectOwners.get(key) ?? new Map();
|
|
@@ -980,9 +1697,15 @@ export class HostRuntimeLeaseManager {
|
|
|
980
1697
|
if (this.admissions.get(dispatchAttemptId)?.fencingToken === fencingToken) {
|
|
981
1698
|
this.admissions.delete(dispatchAttemptId);
|
|
982
1699
|
}
|
|
1700
|
+
this.releaseHostCapacity(dispatchAttemptId, fencingToken);
|
|
983
1701
|
}
|
|
984
1702
|
renewAdmissionWaiterDeadline(dispatchAttemptId, fencingToken, expiresAt) {
|
|
985
|
-
for (const waiter of [
|
|
1703
|
+
for (const waiter of [
|
|
1704
|
+
...this.singletonWaiters,
|
|
1705
|
+
...this.parallelProjectWaiters,
|
|
1706
|
+
...this.capacityWaiters,
|
|
1707
|
+
...this.backgroundAdmissionWaiters,
|
|
1708
|
+
]) {
|
|
986
1709
|
if (!sameSingletonOwner(waiter.owner, { dispatchAttemptId, fencingToken })
|
|
987
1710
|
|| Date.parse(expiresAt) <= Date.parse(waiter.expiresAt))
|
|
988
1711
|
continue;
|
|
@@ -998,13 +1721,20 @@ export class HostRuntimeLeaseManager {
|
|
|
998
1721
|
}
|
|
999
1722
|
cancelAdmissionWaiters(dispatchAttemptId, fencingToken, reason) {
|
|
1000
1723
|
const owner = { dispatchAttemptId, fencingToken };
|
|
1001
|
-
for (const waiters of [
|
|
1724
|
+
for (const waiters of [
|
|
1725
|
+
this.singletonWaiters,
|
|
1726
|
+
this.parallelProjectWaiters,
|
|
1727
|
+
this.capacityWaiters,
|
|
1728
|
+
this.backgroundAdmissionWaiters,
|
|
1729
|
+
]) {
|
|
1002
1730
|
for (let index = waiters.length - 1; index >= 0; index -= 1) {
|
|
1003
1731
|
const waiter = waiters[index];
|
|
1004
1732
|
if (!sameSingletonOwner(waiter.owner, owner))
|
|
1005
1733
|
continue;
|
|
1006
1734
|
waiters.splice(index, 1);
|
|
1007
1735
|
clearAdmissionDeadline(waiter);
|
|
1736
|
+
if ("capacityRecheckTimer" in waiter)
|
|
1737
|
+
clearCapacityRecheck(waiter);
|
|
1008
1738
|
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
1009
1739
|
waiter.reject(new Error(`host_runtime_admission_cancelled: ${reason}`));
|
|
1010
1740
|
}
|
|
@@ -1060,7 +1790,7 @@ export class HostRuntimeLeaseManager {
|
|
|
1060
1790
|
waiter.reject(new HostRuntimeAdmissionExpiredError());
|
|
1061
1791
|
continue;
|
|
1062
1792
|
}
|
|
1063
|
-
if (!this.parallelProjectAvailable(waiter.key)) {
|
|
1793
|
+
if (!this.parallelProjectAvailable(waiter.key, waiter.priority)) {
|
|
1064
1794
|
index += 1;
|
|
1065
1795
|
continue;
|
|
1066
1796
|
}
|
|
@@ -1071,19 +1801,103 @@ export class HostRuntimeLeaseManager {
|
|
|
1071
1801
|
waiter.resolve();
|
|
1072
1802
|
}
|
|
1073
1803
|
}
|
|
1804
|
+
drainBackgroundAdmissionWaiters() {
|
|
1805
|
+
if (this.hasForegroundDemand())
|
|
1806
|
+
return;
|
|
1807
|
+
for (let index = 0; index < this.backgroundAdmissionWaiters.length;) {
|
|
1808
|
+
const waiter = this.backgroundAdmissionWaiters[index];
|
|
1809
|
+
if (waiter.signal?.aborted) {
|
|
1810
|
+
this.backgroundAdmissionWaiters.splice(index, 1);
|
|
1811
|
+
clearAdmissionDeadline(waiter);
|
|
1812
|
+
clearCapacityRecheck(waiter);
|
|
1813
|
+
waiter.signal.removeEventListener("abort", waiter.abort);
|
|
1814
|
+
this.clearPendingRenewal(waiter.owner);
|
|
1815
|
+
waiter.reject(cancellationError(waiter.signal));
|
|
1816
|
+
continue;
|
|
1817
|
+
}
|
|
1818
|
+
if (admissionExpired(waiter.expiresAt)) {
|
|
1819
|
+
this.backgroundAdmissionWaiters.splice(index, 1);
|
|
1820
|
+
clearAdmissionDeadline(waiter);
|
|
1821
|
+
clearCapacityRecheck(waiter);
|
|
1822
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
1823
|
+
this.clearPendingRenewal(waiter.owner);
|
|
1824
|
+
waiter.reject(new HostRuntimeAdmissionExpiredError());
|
|
1825
|
+
continue;
|
|
1826
|
+
}
|
|
1827
|
+
if (!this.singletonKeysAvailable(waiter.keys) || !this.hostCapacityAvailable(waiter.resources)) {
|
|
1828
|
+
index += 1;
|
|
1829
|
+
continue;
|
|
1830
|
+
}
|
|
1831
|
+
this.backgroundAdmissionWaiters.splice(index, 1);
|
|
1832
|
+
clearAdmissionDeadline(waiter);
|
|
1833
|
+
clearCapacityRecheck(waiter);
|
|
1834
|
+
waiter.signal?.removeEventListener("abort", waiter.abort);
|
|
1835
|
+
this.assignSingletonKeys(waiter.keys, waiter.owner);
|
|
1836
|
+
this.capacityReservations.set(waiter.owner.dispatchAttemptId, {
|
|
1837
|
+
owner: waiter.owner,
|
|
1838
|
+
resources: waiter.resources,
|
|
1839
|
+
state: "pending",
|
|
1840
|
+
});
|
|
1841
|
+
waiter.resolve(waiter.keys);
|
|
1842
|
+
}
|
|
1843
|
+
}
|
|
1074
1844
|
async refreshKnownPromotions() {
|
|
1075
1845
|
const promotions = await this.promotionStore.list();
|
|
1076
1846
|
this.knownPromotions.clear();
|
|
1077
1847
|
for (const promotion of promotions)
|
|
1078
1848
|
this.knownPromotions.set(promotion.repoId, promotion);
|
|
1079
1849
|
}
|
|
1080
|
-
async invalidatePromotion(repoId) {
|
|
1081
|
-
await this.promotionStore.invalidate(repoId);
|
|
1850
|
+
async invalidatePromotion(repoId, scope = "all") {
|
|
1851
|
+
await this.promotionStore.invalidate(repoId, scope);
|
|
1082
1852
|
this.knownPromotions.delete(repoId.trim().toLowerCase());
|
|
1083
1853
|
}
|
|
1084
1854
|
drainAdmissionWaiters() {
|
|
1855
|
+
this.reconcileReservationOwners();
|
|
1085
1856
|
this.drainSingletonWaiters();
|
|
1086
1857
|
this.drainParallelProjectWaiters();
|
|
1858
|
+
this.drainCapacityWaiters();
|
|
1859
|
+
this.drainBackgroundAdmissionWaiters();
|
|
1860
|
+
}
|
|
1861
|
+
reconcileReservationOwners() {
|
|
1862
|
+
for (const [key, owner] of this.singletonOwners) {
|
|
1863
|
+
if (!this.reservationOwnerIsLive(owner))
|
|
1864
|
+
this.singletonOwners.delete(key);
|
|
1865
|
+
}
|
|
1866
|
+
for (const [key, owners] of this.parallelProjectOwners) {
|
|
1867
|
+
for (const [dispatchAttemptId, owner] of owners) {
|
|
1868
|
+
const awaitingExclusiveUpgrade = this.singletonWaiters.some(waiter => waiter.keys.includes(key) && sameSingletonOwner(waiter.owner, owner));
|
|
1869
|
+
// A parallel owner cannot legitimately keep the shared claim after the
|
|
1870
|
+
// same fenced attempt has queued its exclusive downgrade. Reconcile the
|
|
1871
|
+
// half-upgraded state before testing singleton availability so one missed
|
|
1872
|
+
// release cannot make an attempt wait behind itself until lease expiry.
|
|
1873
|
+
if (awaitingExclusiveUpgrade || !this.reservationOwnerIsLive(owner)) {
|
|
1874
|
+
owners.delete(dispatchAttemptId);
|
|
1875
|
+
}
|
|
1876
|
+
}
|
|
1877
|
+
if (owners.size === 0)
|
|
1878
|
+
this.parallelProjectOwners.delete(key);
|
|
1879
|
+
}
|
|
1880
|
+
}
|
|
1881
|
+
reservationOwnerIsLive(owner) {
|
|
1882
|
+
if (this.acquiringAdmissions.has(hostRuntimeOwnerKey(owner)))
|
|
1883
|
+
return true;
|
|
1884
|
+
if (this.admissions.get(owner.dispatchAttemptId)?.fencingToken === owner.fencingToken)
|
|
1885
|
+
return true;
|
|
1886
|
+
if (this.active.get(owner.dispatchAttemptId)?.lease.fencingToken === owner.fencingToken)
|
|
1887
|
+
return true;
|
|
1888
|
+
if (this.degraded.get(owner.dispatchAttemptId)?.lease.fencingToken === owner.fencingToken)
|
|
1889
|
+
return true;
|
|
1890
|
+
if (sameSingletonOwner(this.capacityReservations.get(owner.dispatchAttemptId)?.owner, owner))
|
|
1891
|
+
return true;
|
|
1892
|
+
return [
|
|
1893
|
+
...this.singletonWaiters,
|
|
1894
|
+
...this.parallelProjectWaiters,
|
|
1895
|
+
...this.capacityWaiters,
|
|
1896
|
+
...this.backgroundAdmissionWaiters,
|
|
1897
|
+
].some(waiter => sameSingletonOwner(waiter.owner, owner));
|
|
1898
|
+
}
|
|
1899
|
+
hasAttemptRepresentation(dispatchAttemptId, fencingToken) {
|
|
1900
|
+
return this.reservationOwnerIsLive({ dispatchAttemptId, fencingToken });
|
|
1087
1901
|
}
|
|
1088
1902
|
}
|
|
1089
1903
|
class AttemptSupervisedProviderProcessRunner {
|
|
@@ -1091,14 +1905,20 @@ class AttemptSupervisedProviderProcessRunner {
|
|
|
1091
1905
|
owner;
|
|
1092
1906
|
listenerInspector;
|
|
1093
1907
|
enforceNoListeners;
|
|
1094
|
-
|
|
1908
|
+
onProcessTreeStarted;
|
|
1909
|
+
constructor(supervisor, owner, listenerInspector, enforceNoListeners, onProcessTreeStarted) {
|
|
1095
1910
|
this.supervisor = supervisor;
|
|
1096
1911
|
this.owner = owner;
|
|
1097
1912
|
this.listenerInspector = listenerInspector;
|
|
1098
1913
|
this.enforceNoListeners = enforceNoListeners;
|
|
1914
|
+
this.onProcessTreeStarted = onProcessTreeStarted;
|
|
1099
1915
|
}
|
|
1100
1916
|
async run(request, handlers = {}, signal) {
|
|
1917
|
+
const baselineListeners = this.enforceNoListeners
|
|
1918
|
+
? new Set(await this.listenerInspector.inspectLoopbackListeners())
|
|
1919
|
+
: new Set();
|
|
1101
1920
|
const child = await this.supervisor.startChildReady("provider", request, handlers, signal, { maxStartAttempts: request.stdin === undefined ? 1 : 3, retryDelayMs: 100 });
|
|
1921
|
+
this.onProcessTreeStarted();
|
|
1102
1922
|
if (!this.enforceNoListeners)
|
|
1103
1923
|
return await child.result;
|
|
1104
1924
|
let settled = false;
|
|
@@ -1106,7 +1926,8 @@ class AttemptSupervisedProviderProcessRunner {
|
|
|
1106
1926
|
const monitor = (async () => {
|
|
1107
1927
|
while (!settled) {
|
|
1108
1928
|
try {
|
|
1109
|
-
const ports = await this.listenerInspector.inspectOwnedLoopbackListeners(child, this.owner)
|
|
1929
|
+
const ports = (await this.listenerInspector.inspectOwnedLoopbackListeners(child, this.owner))
|
|
1930
|
+
.filter(port => !baselineListeners.has(port));
|
|
1110
1931
|
if (ports.length > 0) {
|
|
1111
1932
|
inspectionFailure = new Error(`host_analysis_listener_violation: analysis-only provider opened undeclared loopback listener(s): ${ports.join(", ")}.`);
|
|
1112
1933
|
await child.stop(this.owner.fencingToken).catch(() => undefined);
|
|
@@ -1170,6 +1991,12 @@ export class HostRuntimeAdmissionExpiredError extends Error {
|
|
|
1170
1991
|
this.name = "HostRuntimeAdmissionExpiredError";
|
|
1171
1992
|
}
|
|
1172
1993
|
}
|
|
1994
|
+
export class HostRuntimeCapacityRequestImpossibleError extends Error {
|
|
1995
|
+
constructor(reason) {
|
|
1996
|
+
super(`host_runtime_capacity_request_impossible: ${reason}`);
|
|
1997
|
+
this.name = "HostRuntimeCapacityRequestImpossibleError";
|
|
1998
|
+
}
|
|
1999
|
+
}
|
|
1173
2000
|
export function hostRuntimeEventResources(lease, runtimeContract, executionClass = "managed_runtime") {
|
|
1174
2001
|
const requestedHostConcurrency = runtimeContract?.contract.runtime.hostConcurrency ?? "serial";
|
|
1175
2002
|
const configuredLaunches = runtimeContract?.contract.runtime.ports.filter(port => port.launch !== null) ?? [];
|
|
@@ -1202,13 +2029,15 @@ export function hostRuntimeEventResources(lease, runtimeContract, executionClass
|
|
|
1202
2029
|
export function hostRuntimeSingletonKeys(dispatch) {
|
|
1203
2030
|
return [projectHostRuntimeSingletonKey(dispatch.repo.id)];
|
|
1204
2031
|
}
|
|
1205
|
-
function hostRuntimeEnvironment(sessionId, lease, runtimeContract, executionClass) {
|
|
2032
|
+
function hostRuntimeEnvironment(sessionId, lease, runtimeContract, executionClass, databaseCapability) {
|
|
1206
2033
|
const pathFor = (role) => lease.resources.paths.find(entry => entry.role === role)?.path ?? "";
|
|
1207
2034
|
const environment = {
|
|
1208
2035
|
FORGE_SESSION_ID: sessionId,
|
|
1209
2036
|
FORGE_RUNTIME_MODE: "host",
|
|
1210
2037
|
FORGE_RESOURCE_LEASE_ID: lease.leaseId,
|
|
1211
2038
|
FORGE_DISPATCH_ATTEMPT_ID: lease.dispatchAttemptId,
|
|
2039
|
+
FORGE_E2E_OWNERSHIP_MODE: "attempt",
|
|
2040
|
+
FORGE_E2E_OWNERSHIP_TOKEN: uuidHex(lease.dispatchAttemptId, "dispatch attempt"),
|
|
1212
2041
|
FORGE_EXECUTION_LEASE_ID: lease.executionLeaseId,
|
|
1213
2042
|
FORGE_FENCING_TOKEN: lease.fencingToken,
|
|
1214
2043
|
FORGE_WORKSPACE: lease.worktreePath,
|
|
@@ -1218,6 +2047,12 @@ function hostRuntimeEnvironment(sessionId, lease, runtimeContract, executionClas
|
|
|
1218
2047
|
FORGE_CACHE_DIR: pathFor("cache"),
|
|
1219
2048
|
FORGE_BROWSER_PROFILE_DIR: pathFor("browser_profile"),
|
|
1220
2049
|
FORGE_TEST_OUTPUT_DIR: pathFor("test_output"),
|
|
2050
|
+
// Project-authored ASP.NET code must not fall back to the durable Forge
|
|
2051
|
+
// database when no brokered attempt database exists. The API policy treats
|
|
2052
|
+
// this marker as a fail-closed isolation requirement.
|
|
2053
|
+
FORGE_ATTEMPT_DATABASE_POLICY: "isolated_only",
|
|
2054
|
+
FORGE_ATTEMPT_DATABASE_SCHEMA: `forge_attempt_${uuidHex(lease.dispatchAttemptId, "dispatch attempt")}`,
|
|
2055
|
+
FORGE_ATTEMPT_DATABASE_READINESS_PATH: path.join(pathFor("logs"), "attempt-database-readiness.json"),
|
|
1221
2056
|
FORGE_PROCESS_SUPERVISOR_ID: lease.resources.processSupervisorId ?? "",
|
|
1222
2057
|
FORGE_PROCESS_SUPERVISOR_METADATA_PATH: pathFor("process_supervisor_metadata"),
|
|
1223
2058
|
FORGE_HOST_CONCURRENCY: effectiveHostConcurrency(lease, runtimeContract, executionClass),
|
|
@@ -1233,6 +2068,8 @@ function hostRuntimeEnvironment(sessionId, lease, runtimeContract, executionClas
|
|
|
1233
2068
|
};
|
|
1234
2069
|
if (lease.resources.singletonKeys.length === 0)
|
|
1235
2070
|
delete environment.FORGE_SINGLETON_KEYS;
|
|
2071
|
+
if (databaseCapability)
|
|
2072
|
+
environment[attemptDatabaseCapabilityEnvironment] = databaseCapability.connectionString;
|
|
1236
2073
|
const resolvedBaseUrls = [];
|
|
1237
2074
|
for (const assigned of lease.resources.ports) {
|
|
1238
2075
|
const configured = runtimeContract.contract.runtime.ports.find(port => port.name === assigned.name);
|
|
@@ -1257,6 +2094,45 @@ function hostRuntimeEnvironment(sessionId, lease, runtimeContract, executionClas
|
|
|
1257
2094
|
environment.FORGE_BASE_URL = resolvedBaseUrls[0];
|
|
1258
2095
|
return environment;
|
|
1259
2096
|
}
|
|
2097
|
+
async function executeAspNetRuntimeBuild(supervisor, logDirectory, plan, signal, exactSecrets = []) {
|
|
2098
|
+
const childName = `runtime-build-${plan.fingerprint.slice(-12)}`;
|
|
2099
|
+
const stdoutLogPath = path.join(logDirectory, `${childName}.stdout.log`);
|
|
2100
|
+
const stderrLogPath = path.join(logDirectory, `${childName}.stderr.log`);
|
|
2101
|
+
let stdoutTail = "";
|
|
2102
|
+
let stderrTail = "";
|
|
2103
|
+
const append = async (target, line) => {
|
|
2104
|
+
await fs.appendFile(target, `${redactText(line, exactSecrets)}\n`, { encoding: "utf8", mode: 0o600 });
|
|
2105
|
+
};
|
|
2106
|
+
const child = await supervisor.startChildReady(childName, {
|
|
2107
|
+
command: plan.command,
|
|
2108
|
+
args: plan.args,
|
|
2109
|
+
cwd: plan.cwd,
|
|
2110
|
+
env: plan.environment,
|
|
2111
|
+
captureOutput: false,
|
|
2112
|
+
}, {
|
|
2113
|
+
onStdoutLine: async (line) => {
|
|
2114
|
+
stdoutTail = appendRuntimeBuildTail(stdoutTail, redactText(line, exactSecrets));
|
|
2115
|
+
await append(stdoutLogPath, line);
|
|
2116
|
+
},
|
|
2117
|
+
onStderrLine: async (line) => {
|
|
2118
|
+
stderrTail = appendRuntimeBuildTail(stderrTail, redactText(line, exactSecrets));
|
|
2119
|
+
await append(stderrLogPath, line);
|
|
2120
|
+
},
|
|
2121
|
+
}, signal, { maxStartAttempts: 2, retryDelayMs: 250 });
|
|
2122
|
+
const result = await child.result;
|
|
2123
|
+
return {
|
|
2124
|
+
exitCode: result.exitCode,
|
|
2125
|
+
stdoutTail: appendRuntimeBuildTail(stdoutTail, redactText(result.stdout, exactSecrets)),
|
|
2126
|
+
stderrTail: appendRuntimeBuildTail(stderrTail, redactText(result.stderr, exactSecrets)),
|
|
2127
|
+
stdoutLogPath,
|
|
2128
|
+
stderrLogPath,
|
|
2129
|
+
};
|
|
2130
|
+
}
|
|
2131
|
+
function appendRuntimeBuildTail(current, value) {
|
|
2132
|
+
if (!value)
|
|
2133
|
+
return current;
|
|
2134
|
+
return `${current}${current ? "\n" : ""}${value}`.slice(-4_000);
|
|
2135
|
+
}
|
|
1260
2136
|
function effectiveHostConcurrency(lease, runtimeContract, executionClass = "managed_runtime") {
|
|
1261
2137
|
if (executionClass === "analysis_only") {
|
|
1262
2138
|
const analysisSingletonHeld = lease.resources.singletonKeys.length > 0;
|
|
@@ -1285,6 +2161,125 @@ function runtimeExecutionClass(dispatch) {
|
|
|
1285
2161
|
? "analysis_only"
|
|
1286
2162
|
: "managed_runtime";
|
|
1287
2163
|
}
|
|
2164
|
+
function runtimeAdmissionPriority(dispatch) {
|
|
2165
|
+
return dispatch.execution.lane.trim().toLowerCase() === "runtime_readiness"
|
|
2166
|
+
|| dispatch.cycleRun.phase.trim().toLowerCase() === "readiness_probe"
|
|
2167
|
+
? "background"
|
|
2168
|
+
: "foreground";
|
|
2169
|
+
}
|
|
2170
|
+
function enqueueAdmissionWaiter(waiters, waiter) {
|
|
2171
|
+
if (waiter.priority === "background") {
|
|
2172
|
+
waiters.push(waiter);
|
|
2173
|
+
return;
|
|
2174
|
+
}
|
|
2175
|
+
const backgroundIndex = waiters.findIndex(candidate => candidate.priority === "background");
|
|
2176
|
+
if (backgroundIndex < 0)
|
|
2177
|
+
waiters.push(waiter);
|
|
2178
|
+
else
|
|
2179
|
+
waiters.splice(backgroundIndex, 0, waiter);
|
|
2180
|
+
}
|
|
2181
|
+
function defaultHostResourceRequest() {
|
|
2182
|
+
const workerMemoryLimitBytes = configuredInteger("FORGE_HOST_DEFAULT_WORKER_MEMORY_LIMIT_BYTES", 2 * 1024 ** 3, 128 * 1024 ** 2, 512 * 1024 ** 3);
|
|
2183
|
+
const runtimeOverheadBytes = 128 * 1024 ** 2;
|
|
2184
|
+
return {
|
|
2185
|
+
sandboxSlots: 1,
|
|
2186
|
+
cpuMillis: 2000,
|
|
2187
|
+
workerMemoryLimitBytes,
|
|
2188
|
+
serviceMemoryLimitBytes: 0,
|
|
2189
|
+
runtimeOverheadBytes,
|
|
2190
|
+
memoryBytes: workerMemoryLimitBytes + runtimeOverheadBytes,
|
|
2191
|
+
workspaceBytes: 10 * 1024 ** 3,
|
|
2192
|
+
pids: 512,
|
|
2193
|
+
buildSlots: 1,
|
|
2194
|
+
};
|
|
2195
|
+
}
|
|
2196
|
+
function minimumSupportedHostResourceRequest() {
|
|
2197
|
+
return {
|
|
2198
|
+
sandboxSlots: 1,
|
|
2199
|
+
cpuMillis: 1000,
|
|
2200
|
+
workerMemoryLimitBytes: 1024 ** 3,
|
|
2201
|
+
serviceMemoryLimitBytes: 0,
|
|
2202
|
+
runtimeOverheadBytes: 128 * 1024 ** 2,
|
|
2203
|
+
memoryBytes: 1152 * 1024 ** 2,
|
|
2204
|
+
workspaceBytes: architectMinimumWorkspaceEstimateBytes,
|
|
2205
|
+
pids: 256,
|
|
2206
|
+
buildSlots: 0,
|
|
2207
|
+
};
|
|
2208
|
+
}
|
|
2209
|
+
function normalizedHostResourceRequest(resources) {
|
|
2210
|
+
const defaults = defaultHostResourceRequest();
|
|
2211
|
+
const workerMemoryLimitBytes = positiveResourceValue(resources?.workerMemoryLimitBytes, defaults.workerMemoryLimitBytes);
|
|
2212
|
+
const serviceMemoryLimitBytes = nonNegativeResourceValue(resources?.serviceMemoryLimitBytes, defaults.serviceMemoryLimitBytes);
|
|
2213
|
+
const runtimeOverheadBytes = nonNegativeResourceValue(resources?.runtimeOverheadBytes, defaults.runtimeOverheadBytes);
|
|
2214
|
+
return {
|
|
2215
|
+
sandboxSlots: positiveResourceValue(resources?.sandboxSlots, defaults.sandboxSlots),
|
|
2216
|
+
cpuMillis: positiveResourceValue(resources?.cpuMillis, defaults.cpuMillis),
|
|
2217
|
+
workerMemoryLimitBytes,
|
|
2218
|
+
serviceMemoryLimitBytes,
|
|
2219
|
+
runtimeOverheadBytes,
|
|
2220
|
+
memoryBytes: workerMemoryLimitBytes + serviceMemoryLimitBytes + runtimeOverheadBytes,
|
|
2221
|
+
workspaceBytes: positiveResourceValue(resources?.workspaceBytes, defaults.workspaceBytes),
|
|
2222
|
+
pids: positiveResourceValue(resources?.pids, defaults.pids),
|
|
2223
|
+
buildSlots: nonNegativeResourceValue(resources?.buildSlots, defaults.buildSlots),
|
|
2224
|
+
};
|
|
2225
|
+
}
|
|
2226
|
+
function positiveResourceValue(value, fallback) {
|
|
2227
|
+
return Number.isFinite(value) && value > 0 ? Math.floor(value) : fallback;
|
|
2228
|
+
}
|
|
2229
|
+
function nonNegativeResourceValue(value, fallback) {
|
|
2230
|
+
return Number.isFinite(value) && value >= 0 ? Math.floor(value) : fallback;
|
|
2231
|
+
}
|
|
2232
|
+
function configuredInteger(name, fallback, minimum, maximum) {
|
|
2233
|
+
const parsed = Number.parseInt(process.env[name] ?? "", 10);
|
|
2234
|
+
return Number.isFinite(parsed) ? Math.max(minimum, Math.min(maximum, parsed)) : fallback;
|
|
2235
|
+
}
|
|
2236
|
+
function loadHostCapacityConfiguration(capacityResourceSample) {
|
|
2237
|
+
const sampledCpuCount = capacityResourceSample
|
|
2238
|
+
? Math.max(1, Math.floor(capacityResourceSample.totalCpuMillis / 1_000))
|
|
2239
|
+
: os.cpus().length;
|
|
2240
|
+
const totalMemoryBytes = capacityResourceSample?.totalMemoryBytes ?? os.totalmem();
|
|
2241
|
+
const reserveFloorBytes = configuredInteger("FORGE_HOST_MEMORY_RESERVE_FLOOR_BYTES", 1024 ** 3, 512 * 1024 ** 2, 128 * 1024 ** 3);
|
|
2242
|
+
const reserveCeilingBytes = configuredInteger("FORGE_HOST_MEMORY_RESERVE_CEILING_BYTES", 4 * 1024 ** 3, reserveFloorBytes, 128 * 1024 ** 3);
|
|
2243
|
+
const hostReserveBytes = resolveAdaptiveMemoryReserveBytes(totalMemoryBytes, {
|
|
2244
|
+
explicitBytes: configuredOptionalInteger("FORGE_HOST_MEMORY_RESERVE_BYTES", 512 * 1024 ** 2, 128 * 1024 ** 3),
|
|
2245
|
+
floorBytes: reserveFloorBytes,
|
|
2246
|
+
ceilingBytes: reserveCeilingBytes,
|
|
2247
|
+
percent: configuredNumber("FORGE_HOST_MEMORY_RESERVE_PERCENT", 12.5, 1, 50),
|
|
2248
|
+
});
|
|
2249
|
+
const minimumIdleReserveBytes = Math.min(hostReserveBytes, reserveFloorBytes);
|
|
2250
|
+
return {
|
|
2251
|
+
hardMaxAttempts: configuredInteger("FORGE_HOST_MAX_CONCURRENT_ATTEMPTS", Math.max(1, Math.min(8, Math.floor(sampledCpuCount / 2))), 1, 32),
|
|
2252
|
+
hostReserveBytes,
|
|
2253
|
+
idleHostReserveBytes: configuredInteger("FORGE_HOST_IDLE_MEMORY_RESERVE_BYTES", minimumIdleReserveBytes, 512 * 1024 ** 2, hostReserveBytes),
|
|
2254
|
+
cpuReserveMillis: configuredInteger("FORGE_HOST_CPU_RESERVE_MILLIS", 1000, 0, 32_000),
|
|
2255
|
+
workspaceReserveBytes: configuredInteger("FORGE_HOST_WORKSPACE_RESERVE_BYTES", 10 * 1024 ** 3, 1024 ** 3, Number.MAX_SAFE_INTEGER),
|
|
2256
|
+
memoryRecoverySamples: configuredInteger("FORGE_HOST_MEMORY_RECOVERY_SAMPLES", 2, 1, 10),
|
|
2257
|
+
memoryRecoverySampleIntervalMs: configuredInteger("FORGE_HOST_MEMORY_RECOVERY_SAMPLE_INTERVAL_MS", 1_000, 100, 30_000),
|
|
2258
|
+
};
|
|
2259
|
+
}
|
|
2260
|
+
function configuredOptionalInteger(name, minimum, maximum) {
|
|
2261
|
+
const raw = process.env[name];
|
|
2262
|
+
if (raw === undefined || raw.trim().length === 0)
|
|
2263
|
+
return null;
|
|
2264
|
+
const parsed = Number.parseInt(raw, 10);
|
|
2265
|
+
return Number.isFinite(parsed) ? Math.max(minimum, Math.min(maximum, parsed)) : null;
|
|
2266
|
+
}
|
|
2267
|
+
function configuredNumber(name, fallback, minimum, maximum) {
|
|
2268
|
+
const parsed = Number(process.env[name] ?? "");
|
|
2269
|
+
return Number.isFinite(parsed) && parsed > 0
|
|
2270
|
+
? Math.max(minimum, Math.min(maximum, parsed))
|
|
2271
|
+
: fallback;
|
|
2272
|
+
}
|
|
2273
|
+
function resourceSlots(available, requested) {
|
|
2274
|
+
if (requested <= 0)
|
|
2275
|
+
return Number.MAX_SAFE_INTEGER;
|
|
2276
|
+
return Math.max(0, Math.floor(available / requested));
|
|
2277
|
+
}
|
|
2278
|
+
function removeWaiter(waiters, waiter) {
|
|
2279
|
+
const index = waiters.indexOf(waiter);
|
|
2280
|
+
if (index >= 0)
|
|
2281
|
+
waiters.splice(index, 1);
|
|
2282
|
+
}
|
|
1288
2283
|
function capacityProject(projects, repoId) {
|
|
1289
2284
|
const normalized = repoId.trim().toLowerCase();
|
|
1290
2285
|
const existing = projects.get(normalized);
|
|
@@ -1327,6 +2322,9 @@ function repoIdFromProjectSingletonKey(key) {
|
|
|
1327
2322
|
function sameSingletonOwner(left, right) {
|
|
1328
2323
|
return left?.dispatchAttemptId === right.dispatchAttemptId && left.fencingToken === right.fencingToken;
|
|
1329
2324
|
}
|
|
2325
|
+
function hostRuntimeOwnerKey(owner) {
|
|
2326
|
+
return `${owner.dispatchAttemptId}:${owner.fencingToken}`;
|
|
2327
|
+
}
|
|
1330
2328
|
function hostPortOwner(lease) {
|
|
1331
2329
|
const processSupervisorId = lease.resources.processSupervisorId;
|
|
1332
2330
|
if (!processSupervisorId) {
|
|
@@ -1375,6 +2373,14 @@ async function releaseAdapterResources(cleanups) {
|
|
|
1375
2373
|
if (firstError !== undefined)
|
|
1376
2374
|
throw firstError;
|
|
1377
2375
|
}
|
|
2376
|
+
async function removeHostAttemptRoot(attemptRoot) {
|
|
2377
|
+
await fs.rm(attemptRoot, {
|
|
2378
|
+
recursive: true,
|
|
2379
|
+
force: true,
|
|
2380
|
+
maxRetries: 10,
|
|
2381
|
+
retryDelay: 100,
|
|
2382
|
+
});
|
|
2383
|
+
}
|
|
1378
2384
|
function cancellationError(signal) {
|
|
1379
2385
|
return new Error(String(signal?.reason ?? "Dispatch cancelled while waiting for host runtime singleton ownership."));
|
|
1380
2386
|
}
|
|
@@ -1384,9 +2390,25 @@ function validHostRuntimeLedger(value) {
|
|
|
1384
2390
|
const ledger = value;
|
|
1385
2391
|
return ledger.schemaVersion === 1
|
|
1386
2392
|
&& (ledger.repoId === undefined || /^[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}$/iu.test(ledger.repoId))
|
|
2393
|
+
&& (ledger.attemptRootIdentity === undefined
|
|
2394
|
+
|| validSerializedHostAttemptRootIdentity(ledger.attemptRootIdentity))
|
|
1387
2395
|
&& validRuntimeIsolationLease(ledger.runtimeLease)
|
|
1388
2396
|
&& ledger.runtimeLease.mode === "host";
|
|
1389
2397
|
}
|
|
2398
|
+
function validSerializedHostAttemptRootIdentity(value) {
|
|
2399
|
+
if (typeof value !== "object" || value === null)
|
|
2400
|
+
return false;
|
|
2401
|
+
const identity = value;
|
|
2402
|
+
return /^-?\d{1,64}$/u.test(identity.dev) && /^-?\d{1,64}$/u.test(identity.ino);
|
|
2403
|
+
}
|
|
2404
|
+
function serializeHostAttemptRootIdentity(identity) {
|
|
2405
|
+
return { dev: identity.dev.toString(), ino: identity.ino.toString() };
|
|
2406
|
+
}
|
|
2407
|
+
function deserializeHostAttemptRootIdentity(identity) {
|
|
2408
|
+
return identity && validSerializedHostAttemptRootIdentity(identity)
|
|
2409
|
+
? { dev: BigInt(identity.dev), ino: BigInt(identity.ino) }
|
|
2410
|
+
: null;
|
|
2411
|
+
}
|
|
1390
2412
|
function validHostProcessSupervisorMetadata(value, lease) {
|
|
1391
2413
|
if (typeof value !== "object" || value === null)
|
|
1392
2414
|
return false;
|
|
@@ -1435,6 +2457,11 @@ function clearAdmissionDeadline(waiter) {
|
|
|
1435
2457
|
clearTimeout(waiter.deadlineTimer);
|
|
1436
2458
|
waiter.deadlineTimer = null;
|
|
1437
2459
|
}
|
|
2460
|
+
function clearCapacityRecheck(waiter) {
|
|
2461
|
+
if (waiter.capacityRecheckTimer)
|
|
2462
|
+
clearInterval(waiter.capacityRecheckTimer);
|
|
2463
|
+
waiter.capacityRecheckTimer = null;
|
|
2464
|
+
}
|
|
1438
2465
|
async function ensurePrivateDirectory(directory) {
|
|
1439
2466
|
await fs.mkdir(directory, { recursive: true, mode: 0o700 });
|
|
1440
2467
|
if (process.platform !== "win32")
|