@loucompanion/forge-bridge 0.1.3-dev.0b2f81d660c0 → 0.1.3-dev.0d26c10244e1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +79 -0
- package/dist/bridge/bridge.base.d.ts +8 -1
- package/dist/bridge/bridge.service.d.ts +77 -3
- package/dist/bridge/bridge.service.js +1113 -34
- package/dist/bridge/bridge.types.d.ts +168 -1
- package/dist/cleanup/cleanup.service.d.ts +14 -2
- package/dist/cleanup/cleanup.service.js +469 -3
- package/dist/cleanup/cleanup.types.d.ts +10 -0
- package/dist/cleanup/index.d.ts +1 -1
- package/dist/cleanup/index.js +1 -1
- package/dist/cleanup/internals.d.ts +2 -2
- package/dist/cleanup/internals.js +1 -1
- package/dist/cli/cli.help.js +47 -1
- package/dist/cli/cli.service.d.ts +42 -2
- package/dist/cli/cli.service.js +357 -26
- package/dist/cli/local-config.d.ts +4 -0
- package/dist/cli/local-config.js +41 -1
- package/dist/runtime/aspnet-runtime-prebuild.d.ts +35 -0
- package/dist/runtime/aspnet-runtime-prebuild.js +168 -0
- package/dist/runtime/attempt-node-runtime-launcher.d.ts +17 -0
- package/dist/runtime/attempt-node-runtime-launcher.js +105 -0
- package/dist/runtime/attempt-process-supervisor-registry.d.ts +20 -0
- package/dist/runtime/attempt-process-supervisor-registry.js +64 -0
- package/dist/runtime/attempt-process-supervisor.d.ts +86 -0
- package/dist/runtime/attempt-process-supervisor.js +350 -0
- package/dist/runtime/generated-output-reset.d.ts +10 -0
- package/dist/runtime/generated-output-reset.js +103 -0
- package/dist/runtime/host-listener-inspector.d.ts +46 -0
- package/dist/runtime/host-listener-inspector.js +349 -0
- package/dist/runtime/host-port-broker.d.ts +76 -0
- package/dist/runtime/host-port-broker.js +333 -0
- package/dist/runtime/host-runtime-contract.d.ts +47 -0
- package/dist/runtime/host-runtime-contract.js +700 -0
- package/dist/runtime/host-runtime-lease.d.ts +179 -0
- package/dist/runtime/host-runtime-lease.js +1990 -0
- package/dist/runtime/host-runtime-promotion.d.ts +23 -0
- package/dist/runtime/host-runtime-promotion.js +159 -0
- package/dist/runtime/node-host-runtime-adapter.d.ts +54 -0
- package/dist/runtime/node-host-runtime-adapter.js +284 -0
- package/dist/runtime/playwright-host-runtime-adapter.d.ts +20 -0
- package/dist/runtime/playwright-host-runtime-adapter.js +220 -0
- package/dist/runtime/runtime-isolation-lease.d.ts +67 -0
- package/dist/runtime/runtime-isolation-lease.js +158 -0
- package/dist/runtime/workspace-resource-estimator.d.ts +16 -0
- package/dist/runtime/workspace-resource-estimator.js +165 -0
- package/dist/sandbox/command-runner.d.ts +4 -0
- package/dist/sandbox/command-runner.js +47 -0
- package/dist/sandbox/contract.d.ts +5 -0
- package/dist/sandbox/contract.js +276 -0
- package/dist/sandbox/event-budget.d.ts +11 -0
- package/dist/sandbox/event-budget.js +27 -0
- package/dist/sandbox/image-policy.d.ts +35 -0
- package/dist/sandbox/image-policy.js +247 -0
- package/dist/sandbox/index.d.ts +5 -0
- package/dist/sandbox/index.js +5 -0
- package/dist/sandbox/lease-watchdog.d.ts +18 -0
- package/dist/sandbox/lease-watchdog.js +67 -0
- package/dist/sandbox/preflight.d.ts +55 -0
- package/dist/sandbox/preflight.js +343 -0
- package/dist/sandbox/process.d.ts +20 -0
- package/dist/sandbox/process.js +209 -0
- package/dist/sandbox/provider-auth/codex.d.ts +26 -0
- package/dist/sandbox/provider-auth/codex.js +212 -0
- package/dist/sandbox/repository-lock.d.ts +5 -0
- package/dist/sandbox/repository-lock.js +119 -0
- package/dist/sandbox/runtime.d.ts +297 -0
- package/dist/sandbox/runtime.js +2338 -0
- package/dist/sandbox/sandbox.types.d.ts +86 -0
- package/dist/sandbox/sandbox.types.js +1 -0
- package/dist/sandbox/smoke.d.ts +76 -0
- package/dist/sandbox/smoke.js +380 -0
- package/dist/sandbox/worker-supervisor.d.ts +13 -0
- package/dist/sandbox/worker-supervisor.js +36 -0
- package/dist/sandbox/worker.d.ts +1 -0
- package/dist/sandbox/worker.js +225 -0
- package/dist/sandbox/workspace.service.d.ts +13 -0
- package/dist/sandbox/workspace.service.js +72 -0
- package/dist/session/managed-attempt-tools.d.ts +18 -0
- package/dist/session/managed-attempt-tools.js +797 -0
- package/dist/session/managed-build-coordinator.d.ts +5 -0
- package/dist/session/managed-build-coordinator.js +105 -0
- package/dist/session/phase-completion.artifacts.js +21 -3
- package/dist/session/phase-completion.d.ts +11 -2
- package/dist/session/phase-completion.helpers.d.ts +5 -0
- package/dist/session/phase-completion.helpers.js +51 -10
- package/dist/session/phase-completion.js +12 -3
- package/dist/session/phase-completion.prompts.d.ts +3 -3
- package/dist/session/phase-completion.prompts.js +21 -6
- package/dist/session/phase-completion.runtime.d.ts +13 -1
- package/dist/session/phase-completion.runtime.js +187 -8
- package/dist/session/phase-completion.schemas.js +108 -3
- package/dist/session/phase-completion.validation.d.ts +4 -1
- package/dist/session/phase-completion.validation.js +150 -21
- package/dist/session/provider-cli/ci-repair-readiness.d.ts +14 -0
- package/dist/session/provider-cli/ci-repair-readiness.js +105 -0
- package/dist/session/provider-cli/provider-cli.artifacts.d.ts +3 -2
- package/dist/session/provider-cli/provider-cli.artifacts.js +65 -7
- package/dist/session/provider-cli/provider-cli.base.d.ts +1 -0
- package/dist/session/provider-cli/provider-cli.deterministic.d.ts +2 -0
- package/dist/session/provider-cli/provider-cli.deterministic.js +66 -9
- package/dist/session/provider-cli/provider-cli.process.d.ts +16 -0
- package/dist/session/provider-cli/provider-cli.process.js +212 -25
- package/dist/session/provider-cli/provider-cli.prompts.d.ts +6 -1
- package/dist/session/provider-cli/provider-cli.prompts.js +129 -18
- package/dist/session/provider-cli/provider-cli.qa-sidecars.js +1 -0
- package/dist/session/provider-cli/provider-cli.runtime.d.ts +1 -1
- package/dist/session/provider-cli/provider-cli.runtime.js +3 -2
- package/dist/session/provider-cli/provider-cli.service.d.ts +23 -2
- package/dist/session/provider-cli/provider-cli.service.js +507 -88
- package/dist/session/provider-cli/provider-cli.types.d.ts +5 -0
- package/dist/session/provider-cli/provider-output-state.d.ts +2 -0
- package/dist/session/provider-cli/provider-output-state.js +69 -7
- package/dist/session/session-redaction.d.ts +3 -0
- package/dist/session/session-redaction.js +38 -0
- package/dist/session/session.base.d.ts +6 -1
- package/dist/session/session.service.d.ts +8 -2
- package/dist/session/session.service.js +490 -42
- package/dist/session/session.types.d.ts +111 -1
- package/dist/session/session.types.js +192 -3
- package/dist/shared/format/format-bytes.d.ts +1 -0
- package/dist/shared/format/format-bytes.js +9 -0
- package/dist/shared/path/path.js +3 -1
- package/dist/shared/process/host-process-recovery.d.ts +12 -0
- package/dist/shared/process/host-process-recovery.js +78 -0
- package/dist/shared/process/host-process-supervisor.d.ts +57 -0
- package/dist/shared/process/host-process-supervisor.js +282 -0
- package/dist/shared/process/process.d.ts +1 -0
- package/dist/shared/process/process.js +1 -0
- package/dist/shared/redaction/redaction.js +6 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/worktree/artifact-injector.d.ts +3 -2
- package/dist/worktree/artifact-injector.js +43 -4
- package/dist/worktree/dependency-warmer.d.ts +10 -1
- package/dist/worktree/dependency-warmer.js +94 -5
- package/dist/worktree/index.d.ts +1 -0
- package/dist/worktree/index.js +1 -0
- package/dist/worktree/prior-phase-artifact-hydrator.d.ts +2 -0
- package/dist/worktree/prior-phase-artifact-hydrator.js +136 -0
- package/dist/worktree/worktree-paths.d.ts +4 -0
- package/dist/worktree/worktree-paths.js +30 -0
- package/dist/worktree/worktree.base.d.ts +3 -0
- package/dist/worktree/worktree.service.d.ts +8 -2
- package/dist/worktree/worktree.service.js +206 -43
- package/dist/worktree/worktree.types.d.ts +4 -1
- package/package.json +6 -3
- package/runtime/windows-job-supervisor.ps1 +356 -0
- package/runtime/windows-listener-owner.ps1 +33 -0
|
@@ -1,48 +1,177 @@
|
|
|
1
1
|
import { HubConnectionBuilder, HttpTransportType, LogLevel } from "@microsoft/signalr";
|
|
2
2
|
import fs from "node:fs";
|
|
3
3
|
import path from "node:path";
|
|
4
|
+
import { randomUUID } from "node:crypto";
|
|
5
|
+
import os from "node:os";
|
|
4
6
|
import { defaultForgeBridgeConfig } from "../config.js";
|
|
5
7
|
import { supportsDraftImageAttachmentsFor } from "../shared/provider-capabilities.js";
|
|
6
|
-
import {
|
|
8
|
+
import { redactText } from "../shared/redaction/redaction.js";
|
|
7
9
|
import { BridgeSessionService } from "../session/session.service.js";
|
|
10
|
+
import { ProcessBackedProviderCliService, RoutingProviderCliService, StreamingProviderProcessRunner, } from "../session/provider-cli/provider-cli.service.js";
|
|
11
|
+
import { redactRunnerDispatchEvent, redactRunOnceResultPacket } from "../session/session-redaction.js";
|
|
12
|
+
import { HostRuntimeLeaseManager } from "../runtime/host-runtime-lease.js";
|
|
13
|
+
import { estimateWorkspaceBytesFromRepo } from "../runtime/workspace-resource-estimator.js";
|
|
14
|
+
import { GitWorktreeService } from "../worktree/worktree.service.js";
|
|
15
|
+
import { RepositoryCoordinatorLock } from "../sandbox/repository-lock.js";
|
|
16
|
+
import { AttemptCoordinatedProviderProcessRunner, AttemptProcessSupervisorRegistry, } from "../runtime/attempt-process-supervisor-registry.js";
|
|
8
17
|
import { failureResult, parseRunOnceDispatchPacket } from "../session/session.types.js";
|
|
9
18
|
import { forgeBridgeVersion } from "../version.js";
|
|
19
|
+
import { canRunDockerMaintenance, DockerPreflight } from "../sandbox/preflight.js";
|
|
20
|
+
import { validateSandboxContract } from "../sandbox/contract.js";
|
|
21
|
+
import { runSandboxProcess } from "../sandbox/process.js";
|
|
22
|
+
import { DockerSandboxRuntime, selectSandboxConfigRefs } from "../sandbox/runtime.js";
|
|
23
|
+
import { codexSandboxProviderAuthAdapter, sandboxProviderAuthAdapterFor, sandboxProviderAuthUnavailableMessage } from "../sandbox/provider-auth/codex.js";
|
|
24
|
+
import { formatBytes } from "../shared/format/format-bytes.js";
|
|
25
|
+
/**
|
|
26
|
+
* Protocol-v2 events are acknowledged one at a time before runOnce returns.
|
|
27
|
+
* Sending the complete transcript again in DispatchResult made long QA and CI
|
|
28
|
+
* sessions exceed SignalR's receive limit, disconnecting the bridge after the
|
|
29
|
+
* provider had already completed. The API combines its acknowledged streamed
|
|
30
|
+
* events with the terminal result, so the callback only needs the outcome and
|
|
31
|
+
* artifacts. Protocol v1 retains the transcript for legacy servers.
|
|
32
|
+
*/
|
|
33
|
+
export function compactTerminalResult(dispatch, result) {
|
|
34
|
+
return dispatch.schemaVersion >= 2 && result.events.length > 0
|
|
35
|
+
? { ...result, events: [] }
|
|
36
|
+
: result;
|
|
37
|
+
}
|
|
10
38
|
export class ForgeBridgeService {
|
|
11
39
|
config;
|
|
12
|
-
sessions;
|
|
13
40
|
connections;
|
|
14
41
|
connection = null;
|
|
15
42
|
heartbeatTimer = null;
|
|
43
|
+
heartbeatInFlight = false;
|
|
44
|
+
consecutiveHeartbeatFailures = 0;
|
|
45
|
+
reconnectingFromHeartbeatFailure = false;
|
|
46
|
+
leaseWatchdogTimer = null;
|
|
47
|
+
sandboxMaintenanceTimer = null;
|
|
48
|
+
sandboxMaintenanceRunning = false;
|
|
16
49
|
closed = null;
|
|
50
|
+
transportRecovery = null;
|
|
51
|
+
transportRecoveryGeneration = 0;
|
|
17
52
|
stopped = false;
|
|
18
53
|
activeSessions = new Map();
|
|
19
|
-
|
|
54
|
+
reservations = new Map();
|
|
55
|
+
activeReadinessProbes = new Map();
|
|
56
|
+
preflight;
|
|
57
|
+
sandboxRuntime;
|
|
58
|
+
sandboxResourceResolver;
|
|
59
|
+
sessions;
|
|
60
|
+
constructor(config, sessions = undefined, connections = new SignalRBridgeConnectionFactory(), preflight, sandboxRuntime, sandboxResourceResolver) {
|
|
20
61
|
this.config = config;
|
|
21
|
-
this.sessions = sessions;
|
|
22
62
|
this.connections = connections;
|
|
63
|
+
this.sessions = sessions ?? createDefaultSessionService(config.sandboxStateRoot);
|
|
64
|
+
this.preflight = preflight ?? new DockerPreflight({
|
|
65
|
+
dockerExecutable: config.dockerExecutable,
|
|
66
|
+
dockerContext: config.dockerContext,
|
|
67
|
+
stateRoot: config.sandboxStateRoot,
|
|
68
|
+
runnerInstanceId: config.runnerInstanceId,
|
|
69
|
+
maxConcurrentSandboxes: config.maxConcurrentSandboxes,
|
|
70
|
+
diskHighWatermarkPercent: config.diskHighWatermarkPercent,
|
|
71
|
+
diskLowWatermarkPercent: config.diskLowWatermarkPercent,
|
|
72
|
+
minimumWorkspaceBytes: config.minimumWorkspaceBytes,
|
|
73
|
+
hostMemoryReserveBytes: config.sandboxHostMemoryReserveBytes,
|
|
74
|
+
defaultWorkerMemoryLimitBytes: config.defaultSandboxWorkerMemoryLimitBytes,
|
|
75
|
+
defaultServiceMemoryLimitBytes: config.defaultSandboxServiceMemoryLimitBytes,
|
|
76
|
+
runtimeOverheadBytes: config.sandboxRuntimeOverheadBytes,
|
|
77
|
+
requireRootless: config.requireRootlessDocker
|
|
78
|
+
});
|
|
79
|
+
this.sandboxRuntime = sandboxRuntime ?? new DockerSandboxRuntime(config, (request, signal) => this.redeemPublishGrant(request, signal));
|
|
80
|
+
this.sandboxResourceResolver = sandboxResourceResolver
|
|
81
|
+
?? (dispatch => resolveSandboxResourcesFromRepo(dispatch, this.config.sandboxRuntimeOverheadBytes));
|
|
23
82
|
}
|
|
24
83
|
async connectOnce() {
|
|
25
84
|
await this.stopConnection();
|
|
26
85
|
this.stopped = false;
|
|
86
|
+
const hostReap = await this.sessions.reapHostRuntime?.({
|
|
87
|
+
runnerId: this.config.runnerId,
|
|
88
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
89
|
+
});
|
|
90
|
+
if (hostReap && (hostReap.reaped > 0 || hostReap.degraded > 0)) {
|
|
91
|
+
logBridgeStatus(`Managed-host recovery reaped ${hostReap.reaped} stale attempt(s), protected ${hostReap.reusedPidsProtected} reused PID(s), and left ${hostReap.degraded} degraded attempt(s) for reconciliation.`);
|
|
92
|
+
}
|
|
93
|
+
if (this.config.dockerEnabled) {
|
|
94
|
+
const preflight = await this.preflight.refresh();
|
|
95
|
+
if (canReapManagedSandboxes(preflight)) {
|
|
96
|
+
await this.sandboxRuntime.reap({ diskLow: preflight.diskLow });
|
|
97
|
+
await this.preflight.refresh();
|
|
98
|
+
}
|
|
99
|
+
}
|
|
27
100
|
const connection = this.connections.create(this.config);
|
|
28
101
|
this.connection = connection;
|
|
102
|
+
this.transportRecovery = null;
|
|
29
103
|
this.closed = new Promise((resolve) => {
|
|
30
|
-
connection.onClose(() => {
|
|
104
|
+
connection.onClose((error) => {
|
|
31
105
|
this.stopHeartbeat();
|
|
106
|
+
this.completeTransportRecovery(error ?? new Error("bridge connection closed"));
|
|
32
107
|
this.abortActiveSessions("bridge connection closed");
|
|
108
|
+
this.abortActiveReadinessProbes("bridge connection closed");
|
|
33
109
|
resolve();
|
|
34
110
|
});
|
|
35
111
|
});
|
|
112
|
+
connection.onReconnecting?.((error) => {
|
|
113
|
+
if (connection !== this.connection || this.stopped)
|
|
114
|
+
return;
|
|
115
|
+
this.stopHeartbeat();
|
|
116
|
+
this.beginTransportRecovery();
|
|
117
|
+
logBridgeStatus(`Forge connection interrupted${error ? `: ${errorMessage(error)}` : ""}. Reconnecting in place; active dispatches remain supervised...`);
|
|
118
|
+
});
|
|
119
|
+
connection.onReconnected?.(async () => {
|
|
120
|
+
if (connection !== this.connection || this.stopped)
|
|
121
|
+
return;
|
|
122
|
+
try {
|
|
123
|
+
await this.registerConnection(connection);
|
|
124
|
+
this.completeTransportRecovery();
|
|
125
|
+
this.startHeartbeat();
|
|
126
|
+
await this.flushCleanupReports().catch(() => undefined);
|
|
127
|
+
logBridgeStatus("Forge connection restored; active dispatches remain attached to this runner process.");
|
|
128
|
+
}
|
|
129
|
+
catch (error) {
|
|
130
|
+
this.completeTransportRecovery(error);
|
|
131
|
+
logBridgeStatus(`Forge reconnect registration failed: ${errorMessage(error)}. Closing the connection for a clean retry...`);
|
|
132
|
+
await connection.stop().catch(() => undefined);
|
|
133
|
+
}
|
|
134
|
+
});
|
|
36
135
|
connection.onDispatch((dispatch) => this.handleDispatch(dispatch));
|
|
136
|
+
connection.onOffer?.((dispatch) => this.handleOffer(dispatch));
|
|
137
|
+
connection.onStart?.((dispatch) => this.handleStart(dispatch));
|
|
138
|
+
connection.onLeaseUpdate?.((request) => this.handleLeaseUpdate(request));
|
|
37
139
|
connection.onCancel((request) => this.handleCancel(request));
|
|
140
|
+
connection.onReap?.((request) => this.handleReap(request));
|
|
141
|
+
connection.onReadinessProbe?.((request) => this.handleReadinessProbe(request));
|
|
38
142
|
await connection.start();
|
|
39
|
-
await this.
|
|
40
|
-
|
|
41
|
-
protocolVersion: this.config.protocolVersion,
|
|
42
|
-
minServerVersion: this.config.minServerVersion ?? null,
|
|
43
|
-
capabilities: this.capabilities()
|
|
44
|
-
});
|
|
143
|
+
await this.registerConnection(connection);
|
|
144
|
+
await this.flushCleanupReports();
|
|
45
145
|
this.startHeartbeat();
|
|
146
|
+
this.startLeaseWatchdog();
|
|
147
|
+
this.startSandboxMaintenance();
|
|
148
|
+
}
|
|
149
|
+
async registerConnection(connection) {
|
|
150
|
+
let storageRecoveryAttempt = 0;
|
|
151
|
+
while (true) {
|
|
152
|
+
try {
|
|
153
|
+
const ack = await connection.invoke("Hello", {
|
|
154
|
+
version: this.config.version,
|
|
155
|
+
protocolVersion: this.config.protocolVersion,
|
|
156
|
+
minServerVersion: this.config.minServerVersion ?? null,
|
|
157
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
158
|
+
capabilities: this.capabilities()
|
|
159
|
+
});
|
|
160
|
+
if (!ack.accepted) {
|
|
161
|
+
throw new BridgeCallbackRejectedError(ack.code ?? null, ack.message ?? ack.code ?? "Hello callback was rejected.");
|
|
162
|
+
}
|
|
163
|
+
return;
|
|
164
|
+
}
|
|
165
|
+
catch (error) {
|
|
166
|
+
if (!isProviderStorageUnavailable(error) || storageRecoveryAttempt >= this.config.reconnectBackoffMs.length) {
|
|
167
|
+
throw error;
|
|
168
|
+
}
|
|
169
|
+
const backoff = this.config.reconnectBackoffMs[storageRecoveryAttempt] ?? 5000;
|
|
170
|
+
storageRecoveryAttempt += 1;
|
|
171
|
+
logBridgeStatus(`Hello reached Forge while storage was recovering. Retrying registration ${storageRecoveryAttempt} in ${formatDelay(backoff)}...`);
|
|
172
|
+
await delay(backoff);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
46
175
|
}
|
|
47
176
|
async run(signal) {
|
|
48
177
|
let attempt = 0;
|
|
@@ -73,6 +202,7 @@ export class ForgeBridgeService {
|
|
|
73
202
|
finally {
|
|
74
203
|
if (signal?.aborted) {
|
|
75
204
|
this.abortActiveSessions(String(signal.reason ?? "bridge stopping"));
|
|
205
|
+
this.abortActiveReadinessProbes(String(signal.reason ?? "bridge stopping"));
|
|
76
206
|
}
|
|
77
207
|
await this.stopConnection();
|
|
78
208
|
}
|
|
@@ -80,7 +210,9 @@ export class ForgeBridgeService {
|
|
|
80
210
|
async stop() {
|
|
81
211
|
this.stopped = true;
|
|
82
212
|
this.abortActiveSessions("bridge stopping");
|
|
213
|
+
this.abortActiveReadinessProbes("bridge stopping");
|
|
83
214
|
await this.stopConnection();
|
|
215
|
+
await this.preflight.release();
|
|
84
216
|
}
|
|
85
217
|
async handleDispatch(rawDispatch) {
|
|
86
218
|
const dispatch = this.parseDispatchOrReport(rawDispatch);
|
|
@@ -90,21 +222,53 @@ export class ForgeBridgeService {
|
|
|
90
222
|
const leaseId = dispatch.lease.id;
|
|
91
223
|
const fencingToken = dispatch.lease.fencingToken;
|
|
92
224
|
const redactionSecrets = exactSecrets(dispatch, this.config);
|
|
93
|
-
|
|
94
|
-
|
|
225
|
+
const existing = this.activeSessions.get(leaseId);
|
|
226
|
+
if (existing) {
|
|
227
|
+
if (existing.fencingToken === fencingToken) {
|
|
228
|
+
return;
|
|
229
|
+
}
|
|
230
|
+
await this.sendResult(dispatch, failureResult("Dispatch fence conflicts with an active bridge session.")).catch(() => undefined);
|
|
95
231
|
return;
|
|
96
232
|
}
|
|
97
233
|
const controller = new AbortController();
|
|
98
|
-
this.activeSessions.set(leaseId, {
|
|
234
|
+
this.activeSessions.set(leaseId, {
|
|
235
|
+
fencingToken,
|
|
236
|
+
dispatchAttemptId: dispatch.sandbox?.dispatchAttemptId ?? null,
|
|
237
|
+
runnerInstanceId: dispatch.sandbox?.runnerInstanceId ?? null,
|
|
238
|
+
leaseExpiresAt: dispatch.lease.expiresAt,
|
|
239
|
+
sandboxed: dispatch.sandbox?.executionMode === "docker",
|
|
240
|
+
resources: dispatch.sandbox?.executionMode === "docker"
|
|
241
|
+
? dispatch.sandbox.requestedResources
|
|
242
|
+
: zeroSandboxResources(),
|
|
243
|
+
controller
|
|
244
|
+
});
|
|
99
245
|
try {
|
|
100
246
|
const normalized = this.normalizeDispatch(dispatch);
|
|
101
|
-
|
|
247
|
+
if (normalized.sandbox) {
|
|
248
|
+
const started = {
|
|
249
|
+
dispatchAttemptId: normalized.sandbox.dispatchAttemptId,
|
|
250
|
+
leaseId,
|
|
251
|
+
fencingToken,
|
|
252
|
+
runnerInstanceId: normalized.sandbox.runnerInstanceId
|
|
253
|
+
};
|
|
254
|
+
await this.invokeAck("DispatchStarted", started);
|
|
255
|
+
}
|
|
256
|
+
const sink = {
|
|
102
257
|
emit: async (event) => {
|
|
258
|
+
if (event.eventType === "provider_cli_started") {
|
|
259
|
+
const active = this.activeSessions.get(leaseId);
|
|
260
|
+
if (active)
|
|
261
|
+
active.resources = { ...active.resources, buildSlots: 0 };
|
|
262
|
+
}
|
|
103
263
|
try {
|
|
104
264
|
await this.invokeAck("DispatchEvent", {
|
|
105
265
|
leaseId,
|
|
106
266
|
fencingToken,
|
|
107
|
-
|
|
267
|
+
...(dispatch.sandbox ? {
|
|
268
|
+
dispatchAttemptId: dispatch.sandbox.dispatchAttemptId,
|
|
269
|
+
runnerInstanceId: dispatch.sandbox.runnerInstanceId
|
|
270
|
+
} : {}),
|
|
271
|
+
event: redactRunnerDispatchEvent(event, redactionSecrets)
|
|
108
272
|
});
|
|
109
273
|
}
|
|
110
274
|
catch (error) {
|
|
@@ -115,19 +279,247 @@ export class ForgeBridgeService {
|
|
|
115
279
|
throw error;
|
|
116
280
|
}
|
|
117
281
|
}
|
|
118
|
-
}
|
|
119
|
-
|
|
282
|
+
};
|
|
283
|
+
const result = normalized.sandbox?.executionMode === "docker"
|
|
284
|
+
? await this.sandboxRuntime.runOnce(normalized, sink, controller.signal)
|
|
285
|
+
: await this.sessions.runOnce(normalized, sink, controller.signal);
|
|
286
|
+
if (normalized.sandbox?.executionMode === "docker") {
|
|
287
|
+
await this.flushCleanupReports().catch(() => undefined);
|
|
288
|
+
}
|
|
289
|
+
const redactedResult = redactRunOnceResultPacket(result, redactionSecrets);
|
|
290
|
+
await this.sendResult(dispatch, compactTerminalResult(dispatch, redactedResult));
|
|
120
291
|
}
|
|
121
292
|
catch (error) {
|
|
122
293
|
const summary = controller.signal.aborted
|
|
123
294
|
? `Dispatch cancelled: ${String(controller.signal.reason ?? "cancelled")}`
|
|
124
295
|
: redactText(error instanceof Error ? error.message : String(error), redactionSecrets);
|
|
125
|
-
|
|
296
|
+
const failureEventType = infrastructureFailureEventType(summary);
|
|
297
|
+
const failureEvents = failureEventType ? [{
|
|
298
|
+
role: "runner",
|
|
299
|
+
kind: "error",
|
|
300
|
+
content: summary,
|
|
301
|
+
eventType: failureEventType,
|
|
302
|
+
}] : [];
|
|
303
|
+
await this.sendResult(dispatch, failureResult(summary, failureEvents)).catch(() => undefined);
|
|
126
304
|
}
|
|
127
305
|
finally {
|
|
128
306
|
this.activeSessions.delete(leaseId);
|
|
129
307
|
}
|
|
130
308
|
}
|
|
309
|
+
async handleOffer(rawDispatch) {
|
|
310
|
+
const envelope = parseOfferEnvelope(rawDispatch);
|
|
311
|
+
if (!envelope) {
|
|
312
|
+
return;
|
|
313
|
+
}
|
|
314
|
+
const dispatch = this.parseDispatchOrReport(envelope.dispatch);
|
|
315
|
+
if (!dispatch?.sandbox
|
|
316
|
+
|| dispatch.sandbox.dispatchAttemptId !== envelope.dispatchAttemptId
|
|
317
|
+
|| dispatch.lease.id !== envelope.leaseId
|
|
318
|
+
|| dispatch.lease.fencingToken !== envelope.fencingToken)
|
|
319
|
+
return;
|
|
320
|
+
const sandbox = dispatch.sandbox;
|
|
321
|
+
let existing = this.reservations.get(sandbox.dispatchAttemptId);
|
|
322
|
+
if (existing && Date.parse(existing.reservedUntil) <= Date.now()) {
|
|
323
|
+
this.reservations.delete(sandbox.dispatchAttemptId);
|
|
324
|
+
existing = undefined;
|
|
325
|
+
}
|
|
326
|
+
if (existing) {
|
|
327
|
+
const sameFence = existing.fencingToken === dispatch.lease.fencingToken;
|
|
328
|
+
await this.sendOfferDecision(dispatch, sameFence, sameFence ? null : "stale_fence", sameFence ? null : "Offer fence conflicts with the existing reservation.", sameFence ? existing.reservedUntil : null, sameFence ? existing.resources : null);
|
|
329
|
+
return;
|
|
330
|
+
}
|
|
331
|
+
if (sandbox.runnerInstanceId !== this.config.runnerInstanceId) {
|
|
332
|
+
await this.sendOfferDecision(dispatch, false, "stale_runner_instance", "Offer targets a different runner process instance.");
|
|
333
|
+
return;
|
|
334
|
+
}
|
|
335
|
+
if (Date.parse(envelope.expiresAt) <= Date.now()) {
|
|
336
|
+
await this.sendOfferDecision(dispatch, false, "lease_expired", "Offer lease is already expired.");
|
|
337
|
+
return;
|
|
338
|
+
}
|
|
339
|
+
const preflight = sandbox.executionMode === "docker" && this.config.dockerEnabled
|
|
340
|
+
? await this.preflight.run()
|
|
341
|
+
: this.preflight.current();
|
|
342
|
+
if (sandbox.executionMode === "docker") {
|
|
343
|
+
if (!this.config.dockerEnabled || !preflight?.healthy) {
|
|
344
|
+
await this.sendOfferDecision(dispatch, false, preflight?.unavailableCode ?? "docker_unavailable", preflight?.unavailableReason ?? "Docker sandbox preflight is unavailable.");
|
|
345
|
+
return;
|
|
346
|
+
}
|
|
347
|
+
const networkFailure = sandboxNetworkPolicyFailure(dispatch, this.config);
|
|
348
|
+
if (networkFailure) {
|
|
349
|
+
await this.sendOfferDecision(dispatch, false, "policy_mismatch", networkFailure);
|
|
350
|
+
return;
|
|
351
|
+
}
|
|
352
|
+
const providerNetworkFailure = sandboxProviderNetworkFailure(dispatch, this.config);
|
|
353
|
+
if (providerNetworkFailure) {
|
|
354
|
+
await this.sendOfferDecision(dispatch, false, "provider_network_policy_denied", providerNetworkFailure);
|
|
355
|
+
return;
|
|
356
|
+
}
|
|
357
|
+
const authFailure = sandboxAuthFailure(dispatch, this.config);
|
|
358
|
+
if (authFailure) {
|
|
359
|
+
await this.sendOfferDecision(dispatch, false, "sandbox_auth_unavailable", authFailure);
|
|
360
|
+
return;
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
const runtimeState = this.sandboxRuntime.snapshot();
|
|
364
|
+
const usedResources = this.usedSandboxResources(runtimeState.degraded);
|
|
365
|
+
const activeResources = this.activeSandboxResources(runtimeState.degraded);
|
|
366
|
+
const capacity = this.sandboxResourceCapacity(preflight, activeResources);
|
|
367
|
+
let requestedResources = sandbox.requestedResources;
|
|
368
|
+
let sandboxConfigCommitSha = sandbox.sandboxConfigCommitSha ?? null;
|
|
369
|
+
const normalizedDispatch = this.normalizeDispatch(dispatch);
|
|
370
|
+
if (sandbox.executionMode === "docker") {
|
|
371
|
+
try {
|
|
372
|
+
const profile = await this.sandboxResourceResolver(normalizedDispatch);
|
|
373
|
+
requestedResources = profile.resources;
|
|
374
|
+
sandboxConfigCommitSha = profile.sandboxConfigCommitSha;
|
|
375
|
+
}
|
|
376
|
+
catch (error) {
|
|
377
|
+
await this.sendOfferDecision(dispatch, false, "sandbox_capacity_profile_unavailable", error instanceof Error ? error.message : "The immutable sandbox contract could not be profiled for admission.");
|
|
378
|
+
return;
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
else if (normalizedDispatch.repo.serverLocalPath) {
|
|
382
|
+
// Host execution uses the same project-derived disk reservation as Docker.
|
|
383
|
+
// The API supplies a placement-time estimate; the runner can safely raise it
|
|
384
|
+
// from its authoritative checkout before admitting the attempt.
|
|
385
|
+
const measuredWorkspaceBytes = await estimateWorkspaceBytesFromRepo(normalizedDispatch.repo.serverLocalPath, { role: normalizedDispatch.execution.role }).catch(() => null);
|
|
386
|
+
if (measuredWorkspaceBytes !== null) {
|
|
387
|
+
requestedResources = {
|
|
388
|
+
...requestedResources,
|
|
389
|
+
workspaceBytes: Math.max(requestedResources.workspaceBytes, measuredWorkspaceBytes),
|
|
390
|
+
};
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
if (sandbox.executionMode === "docker"
|
|
394
|
+
&& !canReserveSandboxResources(usedResources, requestedResources, capacity)) {
|
|
395
|
+
await this.sendOfferDecision(dispatch, false, "capacity_unavailable", sandboxCapacityFailureMessage(usedResources, requestedResources, capacity, preflight, this.config));
|
|
396
|
+
return;
|
|
397
|
+
}
|
|
398
|
+
const reservedUntil = new Date(Math.min(Date.parse(dispatch.lease.expiresAt), Date.now() + 30_000)).toISOString();
|
|
399
|
+
this.reservations.set(sandbox.dispatchAttemptId, {
|
|
400
|
+
leaseId: dispatch.lease.id,
|
|
401
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
402
|
+
runnerInstanceId: sandbox.runnerInstanceId,
|
|
403
|
+
expiresAt: dispatch.lease.expiresAt,
|
|
404
|
+
reservedUntil,
|
|
405
|
+
sandboxed: sandbox.executionMode === "docker",
|
|
406
|
+
resources: requestedResources
|
|
407
|
+
});
|
|
408
|
+
await this.sendOfferDecision(dispatch, true, null, null, reservedUntil, requestedResources, sandboxConfigCommitSha);
|
|
409
|
+
}
|
|
410
|
+
async handleStart(rawDispatch) {
|
|
411
|
+
const envelope = parseStartEnvelope(rawDispatch);
|
|
412
|
+
if (!envelope) {
|
|
413
|
+
return;
|
|
414
|
+
}
|
|
415
|
+
let dispatch = this.parseDispatchOrReport(envelope.dispatch);
|
|
416
|
+
if (!dispatch?.sandbox
|
|
417
|
+
|| dispatch.sandbox.dispatchAttemptId !== envelope.dispatchAttemptId
|
|
418
|
+
|| dispatch.lease.id !== envelope.leaseId
|
|
419
|
+
|| dispatch.lease.fencingToken !== envelope.fencingToken)
|
|
420
|
+
return;
|
|
421
|
+
const reservation = this.reservations.get(dispatch.sandbox.dispatchAttemptId);
|
|
422
|
+
if (!reservation
|
|
423
|
+
|| reservation.leaseId !== dispatch.lease.id
|
|
424
|
+
|| reservation.fencingToken !== dispatch.lease.fencingToken
|
|
425
|
+
|| reservation.runnerInstanceId !== dispatch.sandbox.runnerInstanceId
|
|
426
|
+
|| Date.parse(reservation.reservedUntil) <= Date.now()) {
|
|
427
|
+
await this.sendResult(dispatch, failureResult("Start rejected because its reservation is missing or stale.")).catch(() => undefined);
|
|
428
|
+
return;
|
|
429
|
+
}
|
|
430
|
+
if (dispatch.apiKeySecretGrant) {
|
|
431
|
+
this.reservations.delete(dispatch.sandbox.dispatchAttemptId);
|
|
432
|
+
await this.sendResult(dispatch, failureResult("Start rejected because protocol-v2 packets may not contain a plaintext provider grant.")).catch(() => undefined);
|
|
433
|
+
return;
|
|
434
|
+
}
|
|
435
|
+
if (dispatch.apiKeySecretGrantHandle) {
|
|
436
|
+
const response = await this.redeemSecretGrant(dispatch);
|
|
437
|
+
const grant = response.grant;
|
|
438
|
+
if (!response.accepted
|
|
439
|
+
|| !grant
|
|
440
|
+
|| grant.grantId !== dispatch.apiKeySecretGrantHandle.grantId
|
|
441
|
+
|| grant.provider !== dispatch.apiKeySecretGrantHandle.provider
|
|
442
|
+
|| Date.parse(grant.expiresAt) <= Date.now()) {
|
|
443
|
+
this.reservations.delete(dispatch.sandbox.dispatchAttemptId);
|
|
444
|
+
await this.sendResult(dispatch, failureResult(response.message ?? "Secret grant redemption was rejected.")).catch(() => undefined);
|
|
445
|
+
return;
|
|
446
|
+
}
|
|
447
|
+
dispatch = { ...dispatch, apiKeySecretGrant: grant, apiKeySecretGrantHandle: null };
|
|
448
|
+
}
|
|
449
|
+
else if (dispatch.execution.requiresApiKeySecretGrant) {
|
|
450
|
+
this.reservations.delete(dispatch.sandbox.dispatchAttemptId);
|
|
451
|
+
await this.sendResult(dispatch, failureResult("Start rejected because the provider secret grant handle is missing.")).catch(() => undefined);
|
|
452
|
+
return;
|
|
453
|
+
}
|
|
454
|
+
dispatch = {
|
|
455
|
+
...dispatch,
|
|
456
|
+
sandbox: { ...dispatch.sandbox, requestedResources: reservation.resources },
|
|
457
|
+
};
|
|
458
|
+
this.reservations.delete(envelope.dispatchAttemptId);
|
|
459
|
+
await this.handleDispatch(dispatch);
|
|
460
|
+
}
|
|
461
|
+
async redeemSecretGrant(dispatch) {
|
|
462
|
+
const sandbox = dispatch.sandbox;
|
|
463
|
+
const handle = dispatch.apiKeySecretGrantHandle;
|
|
464
|
+
const connection = this.connection;
|
|
465
|
+
if (!connection)
|
|
466
|
+
return { accepted: false, code: "bridge.disconnected", message: "Bridge is disconnected.", grant: null };
|
|
467
|
+
return await connection.invoke("RedeemSecretGrant", {
|
|
468
|
+
dispatchAttemptId: sandbox.dispatchAttemptId,
|
|
469
|
+
leaseId: dispatch.lease.id,
|
|
470
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
471
|
+
runnerInstanceId: sandbox.runnerInstanceId,
|
|
472
|
+
grantId: handle.grantId
|
|
473
|
+
});
|
|
474
|
+
}
|
|
475
|
+
async redeemPublishGrant(request, signal) {
|
|
476
|
+
if (signal?.aborted)
|
|
477
|
+
throw new Error(String(signal.reason ?? "publication cancelled"));
|
|
478
|
+
const connection = this.connection;
|
|
479
|
+
if (!connection)
|
|
480
|
+
throw new Error("sandbox_output_publish_forbidden: bridge is disconnected.");
|
|
481
|
+
const response = await connection.invoke("RedeemPublishGrant", request);
|
|
482
|
+
if (!response.accepted || !response.credential) {
|
|
483
|
+
throw new Error(`sandbox_output_publish_forbidden: ${response.message ?? response.code ?? "publication grant redemption was rejected"}`);
|
|
484
|
+
}
|
|
485
|
+
return response.credential;
|
|
486
|
+
}
|
|
487
|
+
async handleLeaseUpdate(request) {
|
|
488
|
+
if (request.runnerInstanceId !== this.config.runnerInstanceId) {
|
|
489
|
+
return;
|
|
490
|
+
}
|
|
491
|
+
const reservation = this.reservations.get(request.dispatchAttemptId);
|
|
492
|
+
if (reservation && reservation.leaseId === request.leaseId && reservation.fencingToken === request.fencingToken) {
|
|
493
|
+
reservation.expiresAt = request.expiresAt;
|
|
494
|
+
}
|
|
495
|
+
const active = this.activeSessions.get(request.leaseId);
|
|
496
|
+
if (active && active.fencingToken === request.fencingToken && active.dispatchAttemptId === request.dispatchAttemptId) {
|
|
497
|
+
active.leaseExpiresAt = request.expiresAt;
|
|
498
|
+
if (active.sandboxed) {
|
|
499
|
+
await this.sandboxRuntime.updateLease(request.dispatchAttemptId, request.fencingToken, request.expiresAt);
|
|
500
|
+
}
|
|
501
|
+
else {
|
|
502
|
+
await this.sessions.updateLease?.(request.dispatchAttemptId, request.fencingToken, request.expiresAt);
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
async sendOfferDecision(dispatch, accepted, code = null, message = null, reservedUntil = null, reservedResources = null, sandboxConfigCommitSha = null) {
|
|
507
|
+
if (!dispatch.sandbox)
|
|
508
|
+
return;
|
|
509
|
+
const payload = {
|
|
510
|
+
dispatchAttemptId: dispatch.sandbox.dispatchAttemptId,
|
|
511
|
+
leaseId: dispatch.lease.id,
|
|
512
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
513
|
+
runnerInstanceId: dispatch.sandbox.runnerInstanceId,
|
|
514
|
+
accepted,
|
|
515
|
+
code,
|
|
516
|
+
message,
|
|
517
|
+
reservedUntil: accepted ? reservedUntil ?? dispatch.lease.expiresAt : null,
|
|
518
|
+
reservedResources: accepted ? reservedResources : null,
|
|
519
|
+
sandboxConfigCommitSha: accepted ? sandboxConfigCommitSha : null,
|
|
520
|
+
};
|
|
521
|
+
await this.invokeAck("OfferDecision", payload);
|
|
522
|
+
}
|
|
131
523
|
parseDispatchOrReport(rawDispatch) {
|
|
132
524
|
try {
|
|
133
525
|
return parseRunOnceDispatchPacket(rawDispatch);
|
|
@@ -138,7 +530,7 @@ export class ForgeBridgeService {
|
|
|
138
530
|
return null;
|
|
139
531
|
}
|
|
140
532
|
const summary = invalidDispatchSummary(error);
|
|
141
|
-
void this.
|
|
533
|
+
void this.sendLegacyResult(identifiers.leaseId, identifiers.fencingToken, failureResult(summary, [
|
|
142
534
|
{
|
|
143
535
|
role: "runner",
|
|
144
536
|
kind: "error",
|
|
@@ -156,12 +548,74 @@ export class ForgeBridgeService {
|
|
|
156
548
|
}
|
|
157
549
|
active.controller.abort(request.reason ?? "cancelled by server");
|
|
158
550
|
}
|
|
551
|
+
async handleReap(request) {
|
|
552
|
+
if (request.runnerId !== this.config.runnerId
|
|
553
|
+
|| request.runnerInstanceId !== this.config.runnerInstanceId)
|
|
554
|
+
return;
|
|
555
|
+
await this.runSandboxMaintenance();
|
|
556
|
+
await this.sendHeartbeat();
|
|
557
|
+
}
|
|
558
|
+
async handleReadinessProbe(rawRequest) {
|
|
559
|
+
const request = parseReadinessProbe(rawRequest);
|
|
560
|
+
if (!request || request.runnerInstanceId !== this.config.runnerInstanceId)
|
|
561
|
+
return;
|
|
562
|
+
const existing = this.activeReadinessProbes.get(request.requestId);
|
|
563
|
+
if (existing)
|
|
564
|
+
return;
|
|
565
|
+
const controller = new AbortController();
|
|
566
|
+
this.activeReadinessProbes.set(request.requestId, controller);
|
|
567
|
+
const deadlineMs = Date.parse(request.expiresAt) - Date.now();
|
|
568
|
+
const timer = setTimeout(() => controller.abort("readiness probe expired"), Math.max(0, deadlineMs));
|
|
569
|
+
timer.unref?.();
|
|
570
|
+
let result;
|
|
571
|
+
try {
|
|
572
|
+
if (!this.sessions.probeHostRuntime) {
|
|
573
|
+
throw new Error("host_runtime_probe_unavailable: this bridge does not implement runtime readiness probes.");
|
|
574
|
+
}
|
|
575
|
+
const probe = await this.sessions.probeHostRuntime({
|
|
576
|
+
...request,
|
|
577
|
+
runnerId: this.config.runnerId,
|
|
578
|
+
repoRoot: this.config.defaultRepoRoot,
|
|
579
|
+
cleanupWorktrees: this.config.cleanupWorktrees,
|
|
580
|
+
}, controller.signal);
|
|
581
|
+
result = {
|
|
582
|
+
schemaVersion: 1,
|
|
583
|
+
requestId: request.requestId,
|
|
584
|
+
fencingToken: request.fencingToken,
|
|
585
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
586
|
+
...probe,
|
|
587
|
+
};
|
|
588
|
+
}
|
|
589
|
+
catch (error) {
|
|
590
|
+
result = {
|
|
591
|
+
schemaVersion: 1,
|
|
592
|
+
requestId: request.requestId,
|
|
593
|
+
fencingToken: request.fencingToken,
|
|
594
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
595
|
+
status: "failed",
|
|
596
|
+
reasonCode: "host_runtime_probe_failed",
|
|
597
|
+
reasonMessage: errorMessage(error).slice(0, 1000),
|
|
598
|
+
checkedAt: new Date().toISOString(),
|
|
599
|
+
};
|
|
600
|
+
}
|
|
601
|
+
finally {
|
|
602
|
+
clearTimeout(timer);
|
|
603
|
+
this.activeReadinessProbes.delete(request.requestId);
|
|
604
|
+
}
|
|
605
|
+
await this.invokeAck("ReadinessResult", result).catch(() => undefined);
|
|
606
|
+
await this.sendHeartbeat().catch(() => undefined);
|
|
607
|
+
}
|
|
159
608
|
abortActiveSessions(reason) {
|
|
160
609
|
for (const active of this.activeSessions.values()) {
|
|
161
610
|
active.controller.abort(reason);
|
|
162
611
|
}
|
|
163
612
|
this.activeSessions.clear();
|
|
164
613
|
}
|
|
614
|
+
abortActiveReadinessProbes(reason) {
|
|
615
|
+
for (const controller of this.activeReadinessProbes.values())
|
|
616
|
+
controller.abort(reason);
|
|
617
|
+
this.activeReadinessProbes.clear();
|
|
618
|
+
}
|
|
165
619
|
normalizeDispatch(dispatch) {
|
|
166
620
|
const mapping = findRepoMapping(dispatch, repoMappingsFor(this.config));
|
|
167
621
|
const localPath = mapping?.localPath?.trim();
|
|
@@ -186,12 +640,20 @@ export class ForgeBridgeService {
|
|
|
186
640
|
: dispatch.qaBrowser
|
|
187
641
|
};
|
|
188
642
|
}
|
|
189
|
-
async sendResult(
|
|
643
|
+
async sendResult(dispatch, result) {
|
|
190
644
|
const payload = {
|
|
191
|
-
leaseId,
|
|
192
|
-
fencingToken,
|
|
645
|
+
leaseId: dispatch.lease.id,
|
|
646
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
647
|
+
dispatchAttemptId: dispatch.sandbox?.dispatchAttemptId ?? null,
|
|
648
|
+
runnerInstanceId: dispatch.sandbox?.runnerInstanceId ?? null,
|
|
193
649
|
result
|
|
194
650
|
};
|
|
651
|
+
await this.sendResultPayload(payload);
|
|
652
|
+
}
|
|
653
|
+
async sendLegacyResult(leaseId, fencingToken, result) {
|
|
654
|
+
await this.sendResultPayload({ leaseId, fencingToken, result });
|
|
655
|
+
}
|
|
656
|
+
async sendResultPayload(payload) {
|
|
195
657
|
try {
|
|
196
658
|
await this.invokeAck("DispatchResult", payload);
|
|
197
659
|
}
|
|
@@ -204,18 +666,132 @@ export class ForgeBridgeService {
|
|
|
204
666
|
}
|
|
205
667
|
startHeartbeat() {
|
|
206
668
|
this.stopHeartbeat();
|
|
669
|
+
this.consecutiveHeartbeatFailures = 0;
|
|
670
|
+
this.reconnectingFromHeartbeatFailure = false;
|
|
207
671
|
const intervalMs = positiveFiniteNumber(this.config.heartbeatIntervalMs, 30_000);
|
|
208
672
|
this.heartbeatTimer = setInterval(() => {
|
|
209
|
-
this.
|
|
673
|
+
void this.heartbeatTick();
|
|
210
674
|
}, intervalMs);
|
|
211
675
|
this.heartbeatTimer.unref?.();
|
|
212
676
|
}
|
|
677
|
+
async heartbeatTick() {
|
|
678
|
+
if (this.heartbeatInFlight || this.reconnectingFromHeartbeatFailure)
|
|
679
|
+
return;
|
|
680
|
+
const connection = this.connection;
|
|
681
|
+
if (!connection)
|
|
682
|
+
return;
|
|
683
|
+
this.heartbeatInFlight = true;
|
|
684
|
+
try {
|
|
685
|
+
await this.sendHeartbeat();
|
|
686
|
+
this.consecutiveHeartbeatFailures = 0;
|
|
687
|
+
}
|
|
688
|
+
catch (error) {
|
|
689
|
+
if (connection !== this.connection || this.stopped)
|
|
690
|
+
return;
|
|
691
|
+
this.consecutiveHeartbeatFailures += 1;
|
|
692
|
+
logBridgeStatus(`Heartbeat ${this.consecutiveHeartbeatFailures} failed: ${errorMessage(error)}.`);
|
|
693
|
+
if (this.consecutiveHeartbeatFailures < 2)
|
|
694
|
+
return;
|
|
695
|
+
this.reconnectingFromHeartbeatFailure = true;
|
|
696
|
+
this.stopHeartbeat();
|
|
697
|
+
logBridgeStatus("Heartbeat failed twice. Closing the stale connection so the Bridge can reconnect...");
|
|
698
|
+
await connection.stop().catch(() => undefined);
|
|
699
|
+
}
|
|
700
|
+
finally {
|
|
701
|
+
this.heartbeatInFlight = false;
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
startLeaseWatchdog() {
|
|
705
|
+
this.stopLeaseWatchdog();
|
|
706
|
+
this.leaseWatchdogTimer = setInterval(() => this.expireLeases(), 1000);
|
|
707
|
+
this.leaseWatchdogTimer.unref?.();
|
|
708
|
+
}
|
|
709
|
+
stopLeaseWatchdog() {
|
|
710
|
+
if (this.leaseWatchdogTimer) {
|
|
711
|
+
clearInterval(this.leaseWatchdogTimer);
|
|
712
|
+
this.leaseWatchdogTimer = null;
|
|
713
|
+
}
|
|
714
|
+
}
|
|
715
|
+
expireLeases() {
|
|
716
|
+
const now = Date.now();
|
|
717
|
+
for (const [attemptId, reservation] of this.reservations) {
|
|
718
|
+
if (Date.parse(reservation.expiresAt) <= now || Date.parse(reservation.reservedUntil) <= now) {
|
|
719
|
+
this.reservations.delete(attemptId);
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
for (const active of this.activeSessions.values()) {
|
|
723
|
+
if (Date.parse(active.leaseExpiresAt) <= now && !active.controller.signal.aborted) {
|
|
724
|
+
active.controller.abort("dispatch lease expired");
|
|
725
|
+
}
|
|
726
|
+
}
|
|
727
|
+
}
|
|
213
728
|
stopHeartbeat() {
|
|
214
729
|
if (this.heartbeatTimer) {
|
|
215
730
|
clearInterval(this.heartbeatTimer);
|
|
216
731
|
this.heartbeatTimer = null;
|
|
217
732
|
}
|
|
218
733
|
}
|
|
734
|
+
startSandboxMaintenance() {
|
|
735
|
+
this.stopSandboxMaintenance();
|
|
736
|
+
this.sandboxMaintenanceTimer = setInterval(() => {
|
|
737
|
+
this.runSandboxMaintenance().catch(() => undefined);
|
|
738
|
+
}, 60_000);
|
|
739
|
+
this.sandboxMaintenanceTimer.unref?.();
|
|
740
|
+
}
|
|
741
|
+
stopSandboxMaintenance() {
|
|
742
|
+
if (this.sandboxMaintenanceTimer) {
|
|
743
|
+
clearInterval(this.sandboxMaintenanceTimer);
|
|
744
|
+
this.sandboxMaintenanceTimer = null;
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
async runSandboxMaintenance() {
|
|
748
|
+
if (this.sandboxMaintenanceRunning)
|
|
749
|
+
return;
|
|
750
|
+
this.sandboxMaintenanceRunning = true;
|
|
751
|
+
try {
|
|
752
|
+
const hostReap = await this.sessions.reapHostRuntime?.({
|
|
753
|
+
runnerId: this.config.runnerId,
|
|
754
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
755
|
+
});
|
|
756
|
+
if (hostReap && (hostReap.reaped > 0 || hostReap.degraded > 0)) {
|
|
757
|
+
logBridgeStatus(`Managed-host maintenance reaped ${hostReap.reaped} stale attempt(s) and left ${hostReap.degraded} degraded attempt(s) for reconciliation.`);
|
|
758
|
+
}
|
|
759
|
+
if (this.config.dockerEnabled) {
|
|
760
|
+
const preflight = await this.preflight.refresh();
|
|
761
|
+
if (canReapManagedSandboxes(preflight)) {
|
|
762
|
+
const attemptSnapshot = await this.readAttemptSnapshot();
|
|
763
|
+
await this.sandboxRuntime.reap({ diskLow: preflight.diskLow, attemptSnapshot });
|
|
764
|
+
await this.flushCleanupReports();
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
}
|
|
768
|
+
finally {
|
|
769
|
+
this.sandboxMaintenanceRunning = false;
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
async readAttemptSnapshot() {
|
|
773
|
+
const connection = this.connection;
|
|
774
|
+
if (!connection)
|
|
775
|
+
throw new Error("Bridge is not connected.");
|
|
776
|
+
const response = await connection.invoke("AttemptSnapshot", {
|
|
777
|
+
runnerInstanceId: this.config.runnerInstanceId
|
|
778
|
+
});
|
|
779
|
+
if (!response.accepted) {
|
|
780
|
+
throw new BridgeCallbackRejectedError(response.code ?? null, response.message ?? response.code ?? "Attempt snapshot was rejected.");
|
|
781
|
+
}
|
|
782
|
+
return response.attempts ?? [];
|
|
783
|
+
}
|
|
784
|
+
async flushCleanupReports() {
|
|
785
|
+
if (!this.sandboxRuntime.pendingCleanupReports || !this.sandboxRuntime.acknowledgeCleanupReport)
|
|
786
|
+
return;
|
|
787
|
+
for (const report of await this.sandboxRuntime.pendingCleanupReports()) {
|
|
788
|
+
await this.invokeAck("CleanupReport", {
|
|
789
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
790
|
+
report
|
|
791
|
+
});
|
|
792
|
+
await this.sandboxRuntime.acknowledgeCleanupReport(report.reportId);
|
|
793
|
+
}
|
|
794
|
+
}
|
|
219
795
|
async sendHeartbeat() {
|
|
220
796
|
await this.invokeAck("Heartbeat", {
|
|
221
797
|
status: "online",
|
|
@@ -225,8 +801,14 @@ export class ForgeBridgeService {
|
|
|
225
801
|
});
|
|
226
802
|
}
|
|
227
803
|
capabilities() {
|
|
804
|
+
const preflight = this.preflight.current();
|
|
805
|
+
const runtimeState = this.sandboxRuntime.snapshot();
|
|
806
|
+
const reserved = [...this.reservations.values()].filter(reservation => reservation.sandboxed).length;
|
|
807
|
+
const used = this.usedSandboxResources(runtimeState.degraded);
|
|
808
|
+
const activeResources = this.activeSandboxResources(runtimeState.degraded);
|
|
809
|
+
const capacity = this.sandboxResourceCapacity(preflight, activeResources);
|
|
228
810
|
return {
|
|
229
|
-
schemaVersion:
|
|
811
|
+
schemaVersion: 3,
|
|
230
812
|
activeSessionCount: this.activeSessions.size,
|
|
231
813
|
cliProviders: this.config.cliProviders,
|
|
232
814
|
providerStatuses: this.config.providerStatuses,
|
|
@@ -234,6 +816,37 @@ export class ForgeBridgeService {
|
|
|
234
816
|
repoMappings: repoMappingsFor(this.config),
|
|
235
817
|
os: this.config.os,
|
|
236
818
|
sandboxProfiles: this.config.sandboxProfiles,
|
|
819
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
820
|
+
trustDomainId: this.config.trustDomainId,
|
|
821
|
+
maxConcurrentSandboxes: capacity.sandboxSlots,
|
|
822
|
+
reservedSandboxCount: reserved,
|
|
823
|
+
activeSandboxCount: runtimeState.active,
|
|
824
|
+
startingSandboxCount: runtimeState.starting,
|
|
825
|
+
degradedReservationCount: runtimeState.degraded,
|
|
826
|
+
sandboxRuntimes: preflight?.healthy ? ["docker"] : [],
|
|
827
|
+
sandboxRuntimeVersion: preflight?.runtimeVersion ?? null,
|
|
828
|
+
sandboxFeatures: preflight?.features ?? null,
|
|
829
|
+
allowsOpenPublicSandbox: this.config.allowOpenPublicSandbox,
|
|
830
|
+
sandboxPublicationModes: [this.config.sandboxPublicationMode ?? "shared_filesystem"],
|
|
831
|
+
sandboxAuthModes: sandboxAuthModesFor(this.config),
|
|
832
|
+
availableMemoryBytes: preflight ? Math.max(0, capacity.memoryBytes - used.memoryBytes) : null,
|
|
833
|
+
availableCpuMillis: preflight ? Math.max(0, capacity.cpuMillis - used.cpuMillis) : null,
|
|
834
|
+
availableWorkspaceBytes: preflight ? Math.max(0, capacity.workspaceBytes - used.workspaceBytes) : null,
|
|
835
|
+
diskLow: preflight?.diskLow ?? null,
|
|
836
|
+
preflightGeneration: preflight?.generation ?? 0,
|
|
837
|
+
sandboxUnavailableCode: this.config.dockerEnabled ? preflight?.unavailableCode ?? null : "docker_disabled",
|
|
838
|
+
sandboxUnavailableReason: this.config.dockerEnabled ? preflight?.unavailableReason ?? null : "Docker sandboxing is disabled by runner configuration.",
|
|
839
|
+
sandboxMemoryCapacity: preflight ? {
|
|
840
|
+
defaultWorkerMemoryLimitBytes: this.config.defaultSandboxWorkerMemoryLimitBytes,
|
|
841
|
+
defaultServiceMemoryLimitBytes: this.config.defaultSandboxServiceMemoryLimitBytes,
|
|
842
|
+
runtimeOverheadBytes: this.config.sandboxRuntimeOverheadBytes,
|
|
843
|
+
hostReserveBytes: preflight.hostReserveBytes,
|
|
844
|
+
activeReservationBytes: used.memoryBytes,
|
|
845
|
+
daemonTotalBytes: preflight.totalMemoryBytes,
|
|
846
|
+
daemonAvailableBytes: preflight.availableMemoryBytes,
|
|
847
|
+
availableForAdmissionBytes: Math.max(0, capacity.memoryBytes - used.memoryBytes),
|
|
848
|
+
} : null,
|
|
849
|
+
hostRuntime: this.sessions.runtimeCapacity?.() ?? null,
|
|
237
850
|
supportsApiSecretGrant: this.config.supportsApiSecretGrant,
|
|
238
851
|
supportsManagedWorktrees: this.config.supportsManagedWorktrees,
|
|
239
852
|
supportsManagedForgeArtifacts: this.config.supportsManagedForgeArtifacts,
|
|
@@ -243,16 +856,118 @@ export class ForgeBridgeService {
|
|
|
243
856
|
supportsDraftImageAttachments: this.config.supportsDraftImageAttachments
|
|
244
857
|
};
|
|
245
858
|
}
|
|
859
|
+
usedSandboxResources(degradedReservations) {
|
|
860
|
+
const resources = [
|
|
861
|
+
...[...this.reservations.values()]
|
|
862
|
+
.filter(reservation => reservation.sandboxed)
|
|
863
|
+
.map(reservation => reservation.resources),
|
|
864
|
+
...[...this.activeSessions.values()]
|
|
865
|
+
.filter(session => session.sandboxed)
|
|
866
|
+
.map(session => session.resources),
|
|
867
|
+
...Array.from({ length: degradedReservations }, () => defaultSandboxResources())
|
|
868
|
+
];
|
|
869
|
+
return sumSandboxResources(resources);
|
|
870
|
+
}
|
|
871
|
+
activeSandboxResources(degradedReservations) {
|
|
872
|
+
return sumSandboxResources([
|
|
873
|
+
...[...this.activeSessions.values()]
|
|
874
|
+
.filter(session => session.sandboxed)
|
|
875
|
+
.map(session => session.resources),
|
|
876
|
+
...Array.from({ length: degradedReservations }, () => defaultSandboxResources())
|
|
877
|
+
]);
|
|
878
|
+
}
|
|
879
|
+
sandboxResourceCapacity(preflight, activeResources) {
|
|
880
|
+
if (!preflight)
|
|
881
|
+
return zeroSandboxResources();
|
|
882
|
+
return {
|
|
883
|
+
// Host free memory and disk already reflect active containers. Add their
|
|
884
|
+
// commitments back to form a stable total capacity, then subtract them
|
|
885
|
+
// exactly once in usedSandboxResources.
|
|
886
|
+
sandboxSlots: Math.min(this.config.maxConcurrentSandboxes, preflight.maxConcurrentSandboxes + activeResources.sandboxSlots),
|
|
887
|
+
cpuMillis: preflight.availableCpuMillis,
|
|
888
|
+
workerMemoryLimitBytes: 0,
|
|
889
|
+
serviceMemoryLimitBytes: 0,
|
|
890
|
+
runtimeOverheadBytes: 0,
|
|
891
|
+
memoryBytes: Math.max(0, preflight.availableMemoryBytes
|
|
892
|
+
- (this.config.sandboxHostMemoryReserveBytes ?? 2 * 1024 ** 3)
|
|
893
|
+
+ activeResources.memoryBytes),
|
|
894
|
+
workspaceBytes: Math.max(0, preflight.availableWorkspaceBytes
|
|
895
|
+
- this.config.minimumWorkspaceBytes
|
|
896
|
+
+ activeResources.workspaceBytes),
|
|
897
|
+
pids: this.config.sandboxPidCapacity,
|
|
898
|
+
buildSlots: this.config.maxConcurrentSandboxBuilds
|
|
899
|
+
};
|
|
900
|
+
}
|
|
246
901
|
async invokeAck(methodName, payload) {
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
902
|
+
let storageRecoveryAttempt = 0;
|
|
903
|
+
while (true) {
|
|
904
|
+
const connection = this.connection;
|
|
905
|
+
if (!connection) {
|
|
906
|
+
throw new Error("Bridge is not connected.");
|
|
907
|
+
}
|
|
908
|
+
await this.waitForTransportRecovery(connection, this.transportRecovery);
|
|
909
|
+
const recoveryGeneration = this.transportRecoveryGeneration;
|
|
910
|
+
try {
|
|
911
|
+
const ack = await connection.invoke(methodName, payload);
|
|
912
|
+
if (!ack.accepted) {
|
|
913
|
+
throw new BridgeCallbackRejectedError(ack.code ?? null, ack.message ?? ack.code ?? `${methodName} callback was rejected.`);
|
|
914
|
+
}
|
|
915
|
+
return ack;
|
|
916
|
+
}
|
|
917
|
+
catch (error) {
|
|
918
|
+
if (connection !== this.connection || this.stopped)
|
|
919
|
+
throw error;
|
|
920
|
+
const recovery = this.transportRecovery;
|
|
921
|
+
if (recovery && recovery.generation > recoveryGeneration) {
|
|
922
|
+
await this.waitForTransportRecovery(connection, recovery);
|
|
923
|
+
continue;
|
|
924
|
+
}
|
|
925
|
+
if (isProviderStorageUnavailable(error) && storageRecoveryAttempt < this.config.reconnectBackoffMs.length) {
|
|
926
|
+
const backoff = this.config.reconnectBackoffMs[storageRecoveryAttempt] ?? 5000;
|
|
927
|
+
storageRecoveryAttempt += 1;
|
|
928
|
+
logBridgeStatus(`${methodName} reached Forge while storage was recovering. Retrying callback ${storageRecoveryAttempt} in ${formatDelay(backoff)}...`);
|
|
929
|
+
await delay(backoff);
|
|
930
|
+
continue;
|
|
931
|
+
}
|
|
932
|
+
throw error;
|
|
933
|
+
}
|
|
250
934
|
}
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
935
|
+
}
|
|
936
|
+
beginTransportRecovery() {
|
|
937
|
+
if (this.transportRecovery && !this.transportRecovery.completed)
|
|
938
|
+
return;
|
|
939
|
+
let resolve = () => undefined;
|
|
940
|
+
const promise = new Promise((complete) => { resolve = complete; });
|
|
941
|
+
this.transportRecoveryGeneration += 1;
|
|
942
|
+
this.transportRecovery = {
|
|
943
|
+
generation: this.transportRecoveryGeneration,
|
|
944
|
+
promise,
|
|
945
|
+
resolve,
|
|
946
|
+
completed: false,
|
|
947
|
+
error: null,
|
|
948
|
+
};
|
|
949
|
+
}
|
|
950
|
+
completeTransportRecovery(error = null) {
|
|
951
|
+
const recovery = this.transportRecovery;
|
|
952
|
+
if (!recovery || recovery.completed)
|
|
953
|
+
return;
|
|
954
|
+
recovery.completed = true;
|
|
955
|
+
recovery.error = error;
|
|
956
|
+
recovery.resolve();
|
|
957
|
+
}
|
|
958
|
+
async waitForTransportRecovery(connection, recovery) {
|
|
959
|
+
if (!recovery)
|
|
960
|
+
return;
|
|
961
|
+
if (recovery.completed) {
|
|
962
|
+
if (recovery.error)
|
|
963
|
+
throw recovery.error;
|
|
964
|
+
return;
|
|
254
965
|
}
|
|
255
|
-
|
|
966
|
+
await recovery.promise;
|
|
967
|
+
if (connection !== this.connection)
|
|
968
|
+
throw new Error("Bridge connection changed while recovering.");
|
|
969
|
+
if (recovery.error)
|
|
970
|
+
throw recovery.error;
|
|
256
971
|
}
|
|
257
972
|
async waitForClose(signal) {
|
|
258
973
|
if (!this.closed) {
|
|
@@ -262,6 +977,8 @@ export class ForgeBridgeService {
|
|
|
262
977
|
}
|
|
263
978
|
async stopConnection() {
|
|
264
979
|
this.stopHeartbeat();
|
|
980
|
+
this.stopLeaseWatchdog();
|
|
981
|
+
this.stopSandboxMaintenance();
|
|
265
982
|
if (this.connection) {
|
|
266
983
|
await this.connection.stop().catch(() => undefined);
|
|
267
984
|
this.connection = null;
|
|
@@ -282,6 +999,17 @@ function isDispatchNoLongerPending(error) {
|
|
|
282
999
|
}
|
|
283
1000
|
return error instanceof Error && error.message === "Dispatch is no longer pending.";
|
|
284
1001
|
}
|
|
1002
|
+
function isProviderStorageUnavailable(error) {
|
|
1003
|
+
const message = errorMessage(error).toLowerCase();
|
|
1004
|
+
return message.includes("provider_storage_unavailable")
|
|
1005
|
+
|| (message.includes("storage is recovering") && message.includes("503"));
|
|
1006
|
+
}
|
|
1007
|
+
function createDefaultSessionService(sandboxStateRoot) {
|
|
1008
|
+
const rawProcessRunner = new StreamingProviderProcessRunner();
|
|
1009
|
+
const processSupervisors = new AttemptProcessSupervisorRegistry();
|
|
1010
|
+
const coordinatedProcessRunner = new AttemptCoordinatedProviderProcessRunner(processSupervisors, rawProcessRunner);
|
|
1011
|
+
return new BridgeSessionService(new GitWorktreeService(undefined, new RepositoryCoordinatorLock(sandboxStateRoot)), new RoutingProviderCliService(new ProcessBackedProviderCliService(coordinatedProcessRunner)), new HostRuntimeLeaseManager(sandboxStateRoot, undefined, undefined, processSupervisors, rawProcessRunner));
|
|
1012
|
+
}
|
|
285
1013
|
export class SignalRBridgeConnectionFactory {
|
|
286
1014
|
create(config) {
|
|
287
1015
|
return new SignalRBridgeConnection(config);
|
|
@@ -304,8 +1032,12 @@ export class SignalRBridgeConnection {
|
|
|
304
1032
|
Authorization: `Bearer ${config.deviceToken}`
|
|
305
1033
|
}
|
|
306
1034
|
})
|
|
1035
|
+
.withAutomaticReconnect([...config.reconnectBackoffMs])
|
|
307
1036
|
.configureLogging(LogLevel.Warning)
|
|
308
1037
|
.build();
|
|
1038
|
+
this.connection.on("Superseded", () => {
|
|
1039
|
+
void this.connection.stop().catch(() => undefined);
|
|
1040
|
+
});
|
|
309
1041
|
}
|
|
310
1042
|
start() {
|
|
311
1043
|
return this.connection.start();
|
|
@@ -316,16 +1048,49 @@ export class SignalRBridgeConnection {
|
|
|
316
1048
|
onClose(handler) {
|
|
317
1049
|
this.connection.onclose(handler);
|
|
318
1050
|
}
|
|
1051
|
+
onReconnecting(handler) {
|
|
1052
|
+
this.connection.onreconnecting(handler);
|
|
1053
|
+
}
|
|
1054
|
+
onReconnected(handler) {
|
|
1055
|
+
this.connection.onreconnected((connectionId) => {
|
|
1056
|
+
void Promise.resolve(handler(connectionId)).catch(() => undefined);
|
|
1057
|
+
});
|
|
1058
|
+
}
|
|
319
1059
|
onDispatch(handler) {
|
|
320
1060
|
this.connection.on("Dispatch", (raw) => {
|
|
321
1061
|
void handler(raw).catch(() => undefined);
|
|
322
1062
|
});
|
|
323
1063
|
}
|
|
1064
|
+
onOffer(handler) {
|
|
1065
|
+
this.connection.on("Offer", (raw) => {
|
|
1066
|
+
void handler(raw).catch(() => undefined);
|
|
1067
|
+
});
|
|
1068
|
+
}
|
|
1069
|
+
onStart(handler) {
|
|
1070
|
+
this.connection.on("Start", (raw) => {
|
|
1071
|
+
void handler(raw).catch(() => undefined);
|
|
1072
|
+
});
|
|
1073
|
+
}
|
|
1074
|
+
onLeaseUpdate(handler) {
|
|
1075
|
+
this.connection.on("LeaseUpdate", (request) => {
|
|
1076
|
+
void handler(request).catch(() => undefined);
|
|
1077
|
+
});
|
|
1078
|
+
}
|
|
324
1079
|
onCancel(handler) {
|
|
325
1080
|
this.connection.on("Cancel", (request) => {
|
|
326
1081
|
void handler(request).catch(() => undefined);
|
|
327
1082
|
});
|
|
328
1083
|
}
|
|
1084
|
+
onReap(handler) {
|
|
1085
|
+
this.connection.on("Reap", (request) => {
|
|
1086
|
+
void handler(request).catch(() => undefined);
|
|
1087
|
+
});
|
|
1088
|
+
}
|
|
1089
|
+
onReadinessProbe(handler) {
|
|
1090
|
+
this.connection.on("ReadinessProbe", (request) => {
|
|
1091
|
+
void handler(request).catch(() => undefined);
|
|
1092
|
+
});
|
|
1093
|
+
}
|
|
329
1094
|
invoke(methodName, payload) {
|
|
330
1095
|
return this.connection.invoke(methodName, payload);
|
|
331
1096
|
}
|
|
@@ -333,9 +1098,37 @@ export class SignalRBridgeConnection {
|
|
|
333
1098
|
export function defaultBridgeServiceConfig(overrides = {}) {
|
|
334
1099
|
const defaults = defaultForgeBridgeConfig();
|
|
335
1100
|
const cliProviders = splitCsv(process.env.FORGE_CLI_PROVIDERS ?? "codex");
|
|
1101
|
+
const sandboxHostMemoryReserveDefault = process.platform === "linux"
|
|
1102
|
+
? 2 * 1024 ** 3
|
|
1103
|
+
: 1024 ** 3;
|
|
1104
|
+
const sandboxPublicationMode = process.env.FORGE_SANDBOX_PUBLICATION_MODE === "bridge_push" ? "bridge_push" : "shared_filesystem";
|
|
336
1105
|
const config = {
|
|
337
1106
|
serverUrl: process.env.FORGE_URL ?? "http://localhost:5208",
|
|
338
1107
|
runnerId: process.env.FORGE_RUNNER_ID ?? "",
|
|
1108
|
+
runnerInstanceId: process.env.FORGE_RUNNER_INSTANCE_ID ?? randomUUID(),
|
|
1109
|
+
trustDomainId: process.env.FORGE_TRUST_DOMAIN_ID ?? "default",
|
|
1110
|
+
maxConcurrentSandboxes: positiveFiniteNumber(Number(process.env.FORGE_MAX_CONCURRENT_SANDBOXES ?? 3), 3),
|
|
1111
|
+
maxConcurrentSandboxBuilds: positiveFiniteNumber(Number(process.env.FORGE_MAX_CONCURRENT_SANDBOX_BUILDS ?? 1), 1),
|
|
1112
|
+
sandboxPidCapacity: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_PID_CAPACITY ?? 2048), 2048),
|
|
1113
|
+
dockerEnabled: process.env.FORGE_DOCKER_SANDBOX !== "false",
|
|
1114
|
+
allowOpenPublicSandbox: process.env.FORGE_ALLOW_OPEN_PUBLIC_SANDBOX === "true",
|
|
1115
|
+
sandboxPublicationMode,
|
|
1116
|
+
dockerExecutable: process.env.FORGE_DOCKER_EXECUTABLE ?? "docker",
|
|
1117
|
+
dockerContext: process.env.FORGE_DOCKER_CONTEXT ?? "",
|
|
1118
|
+
sandboxStateRoot: process.env.FORGE_SANDBOX_STATE_ROOT ?? path.join(os.homedir(), ".forge", "bridge", "sandboxes"),
|
|
1119
|
+
requireRootlessDocker: process.env.FORGE_REQUIRE_ROOTLESS_DOCKER === "true" || process.platform === "linux",
|
|
1120
|
+
diskHighWatermarkPercent: positiveFiniteNumber(Number(process.env.FORGE_DISK_HIGH_WATERMARK_PERCENT ?? 85), 85),
|
|
1121
|
+
diskLowWatermarkPercent: positiveFiniteNumber(Number(process.env.FORGE_DISK_LOW_WATERMARK_PERCENT ?? 75), 75),
|
|
1122
|
+
minimumWorkspaceBytes: positiveFiniteNumber(Number(process.env.FORGE_MINIMUM_WORKSPACE_BYTES ?? 10 * 1024 ** 3), 10 * 1024 ** 3),
|
|
1123
|
+
sandboxHostMemoryReserveBytes: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_HOST_MEMORY_RESERVE_BYTES ?? sandboxHostMemoryReserveDefault), sandboxHostMemoryReserveDefault),
|
|
1124
|
+
defaultSandboxWorkerMemoryLimitBytes: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_DEFAULT_WORKER_MEMORY_LIMIT_BYTES ?? 2 * 1024 ** 3), 2 * 1024 ** 3),
|
|
1125
|
+
defaultSandboxServiceMemoryLimitBytes: nonNegativeFiniteNumber(Number(process.env.FORGE_SANDBOX_DEFAULT_SERVICE_MEMORY_LIMIT_BYTES ?? 0), 0),
|
|
1126
|
+
sandboxRuntimeOverheadBytes: Math.max(128 * 1024 ** 2, nonNegativeFiniteNumber(Number(process.env.FORGE_SANDBOX_RUNTIME_OVERHEAD_BYTES ?? 128 * 1024 ** 2), 128 * 1024 ** 2)),
|
|
1127
|
+
sandboxMemorySampleIntervalMs: Math.min(30_000, Math.max(250, positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_MEMORY_SAMPLE_INTERVAL_MS ?? 1000), 1000))),
|
|
1128
|
+
imageRetentionCount: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_IMAGE_RETENTION_COUNT ?? 3), 3),
|
|
1129
|
+
imageRetentionTtlMs: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_IMAGE_RETENTION_TTL_MS ?? 7 * 24 * 60 * 60 * 1000), 7 * 24 * 60 * 60 * 1000),
|
|
1130
|
+
auditRetentionCount: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_AUDIT_RETENTION_COUNT ?? 200), 200),
|
|
1131
|
+
auditRetentionTtlMs: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_AUDIT_RETENTION_TTL_MS ?? 30 * 24 * 60 * 60 * 1000), 30 * 24 * 60 * 60 * 1000),
|
|
339
1132
|
deviceToken: process.env.FORGE_DEVICE_TOKEN ?? "",
|
|
340
1133
|
deviceName: process.env.FORGE_DEVICE_NAME ?? "",
|
|
341
1134
|
defaultRepoRoot: process.env.FORGE_REPO_ROOT ?? defaults.defaultRepoRoot,
|
|
@@ -354,7 +1147,7 @@ export function defaultBridgeServiceConfig(overrides = {}) {
|
|
|
354
1147
|
supportsDraftAttachments: true,
|
|
355
1148
|
supportsDraftImageAttachments: supportsDraftImageAttachmentsFor(cliProviders),
|
|
356
1149
|
version: process.env.FORGE_BRIDGE_VERSION ?? forgeBridgeVersion,
|
|
357
|
-
protocolVersion:
|
|
1150
|
+
protocolVersion: 2,
|
|
358
1151
|
minServerVersion: null,
|
|
359
1152
|
heartbeatIntervalMs: positiveFiniteNumber(Number(process.env.FORGE_HEARTBEAT_INTERVAL_MS ?? 30_000), 30_000),
|
|
360
1153
|
reconnectBackoffMs: [1000, 2000, 5000, 10000, 30000],
|
|
@@ -380,9 +1173,36 @@ function normalizeServerUrl(serverUrl) {
|
|
|
380
1173
|
function positiveFiniteNumber(value, fallback) {
|
|
381
1174
|
return Number.isFinite(value) && value > 0 ? value : fallback;
|
|
382
1175
|
}
|
|
1176
|
+
function nonNegativeFiniteNumber(value, fallback) {
|
|
1177
|
+
return Number.isFinite(value) && value >= 0 ? value : fallback;
|
|
1178
|
+
}
|
|
383
1179
|
function providerStatusesFor(providers) {
|
|
384
1180
|
return Object.fromEntries(providers.map((provider) => [provider, { signedIn: true, version: "", models: configuredModelsFor(provider) }]));
|
|
385
1181
|
}
|
|
1182
|
+
function sandboxAuthModesFor(config) {
|
|
1183
|
+
return Object.fromEntries(config.cliProviders.map((provider) => {
|
|
1184
|
+
const modes = ["api_key"];
|
|
1185
|
+
if (sandboxBridgeCliAuthAvailable(config, provider)) {
|
|
1186
|
+
modes.push("bridge_cli");
|
|
1187
|
+
}
|
|
1188
|
+
return [provider, modes];
|
|
1189
|
+
}));
|
|
1190
|
+
}
|
|
1191
|
+
function sandboxBridgeCliAuthAvailable(config, provider) {
|
|
1192
|
+
if (provider.trim().toLowerCase() !== "codex") {
|
|
1193
|
+
return false;
|
|
1194
|
+
}
|
|
1195
|
+
const status = providerStatus(config, provider);
|
|
1196
|
+
if (status?.signedIn === false) {
|
|
1197
|
+
return false;
|
|
1198
|
+
}
|
|
1199
|
+
return codexSandboxProviderAuthAdapter.doctor().available;
|
|
1200
|
+
}
|
|
1201
|
+
function providerStatus(config, provider) {
|
|
1202
|
+
const normalized = provider.trim().toLowerCase();
|
|
1203
|
+
return Object.entries(config.providerStatuses)
|
|
1204
|
+
.find(([key]) => key.trim().toLowerCase() === normalized)?.[1];
|
|
1205
|
+
}
|
|
386
1206
|
function defaultSandboxProfiles(providers, platform = process.platform) {
|
|
387
1207
|
if (!providers.some((provider) => provider.toLowerCase() === "codex")) {
|
|
388
1208
|
return ["none"];
|
|
@@ -477,10 +1297,11 @@ function normalizeQaAppLaunchTarget(target) {
|
|
|
477
1297
|
command: cleanString(target.command),
|
|
478
1298
|
working_directory: cleanString(target.working_directory),
|
|
479
1299
|
ready_url: cleanString(target.ready_url),
|
|
1300
|
+
managed_runtime_port: cleanString(target.managed_runtime_port),
|
|
480
1301
|
routes,
|
|
481
1302
|
notes: cleanString(target.notes)
|
|
482
1303
|
};
|
|
483
|
-
return normalized.url || normalized.command || normalized.ready_url ||
|
|
1304
|
+
return normalized.url || normalized.command || normalized.ready_url || normalized.managed_runtime_port
|
|
484
1305
|
? normalized
|
|
485
1306
|
: null;
|
|
486
1307
|
}
|
|
@@ -546,6 +1367,264 @@ function abortPromise(signal) {
|
|
|
546
1367
|
}
|
|
547
1368
|
return new Promise((resolve) => signal.addEventListener("abort", () => resolve(), { once: true }));
|
|
548
1369
|
}
|
|
1370
|
+
export function canReapManagedSandboxes(preflight) {
|
|
1371
|
+
return canRunDockerMaintenance(preflight);
|
|
1372
|
+
}
|
|
1373
|
+
function sandboxNetworkPolicyFailure(dispatch, config) {
|
|
1374
|
+
const sandbox = dispatch.sandbox;
|
|
1375
|
+
if (!sandbox)
|
|
1376
|
+
return "Protocol-v2 sandbox context is missing.";
|
|
1377
|
+
if (sandbox.networkPolicy.mode === "open_public"
|
|
1378
|
+
&& (sandbox.executionPolicy === "sandbox_required" || !config.allowOpenPublicSandbox)) {
|
|
1379
|
+
return "open_public sandbox networking is not permitted by this runner policy.";
|
|
1380
|
+
}
|
|
1381
|
+
if (sandbox.networkPolicy.mode === "public_allowlist"
|
|
1382
|
+
&& sandbox.networkPolicy.allowedHosts.length > 0
|
|
1383
|
+
&& process.env.FORGE_SANDBOX_EGRESS_ENFORCED !== "true") {
|
|
1384
|
+
return "public_allowlist requires an operator-enforced Forge egress proxy or firewall.";
|
|
1385
|
+
}
|
|
1386
|
+
return null;
|
|
1387
|
+
}
|
|
1388
|
+
function infrastructureFailureEventType(summary) {
|
|
1389
|
+
if (summary.startsWith("worktree_prepare_failed:"))
|
|
1390
|
+
return "worktree_prepare_failed";
|
|
1391
|
+
if (summary.startsWith("provider_network_unavailable:"))
|
|
1392
|
+
return "provider_network_unavailable";
|
|
1393
|
+
if (summary.startsWith("provider_start_unavailable:"))
|
|
1394
|
+
return "provider_start_unavailable";
|
|
1395
|
+
if (summary.startsWith("host_port_collision:"))
|
|
1396
|
+
return "host_port_collision";
|
|
1397
|
+
if (summary.startsWith("host_runtime_process_exited:"))
|
|
1398
|
+
return "host_runtime_process_exited";
|
|
1399
|
+
if (summary.startsWith("host_runtime_listener_timeout:"))
|
|
1400
|
+
return "host_runtime_listener_timeout";
|
|
1401
|
+
if (summary.startsWith("host_runtime_http_readiness_timeout:"))
|
|
1402
|
+
return "host_runtime_http_readiness_timeout";
|
|
1403
|
+
if (summary.startsWith("attempt_process_supervisor_start_failed:"))
|
|
1404
|
+
return "attempt_process_supervisor_start_failed";
|
|
1405
|
+
if (summary.startsWith("attempt_process_supervisor_start_timeout:"))
|
|
1406
|
+
return "attempt_process_supervisor_start_timeout";
|
|
1407
|
+
return null;
|
|
1408
|
+
}
|
|
1409
|
+
function parseReadinessProbe(value) {
|
|
1410
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
1411
|
+
return null;
|
|
1412
|
+
const request = value;
|
|
1413
|
+
const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
|
|
1414
|
+
const bounded = (candidate, maximum) => typeof candidate === "string" && candidate.trim().length > 0 && candidate.length <= maximum;
|
|
1415
|
+
if (request.schemaVersion !== 1
|
|
1416
|
+
|| !uuid.test(request.requestId ?? "")
|
|
1417
|
+
|| !uuid.test(request.fencingToken ?? "")
|
|
1418
|
+
|| !uuid.test(request.userId ?? "")
|
|
1419
|
+
|| !uuid.test(request.projectId ?? "")
|
|
1420
|
+
|| !uuid.test(request.repoId ?? "")
|
|
1421
|
+
|| !uuid.test(request.runnerInstanceId ?? "")
|
|
1422
|
+
|| !Number.isInteger(request.attempt) || (request.attempt ?? 0) < 1
|
|
1423
|
+
|| !bounded(request.repoName, 160)
|
|
1424
|
+
|| typeof request.remoteUrl !== "string" || request.remoteUrl.length > 2048
|
|
1425
|
+
|| !bounded(request.defaultBranch, 512)
|
|
1426
|
+
|| typeof request.cloneOnDemand !== "boolean"
|
|
1427
|
+
|| !bounded(request.runnerRepoLocalPath, 4096)
|
|
1428
|
+
|| !bounded(request.revision, 512)
|
|
1429
|
+
|| !Number.isFinite(Date.parse(request.expiresAt ?? ""))
|
|
1430
|
+
|| Date.parse(request.expiresAt ?? "") <= Date.now()) {
|
|
1431
|
+
return null;
|
|
1432
|
+
}
|
|
1433
|
+
return request;
|
|
1434
|
+
}
|
|
1435
|
+
function sandboxProviderNetworkFailure(dispatch, config) {
|
|
1436
|
+
const sandbox = dispatch.sandbox;
|
|
1437
|
+
if (!sandbox || sandbox.networkPolicy.mode !== "offline")
|
|
1438
|
+
return null;
|
|
1439
|
+
const provider = dispatch.execution.provider.trim().toLowerCase();
|
|
1440
|
+
const isNetworkProvider = config.cliProviders.some(candidate => candidate.trim().toLowerCase() === provider);
|
|
1441
|
+
if (!isNetworkProvider)
|
|
1442
|
+
return null;
|
|
1443
|
+
return `Provider ${dispatch.execution.provider} requires outbound network access, but this Docker attempt uses the offline network policy.`;
|
|
1444
|
+
}
|
|
1445
|
+
function sandboxAuthFailure(dispatch, config) {
|
|
1446
|
+
const unsupported = sandboxProviderAuthUnavailableMessage(dispatch.execution.provider, dispatch.execution.authKind);
|
|
1447
|
+
if (unsupported) {
|
|
1448
|
+
return unsupported;
|
|
1449
|
+
}
|
|
1450
|
+
if (dispatch.execution.authKind.trim().toLowerCase() !== "bridge_cli") {
|
|
1451
|
+
return null;
|
|
1452
|
+
}
|
|
1453
|
+
const status = providerStatus(config, dispatch.execution.provider);
|
|
1454
|
+
if (status?.signedIn === false) {
|
|
1455
|
+
return `${dispatch.execution.provider} Local CLI is not signed in on this runner.`;
|
|
1456
|
+
}
|
|
1457
|
+
const adapter = sandboxProviderAuthAdapterFor(dispatch.execution.provider, dispatch.execution.authKind);
|
|
1458
|
+
if (!adapter) {
|
|
1459
|
+
return `${dispatch.execution.provider} Local CLI auth is not portable into Docker yet.`;
|
|
1460
|
+
}
|
|
1461
|
+
const doctor = adapter.doctor();
|
|
1462
|
+
return doctor.available ? null : doctor.message;
|
|
1463
|
+
}
|
|
1464
|
+
export function canReserveSandboxResources(used, requested, capacity) {
|
|
1465
|
+
return used.sandboxSlots + requested.sandboxSlots <= capacity.sandboxSlots
|
|
1466
|
+
&& used.cpuMillis + requested.cpuMillis <= capacity.cpuMillis
|
|
1467
|
+
&& used.memoryBytes + requested.memoryBytes <= capacity.memoryBytes
|
|
1468
|
+
&& used.workspaceBytes + requested.workspaceBytes <= capacity.workspaceBytes
|
|
1469
|
+
&& used.pids + requested.pids <= capacity.pids
|
|
1470
|
+
&& used.buildSlots + requested.buildSlots <= capacity.buildSlots;
|
|
1471
|
+
}
|
|
1472
|
+
export function sandboxCapacityFailureMessage(used, requested, capacity, preflight, config) {
|
|
1473
|
+
const availableSlots = Math.max(0, capacity.sandboxSlots - used.sandboxSlots);
|
|
1474
|
+
if (requested.sandboxSlots > availableSlots) {
|
|
1475
|
+
return `Sandbox slot capacity is insufficient: requested ${requested.sandboxSlots}, available ${availableSlots}, ${used.sandboxSlots} already reserved.`;
|
|
1476
|
+
}
|
|
1477
|
+
const availableMemoryBytes = Math.max(0, capacity.memoryBytes - used.memoryBytes);
|
|
1478
|
+
if (requested.memoryBytes > availableMemoryBytes) {
|
|
1479
|
+
return [
|
|
1480
|
+
`Sandbox memory capacity is insufficient: requested ${formatBytes(requested.memoryBytes)} total`,
|
|
1481
|
+
`(worker ${formatBytes(requested.workerMemoryLimitBytes)}, services ${formatBytes(requested.serviceMemoryLimitBytes)}, runtime overhead ${formatBytes(requested.runtimeOverheadBytes)}),`,
|
|
1482
|
+
`but ${formatBytes(availableMemoryBytes)} is available for admission.`,
|
|
1483
|
+
`Docker reports ${formatBytes(preflight?.availableMemoryBytes ?? 0)} available of ${formatBytes(preflight?.totalMemoryBytes ?? 0)};`,
|
|
1484
|
+
`Forge reserves ${formatBytes(preflight?.hostReserveBytes ?? config.sandboxHostMemoryReserveBytes ?? 0)} for the host and tracks ${formatBytes(used.memoryBytes)} in active or reserved attempts.`,
|
|
1485
|
+
].join(" ");
|
|
1486
|
+
}
|
|
1487
|
+
const availableCpuMillis = Math.max(0, capacity.cpuMillis - used.cpuMillis);
|
|
1488
|
+
if (requested.cpuMillis > availableCpuMillis) {
|
|
1489
|
+
return `Sandbox CPU capacity is insufficient: requested ${requested.cpuMillis} millis, available ${availableCpuMillis} millis, ${used.cpuMillis} millis already reserved.`;
|
|
1490
|
+
}
|
|
1491
|
+
const availableWorkspaceBytes = Math.max(0, capacity.workspaceBytes - used.workspaceBytes);
|
|
1492
|
+
if (requested.workspaceBytes > availableWorkspaceBytes) {
|
|
1493
|
+
return `Sandbox workspace capacity is insufficient: requested ${formatBytes(requested.workspaceBytes)}, available ${formatBytes(availableWorkspaceBytes)}, ${formatBytes(used.workspaceBytes)} already reserved.`;
|
|
1494
|
+
}
|
|
1495
|
+
const availablePids = Math.max(0, capacity.pids - used.pids);
|
|
1496
|
+
if (requested.pids > availablePids) {
|
|
1497
|
+
return `Sandbox PID capacity is insufficient: requested ${requested.pids}, available ${availablePids}, ${used.pids} already reserved.`;
|
|
1498
|
+
}
|
|
1499
|
+
const availableBuildSlots = Math.max(0, capacity.buildSlots - used.buildSlots);
|
|
1500
|
+
return `Sandbox build capacity is insufficient: requested ${requested.buildSlots}, available ${availableBuildSlots}, ${used.buildSlots} already reserved.`;
|
|
1501
|
+
}
|
|
1502
|
+
function sumSandboxResources(resources) {
|
|
1503
|
+
return resources.reduce((total, resource) => ({
|
|
1504
|
+
sandboxSlots: total.sandboxSlots + resource.sandboxSlots,
|
|
1505
|
+
cpuMillis: total.cpuMillis + resource.cpuMillis,
|
|
1506
|
+
workerMemoryLimitBytes: total.workerMemoryLimitBytes + resource.workerMemoryLimitBytes,
|
|
1507
|
+
serviceMemoryLimitBytes: total.serviceMemoryLimitBytes + resource.serviceMemoryLimitBytes,
|
|
1508
|
+
runtimeOverheadBytes: total.runtimeOverheadBytes + resource.runtimeOverheadBytes,
|
|
1509
|
+
memoryBytes: total.memoryBytes + resource.memoryBytes,
|
|
1510
|
+
workspaceBytes: total.workspaceBytes + resource.workspaceBytes,
|
|
1511
|
+
pids: total.pids + resource.pids,
|
|
1512
|
+
buildSlots: total.buildSlots + resource.buildSlots
|
|
1513
|
+
}), zeroSandboxResources());
|
|
1514
|
+
}
|
|
1515
|
+
function zeroSandboxResources() {
|
|
1516
|
+
return {
|
|
1517
|
+
sandboxSlots: 0,
|
|
1518
|
+
cpuMillis: 0,
|
|
1519
|
+
workerMemoryLimitBytes: 0,
|
|
1520
|
+
serviceMemoryLimitBytes: 0,
|
|
1521
|
+
runtimeOverheadBytes: 0,
|
|
1522
|
+
memoryBytes: 0,
|
|
1523
|
+
workspaceBytes: 0,
|
|
1524
|
+
pids: 0,
|
|
1525
|
+
buildSlots: 0,
|
|
1526
|
+
};
|
|
1527
|
+
}
|
|
1528
|
+
function defaultSandboxResources() {
|
|
1529
|
+
return {
|
|
1530
|
+
sandboxSlots: 1,
|
|
1531
|
+
cpuMillis: 2000,
|
|
1532
|
+
workerMemoryLimitBytes: 2 * 1024 ** 3,
|
|
1533
|
+
serviceMemoryLimitBytes: 0,
|
|
1534
|
+
runtimeOverheadBytes: 128 * 1024 ** 2,
|
|
1535
|
+
memoryBytes: 2 * 1024 ** 3 + 128 * 1024 ** 2,
|
|
1536
|
+
workspaceBytes: 10 * 1024 ** 3,
|
|
1537
|
+
pids: 512,
|
|
1538
|
+
buildSlots: 0
|
|
1539
|
+
};
|
|
1540
|
+
}
|
|
1541
|
+
export function resolveDynamicSandboxResources(requested, serviceCount, networkMode, baseRuntimeOverheadBytes = 128 * 1024 ** 2) {
|
|
1542
|
+
const serviceMemoryLimitBytes = Math.max(0, serviceCount) * 1024 ** 3;
|
|
1543
|
+
const runtimeOverheadBytes = Math.max(128 * 1024 ** 2, baseRuntimeOverheadBytes)
|
|
1544
|
+
+ (networkMode === "public_allowlist" ? 256 * 1024 ** 2 : 0);
|
|
1545
|
+
return {
|
|
1546
|
+
...requested,
|
|
1547
|
+
serviceMemoryLimitBytes,
|
|
1548
|
+
runtimeOverheadBytes,
|
|
1549
|
+
memoryBytes: requested.workerMemoryLimitBytes + serviceMemoryLimitBytes + runtimeOverheadBytes,
|
|
1550
|
+
};
|
|
1551
|
+
}
|
|
1552
|
+
export async function resolveSandboxResourcesFromRepo(dispatch, baseRuntimeOverheadBytes = 128 * 1024 ** 2) {
|
|
1553
|
+
const repoPath = dispatch.repo.serverLocalPath;
|
|
1554
|
+
const sandbox = dispatch.sandbox;
|
|
1555
|
+
if (!repoPath || !sandbox) {
|
|
1556
|
+
throw new Error("Sandbox admission needs a mapped repository.");
|
|
1557
|
+
}
|
|
1558
|
+
const selection = selectSandboxConfigRefs(sandbox.executionPolicy, sandbox.sandboxConfigCommitSha, dispatch.repo.defaultBranch);
|
|
1559
|
+
const configCommitSha = await resolveFirstGitCommit(repoPath, selection.refs);
|
|
1560
|
+
if (!configCommitSha) {
|
|
1561
|
+
throw new Error("Sandbox admission could not resolve the immutable sandbox config commit.");
|
|
1562
|
+
}
|
|
1563
|
+
const contractResult = await runSandboxProcess({
|
|
1564
|
+
command: "git",
|
|
1565
|
+
args: ["-C", repoPath, "show", `${configCommitSha}:.forge/forge.sandbox.yml`],
|
|
1566
|
+
timeoutMs: 30_000,
|
|
1567
|
+
maxOutputBytes: 1024 * 1024,
|
|
1568
|
+
});
|
|
1569
|
+
if (contractResult.exitCode !== 0) {
|
|
1570
|
+
throw new Error("The approved immutable sandbox contract is unavailable in the mapped repository.");
|
|
1571
|
+
}
|
|
1572
|
+
const contract = validateSandboxContract(contractResult.stdout);
|
|
1573
|
+
const workspaceBytes = await estimateWorkspaceBytesFromRepo(repoPath, {
|
|
1574
|
+
role: dispatch.execution.role,
|
|
1575
|
+
serviceCount: contract.contract.services.length,
|
|
1576
|
+
browser: contract.contract.browser,
|
|
1577
|
+
});
|
|
1578
|
+
return {
|
|
1579
|
+
resources: resolveDynamicSandboxResources({
|
|
1580
|
+
...sandbox.requestedResources,
|
|
1581
|
+
workspaceBytes: Math.max(sandbox.requestedResources.workspaceBytes, workspaceBytes),
|
|
1582
|
+
}, contract.contract.services.length, sandbox.networkPolicy.mode, baseRuntimeOverheadBytes),
|
|
1583
|
+
sandboxConfigCommitSha: configCommitSha,
|
|
1584
|
+
};
|
|
1585
|
+
}
|
|
1586
|
+
async function resolveFirstGitCommit(repoPath, refs) {
|
|
1587
|
+
for (const ref of refs) {
|
|
1588
|
+
const result = await runSandboxProcess({
|
|
1589
|
+
command: "git",
|
|
1590
|
+
args: ["-C", repoPath, "rev-parse", "--verify", `${ref}^{commit}`],
|
|
1591
|
+
timeoutMs: 30_000,
|
|
1592
|
+
maxOutputBytes: 1024 * 1024,
|
|
1593
|
+
});
|
|
1594
|
+
const commitSha = result.stdout.trim().toLowerCase();
|
|
1595
|
+
if (result.exitCode === 0 && /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/u.test(commitSha)) {
|
|
1596
|
+
return commitSha;
|
|
1597
|
+
}
|
|
1598
|
+
}
|
|
1599
|
+
return null;
|
|
1600
|
+
}
|
|
1601
|
+
function parseOfferEnvelope(value) {
|
|
1602
|
+
return parseDispatchEnvelope(value);
|
|
1603
|
+
}
|
|
1604
|
+
function parseStartEnvelope(value) {
|
|
1605
|
+
return parseDispatchEnvelope(value);
|
|
1606
|
+
}
|
|
1607
|
+
function parseDispatchEnvelope(value) {
|
|
1608
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
1609
|
+
return null;
|
|
1610
|
+
const envelope = value;
|
|
1611
|
+
if (envelope.schemaVersion !== 1
|
|
1612
|
+
|| typeof envelope.dispatchAttemptId !== "string"
|
|
1613
|
+
|| typeof envelope.leaseId !== "string"
|
|
1614
|
+
|| typeof envelope.fencingToken !== "string"
|
|
1615
|
+
|| typeof envelope.runnerInstanceId !== "string"
|
|
1616
|
+
|| typeof envelope.expiresAt !== "string"
|
|
1617
|
+
|| !envelope.dispatch)
|
|
1618
|
+
return null;
|
|
1619
|
+
return {
|
|
1620
|
+
dispatchAttemptId: envelope.dispatchAttemptId,
|
|
1621
|
+
leaseId: envelope.leaseId,
|
|
1622
|
+
fencingToken: envelope.fencingToken,
|
|
1623
|
+
runnerInstanceId: envelope.runnerInstanceId,
|
|
1624
|
+
expiresAt: envelope.expiresAt,
|
|
1625
|
+
dispatch: envelope.dispatch
|
|
1626
|
+
};
|
|
1627
|
+
}
|
|
549
1628
|
const maxAttachmentExactSecretLength = 8 * 1024;
|
|
550
1629
|
function exactSecrets(dispatch, config) {
|
|
551
1630
|
return [
|