@loucompanion/forge-bridge 0.1.3-dev.0bd9fb991b79 → 0.1.3-dev.0d26c10244e1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -0
- package/dist/bridge/bridge.base.d.ts +4 -1
- package/dist/bridge/bridge.service.d.ts +39 -3
- package/dist/bridge/bridge.service.js +585 -56
- package/dist/bridge/bridge.types.d.ts +59 -1
- package/dist/cleanup/cleanup.service.d.ts +14 -2
- package/dist/cleanup/cleanup.service.js +469 -3
- package/dist/cleanup/cleanup.types.d.ts +10 -0
- package/dist/cleanup/index.d.ts +1 -1
- package/dist/cleanup/index.js +1 -1
- package/dist/cleanup/internals.d.ts +2 -2
- package/dist/cleanup/internals.js +1 -1
- package/dist/cli/cli.help.js +37 -3
- package/dist/cli/cli.service.d.ts +30 -2
- package/dist/cli/cli.service.js +191 -27
- package/dist/cli/local-config.d.ts +4 -0
- package/dist/cli/local-config.js +41 -1
- package/dist/runtime/aspnet-runtime-prebuild.d.ts +35 -0
- package/dist/runtime/aspnet-runtime-prebuild.js +168 -0
- package/dist/runtime/attempt-node-runtime-launcher.d.ts +17 -0
- package/dist/runtime/attempt-node-runtime-launcher.js +105 -0
- package/dist/runtime/attempt-process-supervisor-registry.d.ts +20 -0
- package/dist/runtime/attempt-process-supervisor-registry.js +64 -0
- package/dist/runtime/attempt-process-supervisor.d.ts +86 -0
- package/dist/runtime/attempt-process-supervisor.js +350 -0
- package/dist/runtime/generated-output-reset.d.ts +10 -0
- package/dist/runtime/generated-output-reset.js +103 -0
- package/dist/runtime/host-listener-inspector.d.ts +46 -0
- package/dist/runtime/host-listener-inspector.js +349 -0
- package/dist/runtime/host-port-broker.d.ts +76 -0
- package/dist/runtime/host-port-broker.js +333 -0
- package/dist/runtime/host-runtime-contract.d.ts +47 -0
- package/dist/runtime/host-runtime-contract.js +700 -0
- package/dist/runtime/host-runtime-lease.d.ts +179 -0
- package/dist/runtime/host-runtime-lease.js +1990 -0
- package/dist/runtime/host-runtime-promotion.d.ts +23 -0
- package/dist/runtime/host-runtime-promotion.js +159 -0
- package/dist/runtime/node-host-runtime-adapter.d.ts +54 -0
- package/dist/runtime/node-host-runtime-adapter.js +284 -0
- package/dist/runtime/playwright-host-runtime-adapter.d.ts +20 -0
- package/dist/runtime/playwright-host-runtime-adapter.js +220 -0
- package/dist/runtime/runtime-isolation-lease.d.ts +67 -0
- package/dist/runtime/runtime-isolation-lease.js +158 -0
- package/dist/runtime/workspace-resource-estimator.d.ts +16 -0
- package/dist/runtime/workspace-resource-estimator.js +165 -0
- package/dist/sandbox/preflight.d.ts +15 -0
- package/dist/sandbox/preflight.js +100 -13
- package/dist/sandbox/process.js +125 -40
- package/dist/sandbox/repository-lock.js +37 -1
- package/dist/sandbox/runtime.d.ts +66 -3
- package/dist/sandbox/runtime.js +670 -105
- package/dist/sandbox/sandbox.types.d.ts +3 -0
- package/dist/sandbox/smoke.d.ts +76 -0
- package/dist/sandbox/smoke.js +380 -0
- package/dist/sandbox/worker-supervisor.js +1 -1
- package/dist/sandbox/worker.js +2 -5
- package/dist/sandbox/workspace.service.d.ts +1 -0
- package/dist/sandbox/workspace.service.js +28 -5
- package/dist/session/managed-attempt-tools.d.ts +18 -0
- package/dist/session/managed-attempt-tools.js +797 -0
- package/dist/session/managed-build-coordinator.d.ts +5 -0
- package/dist/session/managed-build-coordinator.js +105 -0
- package/dist/session/phase-completion.artifacts.js +21 -3
- package/dist/session/phase-completion.d.ts +11 -2
- package/dist/session/phase-completion.helpers.d.ts +5 -0
- package/dist/session/phase-completion.helpers.js +51 -10
- package/dist/session/phase-completion.js +12 -3
- package/dist/session/phase-completion.prompts.d.ts +3 -3
- package/dist/session/phase-completion.prompts.js +21 -6
- package/dist/session/phase-completion.runtime.d.ts +13 -1
- package/dist/session/phase-completion.runtime.js +187 -8
- package/dist/session/phase-completion.schemas.js +108 -3
- package/dist/session/phase-completion.validation.d.ts +4 -1
- package/dist/session/phase-completion.validation.js +150 -21
- package/dist/session/provider-cli/ci-repair-readiness.d.ts +14 -0
- package/dist/session/provider-cli/ci-repair-readiness.js +105 -0
- package/dist/session/provider-cli/provider-cli.artifacts.d.ts +3 -2
- package/dist/session/provider-cli/provider-cli.artifacts.js +65 -7
- package/dist/session/provider-cli/provider-cli.deterministic.js +47 -1
- package/dist/session/provider-cli/provider-cli.process.d.ts +16 -0
- package/dist/session/provider-cli/provider-cli.process.js +212 -25
- package/dist/session/provider-cli/provider-cli.prompts.d.ts +4 -1
- package/dist/session/provider-cli/provider-cli.prompts.js +99 -19
- package/dist/session/provider-cli/provider-cli.qa-sidecars.js +1 -0
- package/dist/session/provider-cli/provider-cli.runtime.d.ts +1 -1
- package/dist/session/provider-cli/provider-cli.runtime.js +3 -2
- package/dist/session/provider-cli/provider-cli.service.d.ts +20 -2
- package/dist/session/provider-cli/provider-cli.service.js +464 -102
- package/dist/session/provider-cli/provider-cli.types.d.ts +5 -0
- package/dist/session/provider-cli/provider-output-state.d.ts +2 -0
- package/dist/session/provider-cli/provider-output-state.js +69 -7
- package/dist/session/session-redaction.d.ts +3 -0
- package/dist/session/session-redaction.js +38 -0
- package/dist/session/session.base.d.ts +6 -1
- package/dist/session/session.service.d.ts +8 -2
- package/dist/session/session.service.js +482 -41
- package/dist/session/session.types.d.ts +70 -0
- package/dist/session/session.types.js +76 -4
- package/dist/shared/format/format-bytes.d.ts +1 -0
- package/dist/shared/format/format-bytes.js +9 -0
- package/dist/shared/path/path.js +3 -1
- package/dist/shared/process/host-process-recovery.d.ts +12 -0
- package/dist/shared/process/host-process-recovery.js +78 -0
- package/dist/shared/process/host-process-supervisor.d.ts +57 -0
- package/dist/shared/process/host-process-supervisor.js +282 -0
- package/dist/shared/process/process.d.ts +1 -0
- package/dist/shared/process/process.js +1 -0
- package/dist/shared/redaction/redaction.js +6 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/worktree/artifact-injector.d.ts +3 -2
- package/dist/worktree/artifact-injector.js +43 -4
- package/dist/worktree/dependency-warmer.d.ts +10 -1
- package/dist/worktree/dependency-warmer.js +94 -5
- package/dist/worktree/index.d.ts +1 -0
- package/dist/worktree/index.js +1 -0
- package/dist/worktree/prior-phase-artifact-hydrator.d.ts +2 -0
- package/dist/worktree/prior-phase-artifact-hydrator.js +136 -0
- package/dist/worktree/worktree-paths.d.ts +4 -0
- package/dist/worktree/worktree-paths.js +30 -0
- package/dist/worktree/worktree.base.d.ts +3 -0
- package/dist/worktree/worktree.service.d.ts +8 -2
- package/dist/worktree/worktree.service.js +206 -43
- package/dist/worktree/worktree.types.d.ts +4 -1
- package/package.json +3 -2
- package/runtime/windows-job-supervisor.ps1 +356 -0
- package/runtime/windows-listener-owner.ps1 +33 -0
|
@@ -5,32 +5,62 @@ import { randomUUID } from "node:crypto";
|
|
|
5
5
|
import os from "node:os";
|
|
6
6
|
import { defaultForgeBridgeConfig } from "../config.js";
|
|
7
7
|
import { supportsDraftImageAttachmentsFor } from "../shared/provider-capabilities.js";
|
|
8
|
-
import {
|
|
8
|
+
import { redactText } from "../shared/redaction/redaction.js";
|
|
9
9
|
import { BridgeSessionService } from "../session/session.service.js";
|
|
10
|
+
import { ProcessBackedProviderCliService, RoutingProviderCliService, StreamingProviderProcessRunner, } from "../session/provider-cli/provider-cli.service.js";
|
|
11
|
+
import { redactRunnerDispatchEvent, redactRunOnceResultPacket } from "../session/session-redaction.js";
|
|
12
|
+
import { HostRuntimeLeaseManager } from "../runtime/host-runtime-lease.js";
|
|
13
|
+
import { estimateWorkspaceBytesFromRepo } from "../runtime/workspace-resource-estimator.js";
|
|
14
|
+
import { GitWorktreeService } from "../worktree/worktree.service.js";
|
|
15
|
+
import { RepositoryCoordinatorLock } from "../sandbox/repository-lock.js";
|
|
16
|
+
import { AttemptCoordinatedProviderProcessRunner, AttemptProcessSupervisorRegistry, } from "../runtime/attempt-process-supervisor-registry.js";
|
|
10
17
|
import { failureResult, parseRunOnceDispatchPacket } from "../session/session.types.js";
|
|
11
18
|
import { forgeBridgeVersion } from "../version.js";
|
|
12
19
|
import { canRunDockerMaintenance, DockerPreflight } from "../sandbox/preflight.js";
|
|
13
|
-
import {
|
|
20
|
+
import { validateSandboxContract } from "../sandbox/contract.js";
|
|
21
|
+
import { runSandboxProcess } from "../sandbox/process.js";
|
|
22
|
+
import { DockerSandboxRuntime, selectSandboxConfigRefs } from "../sandbox/runtime.js";
|
|
14
23
|
import { codexSandboxProviderAuthAdapter, sandboxProviderAuthAdapterFor, sandboxProviderAuthUnavailableMessage } from "../sandbox/provider-auth/codex.js";
|
|
24
|
+
import { formatBytes } from "../shared/format/format-bytes.js";
|
|
25
|
+
/**
|
|
26
|
+
* Protocol-v2 events are acknowledged one at a time before runOnce returns.
|
|
27
|
+
* Sending the complete transcript again in DispatchResult made long QA and CI
|
|
28
|
+
* sessions exceed SignalR's receive limit, disconnecting the bridge after the
|
|
29
|
+
* provider had already completed. The API combines its acknowledged streamed
|
|
30
|
+
* events with the terminal result, so the callback only needs the outcome and
|
|
31
|
+
* artifacts. Protocol v1 retains the transcript for legacy servers.
|
|
32
|
+
*/
|
|
33
|
+
export function compactTerminalResult(dispatch, result) {
|
|
34
|
+
return dispatch.schemaVersion >= 2 && result.events.length > 0
|
|
35
|
+
? { ...result, events: [] }
|
|
36
|
+
: result;
|
|
37
|
+
}
|
|
15
38
|
export class ForgeBridgeService {
|
|
16
39
|
config;
|
|
17
|
-
sessions;
|
|
18
40
|
connections;
|
|
19
41
|
connection = null;
|
|
20
42
|
heartbeatTimer = null;
|
|
43
|
+
heartbeatInFlight = false;
|
|
44
|
+
consecutiveHeartbeatFailures = 0;
|
|
45
|
+
reconnectingFromHeartbeatFailure = false;
|
|
21
46
|
leaseWatchdogTimer = null;
|
|
22
47
|
sandboxMaintenanceTimer = null;
|
|
23
48
|
sandboxMaintenanceRunning = false;
|
|
24
49
|
closed = null;
|
|
50
|
+
transportRecovery = null;
|
|
51
|
+
transportRecoveryGeneration = 0;
|
|
25
52
|
stopped = false;
|
|
26
53
|
activeSessions = new Map();
|
|
27
54
|
reservations = new Map();
|
|
55
|
+
activeReadinessProbes = new Map();
|
|
28
56
|
preflight;
|
|
29
57
|
sandboxRuntime;
|
|
30
|
-
|
|
58
|
+
sandboxResourceResolver;
|
|
59
|
+
sessions;
|
|
60
|
+
constructor(config, sessions = undefined, connections = new SignalRBridgeConnectionFactory(), preflight, sandboxRuntime, sandboxResourceResolver) {
|
|
31
61
|
this.config = config;
|
|
32
|
-
this.sessions = sessions;
|
|
33
62
|
this.connections = connections;
|
|
63
|
+
this.sessions = sessions ?? createDefaultSessionService(config.sandboxStateRoot);
|
|
34
64
|
this.preflight = preflight ?? new DockerPreflight({
|
|
35
65
|
dockerExecutable: config.dockerExecutable,
|
|
36
66
|
dockerContext: config.dockerContext,
|
|
@@ -40,47 +70,109 @@ export class ForgeBridgeService {
|
|
|
40
70
|
diskHighWatermarkPercent: config.diskHighWatermarkPercent,
|
|
41
71
|
diskLowWatermarkPercent: config.diskLowWatermarkPercent,
|
|
42
72
|
minimumWorkspaceBytes: config.minimumWorkspaceBytes,
|
|
73
|
+
hostMemoryReserveBytes: config.sandboxHostMemoryReserveBytes,
|
|
74
|
+
defaultWorkerMemoryLimitBytes: config.defaultSandboxWorkerMemoryLimitBytes,
|
|
75
|
+
defaultServiceMemoryLimitBytes: config.defaultSandboxServiceMemoryLimitBytes,
|
|
76
|
+
runtimeOverheadBytes: config.sandboxRuntimeOverheadBytes,
|
|
43
77
|
requireRootless: config.requireRootlessDocker
|
|
44
78
|
});
|
|
45
79
|
this.sandboxRuntime = sandboxRuntime ?? new DockerSandboxRuntime(config, (request, signal) => this.redeemPublishGrant(request, signal));
|
|
80
|
+
this.sandboxResourceResolver = sandboxResourceResolver
|
|
81
|
+
?? (dispatch => resolveSandboxResourcesFromRepo(dispatch, this.config.sandboxRuntimeOverheadBytes));
|
|
46
82
|
}
|
|
47
83
|
async connectOnce() {
|
|
48
84
|
await this.stopConnection();
|
|
49
85
|
this.stopped = false;
|
|
86
|
+
const hostReap = await this.sessions.reapHostRuntime?.({
|
|
87
|
+
runnerId: this.config.runnerId,
|
|
88
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
89
|
+
});
|
|
90
|
+
if (hostReap && (hostReap.reaped > 0 || hostReap.degraded > 0)) {
|
|
91
|
+
logBridgeStatus(`Managed-host recovery reaped ${hostReap.reaped} stale attempt(s), protected ${hostReap.reusedPidsProtected} reused PID(s), and left ${hostReap.degraded} degraded attempt(s) for reconciliation.`);
|
|
92
|
+
}
|
|
50
93
|
if (this.config.dockerEnabled) {
|
|
51
94
|
const preflight = await this.preflight.refresh();
|
|
52
95
|
if (canReapManagedSandboxes(preflight)) {
|
|
53
96
|
await this.sandboxRuntime.reap({ diskLow: preflight.diskLow });
|
|
97
|
+
await this.preflight.refresh();
|
|
54
98
|
}
|
|
55
99
|
}
|
|
56
100
|
const connection = this.connections.create(this.config);
|
|
57
101
|
this.connection = connection;
|
|
102
|
+
this.transportRecovery = null;
|
|
58
103
|
this.closed = new Promise((resolve) => {
|
|
59
|
-
connection.onClose(() => {
|
|
104
|
+
connection.onClose((error) => {
|
|
60
105
|
this.stopHeartbeat();
|
|
106
|
+
this.completeTransportRecovery(error ?? new Error("bridge connection closed"));
|
|
61
107
|
this.abortActiveSessions("bridge connection closed");
|
|
108
|
+
this.abortActiveReadinessProbes("bridge connection closed");
|
|
62
109
|
resolve();
|
|
63
110
|
});
|
|
64
111
|
});
|
|
112
|
+
connection.onReconnecting?.((error) => {
|
|
113
|
+
if (connection !== this.connection || this.stopped)
|
|
114
|
+
return;
|
|
115
|
+
this.stopHeartbeat();
|
|
116
|
+
this.beginTransportRecovery();
|
|
117
|
+
logBridgeStatus(`Forge connection interrupted${error ? `: ${errorMessage(error)}` : ""}. Reconnecting in place; active dispatches remain supervised...`);
|
|
118
|
+
});
|
|
119
|
+
connection.onReconnected?.(async () => {
|
|
120
|
+
if (connection !== this.connection || this.stopped)
|
|
121
|
+
return;
|
|
122
|
+
try {
|
|
123
|
+
await this.registerConnection(connection);
|
|
124
|
+
this.completeTransportRecovery();
|
|
125
|
+
this.startHeartbeat();
|
|
126
|
+
await this.flushCleanupReports().catch(() => undefined);
|
|
127
|
+
logBridgeStatus("Forge connection restored; active dispatches remain attached to this runner process.");
|
|
128
|
+
}
|
|
129
|
+
catch (error) {
|
|
130
|
+
this.completeTransportRecovery(error);
|
|
131
|
+
logBridgeStatus(`Forge reconnect registration failed: ${errorMessage(error)}. Closing the connection for a clean retry...`);
|
|
132
|
+
await connection.stop().catch(() => undefined);
|
|
133
|
+
}
|
|
134
|
+
});
|
|
65
135
|
connection.onDispatch((dispatch) => this.handleDispatch(dispatch));
|
|
66
136
|
connection.onOffer?.((dispatch) => this.handleOffer(dispatch));
|
|
67
137
|
connection.onStart?.((dispatch) => this.handleStart(dispatch));
|
|
68
138
|
connection.onLeaseUpdate?.((request) => this.handleLeaseUpdate(request));
|
|
69
139
|
connection.onCancel((request) => this.handleCancel(request));
|
|
70
140
|
connection.onReap?.((request) => this.handleReap(request));
|
|
141
|
+
connection.onReadinessProbe?.((request) => this.handleReadinessProbe(request));
|
|
71
142
|
await connection.start();
|
|
72
|
-
await this.
|
|
73
|
-
version: this.config.version,
|
|
74
|
-
protocolVersion: this.config.protocolVersion,
|
|
75
|
-
minServerVersion: this.config.minServerVersion ?? null,
|
|
76
|
-
runnerInstanceId: this.config.runnerInstanceId,
|
|
77
|
-
capabilities: this.capabilities()
|
|
78
|
-
});
|
|
143
|
+
await this.registerConnection(connection);
|
|
79
144
|
await this.flushCleanupReports();
|
|
80
145
|
this.startHeartbeat();
|
|
81
146
|
this.startLeaseWatchdog();
|
|
82
147
|
this.startSandboxMaintenance();
|
|
83
148
|
}
|
|
149
|
+
async registerConnection(connection) {
|
|
150
|
+
let storageRecoveryAttempt = 0;
|
|
151
|
+
while (true) {
|
|
152
|
+
try {
|
|
153
|
+
const ack = await connection.invoke("Hello", {
|
|
154
|
+
version: this.config.version,
|
|
155
|
+
protocolVersion: this.config.protocolVersion,
|
|
156
|
+
minServerVersion: this.config.minServerVersion ?? null,
|
|
157
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
158
|
+
capabilities: this.capabilities()
|
|
159
|
+
});
|
|
160
|
+
if (!ack.accepted) {
|
|
161
|
+
throw new BridgeCallbackRejectedError(ack.code ?? null, ack.message ?? ack.code ?? "Hello callback was rejected.");
|
|
162
|
+
}
|
|
163
|
+
return;
|
|
164
|
+
}
|
|
165
|
+
catch (error) {
|
|
166
|
+
if (!isProviderStorageUnavailable(error) || storageRecoveryAttempt >= this.config.reconnectBackoffMs.length) {
|
|
167
|
+
throw error;
|
|
168
|
+
}
|
|
169
|
+
const backoff = this.config.reconnectBackoffMs[storageRecoveryAttempt] ?? 5000;
|
|
170
|
+
storageRecoveryAttempt += 1;
|
|
171
|
+
logBridgeStatus(`Hello reached Forge while storage was recovering. Retrying registration ${storageRecoveryAttempt} in ${formatDelay(backoff)}...`);
|
|
172
|
+
await delay(backoff);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
}
|
|
84
176
|
async run(signal) {
|
|
85
177
|
let attempt = 0;
|
|
86
178
|
try {
|
|
@@ -110,6 +202,7 @@ export class ForgeBridgeService {
|
|
|
110
202
|
finally {
|
|
111
203
|
if (signal?.aborted) {
|
|
112
204
|
this.abortActiveSessions(String(signal.reason ?? "bridge stopping"));
|
|
205
|
+
this.abortActiveReadinessProbes(String(signal.reason ?? "bridge stopping"));
|
|
113
206
|
}
|
|
114
207
|
await this.stopConnection();
|
|
115
208
|
}
|
|
@@ -117,6 +210,7 @@ export class ForgeBridgeService {
|
|
|
117
210
|
async stop() {
|
|
118
211
|
this.stopped = true;
|
|
119
212
|
this.abortActiveSessions("bridge stopping");
|
|
213
|
+
this.abortActiveReadinessProbes("bridge stopping");
|
|
120
214
|
await this.stopConnection();
|
|
121
215
|
await this.preflight.release();
|
|
122
216
|
}
|
|
@@ -174,7 +268,7 @@ export class ForgeBridgeService {
|
|
|
174
268
|
dispatchAttemptId: dispatch.sandbox.dispatchAttemptId,
|
|
175
269
|
runnerInstanceId: dispatch.sandbox.runnerInstanceId
|
|
176
270
|
} : {}),
|
|
177
|
-
event:
|
|
271
|
+
event: redactRunnerDispatchEvent(event, redactionSecrets)
|
|
178
272
|
});
|
|
179
273
|
}
|
|
180
274
|
catch (error) {
|
|
@@ -192,13 +286,21 @@ export class ForgeBridgeService {
|
|
|
192
286
|
if (normalized.sandbox?.executionMode === "docker") {
|
|
193
287
|
await this.flushCleanupReports().catch(() => undefined);
|
|
194
288
|
}
|
|
195
|
-
|
|
289
|
+
const redactedResult = redactRunOnceResultPacket(result, redactionSecrets);
|
|
290
|
+
await this.sendResult(dispatch, compactTerminalResult(dispatch, redactedResult));
|
|
196
291
|
}
|
|
197
292
|
catch (error) {
|
|
198
293
|
const summary = controller.signal.aborted
|
|
199
294
|
? `Dispatch cancelled: ${String(controller.signal.reason ?? "cancelled")}`
|
|
200
295
|
: redactText(error instanceof Error ? error.message : String(error), redactionSecrets);
|
|
201
|
-
|
|
296
|
+
const failureEventType = infrastructureFailureEventType(summary);
|
|
297
|
+
const failureEvents = failureEventType ? [{
|
|
298
|
+
role: "runner",
|
|
299
|
+
kind: "error",
|
|
300
|
+
content: summary,
|
|
301
|
+
eventType: failureEventType,
|
|
302
|
+
}] : [];
|
|
303
|
+
await this.sendResult(dispatch, failureResult(summary, failureEvents)).catch(() => undefined);
|
|
202
304
|
}
|
|
203
305
|
finally {
|
|
204
306
|
this.activeSessions.delete(leaseId);
|
|
@@ -223,7 +325,7 @@ export class ForgeBridgeService {
|
|
|
223
325
|
}
|
|
224
326
|
if (existing) {
|
|
225
327
|
const sameFence = existing.fencingToken === dispatch.lease.fencingToken;
|
|
226
|
-
await this.sendOfferDecision(dispatch, sameFence, sameFence ? null : "stale_fence", sameFence ? null : "Offer fence conflicts with the existing reservation.", sameFence ? existing.reservedUntil : null);
|
|
328
|
+
await this.sendOfferDecision(dispatch, sameFence, sameFence ? null : "stale_fence", sameFence ? null : "Offer fence conflicts with the existing reservation.", sameFence ? existing.reservedUntil : null, sameFence ? existing.resources : null);
|
|
227
329
|
return;
|
|
228
330
|
}
|
|
229
331
|
if (sandbox.runnerInstanceId !== this.config.runnerInstanceId) {
|
|
@@ -234,17 +336,24 @@ export class ForgeBridgeService {
|
|
|
234
336
|
await this.sendOfferDecision(dispatch, false, "lease_expired", "Offer lease is already expired.");
|
|
235
337
|
return;
|
|
236
338
|
}
|
|
237
|
-
const preflight = this.
|
|
339
|
+
const preflight = sandbox.executionMode === "docker" && this.config.dockerEnabled
|
|
340
|
+
? await this.preflight.run()
|
|
341
|
+
: this.preflight.current();
|
|
238
342
|
if (sandbox.executionMode === "docker") {
|
|
239
343
|
if (!this.config.dockerEnabled || !preflight?.healthy) {
|
|
240
344
|
await this.sendOfferDecision(dispatch, false, preflight?.unavailableCode ?? "docker_unavailable", preflight?.unavailableReason ?? "Docker sandbox preflight is unavailable.");
|
|
241
345
|
return;
|
|
242
346
|
}
|
|
243
|
-
const networkFailure = sandboxNetworkPolicyFailure(dispatch);
|
|
347
|
+
const networkFailure = sandboxNetworkPolicyFailure(dispatch, this.config);
|
|
244
348
|
if (networkFailure) {
|
|
245
349
|
await this.sendOfferDecision(dispatch, false, "policy_mismatch", networkFailure);
|
|
246
350
|
return;
|
|
247
351
|
}
|
|
352
|
+
const providerNetworkFailure = sandboxProviderNetworkFailure(dispatch, this.config);
|
|
353
|
+
if (providerNetworkFailure) {
|
|
354
|
+
await this.sendOfferDecision(dispatch, false, "provider_network_policy_denied", providerNetworkFailure);
|
|
355
|
+
return;
|
|
356
|
+
}
|
|
248
357
|
const authFailure = sandboxAuthFailure(dispatch, this.config);
|
|
249
358
|
if (authFailure) {
|
|
250
359
|
await this.sendOfferDecision(dispatch, false, "sandbox_auth_unavailable", authFailure);
|
|
@@ -253,10 +362,37 @@ export class ForgeBridgeService {
|
|
|
253
362
|
}
|
|
254
363
|
const runtimeState = this.sandboxRuntime.snapshot();
|
|
255
364
|
const usedResources = this.usedSandboxResources(runtimeState.degraded);
|
|
256
|
-
const
|
|
365
|
+
const activeResources = this.activeSandboxResources(runtimeState.degraded);
|
|
366
|
+
const capacity = this.sandboxResourceCapacity(preflight, activeResources);
|
|
367
|
+
let requestedResources = sandbox.requestedResources;
|
|
368
|
+
let sandboxConfigCommitSha = sandbox.sandboxConfigCommitSha ?? null;
|
|
369
|
+
const normalizedDispatch = this.normalizeDispatch(dispatch);
|
|
370
|
+
if (sandbox.executionMode === "docker") {
|
|
371
|
+
try {
|
|
372
|
+
const profile = await this.sandboxResourceResolver(normalizedDispatch);
|
|
373
|
+
requestedResources = profile.resources;
|
|
374
|
+
sandboxConfigCommitSha = profile.sandboxConfigCommitSha;
|
|
375
|
+
}
|
|
376
|
+
catch (error) {
|
|
377
|
+
await this.sendOfferDecision(dispatch, false, "sandbox_capacity_profile_unavailable", error instanceof Error ? error.message : "The immutable sandbox contract could not be profiled for admission.");
|
|
378
|
+
return;
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
else if (normalizedDispatch.repo.serverLocalPath) {
|
|
382
|
+
// Host execution uses the same project-derived disk reservation as Docker.
|
|
383
|
+
// The API supplies a placement-time estimate; the runner can safely raise it
|
|
384
|
+
// from its authoritative checkout before admitting the attempt.
|
|
385
|
+
const measuredWorkspaceBytes = await estimateWorkspaceBytesFromRepo(normalizedDispatch.repo.serverLocalPath, { role: normalizedDispatch.execution.role }).catch(() => null);
|
|
386
|
+
if (measuredWorkspaceBytes !== null) {
|
|
387
|
+
requestedResources = {
|
|
388
|
+
...requestedResources,
|
|
389
|
+
workspaceBytes: Math.max(requestedResources.workspaceBytes, measuredWorkspaceBytes),
|
|
390
|
+
};
|
|
391
|
+
}
|
|
392
|
+
}
|
|
257
393
|
if (sandbox.executionMode === "docker"
|
|
258
|
-
&& !canReserveSandboxResources(usedResources,
|
|
259
|
-
await this.sendOfferDecision(dispatch, false, "capacity_unavailable",
|
|
394
|
+
&& !canReserveSandboxResources(usedResources, requestedResources, capacity)) {
|
|
395
|
+
await this.sendOfferDecision(dispatch, false, "capacity_unavailable", sandboxCapacityFailureMessage(usedResources, requestedResources, capacity, preflight, this.config));
|
|
260
396
|
return;
|
|
261
397
|
}
|
|
262
398
|
const reservedUntil = new Date(Math.min(Date.parse(dispatch.lease.expiresAt), Date.now() + 30_000)).toISOString();
|
|
@@ -267,9 +403,9 @@ export class ForgeBridgeService {
|
|
|
267
403
|
expiresAt: dispatch.lease.expiresAt,
|
|
268
404
|
reservedUntil,
|
|
269
405
|
sandboxed: sandbox.executionMode === "docker",
|
|
270
|
-
resources:
|
|
406
|
+
resources: requestedResources
|
|
271
407
|
});
|
|
272
|
-
await this.sendOfferDecision(dispatch, true, null, null, reservedUntil);
|
|
408
|
+
await this.sendOfferDecision(dispatch, true, null, null, reservedUntil, requestedResources, sandboxConfigCommitSha);
|
|
273
409
|
}
|
|
274
410
|
async handleStart(rawDispatch) {
|
|
275
411
|
const envelope = parseStartEnvelope(rawDispatch);
|
|
@@ -315,6 +451,10 @@ export class ForgeBridgeService {
|
|
|
315
451
|
await this.sendResult(dispatch, failureResult("Start rejected because the provider secret grant handle is missing.")).catch(() => undefined);
|
|
316
452
|
return;
|
|
317
453
|
}
|
|
454
|
+
dispatch = {
|
|
455
|
+
...dispatch,
|
|
456
|
+
sandbox: { ...dispatch.sandbox, requestedResources: reservation.resources },
|
|
457
|
+
};
|
|
318
458
|
this.reservations.delete(envelope.dispatchAttemptId);
|
|
319
459
|
await this.handleDispatch(dispatch);
|
|
320
460
|
}
|
|
@@ -355,10 +495,15 @@ export class ForgeBridgeService {
|
|
|
355
495
|
const active = this.activeSessions.get(request.leaseId);
|
|
356
496
|
if (active && active.fencingToken === request.fencingToken && active.dispatchAttemptId === request.dispatchAttemptId) {
|
|
357
497
|
active.leaseExpiresAt = request.expiresAt;
|
|
358
|
-
|
|
498
|
+
if (active.sandboxed) {
|
|
499
|
+
await this.sandboxRuntime.updateLease(request.dispatchAttemptId, request.fencingToken, request.expiresAt);
|
|
500
|
+
}
|
|
501
|
+
else {
|
|
502
|
+
await this.sessions.updateLease?.(request.dispatchAttemptId, request.fencingToken, request.expiresAt);
|
|
503
|
+
}
|
|
359
504
|
}
|
|
360
505
|
}
|
|
361
|
-
async sendOfferDecision(dispatch, accepted, code = null, message = null, reservedUntil = null) {
|
|
506
|
+
async sendOfferDecision(dispatch, accepted, code = null, message = null, reservedUntil = null, reservedResources = null, sandboxConfigCommitSha = null) {
|
|
362
507
|
if (!dispatch.sandbox)
|
|
363
508
|
return;
|
|
364
509
|
const payload = {
|
|
@@ -369,7 +514,9 @@ export class ForgeBridgeService {
|
|
|
369
514
|
accepted,
|
|
370
515
|
code,
|
|
371
516
|
message,
|
|
372
|
-
reservedUntil: accepted ? reservedUntil ?? dispatch.lease.expiresAt : null
|
|
517
|
+
reservedUntil: accepted ? reservedUntil ?? dispatch.lease.expiresAt : null,
|
|
518
|
+
reservedResources: accepted ? reservedResources : null,
|
|
519
|
+
sandboxConfigCommitSha: accepted ? sandboxConfigCommitSha : null,
|
|
373
520
|
};
|
|
374
521
|
await this.invokeAck("OfferDecision", payload);
|
|
375
522
|
}
|
|
@@ -402,19 +549,73 @@ export class ForgeBridgeService {
|
|
|
402
549
|
active.controller.abort(request.reason ?? "cancelled by server");
|
|
403
550
|
}
|
|
404
551
|
async handleReap(request) {
|
|
405
|
-
if (
|
|
406
|
-
|| request.runnerId !== this.config.runnerId
|
|
552
|
+
if (request.runnerId !== this.config.runnerId
|
|
407
553
|
|| request.runnerInstanceId !== this.config.runnerInstanceId)
|
|
408
554
|
return;
|
|
409
555
|
await this.runSandboxMaintenance();
|
|
410
556
|
await this.sendHeartbeat();
|
|
411
557
|
}
|
|
558
|
+
async handleReadinessProbe(rawRequest) {
|
|
559
|
+
const request = parseReadinessProbe(rawRequest);
|
|
560
|
+
if (!request || request.runnerInstanceId !== this.config.runnerInstanceId)
|
|
561
|
+
return;
|
|
562
|
+
const existing = this.activeReadinessProbes.get(request.requestId);
|
|
563
|
+
if (existing)
|
|
564
|
+
return;
|
|
565
|
+
const controller = new AbortController();
|
|
566
|
+
this.activeReadinessProbes.set(request.requestId, controller);
|
|
567
|
+
const deadlineMs = Date.parse(request.expiresAt) - Date.now();
|
|
568
|
+
const timer = setTimeout(() => controller.abort("readiness probe expired"), Math.max(0, deadlineMs));
|
|
569
|
+
timer.unref?.();
|
|
570
|
+
let result;
|
|
571
|
+
try {
|
|
572
|
+
if (!this.sessions.probeHostRuntime) {
|
|
573
|
+
throw new Error("host_runtime_probe_unavailable: this bridge does not implement runtime readiness probes.");
|
|
574
|
+
}
|
|
575
|
+
const probe = await this.sessions.probeHostRuntime({
|
|
576
|
+
...request,
|
|
577
|
+
runnerId: this.config.runnerId,
|
|
578
|
+
repoRoot: this.config.defaultRepoRoot,
|
|
579
|
+
cleanupWorktrees: this.config.cleanupWorktrees,
|
|
580
|
+
}, controller.signal);
|
|
581
|
+
result = {
|
|
582
|
+
schemaVersion: 1,
|
|
583
|
+
requestId: request.requestId,
|
|
584
|
+
fencingToken: request.fencingToken,
|
|
585
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
586
|
+
...probe,
|
|
587
|
+
};
|
|
588
|
+
}
|
|
589
|
+
catch (error) {
|
|
590
|
+
result = {
|
|
591
|
+
schemaVersion: 1,
|
|
592
|
+
requestId: request.requestId,
|
|
593
|
+
fencingToken: request.fencingToken,
|
|
594
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
595
|
+
status: "failed",
|
|
596
|
+
reasonCode: "host_runtime_probe_failed",
|
|
597
|
+
reasonMessage: errorMessage(error).slice(0, 1000),
|
|
598
|
+
checkedAt: new Date().toISOString(),
|
|
599
|
+
};
|
|
600
|
+
}
|
|
601
|
+
finally {
|
|
602
|
+
clearTimeout(timer);
|
|
603
|
+
this.activeReadinessProbes.delete(request.requestId);
|
|
604
|
+
}
|
|
605
|
+
await this.invokeAck("ReadinessResult", result).catch(() => undefined);
|
|
606
|
+
await this.sendHeartbeat().catch(() => undefined);
|
|
607
|
+
}
|
|
412
608
|
abortActiveSessions(reason) {
|
|
413
609
|
for (const active of this.activeSessions.values()) {
|
|
414
610
|
active.controller.abort(reason);
|
|
415
611
|
}
|
|
416
612
|
this.activeSessions.clear();
|
|
417
613
|
}
|
|
614
|
+
abortActiveReadinessProbes(reason) {
|
|
615
|
+
for (const controller of this.activeReadinessProbes.values())
|
|
616
|
+
controller.abort(reason);
|
|
617
|
+
this.activeReadinessProbes.clear();
|
|
618
|
+
}
|
|
418
619
|
normalizeDispatch(dispatch) {
|
|
419
620
|
const mapping = findRepoMapping(dispatch, repoMappingsFor(this.config));
|
|
420
621
|
const localPath = mapping?.localPath?.trim();
|
|
@@ -465,12 +666,41 @@ export class ForgeBridgeService {
|
|
|
465
666
|
}
|
|
466
667
|
startHeartbeat() {
|
|
467
668
|
this.stopHeartbeat();
|
|
669
|
+
this.consecutiveHeartbeatFailures = 0;
|
|
670
|
+
this.reconnectingFromHeartbeatFailure = false;
|
|
468
671
|
const intervalMs = positiveFiniteNumber(this.config.heartbeatIntervalMs, 30_000);
|
|
469
672
|
this.heartbeatTimer = setInterval(() => {
|
|
470
|
-
this.
|
|
673
|
+
void this.heartbeatTick();
|
|
471
674
|
}, intervalMs);
|
|
472
675
|
this.heartbeatTimer.unref?.();
|
|
473
676
|
}
|
|
677
|
+
async heartbeatTick() {
|
|
678
|
+
if (this.heartbeatInFlight || this.reconnectingFromHeartbeatFailure)
|
|
679
|
+
return;
|
|
680
|
+
const connection = this.connection;
|
|
681
|
+
if (!connection)
|
|
682
|
+
return;
|
|
683
|
+
this.heartbeatInFlight = true;
|
|
684
|
+
try {
|
|
685
|
+
await this.sendHeartbeat();
|
|
686
|
+
this.consecutiveHeartbeatFailures = 0;
|
|
687
|
+
}
|
|
688
|
+
catch (error) {
|
|
689
|
+
if (connection !== this.connection || this.stopped)
|
|
690
|
+
return;
|
|
691
|
+
this.consecutiveHeartbeatFailures += 1;
|
|
692
|
+
logBridgeStatus(`Heartbeat ${this.consecutiveHeartbeatFailures} failed: ${errorMessage(error)}.`);
|
|
693
|
+
if (this.consecutiveHeartbeatFailures < 2)
|
|
694
|
+
return;
|
|
695
|
+
this.reconnectingFromHeartbeatFailure = true;
|
|
696
|
+
this.stopHeartbeat();
|
|
697
|
+
logBridgeStatus("Heartbeat failed twice. Closing the stale connection so the Bridge can reconnect...");
|
|
698
|
+
await connection.stop().catch(() => undefined);
|
|
699
|
+
}
|
|
700
|
+
finally {
|
|
701
|
+
this.heartbeatInFlight = false;
|
|
702
|
+
}
|
|
703
|
+
}
|
|
474
704
|
startLeaseWatchdog() {
|
|
475
705
|
this.stopLeaseWatchdog();
|
|
476
706
|
this.leaseWatchdogTimer = setInterval(() => this.expireLeases(), 1000);
|
|
@@ -503,8 +733,6 @@ export class ForgeBridgeService {
|
|
|
503
733
|
}
|
|
504
734
|
startSandboxMaintenance() {
|
|
505
735
|
this.stopSandboxMaintenance();
|
|
506
|
-
if (!this.config.dockerEnabled)
|
|
507
|
-
return;
|
|
508
736
|
this.sandboxMaintenanceTimer = setInterval(() => {
|
|
509
737
|
this.runSandboxMaintenance().catch(() => undefined);
|
|
510
738
|
}, 60_000);
|
|
@@ -521,11 +749,20 @@ export class ForgeBridgeService {
|
|
|
521
749
|
return;
|
|
522
750
|
this.sandboxMaintenanceRunning = true;
|
|
523
751
|
try {
|
|
524
|
-
const
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
752
|
+
const hostReap = await this.sessions.reapHostRuntime?.({
|
|
753
|
+
runnerId: this.config.runnerId,
|
|
754
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
755
|
+
});
|
|
756
|
+
if (hostReap && (hostReap.reaped > 0 || hostReap.degraded > 0)) {
|
|
757
|
+
logBridgeStatus(`Managed-host maintenance reaped ${hostReap.reaped} stale attempt(s) and left ${hostReap.degraded} degraded attempt(s) for reconciliation.`);
|
|
758
|
+
}
|
|
759
|
+
if (this.config.dockerEnabled) {
|
|
760
|
+
const preflight = await this.preflight.refresh();
|
|
761
|
+
if (canReapManagedSandboxes(preflight)) {
|
|
762
|
+
const attemptSnapshot = await this.readAttemptSnapshot();
|
|
763
|
+
await this.sandboxRuntime.reap({ diskLow: preflight.diskLow, attemptSnapshot });
|
|
764
|
+
await this.flushCleanupReports();
|
|
765
|
+
}
|
|
529
766
|
}
|
|
530
767
|
}
|
|
531
768
|
finally {
|
|
@@ -568,7 +805,8 @@ export class ForgeBridgeService {
|
|
|
568
805
|
const runtimeState = this.sandboxRuntime.snapshot();
|
|
569
806
|
const reserved = [...this.reservations.values()].filter(reservation => reservation.sandboxed).length;
|
|
570
807
|
const used = this.usedSandboxResources(runtimeState.degraded);
|
|
571
|
-
const
|
|
808
|
+
const activeResources = this.activeSandboxResources(runtimeState.degraded);
|
|
809
|
+
const capacity = this.sandboxResourceCapacity(preflight, activeResources);
|
|
572
810
|
return {
|
|
573
811
|
schemaVersion: 3,
|
|
574
812
|
activeSessionCount: this.activeSessions.size,
|
|
@@ -580,7 +818,7 @@ export class ForgeBridgeService {
|
|
|
580
818
|
sandboxProfiles: this.config.sandboxProfiles,
|
|
581
819
|
runnerInstanceId: this.config.runnerInstanceId,
|
|
582
820
|
trustDomainId: this.config.trustDomainId,
|
|
583
|
-
maxConcurrentSandboxes:
|
|
821
|
+
maxConcurrentSandboxes: capacity.sandboxSlots,
|
|
584
822
|
reservedSandboxCount: reserved,
|
|
585
823
|
activeSandboxCount: runtimeState.active,
|
|
586
824
|
startingSandboxCount: runtimeState.starting,
|
|
@@ -588,6 +826,7 @@ export class ForgeBridgeService {
|
|
|
588
826
|
sandboxRuntimes: preflight?.healthy ? ["docker"] : [],
|
|
589
827
|
sandboxRuntimeVersion: preflight?.runtimeVersion ?? null,
|
|
590
828
|
sandboxFeatures: preflight?.features ?? null,
|
|
829
|
+
allowsOpenPublicSandbox: this.config.allowOpenPublicSandbox,
|
|
591
830
|
sandboxPublicationModes: [this.config.sandboxPublicationMode ?? "shared_filesystem"],
|
|
592
831
|
sandboxAuthModes: sandboxAuthModesFor(this.config),
|
|
593
832
|
availableMemoryBytes: preflight ? Math.max(0, capacity.memoryBytes - used.memoryBytes) : null,
|
|
@@ -597,6 +836,17 @@ export class ForgeBridgeService {
|
|
|
597
836
|
preflightGeneration: preflight?.generation ?? 0,
|
|
598
837
|
sandboxUnavailableCode: this.config.dockerEnabled ? preflight?.unavailableCode ?? null : "docker_disabled",
|
|
599
838
|
sandboxUnavailableReason: this.config.dockerEnabled ? preflight?.unavailableReason ?? null : "Docker sandboxing is disabled by runner configuration.",
|
|
839
|
+
sandboxMemoryCapacity: preflight ? {
|
|
840
|
+
defaultWorkerMemoryLimitBytes: this.config.defaultSandboxWorkerMemoryLimitBytes,
|
|
841
|
+
defaultServiceMemoryLimitBytes: this.config.defaultSandboxServiceMemoryLimitBytes,
|
|
842
|
+
runtimeOverheadBytes: this.config.sandboxRuntimeOverheadBytes,
|
|
843
|
+
hostReserveBytes: preflight.hostReserveBytes,
|
|
844
|
+
activeReservationBytes: used.memoryBytes,
|
|
845
|
+
daemonTotalBytes: preflight.totalMemoryBytes,
|
|
846
|
+
daemonAvailableBytes: preflight.availableMemoryBytes,
|
|
847
|
+
availableForAdmissionBytes: Math.max(0, capacity.memoryBytes - used.memoryBytes),
|
|
848
|
+
} : null,
|
|
849
|
+
hostRuntime: this.sessions.runtimeCapacity?.() ?? null,
|
|
600
850
|
supportsApiSecretGrant: this.config.supportsApiSecretGrant,
|
|
601
851
|
supportsManagedWorktrees: this.config.supportsManagedWorktrees,
|
|
602
852
|
supportsManagedForgeArtifacts: this.config.supportsManagedForgeArtifacts,
|
|
@@ -618,28 +868,106 @@ export class ForgeBridgeService {
|
|
|
618
868
|
];
|
|
619
869
|
return sumSandboxResources(resources);
|
|
620
870
|
}
|
|
621
|
-
|
|
871
|
+
activeSandboxResources(degradedReservations) {
|
|
872
|
+
return sumSandboxResources([
|
|
873
|
+
...[...this.activeSessions.values()]
|
|
874
|
+
.filter(session => session.sandboxed)
|
|
875
|
+
.map(session => session.resources),
|
|
876
|
+
...Array.from({ length: degradedReservations }, () => defaultSandboxResources())
|
|
877
|
+
]);
|
|
878
|
+
}
|
|
879
|
+
sandboxResourceCapacity(preflight, activeResources) {
|
|
622
880
|
if (!preflight)
|
|
623
881
|
return zeroSandboxResources();
|
|
624
882
|
return {
|
|
625
|
-
|
|
883
|
+
// Host free memory and disk already reflect active containers. Add their
|
|
884
|
+
// commitments back to form a stable total capacity, then subtract them
|
|
885
|
+
// exactly once in usedSandboxResources.
|
|
886
|
+
sandboxSlots: Math.min(this.config.maxConcurrentSandboxes, preflight.maxConcurrentSandboxes + activeResources.sandboxSlots),
|
|
626
887
|
cpuMillis: preflight.availableCpuMillis,
|
|
627
|
-
|
|
628
|
-
|
|
888
|
+
workerMemoryLimitBytes: 0,
|
|
889
|
+
serviceMemoryLimitBytes: 0,
|
|
890
|
+
runtimeOverheadBytes: 0,
|
|
891
|
+
memoryBytes: Math.max(0, preflight.availableMemoryBytes
|
|
892
|
+
- (this.config.sandboxHostMemoryReserveBytes ?? 2 * 1024 ** 3)
|
|
893
|
+
+ activeResources.memoryBytes),
|
|
894
|
+
workspaceBytes: Math.max(0, preflight.availableWorkspaceBytes
|
|
895
|
+
- this.config.minimumWorkspaceBytes
|
|
896
|
+
+ activeResources.workspaceBytes),
|
|
629
897
|
pids: this.config.sandboxPidCapacity,
|
|
630
898
|
buildSlots: this.config.maxConcurrentSandboxBuilds
|
|
631
899
|
};
|
|
632
900
|
}
|
|
633
901
|
async invokeAck(methodName, payload) {
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
902
|
+
let storageRecoveryAttempt = 0;
|
|
903
|
+
while (true) {
|
|
904
|
+
const connection = this.connection;
|
|
905
|
+
if (!connection) {
|
|
906
|
+
throw new Error("Bridge is not connected.");
|
|
907
|
+
}
|
|
908
|
+
await this.waitForTransportRecovery(connection, this.transportRecovery);
|
|
909
|
+
const recoveryGeneration = this.transportRecoveryGeneration;
|
|
910
|
+
try {
|
|
911
|
+
const ack = await connection.invoke(methodName, payload);
|
|
912
|
+
if (!ack.accepted) {
|
|
913
|
+
throw new BridgeCallbackRejectedError(ack.code ?? null, ack.message ?? ack.code ?? `${methodName} callback was rejected.`);
|
|
914
|
+
}
|
|
915
|
+
return ack;
|
|
916
|
+
}
|
|
917
|
+
catch (error) {
|
|
918
|
+
if (connection !== this.connection || this.stopped)
|
|
919
|
+
throw error;
|
|
920
|
+
const recovery = this.transportRecovery;
|
|
921
|
+
if (recovery && recovery.generation > recoveryGeneration) {
|
|
922
|
+
await this.waitForTransportRecovery(connection, recovery);
|
|
923
|
+
continue;
|
|
924
|
+
}
|
|
925
|
+
if (isProviderStorageUnavailable(error) && storageRecoveryAttempt < this.config.reconnectBackoffMs.length) {
|
|
926
|
+
const backoff = this.config.reconnectBackoffMs[storageRecoveryAttempt] ?? 5000;
|
|
927
|
+
storageRecoveryAttempt += 1;
|
|
928
|
+
logBridgeStatus(`${methodName} reached Forge while storage was recovering. Retrying callback ${storageRecoveryAttempt} in ${formatDelay(backoff)}...`);
|
|
929
|
+
await delay(backoff);
|
|
930
|
+
continue;
|
|
931
|
+
}
|
|
932
|
+
throw error;
|
|
933
|
+
}
|
|
637
934
|
}
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
935
|
+
}
|
|
936
|
+
beginTransportRecovery() {
|
|
937
|
+
if (this.transportRecovery && !this.transportRecovery.completed)
|
|
938
|
+
return;
|
|
939
|
+
let resolve = () => undefined;
|
|
940
|
+
const promise = new Promise((complete) => { resolve = complete; });
|
|
941
|
+
this.transportRecoveryGeneration += 1;
|
|
942
|
+
this.transportRecovery = {
|
|
943
|
+
generation: this.transportRecoveryGeneration,
|
|
944
|
+
promise,
|
|
945
|
+
resolve,
|
|
946
|
+
completed: false,
|
|
947
|
+
error: null,
|
|
948
|
+
};
|
|
949
|
+
}
|
|
950
|
+
completeTransportRecovery(error = null) {
|
|
951
|
+
const recovery = this.transportRecovery;
|
|
952
|
+
if (!recovery || recovery.completed)
|
|
953
|
+
return;
|
|
954
|
+
recovery.completed = true;
|
|
955
|
+
recovery.error = error;
|
|
956
|
+
recovery.resolve();
|
|
957
|
+
}
|
|
958
|
+
async waitForTransportRecovery(connection, recovery) {
|
|
959
|
+
if (!recovery)
|
|
960
|
+
return;
|
|
961
|
+
if (recovery.completed) {
|
|
962
|
+
if (recovery.error)
|
|
963
|
+
throw recovery.error;
|
|
964
|
+
return;
|
|
641
965
|
}
|
|
642
|
-
|
|
966
|
+
await recovery.promise;
|
|
967
|
+
if (connection !== this.connection)
|
|
968
|
+
throw new Error("Bridge connection changed while recovering.");
|
|
969
|
+
if (recovery.error)
|
|
970
|
+
throw recovery.error;
|
|
643
971
|
}
|
|
644
972
|
async waitForClose(signal) {
|
|
645
973
|
if (!this.closed) {
|
|
@@ -671,6 +999,17 @@ function isDispatchNoLongerPending(error) {
|
|
|
671
999
|
}
|
|
672
1000
|
return error instanceof Error && error.message === "Dispatch is no longer pending.";
|
|
673
1001
|
}
|
|
1002
|
+
function isProviderStorageUnavailable(error) {
|
|
1003
|
+
const message = errorMessage(error).toLowerCase();
|
|
1004
|
+
return message.includes("provider_storage_unavailable")
|
|
1005
|
+
|| (message.includes("storage is recovering") && message.includes("503"));
|
|
1006
|
+
}
|
|
1007
|
+
function createDefaultSessionService(sandboxStateRoot) {
|
|
1008
|
+
const rawProcessRunner = new StreamingProviderProcessRunner();
|
|
1009
|
+
const processSupervisors = new AttemptProcessSupervisorRegistry();
|
|
1010
|
+
const coordinatedProcessRunner = new AttemptCoordinatedProviderProcessRunner(processSupervisors, rawProcessRunner);
|
|
1011
|
+
return new BridgeSessionService(new GitWorktreeService(undefined, new RepositoryCoordinatorLock(sandboxStateRoot)), new RoutingProviderCliService(new ProcessBackedProviderCliService(coordinatedProcessRunner)), new HostRuntimeLeaseManager(sandboxStateRoot, undefined, undefined, processSupervisors, rawProcessRunner));
|
|
1012
|
+
}
|
|
674
1013
|
export class SignalRBridgeConnectionFactory {
|
|
675
1014
|
create(config) {
|
|
676
1015
|
return new SignalRBridgeConnection(config);
|
|
@@ -693,10 +1032,11 @@ export class SignalRBridgeConnection {
|
|
|
693
1032
|
Authorization: `Bearer ${config.deviceToken}`
|
|
694
1033
|
}
|
|
695
1034
|
})
|
|
1035
|
+
.withAutomaticReconnect([...config.reconnectBackoffMs])
|
|
696
1036
|
.configureLogging(LogLevel.Warning)
|
|
697
1037
|
.build();
|
|
698
1038
|
this.connection.on("Superseded", () => {
|
|
699
|
-
void this.connection.stop();
|
|
1039
|
+
void this.connection.stop().catch(() => undefined);
|
|
700
1040
|
});
|
|
701
1041
|
}
|
|
702
1042
|
start() {
|
|
@@ -708,6 +1048,14 @@ export class SignalRBridgeConnection {
|
|
|
708
1048
|
onClose(handler) {
|
|
709
1049
|
this.connection.onclose(handler);
|
|
710
1050
|
}
|
|
1051
|
+
onReconnecting(handler) {
|
|
1052
|
+
this.connection.onreconnecting(handler);
|
|
1053
|
+
}
|
|
1054
|
+
onReconnected(handler) {
|
|
1055
|
+
this.connection.onreconnected((connectionId) => {
|
|
1056
|
+
void Promise.resolve(handler(connectionId)).catch(() => undefined);
|
|
1057
|
+
});
|
|
1058
|
+
}
|
|
711
1059
|
onDispatch(handler) {
|
|
712
1060
|
this.connection.on("Dispatch", (raw) => {
|
|
713
1061
|
void handler(raw).catch(() => undefined);
|
|
@@ -738,6 +1086,11 @@ export class SignalRBridgeConnection {
|
|
|
738
1086
|
void handler(request).catch(() => undefined);
|
|
739
1087
|
});
|
|
740
1088
|
}
|
|
1089
|
+
onReadinessProbe(handler) {
|
|
1090
|
+
this.connection.on("ReadinessProbe", (request) => {
|
|
1091
|
+
void handler(request).catch(() => undefined);
|
|
1092
|
+
});
|
|
1093
|
+
}
|
|
741
1094
|
invoke(methodName, payload) {
|
|
742
1095
|
return this.connection.invoke(methodName, payload);
|
|
743
1096
|
}
|
|
@@ -745,6 +1098,9 @@ export class SignalRBridgeConnection {
|
|
|
745
1098
|
export function defaultBridgeServiceConfig(overrides = {}) {
|
|
746
1099
|
const defaults = defaultForgeBridgeConfig();
|
|
747
1100
|
const cliProviders = splitCsv(process.env.FORGE_CLI_PROVIDERS ?? "codex");
|
|
1101
|
+
const sandboxHostMemoryReserveDefault = process.platform === "linux"
|
|
1102
|
+
? 2 * 1024 ** 3
|
|
1103
|
+
: 1024 ** 3;
|
|
748
1104
|
const sandboxPublicationMode = process.env.FORGE_SANDBOX_PUBLICATION_MODE === "bridge_push" ? "bridge_push" : "shared_filesystem";
|
|
749
1105
|
const config = {
|
|
750
1106
|
serverUrl: process.env.FORGE_URL ?? "http://localhost:5208",
|
|
@@ -755,6 +1111,7 @@ export function defaultBridgeServiceConfig(overrides = {}) {
|
|
|
755
1111
|
maxConcurrentSandboxBuilds: positiveFiniteNumber(Number(process.env.FORGE_MAX_CONCURRENT_SANDBOX_BUILDS ?? 1), 1),
|
|
756
1112
|
sandboxPidCapacity: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_PID_CAPACITY ?? 2048), 2048),
|
|
757
1113
|
dockerEnabled: process.env.FORGE_DOCKER_SANDBOX !== "false",
|
|
1114
|
+
allowOpenPublicSandbox: process.env.FORGE_ALLOW_OPEN_PUBLIC_SANDBOX === "true",
|
|
758
1115
|
sandboxPublicationMode,
|
|
759
1116
|
dockerExecutable: process.env.FORGE_DOCKER_EXECUTABLE ?? "docker",
|
|
760
1117
|
dockerContext: process.env.FORGE_DOCKER_CONTEXT ?? "",
|
|
@@ -763,6 +1120,11 @@ export function defaultBridgeServiceConfig(overrides = {}) {
|
|
|
763
1120
|
diskHighWatermarkPercent: positiveFiniteNumber(Number(process.env.FORGE_DISK_HIGH_WATERMARK_PERCENT ?? 85), 85),
|
|
764
1121
|
diskLowWatermarkPercent: positiveFiniteNumber(Number(process.env.FORGE_DISK_LOW_WATERMARK_PERCENT ?? 75), 75),
|
|
765
1122
|
minimumWorkspaceBytes: positiveFiniteNumber(Number(process.env.FORGE_MINIMUM_WORKSPACE_BYTES ?? 10 * 1024 ** 3), 10 * 1024 ** 3),
|
|
1123
|
+
sandboxHostMemoryReserveBytes: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_HOST_MEMORY_RESERVE_BYTES ?? sandboxHostMemoryReserveDefault), sandboxHostMemoryReserveDefault),
|
|
1124
|
+
defaultSandboxWorkerMemoryLimitBytes: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_DEFAULT_WORKER_MEMORY_LIMIT_BYTES ?? 2 * 1024 ** 3), 2 * 1024 ** 3),
|
|
1125
|
+
defaultSandboxServiceMemoryLimitBytes: nonNegativeFiniteNumber(Number(process.env.FORGE_SANDBOX_DEFAULT_SERVICE_MEMORY_LIMIT_BYTES ?? 0), 0),
|
|
1126
|
+
sandboxRuntimeOverheadBytes: Math.max(128 * 1024 ** 2, nonNegativeFiniteNumber(Number(process.env.FORGE_SANDBOX_RUNTIME_OVERHEAD_BYTES ?? 128 * 1024 ** 2), 128 * 1024 ** 2)),
|
|
1127
|
+
sandboxMemorySampleIntervalMs: Math.min(30_000, Math.max(250, positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_MEMORY_SAMPLE_INTERVAL_MS ?? 1000), 1000))),
|
|
766
1128
|
imageRetentionCount: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_IMAGE_RETENTION_COUNT ?? 3), 3),
|
|
767
1129
|
imageRetentionTtlMs: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_IMAGE_RETENTION_TTL_MS ?? 7 * 24 * 60 * 60 * 1000), 7 * 24 * 60 * 60 * 1000),
|
|
768
1130
|
auditRetentionCount: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_AUDIT_RETENTION_COUNT ?? 200), 200),
|
|
@@ -811,6 +1173,9 @@ function normalizeServerUrl(serverUrl) {
|
|
|
811
1173
|
function positiveFiniteNumber(value, fallback) {
|
|
812
1174
|
return Number.isFinite(value) && value > 0 ? value : fallback;
|
|
813
1175
|
}
|
|
1176
|
+
function nonNegativeFiniteNumber(value, fallback) {
|
|
1177
|
+
return Number.isFinite(value) && value >= 0 ? value : fallback;
|
|
1178
|
+
}
|
|
814
1179
|
function providerStatusesFor(providers) {
|
|
815
1180
|
return Object.fromEntries(providers.map((provider) => [provider, { signedIn: true, version: "", models: configuredModelsFor(provider) }]));
|
|
816
1181
|
}
|
|
@@ -932,10 +1297,11 @@ function normalizeQaAppLaunchTarget(target) {
|
|
|
932
1297
|
command: cleanString(target.command),
|
|
933
1298
|
working_directory: cleanString(target.working_directory),
|
|
934
1299
|
ready_url: cleanString(target.ready_url),
|
|
1300
|
+
managed_runtime_port: cleanString(target.managed_runtime_port),
|
|
935
1301
|
routes,
|
|
936
1302
|
notes: cleanString(target.notes)
|
|
937
1303
|
};
|
|
938
|
-
return normalized.url || normalized.command || normalized.ready_url ||
|
|
1304
|
+
return normalized.url || normalized.command || normalized.ready_url || normalized.managed_runtime_port
|
|
939
1305
|
? normalized
|
|
940
1306
|
: null;
|
|
941
1307
|
}
|
|
@@ -1004,12 +1370,12 @@ function abortPromise(signal) {
|
|
|
1004
1370
|
export function canReapManagedSandboxes(preflight) {
|
|
1005
1371
|
return canRunDockerMaintenance(preflight);
|
|
1006
1372
|
}
|
|
1007
|
-
function sandboxNetworkPolicyFailure(dispatch) {
|
|
1373
|
+
function sandboxNetworkPolicyFailure(dispatch, config) {
|
|
1008
1374
|
const sandbox = dispatch.sandbox;
|
|
1009
1375
|
if (!sandbox)
|
|
1010
1376
|
return "Protocol-v2 sandbox context is missing.";
|
|
1011
1377
|
if (sandbox.networkPolicy.mode === "open_public"
|
|
1012
|
-
&& (sandbox.executionPolicy === "sandbox_required" ||
|
|
1378
|
+
&& (sandbox.executionPolicy === "sandbox_required" || !config.allowOpenPublicSandbox)) {
|
|
1013
1379
|
return "open_public sandbox networking is not permitted by this runner policy.";
|
|
1014
1380
|
}
|
|
1015
1381
|
if (sandbox.networkPolicy.mode === "public_allowlist"
|
|
@@ -1019,6 +1385,63 @@ function sandboxNetworkPolicyFailure(dispatch) {
|
|
|
1019
1385
|
}
|
|
1020
1386
|
return null;
|
|
1021
1387
|
}
|
|
1388
|
+
function infrastructureFailureEventType(summary) {
|
|
1389
|
+
if (summary.startsWith("worktree_prepare_failed:"))
|
|
1390
|
+
return "worktree_prepare_failed";
|
|
1391
|
+
if (summary.startsWith("provider_network_unavailable:"))
|
|
1392
|
+
return "provider_network_unavailable";
|
|
1393
|
+
if (summary.startsWith("provider_start_unavailable:"))
|
|
1394
|
+
return "provider_start_unavailable";
|
|
1395
|
+
if (summary.startsWith("host_port_collision:"))
|
|
1396
|
+
return "host_port_collision";
|
|
1397
|
+
if (summary.startsWith("host_runtime_process_exited:"))
|
|
1398
|
+
return "host_runtime_process_exited";
|
|
1399
|
+
if (summary.startsWith("host_runtime_listener_timeout:"))
|
|
1400
|
+
return "host_runtime_listener_timeout";
|
|
1401
|
+
if (summary.startsWith("host_runtime_http_readiness_timeout:"))
|
|
1402
|
+
return "host_runtime_http_readiness_timeout";
|
|
1403
|
+
if (summary.startsWith("attempt_process_supervisor_start_failed:"))
|
|
1404
|
+
return "attempt_process_supervisor_start_failed";
|
|
1405
|
+
if (summary.startsWith("attempt_process_supervisor_start_timeout:"))
|
|
1406
|
+
return "attempt_process_supervisor_start_timeout";
|
|
1407
|
+
return null;
|
|
1408
|
+
}
|
|
1409
|
+
function parseReadinessProbe(value) {
|
|
1410
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
1411
|
+
return null;
|
|
1412
|
+
const request = value;
|
|
1413
|
+
const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
|
|
1414
|
+
const bounded = (candidate, maximum) => typeof candidate === "string" && candidate.trim().length > 0 && candidate.length <= maximum;
|
|
1415
|
+
if (request.schemaVersion !== 1
|
|
1416
|
+
|| !uuid.test(request.requestId ?? "")
|
|
1417
|
+
|| !uuid.test(request.fencingToken ?? "")
|
|
1418
|
+
|| !uuid.test(request.userId ?? "")
|
|
1419
|
+
|| !uuid.test(request.projectId ?? "")
|
|
1420
|
+
|| !uuid.test(request.repoId ?? "")
|
|
1421
|
+
|| !uuid.test(request.runnerInstanceId ?? "")
|
|
1422
|
+
|| !Number.isInteger(request.attempt) || (request.attempt ?? 0) < 1
|
|
1423
|
+
|| !bounded(request.repoName, 160)
|
|
1424
|
+
|| typeof request.remoteUrl !== "string" || request.remoteUrl.length > 2048
|
|
1425
|
+
|| !bounded(request.defaultBranch, 512)
|
|
1426
|
+
|| typeof request.cloneOnDemand !== "boolean"
|
|
1427
|
+
|| !bounded(request.runnerRepoLocalPath, 4096)
|
|
1428
|
+
|| !bounded(request.revision, 512)
|
|
1429
|
+
|| !Number.isFinite(Date.parse(request.expiresAt ?? ""))
|
|
1430
|
+
|| Date.parse(request.expiresAt ?? "") <= Date.now()) {
|
|
1431
|
+
return null;
|
|
1432
|
+
}
|
|
1433
|
+
return request;
|
|
1434
|
+
}
|
|
1435
|
+
function sandboxProviderNetworkFailure(dispatch, config) {
|
|
1436
|
+
const sandbox = dispatch.sandbox;
|
|
1437
|
+
if (!sandbox || sandbox.networkPolicy.mode !== "offline")
|
|
1438
|
+
return null;
|
|
1439
|
+
const provider = dispatch.execution.provider.trim().toLowerCase();
|
|
1440
|
+
const isNetworkProvider = config.cliProviders.some(candidate => candidate.trim().toLowerCase() === provider);
|
|
1441
|
+
if (!isNetworkProvider)
|
|
1442
|
+
return null;
|
|
1443
|
+
return `Provider ${dispatch.execution.provider} requires outbound network access, but this Docker attempt uses the offline network policy.`;
|
|
1444
|
+
}
|
|
1022
1445
|
function sandboxAuthFailure(dispatch, config) {
|
|
1023
1446
|
const unsupported = sandboxProviderAuthUnavailableMessage(dispatch.execution.provider, dispatch.execution.authKind);
|
|
1024
1447
|
if (unsupported) {
|
|
@@ -1046,10 +1469,43 @@ export function canReserveSandboxResources(used, requested, capacity) {
|
|
|
1046
1469
|
&& used.pids + requested.pids <= capacity.pids
|
|
1047
1470
|
&& used.buildSlots + requested.buildSlots <= capacity.buildSlots;
|
|
1048
1471
|
}
|
|
1472
|
+
export function sandboxCapacityFailureMessage(used, requested, capacity, preflight, config) {
|
|
1473
|
+
const availableSlots = Math.max(0, capacity.sandboxSlots - used.sandboxSlots);
|
|
1474
|
+
if (requested.sandboxSlots > availableSlots) {
|
|
1475
|
+
return `Sandbox slot capacity is insufficient: requested ${requested.sandboxSlots}, available ${availableSlots}, ${used.sandboxSlots} already reserved.`;
|
|
1476
|
+
}
|
|
1477
|
+
const availableMemoryBytes = Math.max(0, capacity.memoryBytes - used.memoryBytes);
|
|
1478
|
+
if (requested.memoryBytes > availableMemoryBytes) {
|
|
1479
|
+
return [
|
|
1480
|
+
`Sandbox memory capacity is insufficient: requested ${formatBytes(requested.memoryBytes)} total`,
|
|
1481
|
+
`(worker ${formatBytes(requested.workerMemoryLimitBytes)}, services ${formatBytes(requested.serviceMemoryLimitBytes)}, runtime overhead ${formatBytes(requested.runtimeOverheadBytes)}),`,
|
|
1482
|
+
`but ${formatBytes(availableMemoryBytes)} is available for admission.`,
|
|
1483
|
+
`Docker reports ${formatBytes(preflight?.availableMemoryBytes ?? 0)} available of ${formatBytes(preflight?.totalMemoryBytes ?? 0)};`,
|
|
1484
|
+
`Forge reserves ${formatBytes(preflight?.hostReserveBytes ?? config.sandboxHostMemoryReserveBytes ?? 0)} for the host and tracks ${formatBytes(used.memoryBytes)} in active or reserved attempts.`,
|
|
1485
|
+
].join(" ");
|
|
1486
|
+
}
|
|
1487
|
+
const availableCpuMillis = Math.max(0, capacity.cpuMillis - used.cpuMillis);
|
|
1488
|
+
if (requested.cpuMillis > availableCpuMillis) {
|
|
1489
|
+
return `Sandbox CPU capacity is insufficient: requested ${requested.cpuMillis} millis, available ${availableCpuMillis} millis, ${used.cpuMillis} millis already reserved.`;
|
|
1490
|
+
}
|
|
1491
|
+
const availableWorkspaceBytes = Math.max(0, capacity.workspaceBytes - used.workspaceBytes);
|
|
1492
|
+
if (requested.workspaceBytes > availableWorkspaceBytes) {
|
|
1493
|
+
return `Sandbox workspace capacity is insufficient: requested ${formatBytes(requested.workspaceBytes)}, available ${formatBytes(availableWorkspaceBytes)}, ${formatBytes(used.workspaceBytes)} already reserved.`;
|
|
1494
|
+
}
|
|
1495
|
+
const availablePids = Math.max(0, capacity.pids - used.pids);
|
|
1496
|
+
if (requested.pids > availablePids) {
|
|
1497
|
+
return `Sandbox PID capacity is insufficient: requested ${requested.pids}, available ${availablePids}, ${used.pids} already reserved.`;
|
|
1498
|
+
}
|
|
1499
|
+
const availableBuildSlots = Math.max(0, capacity.buildSlots - used.buildSlots);
|
|
1500
|
+
return `Sandbox build capacity is insufficient: requested ${requested.buildSlots}, available ${availableBuildSlots}, ${used.buildSlots} already reserved.`;
|
|
1501
|
+
}
|
|
1049
1502
|
function sumSandboxResources(resources) {
|
|
1050
1503
|
return resources.reduce((total, resource) => ({
|
|
1051
1504
|
sandboxSlots: total.sandboxSlots + resource.sandboxSlots,
|
|
1052
1505
|
cpuMillis: total.cpuMillis + resource.cpuMillis,
|
|
1506
|
+
workerMemoryLimitBytes: total.workerMemoryLimitBytes + resource.workerMemoryLimitBytes,
|
|
1507
|
+
serviceMemoryLimitBytes: total.serviceMemoryLimitBytes + resource.serviceMemoryLimitBytes,
|
|
1508
|
+
runtimeOverheadBytes: total.runtimeOverheadBytes + resource.runtimeOverheadBytes,
|
|
1053
1509
|
memoryBytes: total.memoryBytes + resource.memoryBytes,
|
|
1054
1510
|
workspaceBytes: total.workspaceBytes + resource.workspaceBytes,
|
|
1055
1511
|
pids: total.pids + resource.pids,
|
|
@@ -1057,18 +1513,91 @@ function sumSandboxResources(resources) {
|
|
|
1057
1513
|
}), zeroSandboxResources());
|
|
1058
1514
|
}
|
|
1059
1515
|
function zeroSandboxResources() {
|
|
1060
|
-
return {
|
|
1516
|
+
return {
|
|
1517
|
+
sandboxSlots: 0,
|
|
1518
|
+
cpuMillis: 0,
|
|
1519
|
+
workerMemoryLimitBytes: 0,
|
|
1520
|
+
serviceMemoryLimitBytes: 0,
|
|
1521
|
+
runtimeOverheadBytes: 0,
|
|
1522
|
+
memoryBytes: 0,
|
|
1523
|
+
workspaceBytes: 0,
|
|
1524
|
+
pids: 0,
|
|
1525
|
+
buildSlots: 0,
|
|
1526
|
+
};
|
|
1061
1527
|
}
|
|
1062
1528
|
function defaultSandboxResources() {
|
|
1063
1529
|
return {
|
|
1064
1530
|
sandboxSlots: 1,
|
|
1065
1531
|
cpuMillis: 2000,
|
|
1066
|
-
|
|
1532
|
+
workerMemoryLimitBytes: 2 * 1024 ** 3,
|
|
1533
|
+
serviceMemoryLimitBytes: 0,
|
|
1534
|
+
runtimeOverheadBytes: 128 * 1024 ** 2,
|
|
1535
|
+
memoryBytes: 2 * 1024 ** 3 + 128 * 1024 ** 2,
|
|
1067
1536
|
workspaceBytes: 10 * 1024 ** 3,
|
|
1068
1537
|
pids: 512,
|
|
1069
1538
|
buildSlots: 0
|
|
1070
1539
|
};
|
|
1071
1540
|
}
|
|
1541
|
+
export function resolveDynamicSandboxResources(requested, serviceCount, networkMode, baseRuntimeOverheadBytes = 128 * 1024 ** 2) {
|
|
1542
|
+
const serviceMemoryLimitBytes = Math.max(0, serviceCount) * 1024 ** 3;
|
|
1543
|
+
const runtimeOverheadBytes = Math.max(128 * 1024 ** 2, baseRuntimeOverheadBytes)
|
|
1544
|
+
+ (networkMode === "public_allowlist" ? 256 * 1024 ** 2 : 0);
|
|
1545
|
+
return {
|
|
1546
|
+
...requested,
|
|
1547
|
+
serviceMemoryLimitBytes,
|
|
1548
|
+
runtimeOverheadBytes,
|
|
1549
|
+
memoryBytes: requested.workerMemoryLimitBytes + serviceMemoryLimitBytes + runtimeOverheadBytes,
|
|
1550
|
+
};
|
|
1551
|
+
}
|
|
1552
|
+
export async function resolveSandboxResourcesFromRepo(dispatch, baseRuntimeOverheadBytes = 128 * 1024 ** 2) {
|
|
1553
|
+
const repoPath = dispatch.repo.serverLocalPath;
|
|
1554
|
+
const sandbox = dispatch.sandbox;
|
|
1555
|
+
if (!repoPath || !sandbox) {
|
|
1556
|
+
throw new Error("Sandbox admission needs a mapped repository.");
|
|
1557
|
+
}
|
|
1558
|
+
const selection = selectSandboxConfigRefs(sandbox.executionPolicy, sandbox.sandboxConfigCommitSha, dispatch.repo.defaultBranch);
|
|
1559
|
+
const configCommitSha = await resolveFirstGitCommit(repoPath, selection.refs);
|
|
1560
|
+
if (!configCommitSha) {
|
|
1561
|
+
throw new Error("Sandbox admission could not resolve the immutable sandbox config commit.");
|
|
1562
|
+
}
|
|
1563
|
+
const contractResult = await runSandboxProcess({
|
|
1564
|
+
command: "git",
|
|
1565
|
+
args: ["-C", repoPath, "show", `${configCommitSha}:.forge/forge.sandbox.yml`],
|
|
1566
|
+
timeoutMs: 30_000,
|
|
1567
|
+
maxOutputBytes: 1024 * 1024,
|
|
1568
|
+
});
|
|
1569
|
+
if (contractResult.exitCode !== 0) {
|
|
1570
|
+
throw new Error("The approved immutable sandbox contract is unavailable in the mapped repository.");
|
|
1571
|
+
}
|
|
1572
|
+
const contract = validateSandboxContract(contractResult.stdout);
|
|
1573
|
+
const workspaceBytes = await estimateWorkspaceBytesFromRepo(repoPath, {
|
|
1574
|
+
role: dispatch.execution.role,
|
|
1575
|
+
serviceCount: contract.contract.services.length,
|
|
1576
|
+
browser: contract.contract.browser,
|
|
1577
|
+
});
|
|
1578
|
+
return {
|
|
1579
|
+
resources: resolveDynamicSandboxResources({
|
|
1580
|
+
...sandbox.requestedResources,
|
|
1581
|
+
workspaceBytes: Math.max(sandbox.requestedResources.workspaceBytes, workspaceBytes),
|
|
1582
|
+
}, contract.contract.services.length, sandbox.networkPolicy.mode, baseRuntimeOverheadBytes),
|
|
1583
|
+
sandboxConfigCommitSha: configCommitSha,
|
|
1584
|
+
};
|
|
1585
|
+
}
|
|
1586
|
+
async function resolveFirstGitCommit(repoPath, refs) {
|
|
1587
|
+
for (const ref of refs) {
|
|
1588
|
+
const result = await runSandboxProcess({
|
|
1589
|
+
command: "git",
|
|
1590
|
+
args: ["-C", repoPath, "rev-parse", "--verify", `${ref}^{commit}`],
|
|
1591
|
+
timeoutMs: 30_000,
|
|
1592
|
+
maxOutputBytes: 1024 * 1024,
|
|
1593
|
+
});
|
|
1594
|
+
const commitSha = result.stdout.trim().toLowerCase();
|
|
1595
|
+
if (result.exitCode === 0 && /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/u.test(commitSha)) {
|
|
1596
|
+
return commitSha;
|
|
1597
|
+
}
|
|
1598
|
+
}
|
|
1599
|
+
return null;
|
|
1600
|
+
}
|
|
1072
1601
|
function parseOfferEnvelope(value) {
|
|
1073
1602
|
return parseDispatchEnvelope(value);
|
|
1074
1603
|
}
|