@loucompanion/forge-bridge 0.1.3-dev.d1ca67076aa9 → 0.1.3-dev.eeefdb0d19c1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +78 -0
- package/dist/bridge/bridge.base.d.ts +6 -1
- package/dist/bridge/bridge.service.d.ts +60 -3
- package/dist/bridge/bridge.service.js +962 -22
- package/dist/bridge/bridge.types.d.ts +172 -1
- package/dist/cleanup/cleanup.service.d.ts +13 -2
- package/dist/cleanup/cleanup.service.js +435 -3
- package/dist/cleanup/cleanup.types.d.ts +10 -0
- package/dist/cleanup/index.d.ts +1 -1
- package/dist/cleanup/index.js +1 -1
- package/dist/cleanup/internals.d.ts +2 -2
- package/dist/cleanup/internals.js +1 -1
- package/dist/cli/cli.help.js +47 -1
- package/dist/cli/cli.service.d.ts +42 -2
- package/dist/cli/cli.service.js +351 -22
- package/dist/cli/local-config.d.ts +6 -0
- package/dist/cli/local-config.js +47 -3
- package/dist/runtime/attempt-node-runtime-launcher.d.ts +17 -0
- package/dist/runtime/attempt-node-runtime-launcher.js +101 -0
- package/dist/runtime/attempt-process-supervisor-registry.d.ts +20 -0
- package/dist/runtime/attempt-process-supervisor-registry.js +64 -0
- package/dist/runtime/attempt-process-supervisor.d.ts +86 -0
- package/dist/runtime/attempt-process-supervisor.js +343 -0
- package/dist/runtime/generated-output-reset.d.ts +10 -0
- package/dist/runtime/generated-output-reset.js +103 -0
- package/dist/runtime/host-listener-inspector.d.ts +44 -0
- package/dist/runtime/host-listener-inspector.js +324 -0
- package/dist/runtime/host-port-broker.d.ts +76 -0
- package/dist/runtime/host-port-broker.js +333 -0
- package/dist/runtime/host-runtime-contract.d.ts +47 -0
- package/dist/runtime/host-runtime-contract.js +700 -0
- package/dist/runtime/host-runtime-lease.d.ts +133 -0
- package/dist/runtime/host-runtime-lease.js +1460 -0
- package/dist/runtime/host-runtime-promotion.d.ts +23 -0
- package/dist/runtime/host-runtime-promotion.js +159 -0
- package/dist/runtime/node-host-runtime-adapter.d.ts +51 -0
- package/dist/runtime/node-host-runtime-adapter.js +268 -0
- package/dist/runtime/playwright-host-runtime-adapter.d.ts +20 -0
- package/dist/runtime/playwright-host-runtime-adapter.js +220 -0
- package/dist/runtime/runtime-isolation-lease.d.ts +67 -0
- package/dist/runtime/runtime-isolation-lease.js +158 -0
- package/dist/sandbox/command-runner.d.ts +4 -0
- package/dist/sandbox/command-runner.js +47 -0
- package/dist/sandbox/contract.d.ts +5 -0
- package/dist/sandbox/contract.js +276 -0
- package/dist/sandbox/event-budget.d.ts +11 -0
- package/dist/sandbox/event-budget.js +27 -0
- package/dist/sandbox/image-policy.d.ts +35 -0
- package/dist/sandbox/image-policy.js +247 -0
- package/dist/sandbox/index.d.ts +5 -0
- package/dist/sandbox/index.js +5 -0
- package/dist/sandbox/lease-watchdog.d.ts +18 -0
- package/dist/sandbox/lease-watchdog.js +67 -0
- package/dist/sandbox/preflight.d.ts +54 -0
- package/dist/sandbox/preflight.js +335 -0
- package/dist/sandbox/process.d.ts +20 -0
- package/dist/sandbox/process.js +209 -0
- package/dist/sandbox/provider-auth/codex.d.ts +26 -0
- package/dist/sandbox/provider-auth/codex.js +212 -0
- package/dist/sandbox/repository-lock.d.ts +5 -0
- package/dist/sandbox/repository-lock.js +119 -0
- package/dist/sandbox/runtime.d.ts +297 -0
- package/dist/sandbox/runtime.js +2338 -0
- package/dist/sandbox/sandbox.types.d.ts +86 -0
- package/dist/sandbox/sandbox.types.js +1 -0
- package/dist/sandbox/smoke.d.ts +76 -0
- package/dist/sandbox/smoke.js +380 -0
- package/dist/sandbox/worker-supervisor.d.ts +13 -0
- package/dist/sandbox/worker-supervisor.js +36 -0
- package/dist/sandbox/worker.d.ts +1 -0
- package/dist/sandbox/worker.js +225 -0
- package/dist/sandbox/workspace.service.d.ts +13 -0
- package/dist/sandbox/workspace.service.js +72 -0
- package/dist/session/managed-attempt-tools.d.ts +12 -0
- package/dist/session/managed-attempt-tools.js +350 -0
- package/dist/session/phase-completion.artifacts.js +32 -4
- package/dist/session/phase-completion.d.ts +4 -2
- package/dist/session/phase-completion.helpers.js +27 -3
- package/dist/session/phase-completion.js +12 -3
- package/dist/session/phase-completion.prompts.d.ts +3 -3
- package/dist/session/phase-completion.prompts.js +19 -5
- package/dist/session/phase-completion.runtime.d.ts +12 -1
- package/dist/session/phase-completion.runtime.js +127 -7
- package/dist/session/phase-completion.schemas.js +108 -3
- package/dist/session/phase-completion.validation.d.ts +4 -1
- package/dist/session/phase-completion.validation.js +106 -2
- package/dist/session/provider-cli/ci-repair-readiness.d.ts +8 -0
- package/dist/session/provider-cli/ci-repair-readiness.js +90 -0
- package/dist/session/provider-cli/provider-cli.artifacts.d.ts +1 -1
- package/dist/session/provider-cli/provider-cli.artifacts.js +28 -4
- package/dist/session/provider-cli/provider-cli.base.d.ts +1 -0
- package/dist/session/provider-cli/provider-cli.deterministic.d.ts +2 -0
- package/dist/session/provider-cli/provider-cli.deterministic.js +66 -9
- package/dist/session/provider-cli/provider-cli.process.d.ts +17 -0
- package/dist/session/provider-cli/provider-cli.process.js +223 -14
- package/dist/session/provider-cli/provider-cli.prompts.d.ts +16 -2
- package/dist/session/provider-cli/provider-cli.prompts.js +202 -18
- package/dist/session/provider-cli/provider-cli.qa-sidecars.js +1 -0
- package/dist/session/provider-cli/provider-cli.runtime.d.ts +1 -1
- package/dist/session/provider-cli/provider-cli.runtime.js +3 -2
- package/dist/session/provider-cli/provider-cli.service.d.ts +7 -3
- package/dist/session/provider-cli/provider-cli.service.js +399 -98
- package/dist/session/provider-cli/provider-cli.types.d.ts +6 -0
- package/dist/session/provider-cli/provider-output-state.d.ts +2 -0
- package/dist/session/provider-cli/provider-output-state.js +30 -2
- package/dist/session/session-redaction.d.ts +3 -0
- package/dist/session/session-redaction.js +23 -0
- package/dist/session/session.base.d.ts +6 -1
- package/dist/session/session.service.d.ts +8 -2
- package/dist/session/session.service.js +463 -58
- package/dist/session/session.types.d.ts +107 -1
- package/dist/session/session.types.js +226 -3
- package/dist/shared/path/path.js +3 -1
- package/dist/shared/process/host-process-recovery.d.ts +12 -0
- package/dist/shared/process/host-process-recovery.js +78 -0
- package/dist/shared/process/host-process-supervisor.d.ts +57 -0
- package/dist/shared/process/host-process-supervisor.js +282 -0
- package/dist/shared/process/process.d.ts +1 -0
- package/dist/shared/process/process.js +1 -0
- package/dist/shared/process/windows-launch.d.ts +1 -1
- package/dist/shared/process/windows-launch.js +6 -3
- package/dist/shared/provider-capabilities.d.ts +1 -0
- package/dist/shared/provider-capabilities.js +3 -0
- package/dist/shared/redaction/redaction.js +10 -3
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/worktree/artifact-injector.d.ts +3 -2
- package/dist/worktree/artifact-injector.js +43 -4
- package/dist/worktree/dependency-warmer.d.ts +21 -1
- package/dist/worktree/dependency-warmer.js +138 -5
- package/dist/worktree/index.d.ts +1 -0
- package/dist/worktree/index.js +1 -0
- package/dist/worktree/prior-phase-artifact-hydrator.d.ts +2 -0
- package/dist/worktree/prior-phase-artifact-hydrator.js +136 -0
- package/dist/worktree/worktree-paths.d.ts +4 -0
- package/dist/worktree/worktree-paths.js +30 -0
- package/dist/worktree/worktree.base.d.ts +3 -0
- package/dist/worktree/worktree.service.d.ts +7 -2
- package/dist/worktree/worktree.service.js +72 -44
- package/dist/worktree/worktree.types.d.ts +3 -1
- package/package.json +6 -3
- package/runtime/windows-job-supervisor.ps1 +356 -0
- package/runtime/windows-listener-owner.ps1 +33 -0
|
@@ -1,47 +1,114 @@
|
|
|
1
1
|
import { HubConnectionBuilder, HttpTransportType, LogLevel } from "@microsoft/signalr";
|
|
2
2
|
import fs from "node:fs";
|
|
3
3
|
import path from "node:path";
|
|
4
|
+
import { randomUUID } from "node:crypto";
|
|
5
|
+
import os from "node:os";
|
|
4
6
|
import { defaultForgeBridgeConfig } from "../config.js";
|
|
5
|
-
import {
|
|
7
|
+
import { supportsDraftImageAttachmentsFor } from "../shared/provider-capabilities.js";
|
|
8
|
+
import { redactText } from "../shared/redaction/redaction.js";
|
|
6
9
|
import { BridgeSessionService } from "../session/session.service.js";
|
|
10
|
+
import { ProcessBackedProviderCliService, RoutingProviderCliService, StreamingProviderProcessRunner, } from "../session/provider-cli/provider-cli.service.js";
|
|
11
|
+
import { redactRunnerDispatchEvent, redactRunOnceResultPacket } from "../session/session-redaction.js";
|
|
12
|
+
import { HostRuntimeLeaseManager } from "../runtime/host-runtime-lease.js";
|
|
13
|
+
import { GitWorktreeService } from "../worktree/worktree.service.js";
|
|
14
|
+
import { RepositoryCoordinatorLock } from "../sandbox/repository-lock.js";
|
|
15
|
+
import { AttemptCoordinatedProviderProcessRunner, AttemptProcessSupervisorRegistry, } from "../runtime/attempt-process-supervisor-registry.js";
|
|
7
16
|
import { failureResult, parseRunOnceDispatchPacket } from "../session/session.types.js";
|
|
8
17
|
import { forgeBridgeVersion } from "../version.js";
|
|
18
|
+
import { canRunDockerMaintenance, DockerPreflight } from "../sandbox/preflight.js";
|
|
19
|
+
import { validateSandboxContract } from "../sandbox/contract.js";
|
|
20
|
+
import { runSandboxProcess } from "../sandbox/process.js";
|
|
21
|
+
import { DockerSandboxRuntime, selectSandboxConfigRefs } from "../sandbox/runtime.js";
|
|
22
|
+
import { codexSandboxProviderAuthAdapter, sandboxProviderAuthAdapterFor, sandboxProviderAuthUnavailableMessage } from "../sandbox/provider-auth/codex.js";
|
|
9
23
|
export class ForgeBridgeService {
|
|
10
24
|
config;
|
|
11
|
-
sessions;
|
|
12
25
|
connections;
|
|
13
26
|
connection = null;
|
|
14
27
|
heartbeatTimer = null;
|
|
28
|
+
heartbeatInFlight = false;
|
|
29
|
+
consecutiveHeartbeatFailures = 0;
|
|
30
|
+
reconnectingFromHeartbeatFailure = false;
|
|
31
|
+
leaseWatchdogTimer = null;
|
|
32
|
+
sandboxMaintenanceTimer = null;
|
|
33
|
+
sandboxMaintenanceRunning = false;
|
|
15
34
|
closed = null;
|
|
16
35
|
stopped = false;
|
|
17
36
|
activeSessions = new Map();
|
|
18
|
-
|
|
37
|
+
reservations = new Map();
|
|
38
|
+
activeReadinessProbes = new Map();
|
|
39
|
+
preflight;
|
|
40
|
+
sandboxRuntime;
|
|
41
|
+
sandboxResourceResolver;
|
|
42
|
+
sessions;
|
|
43
|
+
constructor(config, sessions = undefined, connections = new SignalRBridgeConnectionFactory(), preflight, sandboxRuntime, sandboxResourceResolver) {
|
|
19
44
|
this.config = config;
|
|
20
|
-
this.sessions = sessions;
|
|
21
45
|
this.connections = connections;
|
|
46
|
+
this.sessions = sessions ?? createDefaultSessionService(config.sandboxStateRoot);
|
|
47
|
+
this.preflight = preflight ?? new DockerPreflight({
|
|
48
|
+
dockerExecutable: config.dockerExecutable,
|
|
49
|
+
dockerContext: config.dockerContext,
|
|
50
|
+
stateRoot: config.sandboxStateRoot,
|
|
51
|
+
runnerInstanceId: config.runnerInstanceId,
|
|
52
|
+
maxConcurrentSandboxes: config.maxConcurrentSandboxes,
|
|
53
|
+
diskHighWatermarkPercent: config.diskHighWatermarkPercent,
|
|
54
|
+
diskLowWatermarkPercent: config.diskLowWatermarkPercent,
|
|
55
|
+
minimumWorkspaceBytes: config.minimumWorkspaceBytes,
|
|
56
|
+
hostMemoryReserveBytes: config.sandboxHostMemoryReserveBytes,
|
|
57
|
+
defaultWorkerMemoryLimitBytes: config.defaultSandboxWorkerMemoryLimitBytes,
|
|
58
|
+
defaultServiceMemoryLimitBytes: config.defaultSandboxServiceMemoryLimitBytes,
|
|
59
|
+
runtimeOverheadBytes: config.sandboxRuntimeOverheadBytes,
|
|
60
|
+
requireRootless: config.requireRootlessDocker
|
|
61
|
+
});
|
|
62
|
+
this.sandboxRuntime = sandboxRuntime ?? new DockerSandboxRuntime(config, (request, signal) => this.redeemPublishGrant(request, signal));
|
|
63
|
+
this.sandboxResourceResolver = sandboxResourceResolver
|
|
64
|
+
?? (dispatch => resolveSandboxResourcesFromRepo(dispatch, this.config.sandboxRuntimeOverheadBytes));
|
|
22
65
|
}
|
|
23
66
|
async connectOnce() {
|
|
24
67
|
await this.stopConnection();
|
|
25
68
|
this.stopped = false;
|
|
69
|
+
const hostReap = await this.sessions.reapHostRuntime?.({
|
|
70
|
+
runnerId: this.config.runnerId,
|
|
71
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
72
|
+
});
|
|
73
|
+
if (hostReap && (hostReap.reaped > 0 || hostReap.degraded > 0)) {
|
|
74
|
+
logBridgeStatus(`Managed-host recovery reaped ${hostReap.reaped} stale attempt(s), protected ${hostReap.reusedPidsProtected} reused PID(s), and left ${hostReap.degraded} degraded attempt(s) for reconciliation.`);
|
|
75
|
+
}
|
|
76
|
+
if (this.config.dockerEnabled) {
|
|
77
|
+
const preflight = await this.preflight.refresh();
|
|
78
|
+
if (canReapManagedSandboxes(preflight)) {
|
|
79
|
+
await this.sandboxRuntime.reap({ diskLow: preflight.diskLow });
|
|
80
|
+
await this.preflight.refresh();
|
|
81
|
+
}
|
|
82
|
+
}
|
|
26
83
|
const connection = this.connections.create(this.config);
|
|
27
84
|
this.connection = connection;
|
|
28
85
|
this.closed = new Promise((resolve) => {
|
|
29
86
|
connection.onClose(() => {
|
|
30
87
|
this.stopHeartbeat();
|
|
31
88
|
this.abortActiveSessions("bridge connection closed");
|
|
89
|
+
this.abortActiveReadinessProbes("bridge connection closed");
|
|
32
90
|
resolve();
|
|
33
91
|
});
|
|
34
92
|
});
|
|
35
93
|
connection.onDispatch((dispatch) => this.handleDispatch(dispatch));
|
|
94
|
+
connection.onOffer?.((dispatch) => this.handleOffer(dispatch));
|
|
95
|
+
connection.onStart?.((dispatch) => this.handleStart(dispatch));
|
|
96
|
+
connection.onLeaseUpdate?.((request) => this.handleLeaseUpdate(request));
|
|
36
97
|
connection.onCancel((request) => this.handleCancel(request));
|
|
98
|
+
connection.onReap?.((request) => this.handleReap(request));
|
|
99
|
+
connection.onReadinessProbe?.((request) => this.handleReadinessProbe(request));
|
|
37
100
|
await connection.start();
|
|
38
101
|
await this.invokeAck("Hello", {
|
|
39
102
|
version: this.config.version,
|
|
40
103
|
protocolVersion: this.config.protocolVersion,
|
|
41
104
|
minServerVersion: this.config.minServerVersion ?? null,
|
|
105
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
42
106
|
capabilities: this.capabilities()
|
|
43
107
|
});
|
|
108
|
+
await this.flushCleanupReports();
|
|
44
109
|
this.startHeartbeat();
|
|
110
|
+
this.startLeaseWatchdog();
|
|
111
|
+
this.startSandboxMaintenance();
|
|
45
112
|
}
|
|
46
113
|
async run(signal) {
|
|
47
114
|
let attempt = 0;
|
|
@@ -72,6 +139,7 @@ export class ForgeBridgeService {
|
|
|
72
139
|
finally {
|
|
73
140
|
if (signal?.aborted) {
|
|
74
141
|
this.abortActiveSessions(String(signal.reason ?? "bridge stopping"));
|
|
142
|
+
this.abortActiveReadinessProbes(String(signal.reason ?? "bridge stopping"));
|
|
75
143
|
}
|
|
76
144
|
await this.stopConnection();
|
|
77
145
|
}
|
|
@@ -79,7 +147,9 @@ export class ForgeBridgeService {
|
|
|
79
147
|
async stop() {
|
|
80
148
|
this.stopped = true;
|
|
81
149
|
this.abortActiveSessions("bridge stopping");
|
|
150
|
+
this.abortActiveReadinessProbes("bridge stopping");
|
|
82
151
|
await this.stopConnection();
|
|
152
|
+
await this.preflight.release();
|
|
83
153
|
}
|
|
84
154
|
async handleDispatch(rawDispatch) {
|
|
85
155
|
const dispatch = this.parseDispatchOrReport(rawDispatch);
|
|
@@ -89,21 +159,53 @@ export class ForgeBridgeService {
|
|
|
89
159
|
const leaseId = dispatch.lease.id;
|
|
90
160
|
const fencingToken = dispatch.lease.fencingToken;
|
|
91
161
|
const redactionSecrets = exactSecrets(dispatch, this.config);
|
|
92
|
-
|
|
93
|
-
|
|
162
|
+
const existing = this.activeSessions.get(leaseId);
|
|
163
|
+
if (existing) {
|
|
164
|
+
if (existing.fencingToken === fencingToken) {
|
|
165
|
+
return;
|
|
166
|
+
}
|
|
167
|
+
await this.sendResult(dispatch, failureResult("Dispatch fence conflicts with an active bridge session.")).catch(() => undefined);
|
|
94
168
|
return;
|
|
95
169
|
}
|
|
96
170
|
const controller = new AbortController();
|
|
97
|
-
this.activeSessions.set(leaseId, {
|
|
171
|
+
this.activeSessions.set(leaseId, {
|
|
172
|
+
fencingToken,
|
|
173
|
+
dispatchAttemptId: dispatch.sandbox?.dispatchAttemptId ?? null,
|
|
174
|
+
runnerInstanceId: dispatch.sandbox?.runnerInstanceId ?? null,
|
|
175
|
+
leaseExpiresAt: dispatch.lease.expiresAt,
|
|
176
|
+
sandboxed: dispatch.sandbox?.executionMode === "docker",
|
|
177
|
+
resources: dispatch.sandbox?.executionMode === "docker"
|
|
178
|
+
? dispatch.sandbox.requestedResources
|
|
179
|
+
: zeroSandboxResources(),
|
|
180
|
+
controller
|
|
181
|
+
});
|
|
98
182
|
try {
|
|
99
183
|
const normalized = this.normalizeDispatch(dispatch);
|
|
100
|
-
|
|
184
|
+
if (normalized.sandbox) {
|
|
185
|
+
const started = {
|
|
186
|
+
dispatchAttemptId: normalized.sandbox.dispatchAttemptId,
|
|
187
|
+
leaseId,
|
|
188
|
+
fencingToken,
|
|
189
|
+
runnerInstanceId: normalized.sandbox.runnerInstanceId
|
|
190
|
+
};
|
|
191
|
+
await this.invokeAck("DispatchStarted", started);
|
|
192
|
+
}
|
|
193
|
+
const sink = {
|
|
101
194
|
emit: async (event) => {
|
|
195
|
+
if (event.eventType === "provider_cli_started") {
|
|
196
|
+
const active = this.activeSessions.get(leaseId);
|
|
197
|
+
if (active)
|
|
198
|
+
active.resources = { ...active.resources, buildSlots: 0 };
|
|
199
|
+
}
|
|
102
200
|
try {
|
|
103
201
|
await this.invokeAck("DispatchEvent", {
|
|
104
202
|
leaseId,
|
|
105
203
|
fencingToken,
|
|
106
|
-
|
|
204
|
+
...(dispatch.sandbox ? {
|
|
205
|
+
dispatchAttemptId: dispatch.sandbox.dispatchAttemptId,
|
|
206
|
+
runnerInstanceId: dispatch.sandbox.runnerInstanceId
|
|
207
|
+
} : {}),
|
|
208
|
+
event: redactRunnerDispatchEvent(event, redactionSecrets)
|
|
107
209
|
});
|
|
108
210
|
}
|
|
109
211
|
catch (error) {
|
|
@@ -114,19 +216,233 @@ export class ForgeBridgeService {
|
|
|
114
216
|
throw error;
|
|
115
217
|
}
|
|
116
218
|
}
|
|
117
|
-
}
|
|
118
|
-
|
|
219
|
+
};
|
|
220
|
+
const result = normalized.sandbox?.executionMode === "docker"
|
|
221
|
+
? await this.sandboxRuntime.runOnce(normalized, sink, controller.signal)
|
|
222
|
+
: await this.sessions.runOnce(normalized, sink, controller.signal);
|
|
223
|
+
if (normalized.sandbox?.executionMode === "docker") {
|
|
224
|
+
await this.flushCleanupReports().catch(() => undefined);
|
|
225
|
+
}
|
|
226
|
+
await this.sendResult(dispatch, redactRunOnceResultPacket(result, redactionSecrets));
|
|
119
227
|
}
|
|
120
228
|
catch (error) {
|
|
121
229
|
const summary = controller.signal.aborted
|
|
122
230
|
? `Dispatch cancelled: ${String(controller.signal.reason ?? "cancelled")}`
|
|
123
231
|
: redactText(error instanceof Error ? error.message : String(error), redactionSecrets);
|
|
124
|
-
|
|
232
|
+
const failureEventType = infrastructureFailureEventType(summary);
|
|
233
|
+
const failureEvents = failureEventType ? [{
|
|
234
|
+
role: "runner",
|
|
235
|
+
kind: "error",
|
|
236
|
+
content: summary,
|
|
237
|
+
eventType: failureEventType,
|
|
238
|
+
}] : [];
|
|
239
|
+
await this.sendResult(dispatch, failureResult(summary, failureEvents)).catch(() => undefined);
|
|
125
240
|
}
|
|
126
241
|
finally {
|
|
127
242
|
this.activeSessions.delete(leaseId);
|
|
128
243
|
}
|
|
129
244
|
}
|
|
245
|
+
async handleOffer(rawDispatch) {
|
|
246
|
+
const envelope = parseOfferEnvelope(rawDispatch);
|
|
247
|
+
if (!envelope) {
|
|
248
|
+
return;
|
|
249
|
+
}
|
|
250
|
+
const dispatch = this.parseDispatchOrReport(envelope.dispatch);
|
|
251
|
+
if (!dispatch?.sandbox
|
|
252
|
+
|| dispatch.sandbox.dispatchAttemptId !== envelope.dispatchAttemptId
|
|
253
|
+
|| dispatch.lease.id !== envelope.leaseId
|
|
254
|
+
|| dispatch.lease.fencingToken !== envelope.fencingToken)
|
|
255
|
+
return;
|
|
256
|
+
const sandbox = dispatch.sandbox;
|
|
257
|
+
let existing = this.reservations.get(sandbox.dispatchAttemptId);
|
|
258
|
+
if (existing && Date.parse(existing.reservedUntil) <= Date.now()) {
|
|
259
|
+
this.reservations.delete(sandbox.dispatchAttemptId);
|
|
260
|
+
existing = undefined;
|
|
261
|
+
}
|
|
262
|
+
if (existing) {
|
|
263
|
+
const sameFence = existing.fencingToken === dispatch.lease.fencingToken;
|
|
264
|
+
await this.sendOfferDecision(dispatch, sameFence, sameFence ? null : "stale_fence", sameFence ? null : "Offer fence conflicts with the existing reservation.", sameFence ? existing.reservedUntil : null, sameFence ? existing.resources : null);
|
|
265
|
+
return;
|
|
266
|
+
}
|
|
267
|
+
if (sandbox.runnerInstanceId !== this.config.runnerInstanceId) {
|
|
268
|
+
await this.sendOfferDecision(dispatch, false, "stale_runner_instance", "Offer targets a different runner process instance.");
|
|
269
|
+
return;
|
|
270
|
+
}
|
|
271
|
+
if (Date.parse(envelope.expiresAt) <= Date.now()) {
|
|
272
|
+
await this.sendOfferDecision(dispatch, false, "lease_expired", "Offer lease is already expired.");
|
|
273
|
+
return;
|
|
274
|
+
}
|
|
275
|
+
const preflight = sandbox.executionMode === "docker" && this.config.dockerEnabled
|
|
276
|
+
? await this.preflight.run()
|
|
277
|
+
: this.preflight.current();
|
|
278
|
+
if (sandbox.executionMode === "docker") {
|
|
279
|
+
if (!this.config.dockerEnabled || !preflight?.healthy) {
|
|
280
|
+
await this.sendOfferDecision(dispatch, false, preflight?.unavailableCode ?? "docker_unavailable", preflight?.unavailableReason ?? "Docker sandbox preflight is unavailable.");
|
|
281
|
+
return;
|
|
282
|
+
}
|
|
283
|
+
const networkFailure = sandboxNetworkPolicyFailure(dispatch, this.config);
|
|
284
|
+
if (networkFailure) {
|
|
285
|
+
await this.sendOfferDecision(dispatch, false, "policy_mismatch", networkFailure);
|
|
286
|
+
return;
|
|
287
|
+
}
|
|
288
|
+
const providerNetworkFailure = sandboxProviderNetworkFailure(dispatch, this.config);
|
|
289
|
+
if (providerNetworkFailure) {
|
|
290
|
+
await this.sendOfferDecision(dispatch, false, "provider_network_unavailable", providerNetworkFailure);
|
|
291
|
+
return;
|
|
292
|
+
}
|
|
293
|
+
const authFailure = sandboxAuthFailure(dispatch, this.config);
|
|
294
|
+
if (authFailure) {
|
|
295
|
+
await this.sendOfferDecision(dispatch, false, "sandbox_auth_unavailable", authFailure);
|
|
296
|
+
return;
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
const runtimeState = this.sandboxRuntime.snapshot();
|
|
300
|
+
const usedResources = this.usedSandboxResources(runtimeState.degraded);
|
|
301
|
+
const activeResources = this.activeSandboxResources(runtimeState.degraded);
|
|
302
|
+
const capacity = this.sandboxResourceCapacity(preflight, activeResources);
|
|
303
|
+
let requestedResources = sandbox.requestedResources;
|
|
304
|
+
let sandboxConfigCommitSha = sandbox.sandboxConfigCommitSha ?? null;
|
|
305
|
+
if (sandbox.executionMode === "docker") {
|
|
306
|
+
try {
|
|
307
|
+
const profile = await this.sandboxResourceResolver(this.normalizeDispatch(dispatch));
|
|
308
|
+
requestedResources = profile.resources;
|
|
309
|
+
sandboxConfigCommitSha = profile.sandboxConfigCommitSha;
|
|
310
|
+
}
|
|
311
|
+
catch (error) {
|
|
312
|
+
await this.sendOfferDecision(dispatch, false, "sandbox_capacity_profile_unavailable", error instanceof Error ? error.message : "The immutable sandbox contract could not be profiled for admission.");
|
|
313
|
+
return;
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
if (sandbox.executionMode === "docker"
|
|
317
|
+
&& !canReserveSandboxResources(usedResources, requestedResources, capacity)) {
|
|
318
|
+
await this.sendOfferDecision(dispatch, false, "capacity_unavailable", sandboxCapacityFailureMessage(usedResources, requestedResources, capacity, preflight, this.config));
|
|
319
|
+
return;
|
|
320
|
+
}
|
|
321
|
+
const reservedUntil = new Date(Math.min(Date.parse(dispatch.lease.expiresAt), Date.now() + 30_000)).toISOString();
|
|
322
|
+
this.reservations.set(sandbox.dispatchAttemptId, {
|
|
323
|
+
leaseId: dispatch.lease.id,
|
|
324
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
325
|
+
runnerInstanceId: sandbox.runnerInstanceId,
|
|
326
|
+
expiresAt: dispatch.lease.expiresAt,
|
|
327
|
+
reservedUntil,
|
|
328
|
+
sandboxed: sandbox.executionMode === "docker",
|
|
329
|
+
resources: requestedResources
|
|
330
|
+
});
|
|
331
|
+
await this.sendOfferDecision(dispatch, true, null, null, reservedUntil, requestedResources, sandboxConfigCommitSha);
|
|
332
|
+
}
|
|
333
|
+
async handleStart(rawDispatch) {
|
|
334
|
+
const envelope = parseStartEnvelope(rawDispatch);
|
|
335
|
+
if (!envelope) {
|
|
336
|
+
return;
|
|
337
|
+
}
|
|
338
|
+
let dispatch = this.parseDispatchOrReport(envelope.dispatch);
|
|
339
|
+
if (!dispatch?.sandbox
|
|
340
|
+
|| dispatch.sandbox.dispatchAttemptId !== envelope.dispatchAttemptId
|
|
341
|
+
|| dispatch.lease.id !== envelope.leaseId
|
|
342
|
+
|| dispatch.lease.fencingToken !== envelope.fencingToken)
|
|
343
|
+
return;
|
|
344
|
+
const reservation = this.reservations.get(dispatch.sandbox.dispatchAttemptId);
|
|
345
|
+
if (!reservation
|
|
346
|
+
|| reservation.leaseId !== dispatch.lease.id
|
|
347
|
+
|| reservation.fencingToken !== dispatch.lease.fencingToken
|
|
348
|
+
|| reservation.runnerInstanceId !== dispatch.sandbox.runnerInstanceId
|
|
349
|
+
|| Date.parse(reservation.reservedUntil) <= Date.now()) {
|
|
350
|
+
await this.sendResult(dispatch, failureResult("Start rejected because its reservation is missing or stale.")).catch(() => undefined);
|
|
351
|
+
return;
|
|
352
|
+
}
|
|
353
|
+
if (dispatch.apiKeySecretGrant) {
|
|
354
|
+
this.reservations.delete(dispatch.sandbox.dispatchAttemptId);
|
|
355
|
+
await this.sendResult(dispatch, failureResult("Start rejected because protocol-v2 packets may not contain a plaintext provider grant.")).catch(() => undefined);
|
|
356
|
+
return;
|
|
357
|
+
}
|
|
358
|
+
if (dispatch.apiKeySecretGrantHandle) {
|
|
359
|
+
const response = await this.redeemSecretGrant(dispatch);
|
|
360
|
+
const grant = response.grant;
|
|
361
|
+
if (!response.accepted
|
|
362
|
+
|| !grant
|
|
363
|
+
|| grant.grantId !== dispatch.apiKeySecretGrantHandle.grantId
|
|
364
|
+
|| grant.provider !== dispatch.apiKeySecretGrantHandle.provider
|
|
365
|
+
|| Date.parse(grant.expiresAt) <= Date.now()) {
|
|
366
|
+
this.reservations.delete(dispatch.sandbox.dispatchAttemptId);
|
|
367
|
+
await this.sendResult(dispatch, failureResult(response.message ?? "Secret grant redemption was rejected.")).catch(() => undefined);
|
|
368
|
+
return;
|
|
369
|
+
}
|
|
370
|
+
dispatch = { ...dispatch, apiKeySecretGrant: grant, apiKeySecretGrantHandle: null };
|
|
371
|
+
}
|
|
372
|
+
else if (dispatch.execution.requiresApiKeySecretGrant) {
|
|
373
|
+
this.reservations.delete(dispatch.sandbox.dispatchAttemptId);
|
|
374
|
+
await this.sendResult(dispatch, failureResult("Start rejected because the provider secret grant handle is missing.")).catch(() => undefined);
|
|
375
|
+
return;
|
|
376
|
+
}
|
|
377
|
+
dispatch = {
|
|
378
|
+
...dispatch,
|
|
379
|
+
sandbox: { ...dispatch.sandbox, requestedResources: reservation.resources },
|
|
380
|
+
};
|
|
381
|
+
this.reservations.delete(envelope.dispatchAttemptId);
|
|
382
|
+
await this.handleDispatch(dispatch);
|
|
383
|
+
}
|
|
384
|
+
async redeemSecretGrant(dispatch) {
|
|
385
|
+
const sandbox = dispatch.sandbox;
|
|
386
|
+
const handle = dispatch.apiKeySecretGrantHandle;
|
|
387
|
+
const connection = this.connection;
|
|
388
|
+
if (!connection)
|
|
389
|
+
return { accepted: false, code: "bridge.disconnected", message: "Bridge is disconnected.", grant: null };
|
|
390
|
+
return await connection.invoke("RedeemSecretGrant", {
|
|
391
|
+
dispatchAttemptId: sandbox.dispatchAttemptId,
|
|
392
|
+
leaseId: dispatch.lease.id,
|
|
393
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
394
|
+
runnerInstanceId: sandbox.runnerInstanceId,
|
|
395
|
+
grantId: handle.grantId
|
|
396
|
+
});
|
|
397
|
+
}
|
|
398
|
+
async redeemPublishGrant(request, signal) {
|
|
399
|
+
if (signal?.aborted)
|
|
400
|
+
throw new Error(String(signal.reason ?? "publication cancelled"));
|
|
401
|
+
const connection = this.connection;
|
|
402
|
+
if (!connection)
|
|
403
|
+
throw new Error("sandbox_output_publish_forbidden: bridge is disconnected.");
|
|
404
|
+
const response = await connection.invoke("RedeemPublishGrant", request);
|
|
405
|
+
if (!response.accepted || !response.credential) {
|
|
406
|
+
throw new Error(`sandbox_output_publish_forbidden: ${response.message ?? response.code ?? "publication grant redemption was rejected"}`);
|
|
407
|
+
}
|
|
408
|
+
return response.credential;
|
|
409
|
+
}
|
|
410
|
+
async handleLeaseUpdate(request) {
|
|
411
|
+
if (request.runnerInstanceId !== this.config.runnerInstanceId) {
|
|
412
|
+
return;
|
|
413
|
+
}
|
|
414
|
+
const reservation = this.reservations.get(request.dispatchAttemptId);
|
|
415
|
+
if (reservation && reservation.leaseId === request.leaseId && reservation.fencingToken === request.fencingToken) {
|
|
416
|
+
reservation.expiresAt = request.expiresAt;
|
|
417
|
+
}
|
|
418
|
+
const active = this.activeSessions.get(request.leaseId);
|
|
419
|
+
if (active && active.fencingToken === request.fencingToken && active.dispatchAttemptId === request.dispatchAttemptId) {
|
|
420
|
+
active.leaseExpiresAt = request.expiresAt;
|
|
421
|
+
if (active.sandboxed) {
|
|
422
|
+
await this.sandboxRuntime.updateLease(request.dispatchAttemptId, request.fencingToken, request.expiresAt);
|
|
423
|
+
}
|
|
424
|
+
else {
|
|
425
|
+
await this.sessions.updateLease?.(request.dispatchAttemptId, request.fencingToken, request.expiresAt);
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
async sendOfferDecision(dispatch, accepted, code = null, message = null, reservedUntil = null, reservedResources = null, sandboxConfigCommitSha = null) {
|
|
430
|
+
if (!dispatch.sandbox)
|
|
431
|
+
return;
|
|
432
|
+
const payload = {
|
|
433
|
+
dispatchAttemptId: dispatch.sandbox.dispatchAttemptId,
|
|
434
|
+
leaseId: dispatch.lease.id,
|
|
435
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
436
|
+
runnerInstanceId: dispatch.sandbox.runnerInstanceId,
|
|
437
|
+
accepted,
|
|
438
|
+
code,
|
|
439
|
+
message,
|
|
440
|
+
reservedUntil: accepted ? reservedUntil ?? dispatch.lease.expiresAt : null,
|
|
441
|
+
reservedResources: accepted ? reservedResources : null,
|
|
442
|
+
sandboxConfigCommitSha: accepted ? sandboxConfigCommitSha : null,
|
|
443
|
+
};
|
|
444
|
+
await this.invokeAck("OfferDecision", payload);
|
|
445
|
+
}
|
|
130
446
|
parseDispatchOrReport(rawDispatch) {
|
|
131
447
|
try {
|
|
132
448
|
return parseRunOnceDispatchPacket(rawDispatch);
|
|
@@ -137,7 +453,7 @@ export class ForgeBridgeService {
|
|
|
137
453
|
return null;
|
|
138
454
|
}
|
|
139
455
|
const summary = invalidDispatchSummary(error);
|
|
140
|
-
void this.
|
|
456
|
+
void this.sendLegacyResult(identifiers.leaseId, identifiers.fencingToken, failureResult(summary, [
|
|
141
457
|
{
|
|
142
458
|
role: "runner",
|
|
143
459
|
kind: "error",
|
|
@@ -155,12 +471,74 @@ export class ForgeBridgeService {
|
|
|
155
471
|
}
|
|
156
472
|
active.controller.abort(request.reason ?? "cancelled by server");
|
|
157
473
|
}
|
|
474
|
+
async handleReap(request) {
|
|
475
|
+
if (request.runnerId !== this.config.runnerId
|
|
476
|
+
|| request.runnerInstanceId !== this.config.runnerInstanceId)
|
|
477
|
+
return;
|
|
478
|
+
await this.runSandboxMaintenance();
|
|
479
|
+
await this.sendHeartbeat();
|
|
480
|
+
}
|
|
481
|
+
async handleReadinessProbe(rawRequest) {
|
|
482
|
+
const request = parseReadinessProbe(rawRequest);
|
|
483
|
+
if (!request || request.runnerInstanceId !== this.config.runnerInstanceId)
|
|
484
|
+
return;
|
|
485
|
+
const existing = this.activeReadinessProbes.get(request.requestId);
|
|
486
|
+
if (existing)
|
|
487
|
+
return;
|
|
488
|
+
const controller = new AbortController();
|
|
489
|
+
this.activeReadinessProbes.set(request.requestId, controller);
|
|
490
|
+
const deadlineMs = Date.parse(request.expiresAt) - Date.now();
|
|
491
|
+
const timer = setTimeout(() => controller.abort("readiness probe expired"), Math.max(0, deadlineMs));
|
|
492
|
+
timer.unref?.();
|
|
493
|
+
let result;
|
|
494
|
+
try {
|
|
495
|
+
if (!this.sessions.probeHostRuntime) {
|
|
496
|
+
throw new Error("host_runtime_probe_unavailable: this bridge does not implement runtime readiness probes.");
|
|
497
|
+
}
|
|
498
|
+
const probe = await this.sessions.probeHostRuntime({
|
|
499
|
+
...request,
|
|
500
|
+
runnerId: this.config.runnerId,
|
|
501
|
+
repoRoot: this.config.defaultRepoRoot,
|
|
502
|
+
cleanupWorktrees: this.config.cleanupWorktrees,
|
|
503
|
+
}, controller.signal);
|
|
504
|
+
result = {
|
|
505
|
+
schemaVersion: 1,
|
|
506
|
+
requestId: request.requestId,
|
|
507
|
+
fencingToken: request.fencingToken,
|
|
508
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
509
|
+
...probe,
|
|
510
|
+
};
|
|
511
|
+
}
|
|
512
|
+
catch (error) {
|
|
513
|
+
result = {
|
|
514
|
+
schemaVersion: 1,
|
|
515
|
+
requestId: request.requestId,
|
|
516
|
+
fencingToken: request.fencingToken,
|
|
517
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
518
|
+
status: "failed",
|
|
519
|
+
reasonCode: "host_runtime_probe_failed",
|
|
520
|
+
reasonMessage: errorMessage(error).slice(0, 1000),
|
|
521
|
+
checkedAt: new Date().toISOString(),
|
|
522
|
+
};
|
|
523
|
+
}
|
|
524
|
+
finally {
|
|
525
|
+
clearTimeout(timer);
|
|
526
|
+
this.activeReadinessProbes.delete(request.requestId);
|
|
527
|
+
}
|
|
528
|
+
await this.invokeAck("ReadinessResult", result).catch(() => undefined);
|
|
529
|
+
await this.sendHeartbeat().catch(() => undefined);
|
|
530
|
+
}
|
|
158
531
|
abortActiveSessions(reason) {
|
|
159
532
|
for (const active of this.activeSessions.values()) {
|
|
160
533
|
active.controller.abort(reason);
|
|
161
534
|
}
|
|
162
535
|
this.activeSessions.clear();
|
|
163
536
|
}
|
|
537
|
+
abortActiveReadinessProbes(reason) {
|
|
538
|
+
for (const controller of this.activeReadinessProbes.values())
|
|
539
|
+
controller.abort(reason);
|
|
540
|
+
this.activeReadinessProbes.clear();
|
|
541
|
+
}
|
|
164
542
|
normalizeDispatch(dispatch) {
|
|
165
543
|
const mapping = findRepoMapping(dispatch, repoMappingsFor(this.config));
|
|
166
544
|
const localPath = mapping?.localPath?.trim();
|
|
@@ -185,12 +563,20 @@ export class ForgeBridgeService {
|
|
|
185
563
|
: dispatch.qaBrowser
|
|
186
564
|
};
|
|
187
565
|
}
|
|
188
|
-
async sendResult(
|
|
566
|
+
async sendResult(dispatch, result) {
|
|
189
567
|
const payload = {
|
|
190
|
-
leaseId,
|
|
191
|
-
fencingToken,
|
|
568
|
+
leaseId: dispatch.lease.id,
|
|
569
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
570
|
+
dispatchAttemptId: dispatch.sandbox?.dispatchAttemptId ?? null,
|
|
571
|
+
runnerInstanceId: dispatch.sandbox?.runnerInstanceId ?? null,
|
|
192
572
|
result
|
|
193
573
|
};
|
|
574
|
+
await this.sendResultPayload(payload);
|
|
575
|
+
}
|
|
576
|
+
async sendLegacyResult(leaseId, fencingToken, result) {
|
|
577
|
+
await this.sendResultPayload({ leaseId, fencingToken, result });
|
|
578
|
+
}
|
|
579
|
+
async sendResultPayload(payload) {
|
|
194
580
|
try {
|
|
195
581
|
await this.invokeAck("DispatchResult", payload);
|
|
196
582
|
}
|
|
@@ -203,18 +589,132 @@ export class ForgeBridgeService {
|
|
|
203
589
|
}
|
|
204
590
|
startHeartbeat() {
|
|
205
591
|
this.stopHeartbeat();
|
|
592
|
+
this.consecutiveHeartbeatFailures = 0;
|
|
593
|
+
this.reconnectingFromHeartbeatFailure = false;
|
|
206
594
|
const intervalMs = positiveFiniteNumber(this.config.heartbeatIntervalMs, 30_000);
|
|
207
595
|
this.heartbeatTimer = setInterval(() => {
|
|
208
|
-
this.
|
|
596
|
+
void this.heartbeatTick();
|
|
209
597
|
}, intervalMs);
|
|
210
598
|
this.heartbeatTimer.unref?.();
|
|
211
599
|
}
|
|
600
|
+
async heartbeatTick() {
|
|
601
|
+
if (this.heartbeatInFlight || this.reconnectingFromHeartbeatFailure)
|
|
602
|
+
return;
|
|
603
|
+
const connection = this.connection;
|
|
604
|
+
if (!connection)
|
|
605
|
+
return;
|
|
606
|
+
this.heartbeatInFlight = true;
|
|
607
|
+
try {
|
|
608
|
+
await this.sendHeartbeat();
|
|
609
|
+
this.consecutiveHeartbeatFailures = 0;
|
|
610
|
+
}
|
|
611
|
+
catch (error) {
|
|
612
|
+
if (connection !== this.connection || this.stopped)
|
|
613
|
+
return;
|
|
614
|
+
this.consecutiveHeartbeatFailures += 1;
|
|
615
|
+
logBridgeStatus(`Heartbeat ${this.consecutiveHeartbeatFailures} failed: ${errorMessage(error)}.`);
|
|
616
|
+
if (this.consecutiveHeartbeatFailures < 2)
|
|
617
|
+
return;
|
|
618
|
+
this.reconnectingFromHeartbeatFailure = true;
|
|
619
|
+
this.stopHeartbeat();
|
|
620
|
+
logBridgeStatus("Heartbeat failed twice. Closing the stale connection so the Bridge can reconnect...");
|
|
621
|
+
await connection.stop().catch(() => undefined);
|
|
622
|
+
}
|
|
623
|
+
finally {
|
|
624
|
+
this.heartbeatInFlight = false;
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
startLeaseWatchdog() {
|
|
628
|
+
this.stopLeaseWatchdog();
|
|
629
|
+
this.leaseWatchdogTimer = setInterval(() => this.expireLeases(), 1000);
|
|
630
|
+
this.leaseWatchdogTimer.unref?.();
|
|
631
|
+
}
|
|
632
|
+
stopLeaseWatchdog() {
|
|
633
|
+
if (this.leaseWatchdogTimer) {
|
|
634
|
+
clearInterval(this.leaseWatchdogTimer);
|
|
635
|
+
this.leaseWatchdogTimer = null;
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
expireLeases() {
|
|
639
|
+
const now = Date.now();
|
|
640
|
+
for (const [attemptId, reservation] of this.reservations) {
|
|
641
|
+
if (Date.parse(reservation.expiresAt) <= now || Date.parse(reservation.reservedUntil) <= now) {
|
|
642
|
+
this.reservations.delete(attemptId);
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
for (const active of this.activeSessions.values()) {
|
|
646
|
+
if (Date.parse(active.leaseExpiresAt) <= now && !active.controller.signal.aborted) {
|
|
647
|
+
active.controller.abort("dispatch lease expired");
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
}
|
|
212
651
|
stopHeartbeat() {
|
|
213
652
|
if (this.heartbeatTimer) {
|
|
214
653
|
clearInterval(this.heartbeatTimer);
|
|
215
654
|
this.heartbeatTimer = null;
|
|
216
655
|
}
|
|
217
656
|
}
|
|
657
|
+
startSandboxMaintenance() {
|
|
658
|
+
this.stopSandboxMaintenance();
|
|
659
|
+
this.sandboxMaintenanceTimer = setInterval(() => {
|
|
660
|
+
this.runSandboxMaintenance().catch(() => undefined);
|
|
661
|
+
}, 60_000);
|
|
662
|
+
this.sandboxMaintenanceTimer.unref?.();
|
|
663
|
+
}
|
|
664
|
+
stopSandboxMaintenance() {
|
|
665
|
+
if (this.sandboxMaintenanceTimer) {
|
|
666
|
+
clearInterval(this.sandboxMaintenanceTimer);
|
|
667
|
+
this.sandboxMaintenanceTimer = null;
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
async runSandboxMaintenance() {
|
|
671
|
+
if (this.sandboxMaintenanceRunning)
|
|
672
|
+
return;
|
|
673
|
+
this.sandboxMaintenanceRunning = true;
|
|
674
|
+
try {
|
|
675
|
+
const hostReap = await this.sessions.reapHostRuntime?.({
|
|
676
|
+
runnerId: this.config.runnerId,
|
|
677
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
678
|
+
});
|
|
679
|
+
if (hostReap && (hostReap.reaped > 0 || hostReap.degraded > 0)) {
|
|
680
|
+
logBridgeStatus(`Managed-host maintenance reaped ${hostReap.reaped} stale attempt(s) and left ${hostReap.degraded} degraded attempt(s) for reconciliation.`);
|
|
681
|
+
}
|
|
682
|
+
if (this.config.dockerEnabled) {
|
|
683
|
+
const preflight = await this.preflight.refresh();
|
|
684
|
+
if (canReapManagedSandboxes(preflight)) {
|
|
685
|
+
const attemptSnapshot = await this.readAttemptSnapshot();
|
|
686
|
+
await this.sandboxRuntime.reap({ diskLow: preflight.diskLow, attemptSnapshot });
|
|
687
|
+
await this.flushCleanupReports();
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
finally {
|
|
692
|
+
this.sandboxMaintenanceRunning = false;
|
|
693
|
+
}
|
|
694
|
+
}
|
|
695
|
+
async readAttemptSnapshot() {
|
|
696
|
+
const connection = this.connection;
|
|
697
|
+
if (!connection)
|
|
698
|
+
throw new Error("Bridge is not connected.");
|
|
699
|
+
const response = await connection.invoke("AttemptSnapshot", {
|
|
700
|
+
runnerInstanceId: this.config.runnerInstanceId
|
|
701
|
+
});
|
|
702
|
+
if (!response.accepted) {
|
|
703
|
+
throw new BridgeCallbackRejectedError(response.code ?? null, response.message ?? response.code ?? "Attempt snapshot was rejected.");
|
|
704
|
+
}
|
|
705
|
+
return response.attempts ?? [];
|
|
706
|
+
}
|
|
707
|
+
async flushCleanupReports() {
|
|
708
|
+
if (!this.sandboxRuntime.pendingCleanupReports || !this.sandboxRuntime.acknowledgeCleanupReport)
|
|
709
|
+
return;
|
|
710
|
+
for (const report of await this.sandboxRuntime.pendingCleanupReports()) {
|
|
711
|
+
await this.invokeAck("CleanupReport", {
|
|
712
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
713
|
+
report
|
|
714
|
+
});
|
|
715
|
+
await this.sandboxRuntime.acknowledgeCleanupReport(report.reportId);
|
|
716
|
+
}
|
|
717
|
+
}
|
|
218
718
|
async sendHeartbeat() {
|
|
219
719
|
await this.invokeAck("Heartbeat", {
|
|
220
720
|
status: "online",
|
|
@@ -224,8 +724,14 @@ export class ForgeBridgeService {
|
|
|
224
724
|
});
|
|
225
725
|
}
|
|
226
726
|
capabilities() {
|
|
727
|
+
const preflight = this.preflight.current();
|
|
728
|
+
const runtimeState = this.sandboxRuntime.snapshot();
|
|
729
|
+
const reserved = [...this.reservations.values()].filter(reservation => reservation.sandboxed).length;
|
|
730
|
+
const used = this.usedSandboxResources(runtimeState.degraded);
|
|
731
|
+
const activeResources = this.activeSandboxResources(runtimeState.degraded);
|
|
732
|
+
const capacity = this.sandboxResourceCapacity(preflight, activeResources);
|
|
227
733
|
return {
|
|
228
|
-
schemaVersion:
|
|
734
|
+
schemaVersion: 3,
|
|
229
735
|
activeSessionCount: this.activeSessions.size,
|
|
230
736
|
cliProviders: this.config.cliProviders,
|
|
231
737
|
providerStatuses: this.config.providerStatuses,
|
|
@@ -233,11 +739,86 @@ export class ForgeBridgeService {
|
|
|
233
739
|
repoMappings: repoMappingsFor(this.config),
|
|
234
740
|
os: this.config.os,
|
|
235
741
|
sandboxProfiles: this.config.sandboxProfiles,
|
|
742
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
743
|
+
trustDomainId: this.config.trustDomainId,
|
|
744
|
+
maxConcurrentSandboxes: capacity.sandboxSlots,
|
|
745
|
+
reservedSandboxCount: reserved,
|
|
746
|
+
activeSandboxCount: runtimeState.active,
|
|
747
|
+
startingSandboxCount: runtimeState.starting,
|
|
748
|
+
degradedReservationCount: runtimeState.degraded,
|
|
749
|
+
sandboxRuntimes: preflight?.healthy ? ["docker"] : [],
|
|
750
|
+
sandboxRuntimeVersion: preflight?.runtimeVersion ?? null,
|
|
751
|
+
sandboxFeatures: preflight?.features ?? null,
|
|
752
|
+
allowsOpenPublicSandbox: this.config.allowOpenPublicSandbox,
|
|
753
|
+
sandboxPublicationModes: [this.config.sandboxPublicationMode ?? "shared_filesystem"],
|
|
754
|
+
sandboxAuthModes: sandboxAuthModesFor(this.config),
|
|
755
|
+
availableMemoryBytes: preflight ? Math.max(0, capacity.memoryBytes - used.memoryBytes) : null,
|
|
756
|
+
availableCpuMillis: preflight ? Math.max(0, capacity.cpuMillis - used.cpuMillis) : null,
|
|
757
|
+
availableWorkspaceBytes: preflight ? Math.max(0, capacity.workspaceBytes - used.workspaceBytes) : null,
|
|
758
|
+
diskLow: preflight?.diskLow ?? null,
|
|
759
|
+
preflightGeneration: preflight?.generation ?? 0,
|
|
760
|
+
sandboxUnavailableCode: this.config.dockerEnabled ? preflight?.unavailableCode ?? null : "docker_disabled",
|
|
761
|
+
sandboxUnavailableReason: this.config.dockerEnabled ? preflight?.unavailableReason ?? null : "Docker sandboxing is disabled by runner configuration.",
|
|
762
|
+
sandboxMemoryCapacity: preflight ? {
|
|
763
|
+
defaultWorkerMemoryLimitBytes: this.config.defaultSandboxWorkerMemoryLimitBytes,
|
|
764
|
+
defaultServiceMemoryLimitBytes: this.config.defaultSandboxServiceMemoryLimitBytes,
|
|
765
|
+
runtimeOverheadBytes: this.config.sandboxRuntimeOverheadBytes,
|
|
766
|
+
hostReserveBytes: preflight.hostReserveBytes,
|
|
767
|
+
activeReservationBytes: used.memoryBytes,
|
|
768
|
+
daemonTotalBytes: preflight.totalMemoryBytes,
|
|
769
|
+
daemonAvailableBytes: preflight.availableMemoryBytes,
|
|
770
|
+
availableForAdmissionBytes: Math.max(0, capacity.memoryBytes - used.memoryBytes),
|
|
771
|
+
} : null,
|
|
772
|
+
hostRuntime: this.sessions.runtimeCapacity?.() ?? null,
|
|
236
773
|
supportsApiSecretGrant: this.config.supportsApiSecretGrant,
|
|
237
774
|
supportsManagedWorktrees: this.config.supportsManagedWorktrees,
|
|
238
775
|
supportsManagedForgeArtifacts: this.config.supportsManagedForgeArtifacts,
|
|
239
776
|
supportsRunnerAuthoredPhaseArtifacts: this.config.supportsRunnerAuthoredPhaseArtifacts,
|
|
240
|
-
supportsDraftRunOnce: this.config.supportsDraftRunOnce
|
|
777
|
+
supportsDraftRunOnce: this.config.supportsDraftRunOnce,
|
|
778
|
+
supportsDraftAttachments: this.config.supportsDraftAttachments,
|
|
779
|
+
supportsDraftImageAttachments: this.config.supportsDraftImageAttachments
|
|
780
|
+
};
|
|
781
|
+
}
|
|
782
|
+
usedSandboxResources(degradedReservations) {
|
|
783
|
+
const resources = [
|
|
784
|
+
...[...this.reservations.values()]
|
|
785
|
+
.filter(reservation => reservation.sandboxed)
|
|
786
|
+
.map(reservation => reservation.resources),
|
|
787
|
+
...[...this.activeSessions.values()]
|
|
788
|
+
.filter(session => session.sandboxed)
|
|
789
|
+
.map(session => session.resources),
|
|
790
|
+
...Array.from({ length: degradedReservations }, () => defaultSandboxResources())
|
|
791
|
+
];
|
|
792
|
+
return sumSandboxResources(resources);
|
|
793
|
+
}
|
|
794
|
+
activeSandboxResources(degradedReservations) {
|
|
795
|
+
return sumSandboxResources([
|
|
796
|
+
...[...this.activeSessions.values()]
|
|
797
|
+
.filter(session => session.sandboxed)
|
|
798
|
+
.map(session => session.resources),
|
|
799
|
+
...Array.from({ length: degradedReservations }, () => defaultSandboxResources())
|
|
800
|
+
]);
|
|
801
|
+
}
|
|
802
|
+
sandboxResourceCapacity(preflight, activeResources) {
|
|
803
|
+
if (!preflight)
|
|
804
|
+
return zeroSandboxResources();
|
|
805
|
+
return {
|
|
806
|
+
// Host free memory and disk already reflect active containers. Add their
|
|
807
|
+
// commitments back to form a stable total capacity, then subtract them
|
|
808
|
+
// exactly once in usedSandboxResources.
|
|
809
|
+
sandboxSlots: Math.min(this.config.maxConcurrentSandboxes, preflight.maxConcurrentSandboxes + activeResources.sandboxSlots),
|
|
810
|
+
cpuMillis: preflight.availableCpuMillis,
|
|
811
|
+
workerMemoryLimitBytes: 0,
|
|
812
|
+
serviceMemoryLimitBytes: 0,
|
|
813
|
+
runtimeOverheadBytes: 0,
|
|
814
|
+
memoryBytes: Math.max(0, preflight.availableMemoryBytes
|
|
815
|
+
- (this.config.sandboxHostMemoryReserveBytes ?? 2 * 1024 ** 3)
|
|
816
|
+
+ activeResources.memoryBytes),
|
|
817
|
+
workspaceBytes: Math.max(0, preflight.availableWorkspaceBytes
|
|
818
|
+
- this.config.minimumWorkspaceBytes
|
|
819
|
+
+ activeResources.workspaceBytes),
|
|
820
|
+
pids: this.config.sandboxPidCapacity,
|
|
821
|
+
buildSlots: this.config.maxConcurrentSandboxBuilds
|
|
241
822
|
};
|
|
242
823
|
}
|
|
243
824
|
async invokeAck(methodName, payload) {
|
|
@@ -259,6 +840,8 @@ export class ForgeBridgeService {
|
|
|
259
840
|
}
|
|
260
841
|
async stopConnection() {
|
|
261
842
|
this.stopHeartbeat();
|
|
843
|
+
this.stopLeaseWatchdog();
|
|
844
|
+
this.stopSandboxMaintenance();
|
|
262
845
|
if (this.connection) {
|
|
263
846
|
await this.connection.stop().catch(() => undefined);
|
|
264
847
|
this.connection = null;
|
|
@@ -279,6 +862,12 @@ function isDispatchNoLongerPending(error) {
|
|
|
279
862
|
}
|
|
280
863
|
return error instanceof Error && error.message === "Dispatch is no longer pending.";
|
|
281
864
|
}
|
|
865
|
+
function createDefaultSessionService(sandboxStateRoot) {
|
|
866
|
+
const rawProcessRunner = new StreamingProviderProcessRunner();
|
|
867
|
+
const processSupervisors = new AttemptProcessSupervisorRegistry();
|
|
868
|
+
const coordinatedProcessRunner = new AttemptCoordinatedProviderProcessRunner(processSupervisors, rawProcessRunner);
|
|
869
|
+
return new BridgeSessionService(new GitWorktreeService(undefined, new RepositoryCoordinatorLock(sandboxStateRoot)), new RoutingProviderCliService(new ProcessBackedProviderCliService(coordinatedProcessRunner)), new HostRuntimeLeaseManager(sandboxStateRoot, undefined, undefined, processSupervisors, rawProcessRunner));
|
|
870
|
+
}
|
|
282
871
|
export class SignalRBridgeConnectionFactory {
|
|
283
872
|
create(config) {
|
|
284
873
|
return new SignalRBridgeConnection(config);
|
|
@@ -303,6 +892,9 @@ export class SignalRBridgeConnection {
|
|
|
303
892
|
})
|
|
304
893
|
.configureLogging(LogLevel.Warning)
|
|
305
894
|
.build();
|
|
895
|
+
this.connection.on("Superseded", () => {
|
|
896
|
+
void this.connection.stop();
|
|
897
|
+
});
|
|
306
898
|
}
|
|
307
899
|
start() {
|
|
308
900
|
return this.connection.start();
|
|
@@ -318,11 +910,36 @@ export class SignalRBridgeConnection {
|
|
|
318
910
|
void handler(raw).catch(() => undefined);
|
|
319
911
|
});
|
|
320
912
|
}
|
|
913
|
+
onOffer(handler) {
|
|
914
|
+
this.connection.on("Offer", (raw) => {
|
|
915
|
+
void handler(raw).catch(() => undefined);
|
|
916
|
+
});
|
|
917
|
+
}
|
|
918
|
+
onStart(handler) {
|
|
919
|
+
this.connection.on("Start", (raw) => {
|
|
920
|
+
void handler(raw).catch(() => undefined);
|
|
921
|
+
});
|
|
922
|
+
}
|
|
923
|
+
onLeaseUpdate(handler) {
|
|
924
|
+
this.connection.on("LeaseUpdate", (request) => {
|
|
925
|
+
void handler(request).catch(() => undefined);
|
|
926
|
+
});
|
|
927
|
+
}
|
|
321
928
|
onCancel(handler) {
|
|
322
929
|
this.connection.on("Cancel", (request) => {
|
|
323
930
|
void handler(request).catch(() => undefined);
|
|
324
931
|
});
|
|
325
932
|
}
|
|
933
|
+
onReap(handler) {
|
|
934
|
+
this.connection.on("Reap", (request) => {
|
|
935
|
+
void handler(request).catch(() => undefined);
|
|
936
|
+
});
|
|
937
|
+
}
|
|
938
|
+
onReadinessProbe(handler) {
|
|
939
|
+
this.connection.on("ReadinessProbe", (request) => {
|
|
940
|
+
void handler(request).catch(() => undefined);
|
|
941
|
+
});
|
|
942
|
+
}
|
|
326
943
|
invoke(methodName, payload) {
|
|
327
944
|
return this.connection.invoke(methodName, payload);
|
|
328
945
|
}
|
|
@@ -330,9 +947,37 @@ export class SignalRBridgeConnection {
|
|
|
330
947
|
export function defaultBridgeServiceConfig(overrides = {}) {
|
|
331
948
|
const defaults = defaultForgeBridgeConfig();
|
|
332
949
|
const cliProviders = splitCsv(process.env.FORGE_CLI_PROVIDERS ?? "codex");
|
|
950
|
+
const sandboxHostMemoryReserveDefault = process.platform === "linux"
|
|
951
|
+
? 2 * 1024 ** 3
|
|
952
|
+
: 1024 ** 3;
|
|
953
|
+
const sandboxPublicationMode = process.env.FORGE_SANDBOX_PUBLICATION_MODE === "bridge_push" ? "bridge_push" : "shared_filesystem";
|
|
333
954
|
const config = {
|
|
334
955
|
serverUrl: process.env.FORGE_URL ?? "http://localhost:5208",
|
|
335
956
|
runnerId: process.env.FORGE_RUNNER_ID ?? "",
|
|
957
|
+
runnerInstanceId: process.env.FORGE_RUNNER_INSTANCE_ID ?? randomUUID(),
|
|
958
|
+
trustDomainId: process.env.FORGE_TRUST_DOMAIN_ID ?? "default",
|
|
959
|
+
maxConcurrentSandboxes: positiveFiniteNumber(Number(process.env.FORGE_MAX_CONCURRENT_SANDBOXES ?? 3), 3),
|
|
960
|
+
maxConcurrentSandboxBuilds: positiveFiniteNumber(Number(process.env.FORGE_MAX_CONCURRENT_SANDBOX_BUILDS ?? 1), 1),
|
|
961
|
+
sandboxPidCapacity: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_PID_CAPACITY ?? 2048), 2048),
|
|
962
|
+
dockerEnabled: process.env.FORGE_DOCKER_SANDBOX !== "false",
|
|
963
|
+
allowOpenPublicSandbox: process.env.FORGE_ALLOW_OPEN_PUBLIC_SANDBOX === "true",
|
|
964
|
+
sandboxPublicationMode,
|
|
965
|
+
dockerExecutable: process.env.FORGE_DOCKER_EXECUTABLE ?? "docker",
|
|
966
|
+
dockerContext: process.env.FORGE_DOCKER_CONTEXT ?? "",
|
|
967
|
+
sandboxStateRoot: process.env.FORGE_SANDBOX_STATE_ROOT ?? path.join(os.homedir(), ".forge", "bridge", "sandboxes"),
|
|
968
|
+
requireRootlessDocker: process.env.FORGE_REQUIRE_ROOTLESS_DOCKER === "true" || process.platform === "linux",
|
|
969
|
+
diskHighWatermarkPercent: positiveFiniteNumber(Number(process.env.FORGE_DISK_HIGH_WATERMARK_PERCENT ?? 85), 85),
|
|
970
|
+
diskLowWatermarkPercent: positiveFiniteNumber(Number(process.env.FORGE_DISK_LOW_WATERMARK_PERCENT ?? 75), 75),
|
|
971
|
+
minimumWorkspaceBytes: positiveFiniteNumber(Number(process.env.FORGE_MINIMUM_WORKSPACE_BYTES ?? 10 * 1024 ** 3), 10 * 1024 ** 3),
|
|
972
|
+
sandboxHostMemoryReserveBytes: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_HOST_MEMORY_RESERVE_BYTES ?? sandboxHostMemoryReserveDefault), sandboxHostMemoryReserveDefault),
|
|
973
|
+
defaultSandboxWorkerMemoryLimitBytes: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_DEFAULT_WORKER_MEMORY_LIMIT_BYTES ?? 2 * 1024 ** 3), 2 * 1024 ** 3),
|
|
974
|
+
defaultSandboxServiceMemoryLimitBytes: nonNegativeFiniteNumber(Number(process.env.FORGE_SANDBOX_DEFAULT_SERVICE_MEMORY_LIMIT_BYTES ?? 0), 0),
|
|
975
|
+
sandboxRuntimeOverheadBytes: Math.max(128 * 1024 ** 2, nonNegativeFiniteNumber(Number(process.env.FORGE_SANDBOX_RUNTIME_OVERHEAD_BYTES ?? 128 * 1024 ** 2), 128 * 1024 ** 2)),
|
|
976
|
+
sandboxMemorySampleIntervalMs: Math.min(30_000, Math.max(250, positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_MEMORY_SAMPLE_INTERVAL_MS ?? 1000), 1000))),
|
|
977
|
+
imageRetentionCount: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_IMAGE_RETENTION_COUNT ?? 3), 3),
|
|
978
|
+
imageRetentionTtlMs: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_IMAGE_RETENTION_TTL_MS ?? 7 * 24 * 60 * 60 * 1000), 7 * 24 * 60 * 60 * 1000),
|
|
979
|
+
auditRetentionCount: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_AUDIT_RETENTION_COUNT ?? 200), 200),
|
|
980
|
+
auditRetentionTtlMs: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_AUDIT_RETENTION_TTL_MS ?? 30 * 24 * 60 * 60 * 1000), 30 * 24 * 60 * 60 * 1000),
|
|
336
981
|
deviceToken: process.env.FORGE_DEVICE_TOKEN ?? "",
|
|
337
982
|
deviceName: process.env.FORGE_DEVICE_NAME ?? "",
|
|
338
983
|
defaultRepoRoot: process.env.FORGE_REPO_ROOT ?? defaults.defaultRepoRoot,
|
|
@@ -348,8 +993,10 @@ export function defaultBridgeServiceConfig(overrides = {}) {
|
|
|
348
993
|
supportsManagedForgeArtifacts: true,
|
|
349
994
|
supportsRunnerAuthoredPhaseArtifacts: true,
|
|
350
995
|
supportsDraftRunOnce: true,
|
|
996
|
+
supportsDraftAttachments: true,
|
|
997
|
+
supportsDraftImageAttachments: supportsDraftImageAttachmentsFor(cliProviders),
|
|
351
998
|
version: process.env.FORGE_BRIDGE_VERSION ?? forgeBridgeVersion,
|
|
352
|
-
protocolVersion:
|
|
999
|
+
protocolVersion: 2,
|
|
353
1000
|
minServerVersion: null,
|
|
354
1001
|
heartbeatIntervalMs: positiveFiniteNumber(Number(process.env.FORGE_HEARTBEAT_INTERVAL_MS ?? 30_000), 30_000),
|
|
355
1002
|
reconnectBackoffMs: [1000, 2000, 5000, 10000, 30000],
|
|
@@ -357,7 +1004,9 @@ export function defaultBridgeServiceConfig(overrides = {}) {
|
|
|
357
1004
|
};
|
|
358
1005
|
return {
|
|
359
1006
|
...config,
|
|
1007
|
+
providerStatuses: overrides.providerStatuses ?? providerStatusesFor(config.cliProviders),
|
|
360
1008
|
sandboxProfiles: config.sandboxProfiles ?? defaultSandboxProfiles(config.cliProviders),
|
|
1009
|
+
supportsDraftImageAttachments: (overrides.supportsDraftImageAttachments ?? true) && supportsDraftImageAttachmentsFor(config.cliProviders),
|
|
361
1010
|
heartbeatIntervalMs: positiveFiniteNumber(config.heartbeatIntervalMs, 30_000)
|
|
362
1011
|
};
|
|
363
1012
|
}
|
|
@@ -373,9 +1022,36 @@ function normalizeServerUrl(serverUrl) {
|
|
|
373
1022
|
function positiveFiniteNumber(value, fallback) {
|
|
374
1023
|
return Number.isFinite(value) && value > 0 ? value : fallback;
|
|
375
1024
|
}
|
|
1025
|
+
function nonNegativeFiniteNumber(value, fallback) {
|
|
1026
|
+
return Number.isFinite(value) && value >= 0 ? value : fallback;
|
|
1027
|
+
}
|
|
376
1028
|
function providerStatusesFor(providers) {
|
|
377
1029
|
return Object.fromEntries(providers.map((provider) => [provider, { signedIn: true, version: "", models: configuredModelsFor(provider) }]));
|
|
378
1030
|
}
|
|
1031
|
+
function sandboxAuthModesFor(config) {
|
|
1032
|
+
return Object.fromEntries(config.cliProviders.map((provider) => {
|
|
1033
|
+
const modes = ["api_key"];
|
|
1034
|
+
if (sandboxBridgeCliAuthAvailable(config, provider)) {
|
|
1035
|
+
modes.push("bridge_cli");
|
|
1036
|
+
}
|
|
1037
|
+
return [provider, modes];
|
|
1038
|
+
}));
|
|
1039
|
+
}
|
|
1040
|
+
function sandboxBridgeCliAuthAvailable(config, provider) {
|
|
1041
|
+
if (provider.trim().toLowerCase() !== "codex") {
|
|
1042
|
+
return false;
|
|
1043
|
+
}
|
|
1044
|
+
const status = providerStatus(config, provider);
|
|
1045
|
+
if (status?.signedIn === false) {
|
|
1046
|
+
return false;
|
|
1047
|
+
}
|
|
1048
|
+
return codexSandboxProviderAuthAdapter.doctor().available;
|
|
1049
|
+
}
|
|
1050
|
+
function providerStatus(config, provider) {
|
|
1051
|
+
const normalized = provider.trim().toLowerCase();
|
|
1052
|
+
return Object.entries(config.providerStatuses)
|
|
1053
|
+
.find(([key]) => key.trim().toLowerCase() === normalized)?.[1];
|
|
1054
|
+
}
|
|
379
1055
|
function defaultSandboxProfiles(providers, platform = process.platform) {
|
|
380
1056
|
if (!providers.some((provider) => provider.toLowerCase() === "codex")) {
|
|
381
1057
|
return ["none"];
|
|
@@ -470,10 +1146,11 @@ function normalizeQaAppLaunchTarget(target) {
|
|
|
470
1146
|
command: cleanString(target.command),
|
|
471
1147
|
working_directory: cleanString(target.working_directory),
|
|
472
1148
|
ready_url: cleanString(target.ready_url),
|
|
1149
|
+
managed_runtime_port: cleanString(target.managed_runtime_port),
|
|
473
1150
|
routes,
|
|
474
1151
|
notes: cleanString(target.notes)
|
|
475
1152
|
};
|
|
476
|
-
return normalized.url || normalized.command || normalized.ready_url ||
|
|
1153
|
+
return normalized.url || normalized.command || normalized.ready_url || normalized.managed_runtime_port
|
|
477
1154
|
? normalized
|
|
478
1155
|
: null;
|
|
479
1156
|
}
|
|
@@ -539,9 +1216,272 @@ function abortPromise(signal) {
|
|
|
539
1216
|
}
|
|
540
1217
|
return new Promise((resolve) => signal.addEventListener("abort", () => resolve(), { once: true }));
|
|
541
1218
|
}
|
|
1219
|
+
export function canReapManagedSandboxes(preflight) {
|
|
1220
|
+
return canRunDockerMaintenance(preflight);
|
|
1221
|
+
}
|
|
1222
|
+
function sandboxNetworkPolicyFailure(dispatch, config) {
|
|
1223
|
+
const sandbox = dispatch.sandbox;
|
|
1224
|
+
if (!sandbox)
|
|
1225
|
+
return "Protocol-v2 sandbox context is missing.";
|
|
1226
|
+
if (sandbox.networkPolicy.mode === "open_public"
|
|
1227
|
+
&& (sandbox.executionPolicy === "sandbox_required" || !config.allowOpenPublicSandbox)) {
|
|
1228
|
+
return "open_public sandbox networking is not permitted by this runner policy.";
|
|
1229
|
+
}
|
|
1230
|
+
if (sandbox.networkPolicy.mode === "public_allowlist"
|
|
1231
|
+
&& sandbox.networkPolicy.allowedHosts.length > 0
|
|
1232
|
+
&& process.env.FORGE_SANDBOX_EGRESS_ENFORCED !== "true") {
|
|
1233
|
+
return "public_allowlist requires an operator-enforced Forge egress proxy or firewall.";
|
|
1234
|
+
}
|
|
1235
|
+
return null;
|
|
1236
|
+
}
|
|
1237
|
+
function infrastructureFailureEventType(summary) {
|
|
1238
|
+
if (summary.startsWith("worktree_prepare_failed:"))
|
|
1239
|
+
return "worktree_prepare_failed";
|
|
1240
|
+
if (summary.startsWith("provider_start_unavailable:"))
|
|
1241
|
+
return "provider_start_unavailable";
|
|
1242
|
+
if (summary.startsWith("host_port_collision:"))
|
|
1243
|
+
return "host_port_collision";
|
|
1244
|
+
if (summary.startsWith("host_runtime_process_exited:"))
|
|
1245
|
+
return "host_runtime_process_exited";
|
|
1246
|
+
if (summary.startsWith("host_runtime_listener_timeout:"))
|
|
1247
|
+
return "host_runtime_listener_timeout";
|
|
1248
|
+
if (summary.startsWith("host_runtime_http_readiness_timeout:"))
|
|
1249
|
+
return "host_runtime_http_readiness_timeout";
|
|
1250
|
+
if (summary.startsWith("attempt_process_supervisor_start_failed:"))
|
|
1251
|
+
return "attempt_process_supervisor_start_failed";
|
|
1252
|
+
if (summary.startsWith("attempt_process_supervisor_start_timeout:"))
|
|
1253
|
+
return "attempt_process_supervisor_start_timeout";
|
|
1254
|
+
return null;
|
|
1255
|
+
}
|
|
1256
|
+
function parseReadinessProbe(value) {
|
|
1257
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
1258
|
+
return null;
|
|
1259
|
+
const request = value;
|
|
1260
|
+
const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
|
|
1261
|
+
const bounded = (candidate, maximum) => typeof candidate === "string" && candidate.trim().length > 0 && candidate.length <= maximum;
|
|
1262
|
+
if (request.schemaVersion !== 1
|
|
1263
|
+
|| !uuid.test(request.requestId ?? "")
|
|
1264
|
+
|| !uuid.test(request.fencingToken ?? "")
|
|
1265
|
+
|| !uuid.test(request.userId ?? "")
|
|
1266
|
+
|| !uuid.test(request.projectId ?? "")
|
|
1267
|
+
|| !uuid.test(request.repoId ?? "")
|
|
1268
|
+
|| !uuid.test(request.runnerInstanceId ?? "")
|
|
1269
|
+
|| !Number.isInteger(request.attempt) || (request.attempt ?? 0) < 1
|
|
1270
|
+
|| !bounded(request.repoName, 160)
|
|
1271
|
+
|| typeof request.remoteUrl !== "string" || request.remoteUrl.length > 2048
|
|
1272
|
+
|| !bounded(request.defaultBranch, 512)
|
|
1273
|
+
|| typeof request.cloneOnDemand !== "boolean"
|
|
1274
|
+
|| !bounded(request.runnerRepoLocalPath, 4096)
|
|
1275
|
+
|| !bounded(request.revision, 512)
|
|
1276
|
+
|| !Number.isFinite(Date.parse(request.expiresAt ?? ""))
|
|
1277
|
+
|| Date.parse(request.expiresAt ?? "") <= Date.now()) {
|
|
1278
|
+
return null;
|
|
1279
|
+
}
|
|
1280
|
+
return request;
|
|
1281
|
+
}
|
|
1282
|
+
function sandboxProviderNetworkFailure(dispatch, config) {
|
|
1283
|
+
const sandbox = dispatch.sandbox;
|
|
1284
|
+
if (!sandbox || sandbox.networkPolicy.mode !== "offline")
|
|
1285
|
+
return null;
|
|
1286
|
+
const provider = dispatch.execution.provider.trim().toLowerCase();
|
|
1287
|
+
const isNetworkProvider = config.cliProviders.some(candidate => candidate.trim().toLowerCase() === provider);
|
|
1288
|
+
if (!isNetworkProvider)
|
|
1289
|
+
return null;
|
|
1290
|
+
return `Provider ${dispatch.execution.provider} requires outbound network access, but this Docker attempt uses the offline network policy.`;
|
|
1291
|
+
}
|
|
1292
|
+
function sandboxAuthFailure(dispatch, config) {
|
|
1293
|
+
const unsupported = sandboxProviderAuthUnavailableMessage(dispatch.execution.provider, dispatch.execution.authKind);
|
|
1294
|
+
if (unsupported) {
|
|
1295
|
+
return unsupported;
|
|
1296
|
+
}
|
|
1297
|
+
if (dispatch.execution.authKind.trim().toLowerCase() !== "bridge_cli") {
|
|
1298
|
+
return null;
|
|
1299
|
+
}
|
|
1300
|
+
const status = providerStatus(config, dispatch.execution.provider);
|
|
1301
|
+
if (status?.signedIn === false) {
|
|
1302
|
+
return `${dispatch.execution.provider} Local CLI is not signed in on this runner.`;
|
|
1303
|
+
}
|
|
1304
|
+
const adapter = sandboxProviderAuthAdapterFor(dispatch.execution.provider, dispatch.execution.authKind);
|
|
1305
|
+
if (!adapter) {
|
|
1306
|
+
return `${dispatch.execution.provider} Local CLI auth is not portable into Docker yet.`;
|
|
1307
|
+
}
|
|
1308
|
+
const doctor = adapter.doctor();
|
|
1309
|
+
return doctor.available ? null : doctor.message;
|
|
1310
|
+
}
|
|
1311
|
+
export function canReserveSandboxResources(used, requested, capacity) {
|
|
1312
|
+
return used.sandboxSlots + requested.sandboxSlots <= capacity.sandboxSlots
|
|
1313
|
+
&& used.cpuMillis + requested.cpuMillis <= capacity.cpuMillis
|
|
1314
|
+
&& used.memoryBytes + requested.memoryBytes <= capacity.memoryBytes
|
|
1315
|
+
&& used.workspaceBytes + requested.workspaceBytes <= capacity.workspaceBytes
|
|
1316
|
+
&& used.pids + requested.pids <= capacity.pids
|
|
1317
|
+
&& used.buildSlots + requested.buildSlots <= capacity.buildSlots;
|
|
1318
|
+
}
|
|
1319
|
+
export function sandboxCapacityFailureMessage(used, requested, capacity, preflight, config) {
|
|
1320
|
+
const availableSlots = Math.max(0, capacity.sandboxSlots - used.sandboxSlots);
|
|
1321
|
+
if (requested.sandboxSlots > availableSlots) {
|
|
1322
|
+
return `Sandbox slot capacity is insufficient: requested ${requested.sandboxSlots}, available ${availableSlots}, ${used.sandboxSlots} already reserved.`;
|
|
1323
|
+
}
|
|
1324
|
+
const availableMemoryBytes = Math.max(0, capacity.memoryBytes - used.memoryBytes);
|
|
1325
|
+
if (requested.memoryBytes > availableMemoryBytes) {
|
|
1326
|
+
return [
|
|
1327
|
+
`Sandbox memory capacity is insufficient: requested ${formatBytes(requested.memoryBytes)} total`,
|
|
1328
|
+
`(worker ${formatBytes(requested.workerMemoryLimitBytes)}, services ${formatBytes(requested.serviceMemoryLimitBytes)}, runtime overhead ${formatBytes(requested.runtimeOverheadBytes)}),`,
|
|
1329
|
+
`but ${formatBytes(availableMemoryBytes)} is available for admission.`,
|
|
1330
|
+
`Docker reports ${formatBytes(preflight?.availableMemoryBytes ?? 0)} available of ${formatBytes(preflight?.totalMemoryBytes ?? 0)};`,
|
|
1331
|
+
`Forge reserves ${formatBytes(preflight?.hostReserveBytes ?? config.sandboxHostMemoryReserveBytes ?? 0)} for the host and tracks ${formatBytes(used.memoryBytes)} in active or reserved attempts.`,
|
|
1332
|
+
].join(" ");
|
|
1333
|
+
}
|
|
1334
|
+
const availableCpuMillis = Math.max(0, capacity.cpuMillis - used.cpuMillis);
|
|
1335
|
+
if (requested.cpuMillis > availableCpuMillis) {
|
|
1336
|
+
return `Sandbox CPU capacity is insufficient: requested ${requested.cpuMillis} millis, available ${availableCpuMillis} millis, ${used.cpuMillis} millis already reserved.`;
|
|
1337
|
+
}
|
|
1338
|
+
const availableWorkspaceBytes = Math.max(0, capacity.workspaceBytes - used.workspaceBytes);
|
|
1339
|
+
if (requested.workspaceBytes > availableWorkspaceBytes) {
|
|
1340
|
+
return `Sandbox workspace capacity is insufficient: requested ${formatBytes(requested.workspaceBytes)}, available ${formatBytes(availableWorkspaceBytes)}, ${formatBytes(used.workspaceBytes)} already reserved.`;
|
|
1341
|
+
}
|
|
1342
|
+
const availablePids = Math.max(0, capacity.pids - used.pids);
|
|
1343
|
+
if (requested.pids > availablePids) {
|
|
1344
|
+
return `Sandbox PID capacity is insufficient: requested ${requested.pids}, available ${availablePids}, ${used.pids} already reserved.`;
|
|
1345
|
+
}
|
|
1346
|
+
const availableBuildSlots = Math.max(0, capacity.buildSlots - used.buildSlots);
|
|
1347
|
+
return `Sandbox build capacity is insufficient: requested ${requested.buildSlots}, available ${availableBuildSlots}, ${used.buildSlots} already reserved.`;
|
|
1348
|
+
}
|
|
1349
|
+
function formatBytes(bytes) {
|
|
1350
|
+
if (!Number.isFinite(bytes) || bytes <= 0)
|
|
1351
|
+
return "0 B";
|
|
1352
|
+
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
1353
|
+
const power = Math.min(units.length - 1, Math.floor(Math.log(bytes) / Math.log(1024)));
|
|
1354
|
+
const value = bytes / 1024 ** power;
|
|
1355
|
+
return `${value >= 10 || power === 0 ? value.toFixed(0) : value.toFixed(2)} ${units[power]}`;
|
|
1356
|
+
}
|
|
1357
|
+
function sumSandboxResources(resources) {
|
|
1358
|
+
return resources.reduce((total, resource) => ({
|
|
1359
|
+
sandboxSlots: total.sandboxSlots + resource.sandboxSlots,
|
|
1360
|
+
cpuMillis: total.cpuMillis + resource.cpuMillis,
|
|
1361
|
+
workerMemoryLimitBytes: total.workerMemoryLimitBytes + resource.workerMemoryLimitBytes,
|
|
1362
|
+
serviceMemoryLimitBytes: total.serviceMemoryLimitBytes + resource.serviceMemoryLimitBytes,
|
|
1363
|
+
runtimeOverheadBytes: total.runtimeOverheadBytes + resource.runtimeOverheadBytes,
|
|
1364
|
+
memoryBytes: total.memoryBytes + resource.memoryBytes,
|
|
1365
|
+
workspaceBytes: total.workspaceBytes + resource.workspaceBytes,
|
|
1366
|
+
pids: total.pids + resource.pids,
|
|
1367
|
+
buildSlots: total.buildSlots + resource.buildSlots
|
|
1368
|
+
}), zeroSandboxResources());
|
|
1369
|
+
}
|
|
1370
|
+
function zeroSandboxResources() {
|
|
1371
|
+
return {
|
|
1372
|
+
sandboxSlots: 0,
|
|
1373
|
+
cpuMillis: 0,
|
|
1374
|
+
workerMemoryLimitBytes: 0,
|
|
1375
|
+
serviceMemoryLimitBytes: 0,
|
|
1376
|
+
runtimeOverheadBytes: 0,
|
|
1377
|
+
memoryBytes: 0,
|
|
1378
|
+
workspaceBytes: 0,
|
|
1379
|
+
pids: 0,
|
|
1380
|
+
buildSlots: 0,
|
|
1381
|
+
};
|
|
1382
|
+
}
|
|
1383
|
+
function defaultSandboxResources() {
|
|
1384
|
+
return {
|
|
1385
|
+
sandboxSlots: 1,
|
|
1386
|
+
cpuMillis: 2000,
|
|
1387
|
+
workerMemoryLimitBytes: 2 * 1024 ** 3,
|
|
1388
|
+
serviceMemoryLimitBytes: 0,
|
|
1389
|
+
runtimeOverheadBytes: 128 * 1024 ** 2,
|
|
1390
|
+
memoryBytes: 2 * 1024 ** 3 + 128 * 1024 ** 2,
|
|
1391
|
+
workspaceBytes: 10 * 1024 ** 3,
|
|
1392
|
+
pids: 512,
|
|
1393
|
+
buildSlots: 0
|
|
1394
|
+
};
|
|
1395
|
+
}
|
|
1396
|
+
export function resolveDynamicSandboxResources(requested, serviceCount, networkMode, baseRuntimeOverheadBytes = 128 * 1024 ** 2) {
|
|
1397
|
+
const serviceMemoryLimitBytes = Math.max(0, serviceCount) * 1024 ** 3;
|
|
1398
|
+
const runtimeOverheadBytes = Math.max(128 * 1024 ** 2, baseRuntimeOverheadBytes)
|
|
1399
|
+
+ (networkMode === "public_allowlist" ? 256 * 1024 ** 2 : 0);
|
|
1400
|
+
return {
|
|
1401
|
+
...requested,
|
|
1402
|
+
serviceMemoryLimitBytes,
|
|
1403
|
+
runtimeOverheadBytes,
|
|
1404
|
+
memoryBytes: requested.workerMemoryLimitBytes + serviceMemoryLimitBytes + runtimeOverheadBytes,
|
|
1405
|
+
};
|
|
1406
|
+
}
|
|
1407
|
+
export async function resolveSandboxResourcesFromRepo(dispatch, baseRuntimeOverheadBytes = 128 * 1024 ** 2) {
|
|
1408
|
+
const repoPath = dispatch.repo.serverLocalPath;
|
|
1409
|
+
const sandbox = dispatch.sandbox;
|
|
1410
|
+
if (!repoPath || !sandbox) {
|
|
1411
|
+
throw new Error("Sandbox admission needs a mapped repository.");
|
|
1412
|
+
}
|
|
1413
|
+
const selection = selectSandboxConfigRefs(sandbox.executionPolicy, sandbox.sandboxConfigCommitSha, dispatch.repo.defaultBranch);
|
|
1414
|
+
const configCommitSha = await resolveFirstGitCommit(repoPath, selection.refs);
|
|
1415
|
+
if (!configCommitSha) {
|
|
1416
|
+
throw new Error("Sandbox admission could not resolve the immutable sandbox config commit.");
|
|
1417
|
+
}
|
|
1418
|
+
const contractResult = await runSandboxProcess({
|
|
1419
|
+
command: "git",
|
|
1420
|
+
args: ["-C", repoPath, "show", `${configCommitSha}:.forge/forge.sandbox.yml`],
|
|
1421
|
+
timeoutMs: 30_000,
|
|
1422
|
+
maxOutputBytes: 1024 * 1024,
|
|
1423
|
+
});
|
|
1424
|
+
if (contractResult.exitCode !== 0) {
|
|
1425
|
+
throw new Error("The approved immutable sandbox contract is unavailable in the mapped repository.");
|
|
1426
|
+
}
|
|
1427
|
+
const contract = validateSandboxContract(contractResult.stdout);
|
|
1428
|
+
return {
|
|
1429
|
+
resources: resolveDynamicSandboxResources(sandbox.requestedResources, contract.contract.services.length, sandbox.networkPolicy.mode, baseRuntimeOverheadBytes),
|
|
1430
|
+
sandboxConfigCommitSha: configCommitSha,
|
|
1431
|
+
};
|
|
1432
|
+
}
|
|
1433
|
+
async function resolveFirstGitCommit(repoPath, refs) {
|
|
1434
|
+
for (const ref of refs) {
|
|
1435
|
+
const result = await runSandboxProcess({
|
|
1436
|
+
command: "git",
|
|
1437
|
+
args: ["-C", repoPath, "rev-parse", "--verify", `${ref}^{commit}`],
|
|
1438
|
+
timeoutMs: 30_000,
|
|
1439
|
+
maxOutputBytes: 1024 * 1024,
|
|
1440
|
+
});
|
|
1441
|
+
const commitSha = result.stdout.trim().toLowerCase();
|
|
1442
|
+
if (result.exitCode === 0 && /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/u.test(commitSha)) {
|
|
1443
|
+
return commitSha;
|
|
1444
|
+
}
|
|
1445
|
+
}
|
|
1446
|
+
return null;
|
|
1447
|
+
}
|
|
1448
|
+
function parseOfferEnvelope(value) {
|
|
1449
|
+
return parseDispatchEnvelope(value);
|
|
1450
|
+
}
|
|
1451
|
+
function parseStartEnvelope(value) {
|
|
1452
|
+
return parseDispatchEnvelope(value);
|
|
1453
|
+
}
|
|
1454
|
+
function parseDispatchEnvelope(value) {
|
|
1455
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
1456
|
+
return null;
|
|
1457
|
+
const envelope = value;
|
|
1458
|
+
if (envelope.schemaVersion !== 1
|
|
1459
|
+
|| typeof envelope.dispatchAttemptId !== "string"
|
|
1460
|
+
|| typeof envelope.leaseId !== "string"
|
|
1461
|
+
|| typeof envelope.fencingToken !== "string"
|
|
1462
|
+
|| typeof envelope.runnerInstanceId !== "string"
|
|
1463
|
+
|| typeof envelope.expiresAt !== "string"
|
|
1464
|
+
|| !envelope.dispatch)
|
|
1465
|
+
return null;
|
|
1466
|
+
return {
|
|
1467
|
+
dispatchAttemptId: envelope.dispatchAttemptId,
|
|
1468
|
+
leaseId: envelope.leaseId,
|
|
1469
|
+
fencingToken: envelope.fencingToken,
|
|
1470
|
+
runnerInstanceId: envelope.runnerInstanceId,
|
|
1471
|
+
expiresAt: envelope.expiresAt,
|
|
1472
|
+
dispatch: envelope.dispatch
|
|
1473
|
+
};
|
|
1474
|
+
}
|
|
1475
|
+
const maxAttachmentExactSecretLength = 8 * 1024;
|
|
542
1476
|
function exactSecrets(dispatch, config) {
|
|
543
1477
|
return [
|
|
544
1478
|
dispatch.apiKeySecretGrant?.secret,
|
|
545
|
-
config.deviceToken
|
|
1479
|
+
config.deviceToken,
|
|
1480
|
+
...boundedAttachmentExactSecrets(dispatch)
|
|
546
1481
|
].filter((secret) => typeof secret === "string" && secret.length > 0);
|
|
547
1482
|
}
|
|
1483
|
+
function boundedAttachmentExactSecrets(dispatch) {
|
|
1484
|
+
return (dispatch.draft?.attachments ?? [])
|
|
1485
|
+
.flatMap((attachment) => [attachment.textContent, attachment.contentBase64])
|
|
1486
|
+
.filter((secret) => typeof secret === "string" && secret.length > 0 && secret.length <= maxAttachmentExactSecretLength);
|
|
1487
|
+
}
|