@loucompanion/forge-bridge 0.1.3-dev.0b2f81d660c0 → 0.1.3-dev.0e5564284725
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +78 -0
- package/dist/bridge/bridge.base.d.ts +6 -1
- package/dist/bridge/bridge.service.d.ts +60 -3
- package/dist/bridge/bridge.service.js +946 -20
- package/dist/bridge/bridge.types.d.ts +168 -1
- package/dist/cleanup/cleanup.service.d.ts +13 -2
- package/dist/cleanup/cleanup.service.js +435 -3
- package/dist/cleanup/cleanup.types.d.ts +10 -0
- package/dist/cleanup/index.d.ts +1 -1
- package/dist/cleanup/index.js +1 -1
- package/dist/cleanup/internals.d.ts +2 -2
- package/dist/cleanup/internals.js +1 -1
- package/dist/cli/cli.help.js +47 -1
- package/dist/cli/cli.service.d.ts +42 -2
- package/dist/cli/cli.service.js +348 -22
- package/dist/cli/local-config.d.ts +4 -0
- package/dist/cli/local-config.js +41 -1
- package/dist/runtime/attempt-node-runtime-launcher.d.ts +17 -0
- package/dist/runtime/attempt-node-runtime-launcher.js +101 -0
- package/dist/runtime/attempt-process-supervisor-registry.d.ts +20 -0
- package/dist/runtime/attempt-process-supervisor-registry.js +64 -0
- package/dist/runtime/attempt-process-supervisor.d.ts +86 -0
- package/dist/runtime/attempt-process-supervisor.js +343 -0
- package/dist/runtime/generated-output-reset.d.ts +10 -0
- package/dist/runtime/generated-output-reset.js +103 -0
- package/dist/runtime/host-listener-inspector.d.ts +44 -0
- package/dist/runtime/host-listener-inspector.js +324 -0
- package/dist/runtime/host-port-broker.d.ts +76 -0
- package/dist/runtime/host-port-broker.js +333 -0
- package/dist/runtime/host-runtime-contract.d.ts +47 -0
- package/dist/runtime/host-runtime-contract.js +700 -0
- package/dist/runtime/host-runtime-lease.d.ts +133 -0
- package/dist/runtime/host-runtime-lease.js +1460 -0
- package/dist/runtime/host-runtime-promotion.d.ts +23 -0
- package/dist/runtime/host-runtime-promotion.js +159 -0
- package/dist/runtime/node-host-runtime-adapter.d.ts +51 -0
- package/dist/runtime/node-host-runtime-adapter.js +268 -0
- package/dist/runtime/playwright-host-runtime-adapter.d.ts +20 -0
- package/dist/runtime/playwright-host-runtime-adapter.js +220 -0
- package/dist/runtime/runtime-isolation-lease.d.ts +67 -0
- package/dist/runtime/runtime-isolation-lease.js +158 -0
- package/dist/sandbox/command-runner.d.ts +4 -0
- package/dist/sandbox/command-runner.js +47 -0
- package/dist/sandbox/contract.d.ts +5 -0
- package/dist/sandbox/contract.js +276 -0
- package/dist/sandbox/event-budget.d.ts +11 -0
- package/dist/sandbox/event-budget.js +27 -0
- package/dist/sandbox/image-policy.d.ts +35 -0
- package/dist/sandbox/image-policy.js +247 -0
- package/dist/sandbox/index.d.ts +5 -0
- package/dist/sandbox/index.js +5 -0
- package/dist/sandbox/lease-watchdog.d.ts +18 -0
- package/dist/sandbox/lease-watchdog.js +67 -0
- package/dist/sandbox/preflight.d.ts +54 -0
- package/dist/sandbox/preflight.js +335 -0
- package/dist/sandbox/process.d.ts +20 -0
- package/dist/sandbox/process.js +209 -0
- package/dist/sandbox/provider-auth/codex.d.ts +26 -0
- package/dist/sandbox/provider-auth/codex.js +212 -0
- package/dist/sandbox/repository-lock.d.ts +5 -0
- package/dist/sandbox/repository-lock.js +119 -0
- package/dist/sandbox/runtime.d.ts +297 -0
- package/dist/sandbox/runtime.js +2338 -0
- package/dist/sandbox/sandbox.types.d.ts +86 -0
- package/dist/sandbox/sandbox.types.js +1 -0
- package/dist/sandbox/smoke.d.ts +76 -0
- package/dist/sandbox/smoke.js +380 -0
- package/dist/sandbox/worker-supervisor.d.ts +13 -0
- package/dist/sandbox/worker-supervisor.js +36 -0
- package/dist/sandbox/worker.d.ts +1 -0
- package/dist/sandbox/worker.js +225 -0
- package/dist/sandbox/workspace.service.d.ts +13 -0
- package/dist/sandbox/workspace.service.js +72 -0
- package/dist/session/managed-attempt-tools.d.ts +12 -0
- package/dist/session/managed-attempt-tools.js +490 -0
- package/dist/session/phase-completion.artifacts.js +9 -2
- package/dist/session/phase-completion.d.ts +4 -2
- package/dist/session/phase-completion.helpers.js +27 -3
- package/dist/session/phase-completion.js +12 -3
- package/dist/session/phase-completion.prompts.d.ts +3 -3
- package/dist/session/phase-completion.prompts.js +20 -6
- package/dist/session/phase-completion.runtime.d.ts +12 -1
- package/dist/session/phase-completion.runtime.js +129 -7
- package/dist/session/phase-completion.schemas.js +108 -3
- package/dist/session/phase-completion.validation.d.ts +4 -1
- package/dist/session/phase-completion.validation.js +106 -2
- package/dist/session/provider-cli/ci-repair-readiness.d.ts +8 -0
- package/dist/session/provider-cli/ci-repair-readiness.js +90 -0
- package/dist/session/provider-cli/provider-cli.artifacts.d.ts +1 -1
- package/dist/session/provider-cli/provider-cli.artifacts.js +28 -4
- package/dist/session/provider-cli/provider-cli.base.d.ts +1 -0
- package/dist/session/provider-cli/provider-cli.deterministic.d.ts +2 -0
- package/dist/session/provider-cli/provider-cli.deterministic.js +66 -9
- package/dist/session/provider-cli/provider-cli.process.d.ts +16 -0
- package/dist/session/provider-cli/provider-cli.process.js +212 -25
- package/dist/session/provider-cli/provider-cli.prompts.d.ts +6 -1
- package/dist/session/provider-cli/provider-cli.prompts.js +114 -16
- package/dist/session/provider-cli/provider-cli.qa-sidecars.js +1 -0
- package/dist/session/provider-cli/provider-cli.runtime.d.ts +1 -1
- package/dist/session/provider-cli/provider-cli.runtime.js +3 -2
- package/dist/session/provider-cli/provider-cli.service.d.ts +18 -2
- package/dist/session/provider-cli/provider-cli.service.js +422 -78
- package/dist/session/provider-cli/provider-cli.types.d.ts +5 -0
- package/dist/session/provider-cli/provider-output-state.d.ts +2 -0
- package/dist/session/provider-cli/provider-output-state.js +30 -2
- package/dist/session/session-redaction.d.ts +3 -0
- package/dist/session/session-redaction.js +23 -0
- package/dist/session/session.base.d.ts +6 -1
- package/dist/session/session.service.d.ts +8 -2
- package/dist/session/session.service.js +444 -42
- package/dist/session/session.types.d.ts +96 -1
- package/dist/session/session.types.js +189 -2
- package/dist/shared/path/path.js +3 -1
- package/dist/shared/process/host-process-recovery.d.ts +12 -0
- package/dist/shared/process/host-process-recovery.js +78 -0
- package/dist/shared/process/host-process-supervisor.d.ts +57 -0
- package/dist/shared/process/host-process-supervisor.js +282 -0
- package/dist/shared/process/process.d.ts +1 -0
- package/dist/shared/process/process.js +1 -0
- package/dist/shared/redaction/redaction.js +6 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/worktree/artifact-injector.d.ts +3 -2
- package/dist/worktree/artifact-injector.js +43 -4
- package/dist/worktree/dependency-warmer.d.ts +10 -1
- package/dist/worktree/dependency-warmer.js +94 -5
- package/dist/worktree/index.d.ts +1 -0
- package/dist/worktree/index.js +1 -0
- package/dist/worktree/prior-phase-artifact-hydrator.d.ts +2 -0
- package/dist/worktree/prior-phase-artifact-hydrator.js +136 -0
- package/dist/worktree/worktree-paths.d.ts +4 -0
- package/dist/worktree/worktree-paths.js +30 -0
- package/dist/worktree/worktree.base.d.ts +3 -0
- package/dist/worktree/worktree.service.d.ts +7 -2
- package/dist/worktree/worktree.service.js +72 -44
- package/dist/worktree/worktree.types.d.ts +3 -1
- package/package.json +6 -3
- package/runtime/windows-job-supervisor.ps1 +356 -0
- package/runtime/windows-listener-owner.ps1 +33 -0
|
@@ -1,48 +1,114 @@
|
|
|
1
1
|
import { HubConnectionBuilder, HttpTransportType, LogLevel } from "@microsoft/signalr";
|
|
2
2
|
import fs from "node:fs";
|
|
3
3
|
import path from "node:path";
|
|
4
|
+
import { randomUUID } from "node:crypto";
|
|
5
|
+
import os from "node:os";
|
|
4
6
|
import { defaultForgeBridgeConfig } from "../config.js";
|
|
5
7
|
import { supportsDraftImageAttachmentsFor } from "../shared/provider-capabilities.js";
|
|
6
|
-
import {
|
|
8
|
+
import { redactText } from "../shared/redaction/redaction.js";
|
|
7
9
|
import { BridgeSessionService } from "../session/session.service.js";
|
|
10
|
+
import { ProcessBackedProviderCliService, RoutingProviderCliService, StreamingProviderProcessRunner, } from "../session/provider-cli/provider-cli.service.js";
|
|
11
|
+
import { redactRunnerDispatchEvent, redactRunOnceResultPacket } from "../session/session-redaction.js";
|
|
12
|
+
import { HostRuntimeLeaseManager } from "../runtime/host-runtime-lease.js";
|
|
13
|
+
import { GitWorktreeService } from "../worktree/worktree.service.js";
|
|
14
|
+
import { RepositoryCoordinatorLock } from "../sandbox/repository-lock.js";
|
|
15
|
+
import { AttemptCoordinatedProviderProcessRunner, AttemptProcessSupervisorRegistry, } from "../runtime/attempt-process-supervisor-registry.js";
|
|
8
16
|
import { failureResult, parseRunOnceDispatchPacket } from "../session/session.types.js";
|
|
9
17
|
import { forgeBridgeVersion } from "../version.js";
|
|
18
|
+
import { canRunDockerMaintenance, DockerPreflight } from "../sandbox/preflight.js";
|
|
19
|
+
import { validateSandboxContract } from "../sandbox/contract.js";
|
|
20
|
+
import { runSandboxProcess } from "../sandbox/process.js";
|
|
21
|
+
import { DockerSandboxRuntime, selectSandboxConfigRefs } from "../sandbox/runtime.js";
|
|
22
|
+
import { codexSandboxProviderAuthAdapter, sandboxProviderAuthAdapterFor, sandboxProviderAuthUnavailableMessage } from "../sandbox/provider-auth/codex.js";
|
|
10
23
|
export class ForgeBridgeService {
|
|
11
24
|
config;
|
|
12
|
-
sessions;
|
|
13
25
|
connections;
|
|
14
26
|
connection = null;
|
|
15
27
|
heartbeatTimer = null;
|
|
28
|
+
heartbeatInFlight = false;
|
|
29
|
+
consecutiveHeartbeatFailures = 0;
|
|
30
|
+
reconnectingFromHeartbeatFailure = false;
|
|
31
|
+
leaseWatchdogTimer = null;
|
|
32
|
+
sandboxMaintenanceTimer = null;
|
|
33
|
+
sandboxMaintenanceRunning = false;
|
|
16
34
|
closed = null;
|
|
17
35
|
stopped = false;
|
|
18
36
|
activeSessions = new Map();
|
|
19
|
-
|
|
37
|
+
reservations = new Map();
|
|
38
|
+
activeReadinessProbes = new Map();
|
|
39
|
+
preflight;
|
|
40
|
+
sandboxRuntime;
|
|
41
|
+
sandboxResourceResolver;
|
|
42
|
+
sessions;
|
|
43
|
+
constructor(config, sessions = undefined, connections = new SignalRBridgeConnectionFactory(), preflight, sandboxRuntime, sandboxResourceResolver) {
|
|
20
44
|
this.config = config;
|
|
21
|
-
this.sessions = sessions;
|
|
22
45
|
this.connections = connections;
|
|
46
|
+
this.sessions = sessions ?? createDefaultSessionService(config.sandboxStateRoot);
|
|
47
|
+
this.preflight = preflight ?? new DockerPreflight({
|
|
48
|
+
dockerExecutable: config.dockerExecutable,
|
|
49
|
+
dockerContext: config.dockerContext,
|
|
50
|
+
stateRoot: config.sandboxStateRoot,
|
|
51
|
+
runnerInstanceId: config.runnerInstanceId,
|
|
52
|
+
maxConcurrentSandboxes: config.maxConcurrentSandboxes,
|
|
53
|
+
diskHighWatermarkPercent: config.diskHighWatermarkPercent,
|
|
54
|
+
diskLowWatermarkPercent: config.diskLowWatermarkPercent,
|
|
55
|
+
minimumWorkspaceBytes: config.minimumWorkspaceBytes,
|
|
56
|
+
hostMemoryReserveBytes: config.sandboxHostMemoryReserveBytes,
|
|
57
|
+
defaultWorkerMemoryLimitBytes: config.defaultSandboxWorkerMemoryLimitBytes,
|
|
58
|
+
defaultServiceMemoryLimitBytes: config.defaultSandboxServiceMemoryLimitBytes,
|
|
59
|
+
runtimeOverheadBytes: config.sandboxRuntimeOverheadBytes,
|
|
60
|
+
requireRootless: config.requireRootlessDocker
|
|
61
|
+
});
|
|
62
|
+
this.sandboxRuntime = sandboxRuntime ?? new DockerSandboxRuntime(config, (request, signal) => this.redeemPublishGrant(request, signal));
|
|
63
|
+
this.sandboxResourceResolver = sandboxResourceResolver
|
|
64
|
+
?? (dispatch => resolveSandboxResourcesFromRepo(dispatch, this.config.sandboxRuntimeOverheadBytes));
|
|
23
65
|
}
|
|
24
66
|
async connectOnce() {
|
|
25
67
|
await this.stopConnection();
|
|
26
68
|
this.stopped = false;
|
|
69
|
+
const hostReap = await this.sessions.reapHostRuntime?.({
|
|
70
|
+
runnerId: this.config.runnerId,
|
|
71
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
72
|
+
});
|
|
73
|
+
if (hostReap && (hostReap.reaped > 0 || hostReap.degraded > 0)) {
|
|
74
|
+
logBridgeStatus(`Managed-host recovery reaped ${hostReap.reaped} stale attempt(s), protected ${hostReap.reusedPidsProtected} reused PID(s), and left ${hostReap.degraded} degraded attempt(s) for reconciliation.`);
|
|
75
|
+
}
|
|
76
|
+
if (this.config.dockerEnabled) {
|
|
77
|
+
const preflight = await this.preflight.refresh();
|
|
78
|
+
if (canReapManagedSandboxes(preflight)) {
|
|
79
|
+
await this.sandboxRuntime.reap({ diskLow: preflight.diskLow });
|
|
80
|
+
await this.preflight.refresh();
|
|
81
|
+
}
|
|
82
|
+
}
|
|
27
83
|
const connection = this.connections.create(this.config);
|
|
28
84
|
this.connection = connection;
|
|
29
85
|
this.closed = new Promise((resolve) => {
|
|
30
86
|
connection.onClose(() => {
|
|
31
87
|
this.stopHeartbeat();
|
|
32
88
|
this.abortActiveSessions("bridge connection closed");
|
|
89
|
+
this.abortActiveReadinessProbes("bridge connection closed");
|
|
33
90
|
resolve();
|
|
34
91
|
});
|
|
35
92
|
});
|
|
36
93
|
connection.onDispatch((dispatch) => this.handleDispatch(dispatch));
|
|
94
|
+
connection.onOffer?.((dispatch) => this.handleOffer(dispatch));
|
|
95
|
+
connection.onStart?.((dispatch) => this.handleStart(dispatch));
|
|
96
|
+
connection.onLeaseUpdate?.((request) => this.handleLeaseUpdate(request));
|
|
37
97
|
connection.onCancel((request) => this.handleCancel(request));
|
|
98
|
+
connection.onReap?.((request) => this.handleReap(request));
|
|
99
|
+
connection.onReadinessProbe?.((request) => this.handleReadinessProbe(request));
|
|
38
100
|
await connection.start();
|
|
39
101
|
await this.invokeAck("Hello", {
|
|
40
102
|
version: this.config.version,
|
|
41
103
|
protocolVersion: this.config.protocolVersion,
|
|
42
104
|
minServerVersion: this.config.minServerVersion ?? null,
|
|
105
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
43
106
|
capabilities: this.capabilities()
|
|
44
107
|
});
|
|
108
|
+
await this.flushCleanupReports();
|
|
45
109
|
this.startHeartbeat();
|
|
110
|
+
this.startLeaseWatchdog();
|
|
111
|
+
this.startSandboxMaintenance();
|
|
46
112
|
}
|
|
47
113
|
async run(signal) {
|
|
48
114
|
let attempt = 0;
|
|
@@ -73,6 +139,7 @@ export class ForgeBridgeService {
|
|
|
73
139
|
finally {
|
|
74
140
|
if (signal?.aborted) {
|
|
75
141
|
this.abortActiveSessions(String(signal.reason ?? "bridge stopping"));
|
|
142
|
+
this.abortActiveReadinessProbes(String(signal.reason ?? "bridge stopping"));
|
|
76
143
|
}
|
|
77
144
|
await this.stopConnection();
|
|
78
145
|
}
|
|
@@ -80,7 +147,9 @@ export class ForgeBridgeService {
|
|
|
80
147
|
async stop() {
|
|
81
148
|
this.stopped = true;
|
|
82
149
|
this.abortActiveSessions("bridge stopping");
|
|
150
|
+
this.abortActiveReadinessProbes("bridge stopping");
|
|
83
151
|
await this.stopConnection();
|
|
152
|
+
await this.preflight.release();
|
|
84
153
|
}
|
|
85
154
|
async handleDispatch(rawDispatch) {
|
|
86
155
|
const dispatch = this.parseDispatchOrReport(rawDispatch);
|
|
@@ -90,21 +159,53 @@ export class ForgeBridgeService {
|
|
|
90
159
|
const leaseId = dispatch.lease.id;
|
|
91
160
|
const fencingToken = dispatch.lease.fencingToken;
|
|
92
161
|
const redactionSecrets = exactSecrets(dispatch, this.config);
|
|
93
|
-
|
|
94
|
-
|
|
162
|
+
const existing = this.activeSessions.get(leaseId);
|
|
163
|
+
if (existing) {
|
|
164
|
+
if (existing.fencingToken === fencingToken) {
|
|
165
|
+
return;
|
|
166
|
+
}
|
|
167
|
+
await this.sendResult(dispatch, failureResult("Dispatch fence conflicts with an active bridge session.")).catch(() => undefined);
|
|
95
168
|
return;
|
|
96
169
|
}
|
|
97
170
|
const controller = new AbortController();
|
|
98
|
-
this.activeSessions.set(leaseId, {
|
|
171
|
+
this.activeSessions.set(leaseId, {
|
|
172
|
+
fencingToken,
|
|
173
|
+
dispatchAttemptId: dispatch.sandbox?.dispatchAttemptId ?? null,
|
|
174
|
+
runnerInstanceId: dispatch.sandbox?.runnerInstanceId ?? null,
|
|
175
|
+
leaseExpiresAt: dispatch.lease.expiresAt,
|
|
176
|
+
sandboxed: dispatch.sandbox?.executionMode === "docker",
|
|
177
|
+
resources: dispatch.sandbox?.executionMode === "docker"
|
|
178
|
+
? dispatch.sandbox.requestedResources
|
|
179
|
+
: zeroSandboxResources(),
|
|
180
|
+
controller
|
|
181
|
+
});
|
|
99
182
|
try {
|
|
100
183
|
const normalized = this.normalizeDispatch(dispatch);
|
|
101
|
-
|
|
184
|
+
if (normalized.sandbox) {
|
|
185
|
+
const started = {
|
|
186
|
+
dispatchAttemptId: normalized.sandbox.dispatchAttemptId,
|
|
187
|
+
leaseId,
|
|
188
|
+
fencingToken,
|
|
189
|
+
runnerInstanceId: normalized.sandbox.runnerInstanceId
|
|
190
|
+
};
|
|
191
|
+
await this.invokeAck("DispatchStarted", started);
|
|
192
|
+
}
|
|
193
|
+
const sink = {
|
|
102
194
|
emit: async (event) => {
|
|
195
|
+
if (event.eventType === "provider_cli_started") {
|
|
196
|
+
const active = this.activeSessions.get(leaseId);
|
|
197
|
+
if (active)
|
|
198
|
+
active.resources = { ...active.resources, buildSlots: 0 };
|
|
199
|
+
}
|
|
103
200
|
try {
|
|
104
201
|
await this.invokeAck("DispatchEvent", {
|
|
105
202
|
leaseId,
|
|
106
203
|
fencingToken,
|
|
107
|
-
|
|
204
|
+
...(dispatch.sandbox ? {
|
|
205
|
+
dispatchAttemptId: dispatch.sandbox.dispatchAttemptId,
|
|
206
|
+
runnerInstanceId: dispatch.sandbox.runnerInstanceId
|
|
207
|
+
} : {}),
|
|
208
|
+
event: redactRunnerDispatchEvent(event, redactionSecrets)
|
|
108
209
|
});
|
|
109
210
|
}
|
|
110
211
|
catch (error) {
|
|
@@ -115,19 +216,233 @@ export class ForgeBridgeService {
|
|
|
115
216
|
throw error;
|
|
116
217
|
}
|
|
117
218
|
}
|
|
118
|
-
}
|
|
119
|
-
|
|
219
|
+
};
|
|
220
|
+
const result = normalized.sandbox?.executionMode === "docker"
|
|
221
|
+
? await this.sandboxRuntime.runOnce(normalized, sink, controller.signal)
|
|
222
|
+
: await this.sessions.runOnce(normalized, sink, controller.signal);
|
|
223
|
+
if (normalized.sandbox?.executionMode === "docker") {
|
|
224
|
+
await this.flushCleanupReports().catch(() => undefined);
|
|
225
|
+
}
|
|
226
|
+
await this.sendResult(dispatch, redactRunOnceResultPacket(result, redactionSecrets));
|
|
120
227
|
}
|
|
121
228
|
catch (error) {
|
|
122
229
|
const summary = controller.signal.aborted
|
|
123
230
|
? `Dispatch cancelled: ${String(controller.signal.reason ?? "cancelled")}`
|
|
124
231
|
: redactText(error instanceof Error ? error.message : String(error), redactionSecrets);
|
|
125
|
-
|
|
232
|
+
const failureEventType = infrastructureFailureEventType(summary);
|
|
233
|
+
const failureEvents = failureEventType ? [{
|
|
234
|
+
role: "runner",
|
|
235
|
+
kind: "error",
|
|
236
|
+
content: summary,
|
|
237
|
+
eventType: failureEventType,
|
|
238
|
+
}] : [];
|
|
239
|
+
await this.sendResult(dispatch, failureResult(summary, failureEvents)).catch(() => undefined);
|
|
126
240
|
}
|
|
127
241
|
finally {
|
|
128
242
|
this.activeSessions.delete(leaseId);
|
|
129
243
|
}
|
|
130
244
|
}
|
|
245
|
+
async handleOffer(rawDispatch) {
|
|
246
|
+
const envelope = parseOfferEnvelope(rawDispatch);
|
|
247
|
+
if (!envelope) {
|
|
248
|
+
return;
|
|
249
|
+
}
|
|
250
|
+
const dispatch = this.parseDispatchOrReport(envelope.dispatch);
|
|
251
|
+
if (!dispatch?.sandbox
|
|
252
|
+
|| dispatch.sandbox.dispatchAttemptId !== envelope.dispatchAttemptId
|
|
253
|
+
|| dispatch.lease.id !== envelope.leaseId
|
|
254
|
+
|| dispatch.lease.fencingToken !== envelope.fencingToken)
|
|
255
|
+
return;
|
|
256
|
+
const sandbox = dispatch.sandbox;
|
|
257
|
+
let existing = this.reservations.get(sandbox.dispatchAttemptId);
|
|
258
|
+
if (existing && Date.parse(existing.reservedUntil) <= Date.now()) {
|
|
259
|
+
this.reservations.delete(sandbox.dispatchAttemptId);
|
|
260
|
+
existing = undefined;
|
|
261
|
+
}
|
|
262
|
+
if (existing) {
|
|
263
|
+
const sameFence = existing.fencingToken === dispatch.lease.fencingToken;
|
|
264
|
+
await this.sendOfferDecision(dispatch, sameFence, sameFence ? null : "stale_fence", sameFence ? null : "Offer fence conflicts with the existing reservation.", sameFence ? existing.reservedUntil : null, sameFence ? existing.resources : null);
|
|
265
|
+
return;
|
|
266
|
+
}
|
|
267
|
+
if (sandbox.runnerInstanceId !== this.config.runnerInstanceId) {
|
|
268
|
+
await this.sendOfferDecision(dispatch, false, "stale_runner_instance", "Offer targets a different runner process instance.");
|
|
269
|
+
return;
|
|
270
|
+
}
|
|
271
|
+
if (Date.parse(envelope.expiresAt) <= Date.now()) {
|
|
272
|
+
await this.sendOfferDecision(dispatch, false, "lease_expired", "Offer lease is already expired.");
|
|
273
|
+
return;
|
|
274
|
+
}
|
|
275
|
+
const preflight = sandbox.executionMode === "docker" && this.config.dockerEnabled
|
|
276
|
+
? await this.preflight.run()
|
|
277
|
+
: this.preflight.current();
|
|
278
|
+
if (sandbox.executionMode === "docker") {
|
|
279
|
+
if (!this.config.dockerEnabled || !preflight?.healthy) {
|
|
280
|
+
await this.sendOfferDecision(dispatch, false, preflight?.unavailableCode ?? "docker_unavailable", preflight?.unavailableReason ?? "Docker sandbox preflight is unavailable.");
|
|
281
|
+
return;
|
|
282
|
+
}
|
|
283
|
+
const networkFailure = sandboxNetworkPolicyFailure(dispatch, this.config);
|
|
284
|
+
if (networkFailure) {
|
|
285
|
+
await this.sendOfferDecision(dispatch, false, "policy_mismatch", networkFailure);
|
|
286
|
+
return;
|
|
287
|
+
}
|
|
288
|
+
const providerNetworkFailure = sandboxProviderNetworkFailure(dispatch, this.config);
|
|
289
|
+
if (providerNetworkFailure) {
|
|
290
|
+
await this.sendOfferDecision(dispatch, false, "provider_network_unavailable", providerNetworkFailure);
|
|
291
|
+
return;
|
|
292
|
+
}
|
|
293
|
+
const authFailure = sandboxAuthFailure(dispatch, this.config);
|
|
294
|
+
if (authFailure) {
|
|
295
|
+
await this.sendOfferDecision(dispatch, false, "sandbox_auth_unavailable", authFailure);
|
|
296
|
+
return;
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
const runtimeState = this.sandboxRuntime.snapshot();
|
|
300
|
+
const usedResources = this.usedSandboxResources(runtimeState.degraded);
|
|
301
|
+
const activeResources = this.activeSandboxResources(runtimeState.degraded);
|
|
302
|
+
const capacity = this.sandboxResourceCapacity(preflight, activeResources);
|
|
303
|
+
let requestedResources = sandbox.requestedResources;
|
|
304
|
+
let sandboxConfigCommitSha = sandbox.sandboxConfigCommitSha ?? null;
|
|
305
|
+
if (sandbox.executionMode === "docker") {
|
|
306
|
+
try {
|
|
307
|
+
const profile = await this.sandboxResourceResolver(this.normalizeDispatch(dispatch));
|
|
308
|
+
requestedResources = profile.resources;
|
|
309
|
+
sandboxConfigCommitSha = profile.sandboxConfigCommitSha;
|
|
310
|
+
}
|
|
311
|
+
catch (error) {
|
|
312
|
+
await this.sendOfferDecision(dispatch, false, "sandbox_capacity_profile_unavailable", error instanceof Error ? error.message : "The immutable sandbox contract could not be profiled for admission.");
|
|
313
|
+
return;
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
if (sandbox.executionMode === "docker"
|
|
317
|
+
&& !canReserveSandboxResources(usedResources, requestedResources, capacity)) {
|
|
318
|
+
await this.sendOfferDecision(dispatch, false, "capacity_unavailable", sandboxCapacityFailureMessage(usedResources, requestedResources, capacity, preflight, this.config));
|
|
319
|
+
return;
|
|
320
|
+
}
|
|
321
|
+
const reservedUntil = new Date(Math.min(Date.parse(dispatch.lease.expiresAt), Date.now() + 30_000)).toISOString();
|
|
322
|
+
this.reservations.set(sandbox.dispatchAttemptId, {
|
|
323
|
+
leaseId: dispatch.lease.id,
|
|
324
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
325
|
+
runnerInstanceId: sandbox.runnerInstanceId,
|
|
326
|
+
expiresAt: dispatch.lease.expiresAt,
|
|
327
|
+
reservedUntil,
|
|
328
|
+
sandboxed: sandbox.executionMode === "docker",
|
|
329
|
+
resources: requestedResources
|
|
330
|
+
});
|
|
331
|
+
await this.sendOfferDecision(dispatch, true, null, null, reservedUntil, requestedResources, sandboxConfigCommitSha);
|
|
332
|
+
}
|
|
333
|
+
async handleStart(rawDispatch) {
|
|
334
|
+
const envelope = parseStartEnvelope(rawDispatch);
|
|
335
|
+
if (!envelope) {
|
|
336
|
+
return;
|
|
337
|
+
}
|
|
338
|
+
let dispatch = this.parseDispatchOrReport(envelope.dispatch);
|
|
339
|
+
if (!dispatch?.sandbox
|
|
340
|
+
|| dispatch.sandbox.dispatchAttemptId !== envelope.dispatchAttemptId
|
|
341
|
+
|| dispatch.lease.id !== envelope.leaseId
|
|
342
|
+
|| dispatch.lease.fencingToken !== envelope.fencingToken)
|
|
343
|
+
return;
|
|
344
|
+
const reservation = this.reservations.get(dispatch.sandbox.dispatchAttemptId);
|
|
345
|
+
if (!reservation
|
|
346
|
+
|| reservation.leaseId !== dispatch.lease.id
|
|
347
|
+
|| reservation.fencingToken !== dispatch.lease.fencingToken
|
|
348
|
+
|| reservation.runnerInstanceId !== dispatch.sandbox.runnerInstanceId
|
|
349
|
+
|| Date.parse(reservation.reservedUntil) <= Date.now()) {
|
|
350
|
+
await this.sendResult(dispatch, failureResult("Start rejected because its reservation is missing or stale.")).catch(() => undefined);
|
|
351
|
+
return;
|
|
352
|
+
}
|
|
353
|
+
if (dispatch.apiKeySecretGrant) {
|
|
354
|
+
this.reservations.delete(dispatch.sandbox.dispatchAttemptId);
|
|
355
|
+
await this.sendResult(dispatch, failureResult("Start rejected because protocol-v2 packets may not contain a plaintext provider grant.")).catch(() => undefined);
|
|
356
|
+
return;
|
|
357
|
+
}
|
|
358
|
+
if (dispatch.apiKeySecretGrantHandle) {
|
|
359
|
+
const response = await this.redeemSecretGrant(dispatch);
|
|
360
|
+
const grant = response.grant;
|
|
361
|
+
if (!response.accepted
|
|
362
|
+
|| !grant
|
|
363
|
+
|| grant.grantId !== dispatch.apiKeySecretGrantHandle.grantId
|
|
364
|
+
|| grant.provider !== dispatch.apiKeySecretGrantHandle.provider
|
|
365
|
+
|| Date.parse(grant.expiresAt) <= Date.now()) {
|
|
366
|
+
this.reservations.delete(dispatch.sandbox.dispatchAttemptId);
|
|
367
|
+
await this.sendResult(dispatch, failureResult(response.message ?? "Secret grant redemption was rejected.")).catch(() => undefined);
|
|
368
|
+
return;
|
|
369
|
+
}
|
|
370
|
+
dispatch = { ...dispatch, apiKeySecretGrant: grant, apiKeySecretGrantHandle: null };
|
|
371
|
+
}
|
|
372
|
+
else if (dispatch.execution.requiresApiKeySecretGrant) {
|
|
373
|
+
this.reservations.delete(dispatch.sandbox.dispatchAttemptId);
|
|
374
|
+
await this.sendResult(dispatch, failureResult("Start rejected because the provider secret grant handle is missing.")).catch(() => undefined);
|
|
375
|
+
return;
|
|
376
|
+
}
|
|
377
|
+
dispatch = {
|
|
378
|
+
...dispatch,
|
|
379
|
+
sandbox: { ...dispatch.sandbox, requestedResources: reservation.resources },
|
|
380
|
+
};
|
|
381
|
+
this.reservations.delete(envelope.dispatchAttemptId);
|
|
382
|
+
await this.handleDispatch(dispatch);
|
|
383
|
+
}
|
|
384
|
+
async redeemSecretGrant(dispatch) {
|
|
385
|
+
const sandbox = dispatch.sandbox;
|
|
386
|
+
const handle = dispatch.apiKeySecretGrantHandle;
|
|
387
|
+
const connection = this.connection;
|
|
388
|
+
if (!connection)
|
|
389
|
+
return { accepted: false, code: "bridge.disconnected", message: "Bridge is disconnected.", grant: null };
|
|
390
|
+
return await connection.invoke("RedeemSecretGrant", {
|
|
391
|
+
dispatchAttemptId: sandbox.dispatchAttemptId,
|
|
392
|
+
leaseId: dispatch.lease.id,
|
|
393
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
394
|
+
runnerInstanceId: sandbox.runnerInstanceId,
|
|
395
|
+
grantId: handle.grantId
|
|
396
|
+
});
|
|
397
|
+
}
|
|
398
|
+
async redeemPublishGrant(request, signal) {
|
|
399
|
+
if (signal?.aborted)
|
|
400
|
+
throw new Error(String(signal.reason ?? "publication cancelled"));
|
|
401
|
+
const connection = this.connection;
|
|
402
|
+
if (!connection)
|
|
403
|
+
throw new Error("sandbox_output_publish_forbidden: bridge is disconnected.");
|
|
404
|
+
const response = await connection.invoke("RedeemPublishGrant", request);
|
|
405
|
+
if (!response.accepted || !response.credential) {
|
|
406
|
+
throw new Error(`sandbox_output_publish_forbidden: ${response.message ?? response.code ?? "publication grant redemption was rejected"}`);
|
|
407
|
+
}
|
|
408
|
+
return response.credential;
|
|
409
|
+
}
|
|
410
|
+
async handleLeaseUpdate(request) {
|
|
411
|
+
if (request.runnerInstanceId !== this.config.runnerInstanceId) {
|
|
412
|
+
return;
|
|
413
|
+
}
|
|
414
|
+
const reservation = this.reservations.get(request.dispatchAttemptId);
|
|
415
|
+
if (reservation && reservation.leaseId === request.leaseId && reservation.fencingToken === request.fencingToken) {
|
|
416
|
+
reservation.expiresAt = request.expiresAt;
|
|
417
|
+
}
|
|
418
|
+
const active = this.activeSessions.get(request.leaseId);
|
|
419
|
+
if (active && active.fencingToken === request.fencingToken && active.dispatchAttemptId === request.dispatchAttemptId) {
|
|
420
|
+
active.leaseExpiresAt = request.expiresAt;
|
|
421
|
+
if (active.sandboxed) {
|
|
422
|
+
await this.sandboxRuntime.updateLease(request.dispatchAttemptId, request.fencingToken, request.expiresAt);
|
|
423
|
+
}
|
|
424
|
+
else {
|
|
425
|
+
await this.sessions.updateLease?.(request.dispatchAttemptId, request.fencingToken, request.expiresAt);
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
async sendOfferDecision(dispatch, accepted, code = null, message = null, reservedUntil = null, reservedResources = null, sandboxConfigCommitSha = null) {
|
|
430
|
+
if (!dispatch.sandbox)
|
|
431
|
+
return;
|
|
432
|
+
const payload = {
|
|
433
|
+
dispatchAttemptId: dispatch.sandbox.dispatchAttemptId,
|
|
434
|
+
leaseId: dispatch.lease.id,
|
|
435
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
436
|
+
runnerInstanceId: dispatch.sandbox.runnerInstanceId,
|
|
437
|
+
accepted,
|
|
438
|
+
code,
|
|
439
|
+
message,
|
|
440
|
+
reservedUntil: accepted ? reservedUntil ?? dispatch.lease.expiresAt : null,
|
|
441
|
+
reservedResources: accepted ? reservedResources : null,
|
|
442
|
+
sandboxConfigCommitSha: accepted ? sandboxConfigCommitSha : null,
|
|
443
|
+
};
|
|
444
|
+
await this.invokeAck("OfferDecision", payload);
|
|
445
|
+
}
|
|
131
446
|
parseDispatchOrReport(rawDispatch) {
|
|
132
447
|
try {
|
|
133
448
|
return parseRunOnceDispatchPacket(rawDispatch);
|
|
@@ -138,7 +453,7 @@ export class ForgeBridgeService {
|
|
|
138
453
|
return null;
|
|
139
454
|
}
|
|
140
455
|
const summary = invalidDispatchSummary(error);
|
|
141
|
-
void this.
|
|
456
|
+
void this.sendLegacyResult(identifiers.leaseId, identifiers.fencingToken, failureResult(summary, [
|
|
142
457
|
{
|
|
143
458
|
role: "runner",
|
|
144
459
|
kind: "error",
|
|
@@ -156,12 +471,74 @@ export class ForgeBridgeService {
|
|
|
156
471
|
}
|
|
157
472
|
active.controller.abort(request.reason ?? "cancelled by server");
|
|
158
473
|
}
|
|
474
|
+
async handleReap(request) {
|
|
475
|
+
if (request.runnerId !== this.config.runnerId
|
|
476
|
+
|| request.runnerInstanceId !== this.config.runnerInstanceId)
|
|
477
|
+
return;
|
|
478
|
+
await this.runSandboxMaintenance();
|
|
479
|
+
await this.sendHeartbeat();
|
|
480
|
+
}
|
|
481
|
+
async handleReadinessProbe(rawRequest) {
|
|
482
|
+
const request = parseReadinessProbe(rawRequest);
|
|
483
|
+
if (!request || request.runnerInstanceId !== this.config.runnerInstanceId)
|
|
484
|
+
return;
|
|
485
|
+
const existing = this.activeReadinessProbes.get(request.requestId);
|
|
486
|
+
if (existing)
|
|
487
|
+
return;
|
|
488
|
+
const controller = new AbortController();
|
|
489
|
+
this.activeReadinessProbes.set(request.requestId, controller);
|
|
490
|
+
const deadlineMs = Date.parse(request.expiresAt) - Date.now();
|
|
491
|
+
const timer = setTimeout(() => controller.abort("readiness probe expired"), Math.max(0, deadlineMs));
|
|
492
|
+
timer.unref?.();
|
|
493
|
+
let result;
|
|
494
|
+
try {
|
|
495
|
+
if (!this.sessions.probeHostRuntime) {
|
|
496
|
+
throw new Error("host_runtime_probe_unavailable: this bridge does not implement runtime readiness probes.");
|
|
497
|
+
}
|
|
498
|
+
const probe = await this.sessions.probeHostRuntime({
|
|
499
|
+
...request,
|
|
500
|
+
runnerId: this.config.runnerId,
|
|
501
|
+
repoRoot: this.config.defaultRepoRoot,
|
|
502
|
+
cleanupWorktrees: this.config.cleanupWorktrees,
|
|
503
|
+
}, controller.signal);
|
|
504
|
+
result = {
|
|
505
|
+
schemaVersion: 1,
|
|
506
|
+
requestId: request.requestId,
|
|
507
|
+
fencingToken: request.fencingToken,
|
|
508
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
509
|
+
...probe,
|
|
510
|
+
};
|
|
511
|
+
}
|
|
512
|
+
catch (error) {
|
|
513
|
+
result = {
|
|
514
|
+
schemaVersion: 1,
|
|
515
|
+
requestId: request.requestId,
|
|
516
|
+
fencingToken: request.fencingToken,
|
|
517
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
518
|
+
status: "failed",
|
|
519
|
+
reasonCode: "host_runtime_probe_failed",
|
|
520
|
+
reasonMessage: errorMessage(error).slice(0, 1000),
|
|
521
|
+
checkedAt: new Date().toISOString(),
|
|
522
|
+
};
|
|
523
|
+
}
|
|
524
|
+
finally {
|
|
525
|
+
clearTimeout(timer);
|
|
526
|
+
this.activeReadinessProbes.delete(request.requestId);
|
|
527
|
+
}
|
|
528
|
+
await this.invokeAck("ReadinessResult", result).catch(() => undefined);
|
|
529
|
+
await this.sendHeartbeat().catch(() => undefined);
|
|
530
|
+
}
|
|
159
531
|
abortActiveSessions(reason) {
|
|
160
532
|
for (const active of this.activeSessions.values()) {
|
|
161
533
|
active.controller.abort(reason);
|
|
162
534
|
}
|
|
163
535
|
this.activeSessions.clear();
|
|
164
536
|
}
|
|
537
|
+
abortActiveReadinessProbes(reason) {
|
|
538
|
+
for (const controller of this.activeReadinessProbes.values())
|
|
539
|
+
controller.abort(reason);
|
|
540
|
+
this.activeReadinessProbes.clear();
|
|
541
|
+
}
|
|
165
542
|
normalizeDispatch(dispatch) {
|
|
166
543
|
const mapping = findRepoMapping(dispatch, repoMappingsFor(this.config));
|
|
167
544
|
const localPath = mapping?.localPath?.trim();
|
|
@@ -186,12 +563,20 @@ export class ForgeBridgeService {
|
|
|
186
563
|
: dispatch.qaBrowser
|
|
187
564
|
};
|
|
188
565
|
}
|
|
189
|
-
async sendResult(
|
|
566
|
+
async sendResult(dispatch, result) {
|
|
190
567
|
const payload = {
|
|
191
|
-
leaseId,
|
|
192
|
-
fencingToken,
|
|
568
|
+
leaseId: dispatch.lease.id,
|
|
569
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
570
|
+
dispatchAttemptId: dispatch.sandbox?.dispatchAttemptId ?? null,
|
|
571
|
+
runnerInstanceId: dispatch.sandbox?.runnerInstanceId ?? null,
|
|
193
572
|
result
|
|
194
573
|
};
|
|
574
|
+
await this.sendResultPayload(payload);
|
|
575
|
+
}
|
|
576
|
+
async sendLegacyResult(leaseId, fencingToken, result) {
|
|
577
|
+
await this.sendResultPayload({ leaseId, fencingToken, result });
|
|
578
|
+
}
|
|
579
|
+
async sendResultPayload(payload) {
|
|
195
580
|
try {
|
|
196
581
|
await this.invokeAck("DispatchResult", payload);
|
|
197
582
|
}
|
|
@@ -204,18 +589,132 @@ export class ForgeBridgeService {
|
|
|
204
589
|
}
|
|
205
590
|
startHeartbeat() {
|
|
206
591
|
this.stopHeartbeat();
|
|
592
|
+
this.consecutiveHeartbeatFailures = 0;
|
|
593
|
+
this.reconnectingFromHeartbeatFailure = false;
|
|
207
594
|
const intervalMs = positiveFiniteNumber(this.config.heartbeatIntervalMs, 30_000);
|
|
208
595
|
this.heartbeatTimer = setInterval(() => {
|
|
209
|
-
this.
|
|
596
|
+
void this.heartbeatTick();
|
|
210
597
|
}, intervalMs);
|
|
211
598
|
this.heartbeatTimer.unref?.();
|
|
212
599
|
}
|
|
600
|
+
async heartbeatTick() {
|
|
601
|
+
if (this.heartbeatInFlight || this.reconnectingFromHeartbeatFailure)
|
|
602
|
+
return;
|
|
603
|
+
const connection = this.connection;
|
|
604
|
+
if (!connection)
|
|
605
|
+
return;
|
|
606
|
+
this.heartbeatInFlight = true;
|
|
607
|
+
try {
|
|
608
|
+
await this.sendHeartbeat();
|
|
609
|
+
this.consecutiveHeartbeatFailures = 0;
|
|
610
|
+
}
|
|
611
|
+
catch (error) {
|
|
612
|
+
if (connection !== this.connection || this.stopped)
|
|
613
|
+
return;
|
|
614
|
+
this.consecutiveHeartbeatFailures += 1;
|
|
615
|
+
logBridgeStatus(`Heartbeat ${this.consecutiveHeartbeatFailures} failed: ${errorMessage(error)}.`);
|
|
616
|
+
if (this.consecutiveHeartbeatFailures < 2)
|
|
617
|
+
return;
|
|
618
|
+
this.reconnectingFromHeartbeatFailure = true;
|
|
619
|
+
this.stopHeartbeat();
|
|
620
|
+
logBridgeStatus("Heartbeat failed twice. Closing the stale connection so the Bridge can reconnect...");
|
|
621
|
+
await connection.stop().catch(() => undefined);
|
|
622
|
+
}
|
|
623
|
+
finally {
|
|
624
|
+
this.heartbeatInFlight = false;
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
startLeaseWatchdog() {
|
|
628
|
+
this.stopLeaseWatchdog();
|
|
629
|
+
this.leaseWatchdogTimer = setInterval(() => this.expireLeases(), 1000);
|
|
630
|
+
this.leaseWatchdogTimer.unref?.();
|
|
631
|
+
}
|
|
632
|
+
stopLeaseWatchdog() {
|
|
633
|
+
if (this.leaseWatchdogTimer) {
|
|
634
|
+
clearInterval(this.leaseWatchdogTimer);
|
|
635
|
+
this.leaseWatchdogTimer = null;
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
expireLeases() {
|
|
639
|
+
const now = Date.now();
|
|
640
|
+
for (const [attemptId, reservation] of this.reservations) {
|
|
641
|
+
if (Date.parse(reservation.expiresAt) <= now || Date.parse(reservation.reservedUntil) <= now) {
|
|
642
|
+
this.reservations.delete(attemptId);
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
for (const active of this.activeSessions.values()) {
|
|
646
|
+
if (Date.parse(active.leaseExpiresAt) <= now && !active.controller.signal.aborted) {
|
|
647
|
+
active.controller.abort("dispatch lease expired");
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
}
|
|
213
651
|
stopHeartbeat() {
|
|
214
652
|
if (this.heartbeatTimer) {
|
|
215
653
|
clearInterval(this.heartbeatTimer);
|
|
216
654
|
this.heartbeatTimer = null;
|
|
217
655
|
}
|
|
218
656
|
}
|
|
657
|
+
startSandboxMaintenance() {
|
|
658
|
+
this.stopSandboxMaintenance();
|
|
659
|
+
this.sandboxMaintenanceTimer = setInterval(() => {
|
|
660
|
+
this.runSandboxMaintenance().catch(() => undefined);
|
|
661
|
+
}, 60_000);
|
|
662
|
+
this.sandboxMaintenanceTimer.unref?.();
|
|
663
|
+
}
|
|
664
|
+
stopSandboxMaintenance() {
|
|
665
|
+
if (this.sandboxMaintenanceTimer) {
|
|
666
|
+
clearInterval(this.sandboxMaintenanceTimer);
|
|
667
|
+
this.sandboxMaintenanceTimer = null;
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
async runSandboxMaintenance() {
|
|
671
|
+
if (this.sandboxMaintenanceRunning)
|
|
672
|
+
return;
|
|
673
|
+
this.sandboxMaintenanceRunning = true;
|
|
674
|
+
try {
|
|
675
|
+
const hostReap = await this.sessions.reapHostRuntime?.({
|
|
676
|
+
runnerId: this.config.runnerId,
|
|
677
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
678
|
+
});
|
|
679
|
+
if (hostReap && (hostReap.reaped > 0 || hostReap.degraded > 0)) {
|
|
680
|
+
logBridgeStatus(`Managed-host maintenance reaped ${hostReap.reaped} stale attempt(s) and left ${hostReap.degraded} degraded attempt(s) for reconciliation.`);
|
|
681
|
+
}
|
|
682
|
+
if (this.config.dockerEnabled) {
|
|
683
|
+
const preflight = await this.preflight.refresh();
|
|
684
|
+
if (canReapManagedSandboxes(preflight)) {
|
|
685
|
+
const attemptSnapshot = await this.readAttemptSnapshot();
|
|
686
|
+
await this.sandboxRuntime.reap({ diskLow: preflight.diskLow, attemptSnapshot });
|
|
687
|
+
await this.flushCleanupReports();
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
finally {
|
|
692
|
+
this.sandboxMaintenanceRunning = false;
|
|
693
|
+
}
|
|
694
|
+
}
|
|
695
|
+
async readAttemptSnapshot() {
|
|
696
|
+
const connection = this.connection;
|
|
697
|
+
if (!connection)
|
|
698
|
+
throw new Error("Bridge is not connected.");
|
|
699
|
+
const response = await connection.invoke("AttemptSnapshot", {
|
|
700
|
+
runnerInstanceId: this.config.runnerInstanceId
|
|
701
|
+
});
|
|
702
|
+
if (!response.accepted) {
|
|
703
|
+
throw new BridgeCallbackRejectedError(response.code ?? null, response.message ?? response.code ?? "Attempt snapshot was rejected.");
|
|
704
|
+
}
|
|
705
|
+
return response.attempts ?? [];
|
|
706
|
+
}
|
|
707
|
+
async flushCleanupReports() {
|
|
708
|
+
if (!this.sandboxRuntime.pendingCleanupReports || !this.sandboxRuntime.acknowledgeCleanupReport)
|
|
709
|
+
return;
|
|
710
|
+
for (const report of await this.sandboxRuntime.pendingCleanupReports()) {
|
|
711
|
+
await this.invokeAck("CleanupReport", {
|
|
712
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
713
|
+
report
|
|
714
|
+
});
|
|
715
|
+
await this.sandboxRuntime.acknowledgeCleanupReport(report.reportId);
|
|
716
|
+
}
|
|
717
|
+
}
|
|
219
718
|
async sendHeartbeat() {
|
|
220
719
|
await this.invokeAck("Heartbeat", {
|
|
221
720
|
status: "online",
|
|
@@ -225,8 +724,14 @@ export class ForgeBridgeService {
|
|
|
225
724
|
});
|
|
226
725
|
}
|
|
227
726
|
capabilities() {
|
|
727
|
+
const preflight = this.preflight.current();
|
|
728
|
+
const runtimeState = this.sandboxRuntime.snapshot();
|
|
729
|
+
const reserved = [...this.reservations.values()].filter(reservation => reservation.sandboxed).length;
|
|
730
|
+
const used = this.usedSandboxResources(runtimeState.degraded);
|
|
731
|
+
const activeResources = this.activeSandboxResources(runtimeState.degraded);
|
|
732
|
+
const capacity = this.sandboxResourceCapacity(preflight, activeResources);
|
|
228
733
|
return {
|
|
229
|
-
schemaVersion:
|
|
734
|
+
schemaVersion: 3,
|
|
230
735
|
activeSessionCount: this.activeSessions.size,
|
|
231
736
|
cliProviders: this.config.cliProviders,
|
|
232
737
|
providerStatuses: this.config.providerStatuses,
|
|
@@ -234,6 +739,37 @@ export class ForgeBridgeService {
|
|
|
234
739
|
repoMappings: repoMappingsFor(this.config),
|
|
235
740
|
os: this.config.os,
|
|
236
741
|
sandboxProfiles: this.config.sandboxProfiles,
|
|
742
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
743
|
+
trustDomainId: this.config.trustDomainId,
|
|
744
|
+
maxConcurrentSandboxes: capacity.sandboxSlots,
|
|
745
|
+
reservedSandboxCount: reserved,
|
|
746
|
+
activeSandboxCount: runtimeState.active,
|
|
747
|
+
startingSandboxCount: runtimeState.starting,
|
|
748
|
+
degradedReservationCount: runtimeState.degraded,
|
|
749
|
+
sandboxRuntimes: preflight?.healthy ? ["docker"] : [],
|
|
750
|
+
sandboxRuntimeVersion: preflight?.runtimeVersion ?? null,
|
|
751
|
+
sandboxFeatures: preflight?.features ?? null,
|
|
752
|
+
allowsOpenPublicSandbox: this.config.allowOpenPublicSandbox,
|
|
753
|
+
sandboxPublicationModes: [this.config.sandboxPublicationMode ?? "shared_filesystem"],
|
|
754
|
+
sandboxAuthModes: sandboxAuthModesFor(this.config),
|
|
755
|
+
availableMemoryBytes: preflight ? Math.max(0, capacity.memoryBytes - used.memoryBytes) : null,
|
|
756
|
+
availableCpuMillis: preflight ? Math.max(0, capacity.cpuMillis - used.cpuMillis) : null,
|
|
757
|
+
availableWorkspaceBytes: preflight ? Math.max(0, capacity.workspaceBytes - used.workspaceBytes) : null,
|
|
758
|
+
diskLow: preflight?.diskLow ?? null,
|
|
759
|
+
preflightGeneration: preflight?.generation ?? 0,
|
|
760
|
+
sandboxUnavailableCode: this.config.dockerEnabled ? preflight?.unavailableCode ?? null : "docker_disabled",
|
|
761
|
+
sandboxUnavailableReason: this.config.dockerEnabled ? preflight?.unavailableReason ?? null : "Docker sandboxing is disabled by runner configuration.",
|
|
762
|
+
sandboxMemoryCapacity: preflight ? {
|
|
763
|
+
defaultWorkerMemoryLimitBytes: this.config.defaultSandboxWorkerMemoryLimitBytes,
|
|
764
|
+
defaultServiceMemoryLimitBytes: this.config.defaultSandboxServiceMemoryLimitBytes,
|
|
765
|
+
runtimeOverheadBytes: this.config.sandboxRuntimeOverheadBytes,
|
|
766
|
+
hostReserveBytes: preflight.hostReserveBytes,
|
|
767
|
+
activeReservationBytes: used.memoryBytes,
|
|
768
|
+
daemonTotalBytes: preflight.totalMemoryBytes,
|
|
769
|
+
daemonAvailableBytes: preflight.availableMemoryBytes,
|
|
770
|
+
availableForAdmissionBytes: Math.max(0, capacity.memoryBytes - used.memoryBytes),
|
|
771
|
+
} : null,
|
|
772
|
+
hostRuntime: this.sessions.runtimeCapacity?.() ?? null,
|
|
237
773
|
supportsApiSecretGrant: this.config.supportsApiSecretGrant,
|
|
238
774
|
supportsManagedWorktrees: this.config.supportsManagedWorktrees,
|
|
239
775
|
supportsManagedForgeArtifacts: this.config.supportsManagedForgeArtifacts,
|
|
@@ -243,6 +779,48 @@ export class ForgeBridgeService {
|
|
|
243
779
|
supportsDraftImageAttachments: this.config.supportsDraftImageAttachments
|
|
244
780
|
};
|
|
245
781
|
}
|
|
782
|
+
usedSandboxResources(degradedReservations) {
|
|
783
|
+
const resources = [
|
|
784
|
+
...[...this.reservations.values()]
|
|
785
|
+
.filter(reservation => reservation.sandboxed)
|
|
786
|
+
.map(reservation => reservation.resources),
|
|
787
|
+
...[...this.activeSessions.values()]
|
|
788
|
+
.filter(session => session.sandboxed)
|
|
789
|
+
.map(session => session.resources),
|
|
790
|
+
...Array.from({ length: degradedReservations }, () => defaultSandboxResources())
|
|
791
|
+
];
|
|
792
|
+
return sumSandboxResources(resources);
|
|
793
|
+
}
|
|
794
|
+
activeSandboxResources(degradedReservations) {
|
|
795
|
+
return sumSandboxResources([
|
|
796
|
+
...[...this.activeSessions.values()]
|
|
797
|
+
.filter(session => session.sandboxed)
|
|
798
|
+
.map(session => session.resources),
|
|
799
|
+
...Array.from({ length: degradedReservations }, () => defaultSandboxResources())
|
|
800
|
+
]);
|
|
801
|
+
}
|
|
802
|
+
sandboxResourceCapacity(preflight, activeResources) {
|
|
803
|
+
if (!preflight)
|
|
804
|
+
return zeroSandboxResources();
|
|
805
|
+
return {
|
|
806
|
+
// Host free memory and disk already reflect active containers. Add their
|
|
807
|
+
// commitments back to form a stable total capacity, then subtract them
|
|
808
|
+
// exactly once in usedSandboxResources.
|
|
809
|
+
sandboxSlots: Math.min(this.config.maxConcurrentSandboxes, preflight.maxConcurrentSandboxes + activeResources.sandboxSlots),
|
|
810
|
+
cpuMillis: preflight.availableCpuMillis,
|
|
811
|
+
workerMemoryLimitBytes: 0,
|
|
812
|
+
serviceMemoryLimitBytes: 0,
|
|
813
|
+
runtimeOverheadBytes: 0,
|
|
814
|
+
memoryBytes: Math.max(0, preflight.availableMemoryBytes
|
|
815
|
+
- (this.config.sandboxHostMemoryReserveBytes ?? 2 * 1024 ** 3)
|
|
816
|
+
+ activeResources.memoryBytes),
|
|
817
|
+
workspaceBytes: Math.max(0, preflight.availableWorkspaceBytes
|
|
818
|
+
- this.config.minimumWorkspaceBytes
|
|
819
|
+
+ activeResources.workspaceBytes),
|
|
820
|
+
pids: this.config.sandboxPidCapacity,
|
|
821
|
+
buildSlots: this.config.maxConcurrentSandboxBuilds
|
|
822
|
+
};
|
|
823
|
+
}
|
|
246
824
|
async invokeAck(methodName, payload) {
|
|
247
825
|
const connection = this.connection;
|
|
248
826
|
if (!connection) {
|
|
@@ -262,6 +840,8 @@ export class ForgeBridgeService {
|
|
|
262
840
|
}
|
|
263
841
|
async stopConnection() {
|
|
264
842
|
this.stopHeartbeat();
|
|
843
|
+
this.stopLeaseWatchdog();
|
|
844
|
+
this.stopSandboxMaintenance();
|
|
265
845
|
if (this.connection) {
|
|
266
846
|
await this.connection.stop().catch(() => undefined);
|
|
267
847
|
this.connection = null;
|
|
@@ -282,6 +862,12 @@ function isDispatchNoLongerPending(error) {
|
|
|
282
862
|
}
|
|
283
863
|
return error instanceof Error && error.message === "Dispatch is no longer pending.";
|
|
284
864
|
}
|
|
865
|
+
function createDefaultSessionService(sandboxStateRoot) {
|
|
866
|
+
const rawProcessRunner = new StreamingProviderProcessRunner();
|
|
867
|
+
const processSupervisors = new AttemptProcessSupervisorRegistry();
|
|
868
|
+
const coordinatedProcessRunner = new AttemptCoordinatedProviderProcessRunner(processSupervisors, rawProcessRunner);
|
|
869
|
+
return new BridgeSessionService(new GitWorktreeService(undefined, new RepositoryCoordinatorLock(sandboxStateRoot)), new RoutingProviderCliService(new ProcessBackedProviderCliService(coordinatedProcessRunner)), new HostRuntimeLeaseManager(sandboxStateRoot, undefined, undefined, processSupervisors, rawProcessRunner));
|
|
870
|
+
}
|
|
285
871
|
export class SignalRBridgeConnectionFactory {
|
|
286
872
|
create(config) {
|
|
287
873
|
return new SignalRBridgeConnection(config);
|
|
@@ -306,6 +892,9 @@ export class SignalRBridgeConnection {
|
|
|
306
892
|
})
|
|
307
893
|
.configureLogging(LogLevel.Warning)
|
|
308
894
|
.build();
|
|
895
|
+
this.connection.on("Superseded", () => {
|
|
896
|
+
void this.connection.stop();
|
|
897
|
+
});
|
|
309
898
|
}
|
|
310
899
|
start() {
|
|
311
900
|
return this.connection.start();
|
|
@@ -321,11 +910,36 @@ export class SignalRBridgeConnection {
|
|
|
321
910
|
void handler(raw).catch(() => undefined);
|
|
322
911
|
});
|
|
323
912
|
}
|
|
913
|
+
onOffer(handler) {
|
|
914
|
+
this.connection.on("Offer", (raw) => {
|
|
915
|
+
void handler(raw).catch(() => undefined);
|
|
916
|
+
});
|
|
917
|
+
}
|
|
918
|
+
onStart(handler) {
|
|
919
|
+
this.connection.on("Start", (raw) => {
|
|
920
|
+
void handler(raw).catch(() => undefined);
|
|
921
|
+
});
|
|
922
|
+
}
|
|
923
|
+
onLeaseUpdate(handler) {
|
|
924
|
+
this.connection.on("LeaseUpdate", (request) => {
|
|
925
|
+
void handler(request).catch(() => undefined);
|
|
926
|
+
});
|
|
927
|
+
}
|
|
324
928
|
onCancel(handler) {
|
|
325
929
|
this.connection.on("Cancel", (request) => {
|
|
326
930
|
void handler(request).catch(() => undefined);
|
|
327
931
|
});
|
|
328
932
|
}
|
|
933
|
+
onReap(handler) {
|
|
934
|
+
this.connection.on("Reap", (request) => {
|
|
935
|
+
void handler(request).catch(() => undefined);
|
|
936
|
+
});
|
|
937
|
+
}
|
|
938
|
+
onReadinessProbe(handler) {
|
|
939
|
+
this.connection.on("ReadinessProbe", (request) => {
|
|
940
|
+
void handler(request).catch(() => undefined);
|
|
941
|
+
});
|
|
942
|
+
}
|
|
329
943
|
invoke(methodName, payload) {
|
|
330
944
|
return this.connection.invoke(methodName, payload);
|
|
331
945
|
}
|
|
@@ -333,9 +947,37 @@ export class SignalRBridgeConnection {
|
|
|
333
947
|
export function defaultBridgeServiceConfig(overrides = {}) {
|
|
334
948
|
const defaults = defaultForgeBridgeConfig();
|
|
335
949
|
const cliProviders = splitCsv(process.env.FORGE_CLI_PROVIDERS ?? "codex");
|
|
950
|
+
const sandboxHostMemoryReserveDefault = process.platform === "linux"
|
|
951
|
+
? 2 * 1024 ** 3
|
|
952
|
+
: 1024 ** 3;
|
|
953
|
+
const sandboxPublicationMode = process.env.FORGE_SANDBOX_PUBLICATION_MODE === "bridge_push" ? "bridge_push" : "shared_filesystem";
|
|
336
954
|
const config = {
|
|
337
955
|
serverUrl: process.env.FORGE_URL ?? "http://localhost:5208",
|
|
338
956
|
runnerId: process.env.FORGE_RUNNER_ID ?? "",
|
|
957
|
+
runnerInstanceId: process.env.FORGE_RUNNER_INSTANCE_ID ?? randomUUID(),
|
|
958
|
+
trustDomainId: process.env.FORGE_TRUST_DOMAIN_ID ?? "default",
|
|
959
|
+
maxConcurrentSandboxes: positiveFiniteNumber(Number(process.env.FORGE_MAX_CONCURRENT_SANDBOXES ?? 3), 3),
|
|
960
|
+
maxConcurrentSandboxBuilds: positiveFiniteNumber(Number(process.env.FORGE_MAX_CONCURRENT_SANDBOX_BUILDS ?? 1), 1),
|
|
961
|
+
sandboxPidCapacity: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_PID_CAPACITY ?? 2048), 2048),
|
|
962
|
+
dockerEnabled: process.env.FORGE_DOCKER_SANDBOX !== "false",
|
|
963
|
+
allowOpenPublicSandbox: process.env.FORGE_ALLOW_OPEN_PUBLIC_SANDBOX === "true",
|
|
964
|
+
sandboxPublicationMode,
|
|
965
|
+
dockerExecutable: process.env.FORGE_DOCKER_EXECUTABLE ?? "docker",
|
|
966
|
+
dockerContext: process.env.FORGE_DOCKER_CONTEXT ?? "",
|
|
967
|
+
sandboxStateRoot: process.env.FORGE_SANDBOX_STATE_ROOT ?? path.join(os.homedir(), ".forge", "bridge", "sandboxes"),
|
|
968
|
+
requireRootlessDocker: process.env.FORGE_REQUIRE_ROOTLESS_DOCKER === "true" || process.platform === "linux",
|
|
969
|
+
diskHighWatermarkPercent: positiveFiniteNumber(Number(process.env.FORGE_DISK_HIGH_WATERMARK_PERCENT ?? 85), 85),
|
|
970
|
+
diskLowWatermarkPercent: positiveFiniteNumber(Number(process.env.FORGE_DISK_LOW_WATERMARK_PERCENT ?? 75), 75),
|
|
971
|
+
minimumWorkspaceBytes: positiveFiniteNumber(Number(process.env.FORGE_MINIMUM_WORKSPACE_BYTES ?? 10 * 1024 ** 3), 10 * 1024 ** 3),
|
|
972
|
+
sandboxHostMemoryReserveBytes: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_HOST_MEMORY_RESERVE_BYTES ?? sandboxHostMemoryReserveDefault), sandboxHostMemoryReserveDefault),
|
|
973
|
+
defaultSandboxWorkerMemoryLimitBytes: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_DEFAULT_WORKER_MEMORY_LIMIT_BYTES ?? 2 * 1024 ** 3), 2 * 1024 ** 3),
|
|
974
|
+
defaultSandboxServiceMemoryLimitBytes: nonNegativeFiniteNumber(Number(process.env.FORGE_SANDBOX_DEFAULT_SERVICE_MEMORY_LIMIT_BYTES ?? 0), 0),
|
|
975
|
+
sandboxRuntimeOverheadBytes: Math.max(128 * 1024 ** 2, nonNegativeFiniteNumber(Number(process.env.FORGE_SANDBOX_RUNTIME_OVERHEAD_BYTES ?? 128 * 1024 ** 2), 128 * 1024 ** 2)),
|
|
976
|
+
sandboxMemorySampleIntervalMs: Math.min(30_000, Math.max(250, positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_MEMORY_SAMPLE_INTERVAL_MS ?? 1000), 1000))),
|
|
977
|
+
imageRetentionCount: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_IMAGE_RETENTION_COUNT ?? 3), 3),
|
|
978
|
+
imageRetentionTtlMs: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_IMAGE_RETENTION_TTL_MS ?? 7 * 24 * 60 * 60 * 1000), 7 * 24 * 60 * 60 * 1000),
|
|
979
|
+
auditRetentionCount: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_AUDIT_RETENTION_COUNT ?? 200), 200),
|
|
980
|
+
auditRetentionTtlMs: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_AUDIT_RETENTION_TTL_MS ?? 30 * 24 * 60 * 60 * 1000), 30 * 24 * 60 * 60 * 1000),
|
|
339
981
|
deviceToken: process.env.FORGE_DEVICE_TOKEN ?? "",
|
|
340
982
|
deviceName: process.env.FORGE_DEVICE_NAME ?? "",
|
|
341
983
|
defaultRepoRoot: process.env.FORGE_REPO_ROOT ?? defaults.defaultRepoRoot,
|
|
@@ -354,7 +996,7 @@ export function defaultBridgeServiceConfig(overrides = {}) {
|
|
|
354
996
|
supportsDraftAttachments: true,
|
|
355
997
|
supportsDraftImageAttachments: supportsDraftImageAttachmentsFor(cliProviders),
|
|
356
998
|
version: process.env.FORGE_BRIDGE_VERSION ?? forgeBridgeVersion,
|
|
357
|
-
protocolVersion:
|
|
999
|
+
protocolVersion: 2,
|
|
358
1000
|
minServerVersion: null,
|
|
359
1001
|
heartbeatIntervalMs: positiveFiniteNumber(Number(process.env.FORGE_HEARTBEAT_INTERVAL_MS ?? 30_000), 30_000),
|
|
360
1002
|
reconnectBackoffMs: [1000, 2000, 5000, 10000, 30000],
|
|
@@ -380,9 +1022,36 @@ function normalizeServerUrl(serverUrl) {
|
|
|
380
1022
|
function positiveFiniteNumber(value, fallback) {
|
|
381
1023
|
return Number.isFinite(value) && value > 0 ? value : fallback;
|
|
382
1024
|
}
|
|
1025
|
+
function nonNegativeFiniteNumber(value, fallback) {
|
|
1026
|
+
return Number.isFinite(value) && value >= 0 ? value : fallback;
|
|
1027
|
+
}
|
|
383
1028
|
function providerStatusesFor(providers) {
|
|
384
1029
|
return Object.fromEntries(providers.map((provider) => [provider, { signedIn: true, version: "", models: configuredModelsFor(provider) }]));
|
|
385
1030
|
}
|
|
1031
|
+
function sandboxAuthModesFor(config) {
|
|
1032
|
+
return Object.fromEntries(config.cliProviders.map((provider) => {
|
|
1033
|
+
const modes = ["api_key"];
|
|
1034
|
+
if (sandboxBridgeCliAuthAvailable(config, provider)) {
|
|
1035
|
+
modes.push("bridge_cli");
|
|
1036
|
+
}
|
|
1037
|
+
return [provider, modes];
|
|
1038
|
+
}));
|
|
1039
|
+
}
|
|
1040
|
+
function sandboxBridgeCliAuthAvailable(config, provider) {
|
|
1041
|
+
if (provider.trim().toLowerCase() !== "codex") {
|
|
1042
|
+
return false;
|
|
1043
|
+
}
|
|
1044
|
+
const status = providerStatus(config, provider);
|
|
1045
|
+
if (status?.signedIn === false) {
|
|
1046
|
+
return false;
|
|
1047
|
+
}
|
|
1048
|
+
return codexSandboxProviderAuthAdapter.doctor().available;
|
|
1049
|
+
}
|
|
1050
|
+
function providerStatus(config, provider) {
|
|
1051
|
+
const normalized = provider.trim().toLowerCase();
|
|
1052
|
+
return Object.entries(config.providerStatuses)
|
|
1053
|
+
.find(([key]) => key.trim().toLowerCase() === normalized)?.[1];
|
|
1054
|
+
}
|
|
386
1055
|
function defaultSandboxProfiles(providers, platform = process.platform) {
|
|
387
1056
|
if (!providers.some((provider) => provider.toLowerCase() === "codex")) {
|
|
388
1057
|
return ["none"];
|
|
@@ -477,10 +1146,11 @@ function normalizeQaAppLaunchTarget(target) {
|
|
|
477
1146
|
command: cleanString(target.command),
|
|
478
1147
|
working_directory: cleanString(target.working_directory),
|
|
479
1148
|
ready_url: cleanString(target.ready_url),
|
|
1149
|
+
managed_runtime_port: cleanString(target.managed_runtime_port),
|
|
480
1150
|
routes,
|
|
481
1151
|
notes: cleanString(target.notes)
|
|
482
1152
|
};
|
|
483
|
-
return normalized.url || normalized.command || normalized.ready_url ||
|
|
1153
|
+
return normalized.url || normalized.command || normalized.ready_url || normalized.managed_runtime_port
|
|
484
1154
|
? normalized
|
|
485
1155
|
: null;
|
|
486
1156
|
}
|
|
@@ -546,6 +1216,262 @@ function abortPromise(signal) {
|
|
|
546
1216
|
}
|
|
547
1217
|
return new Promise((resolve) => signal.addEventListener("abort", () => resolve(), { once: true }));
|
|
548
1218
|
}
|
|
1219
|
+
export function canReapManagedSandboxes(preflight) {
|
|
1220
|
+
return canRunDockerMaintenance(preflight);
|
|
1221
|
+
}
|
|
1222
|
+
function sandboxNetworkPolicyFailure(dispatch, config) {
|
|
1223
|
+
const sandbox = dispatch.sandbox;
|
|
1224
|
+
if (!sandbox)
|
|
1225
|
+
return "Protocol-v2 sandbox context is missing.";
|
|
1226
|
+
if (sandbox.networkPolicy.mode === "open_public"
|
|
1227
|
+
&& (sandbox.executionPolicy === "sandbox_required" || !config.allowOpenPublicSandbox)) {
|
|
1228
|
+
return "open_public sandbox networking is not permitted by this runner policy.";
|
|
1229
|
+
}
|
|
1230
|
+
if (sandbox.networkPolicy.mode === "public_allowlist"
|
|
1231
|
+
&& sandbox.networkPolicy.allowedHosts.length > 0
|
|
1232
|
+
&& process.env.FORGE_SANDBOX_EGRESS_ENFORCED !== "true") {
|
|
1233
|
+
return "public_allowlist requires an operator-enforced Forge egress proxy or firewall.";
|
|
1234
|
+
}
|
|
1235
|
+
return null;
|
|
1236
|
+
}
|
|
1237
|
+
function infrastructureFailureEventType(summary) {
|
|
1238
|
+
if (summary.startsWith("worktree_prepare_failed:"))
|
|
1239
|
+
return "worktree_prepare_failed";
|
|
1240
|
+
if (summary.startsWith("provider_start_unavailable:"))
|
|
1241
|
+
return "provider_start_unavailable";
|
|
1242
|
+
if (summary.startsWith("host_port_collision:"))
|
|
1243
|
+
return "host_port_collision";
|
|
1244
|
+
if (summary.startsWith("host_runtime_process_exited:"))
|
|
1245
|
+
return "host_runtime_process_exited";
|
|
1246
|
+
if (summary.startsWith("host_runtime_listener_timeout:"))
|
|
1247
|
+
return "host_runtime_listener_timeout";
|
|
1248
|
+
if (summary.startsWith("host_runtime_http_readiness_timeout:"))
|
|
1249
|
+
return "host_runtime_http_readiness_timeout";
|
|
1250
|
+
if (summary.startsWith("attempt_process_supervisor_start_failed:"))
|
|
1251
|
+
return "attempt_process_supervisor_start_failed";
|
|
1252
|
+
if (summary.startsWith("attempt_process_supervisor_start_timeout:"))
|
|
1253
|
+
return "attempt_process_supervisor_start_timeout";
|
|
1254
|
+
return null;
|
|
1255
|
+
}
|
|
1256
|
+
function parseReadinessProbe(value) {
|
|
1257
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
1258
|
+
return null;
|
|
1259
|
+
const request = value;
|
|
1260
|
+
const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
|
|
1261
|
+
const bounded = (candidate, maximum) => typeof candidate === "string" && candidate.trim().length > 0 && candidate.length <= maximum;
|
|
1262
|
+
if (request.schemaVersion !== 1
|
|
1263
|
+
|| !uuid.test(request.requestId ?? "")
|
|
1264
|
+
|| !uuid.test(request.fencingToken ?? "")
|
|
1265
|
+
|| !uuid.test(request.userId ?? "")
|
|
1266
|
+
|| !uuid.test(request.projectId ?? "")
|
|
1267
|
+
|| !uuid.test(request.repoId ?? "")
|
|
1268
|
+
|| !uuid.test(request.runnerInstanceId ?? "")
|
|
1269
|
+
|| !Number.isInteger(request.attempt) || (request.attempt ?? 0) < 1
|
|
1270
|
+
|| !bounded(request.repoName, 160)
|
|
1271
|
+
|| typeof request.remoteUrl !== "string" || request.remoteUrl.length > 2048
|
|
1272
|
+
|| !bounded(request.defaultBranch, 512)
|
|
1273
|
+
|| typeof request.cloneOnDemand !== "boolean"
|
|
1274
|
+
|| !bounded(request.runnerRepoLocalPath, 4096)
|
|
1275
|
+
|| !bounded(request.revision, 512)
|
|
1276
|
+
|| !Number.isFinite(Date.parse(request.expiresAt ?? ""))
|
|
1277
|
+
|| Date.parse(request.expiresAt ?? "") <= Date.now()) {
|
|
1278
|
+
return null;
|
|
1279
|
+
}
|
|
1280
|
+
return request;
|
|
1281
|
+
}
|
|
1282
|
+
function sandboxProviderNetworkFailure(dispatch, config) {
|
|
1283
|
+
const sandbox = dispatch.sandbox;
|
|
1284
|
+
if (!sandbox || sandbox.networkPolicy.mode !== "offline")
|
|
1285
|
+
return null;
|
|
1286
|
+
const provider = dispatch.execution.provider.trim().toLowerCase();
|
|
1287
|
+
const isNetworkProvider = config.cliProviders.some(candidate => candidate.trim().toLowerCase() === provider);
|
|
1288
|
+
if (!isNetworkProvider)
|
|
1289
|
+
return null;
|
|
1290
|
+
return `Provider ${dispatch.execution.provider} requires outbound network access, but this Docker attempt uses the offline network policy.`;
|
|
1291
|
+
}
|
|
1292
|
+
function sandboxAuthFailure(dispatch, config) {
|
|
1293
|
+
const unsupported = sandboxProviderAuthUnavailableMessage(dispatch.execution.provider, dispatch.execution.authKind);
|
|
1294
|
+
if (unsupported) {
|
|
1295
|
+
return unsupported;
|
|
1296
|
+
}
|
|
1297
|
+
if (dispatch.execution.authKind.trim().toLowerCase() !== "bridge_cli") {
|
|
1298
|
+
return null;
|
|
1299
|
+
}
|
|
1300
|
+
const status = providerStatus(config, dispatch.execution.provider);
|
|
1301
|
+
if (status?.signedIn === false) {
|
|
1302
|
+
return `${dispatch.execution.provider} Local CLI is not signed in on this runner.`;
|
|
1303
|
+
}
|
|
1304
|
+
const adapter = sandboxProviderAuthAdapterFor(dispatch.execution.provider, dispatch.execution.authKind);
|
|
1305
|
+
if (!adapter) {
|
|
1306
|
+
return `${dispatch.execution.provider} Local CLI auth is not portable into Docker yet.`;
|
|
1307
|
+
}
|
|
1308
|
+
const doctor = adapter.doctor();
|
|
1309
|
+
return doctor.available ? null : doctor.message;
|
|
1310
|
+
}
|
|
1311
|
+
export function canReserveSandboxResources(used, requested, capacity) {
|
|
1312
|
+
return used.sandboxSlots + requested.sandboxSlots <= capacity.sandboxSlots
|
|
1313
|
+
&& used.cpuMillis + requested.cpuMillis <= capacity.cpuMillis
|
|
1314
|
+
&& used.memoryBytes + requested.memoryBytes <= capacity.memoryBytes
|
|
1315
|
+
&& used.workspaceBytes + requested.workspaceBytes <= capacity.workspaceBytes
|
|
1316
|
+
&& used.pids + requested.pids <= capacity.pids
|
|
1317
|
+
&& used.buildSlots + requested.buildSlots <= capacity.buildSlots;
|
|
1318
|
+
}
|
|
1319
|
+
export function sandboxCapacityFailureMessage(used, requested, capacity, preflight, config) {
|
|
1320
|
+
const availableSlots = Math.max(0, capacity.sandboxSlots - used.sandboxSlots);
|
|
1321
|
+
if (requested.sandboxSlots > availableSlots) {
|
|
1322
|
+
return `Sandbox slot capacity is insufficient: requested ${requested.sandboxSlots}, available ${availableSlots}, ${used.sandboxSlots} already reserved.`;
|
|
1323
|
+
}
|
|
1324
|
+
const availableMemoryBytes = Math.max(0, capacity.memoryBytes - used.memoryBytes);
|
|
1325
|
+
if (requested.memoryBytes > availableMemoryBytes) {
|
|
1326
|
+
return [
|
|
1327
|
+
`Sandbox memory capacity is insufficient: requested ${formatBytes(requested.memoryBytes)} total`,
|
|
1328
|
+
`(worker ${formatBytes(requested.workerMemoryLimitBytes)}, services ${formatBytes(requested.serviceMemoryLimitBytes)}, runtime overhead ${formatBytes(requested.runtimeOverheadBytes)}),`,
|
|
1329
|
+
`but ${formatBytes(availableMemoryBytes)} is available for admission.`,
|
|
1330
|
+
`Docker reports ${formatBytes(preflight?.availableMemoryBytes ?? 0)} available of ${formatBytes(preflight?.totalMemoryBytes ?? 0)};`,
|
|
1331
|
+
`Forge reserves ${formatBytes(preflight?.hostReserveBytes ?? config.sandboxHostMemoryReserveBytes ?? 0)} for the host and tracks ${formatBytes(used.memoryBytes)} in active or reserved attempts.`,
|
|
1332
|
+
].join(" ");
|
|
1333
|
+
}
|
|
1334
|
+
const availableCpuMillis = Math.max(0, capacity.cpuMillis - used.cpuMillis);
|
|
1335
|
+
if (requested.cpuMillis > availableCpuMillis) {
|
|
1336
|
+
return `Sandbox CPU capacity is insufficient: requested ${requested.cpuMillis} millis, available ${availableCpuMillis} millis, ${used.cpuMillis} millis already reserved.`;
|
|
1337
|
+
}
|
|
1338
|
+
const availableWorkspaceBytes = Math.max(0, capacity.workspaceBytes - used.workspaceBytes);
|
|
1339
|
+
if (requested.workspaceBytes > availableWorkspaceBytes) {
|
|
1340
|
+
return `Sandbox workspace capacity is insufficient: requested ${formatBytes(requested.workspaceBytes)}, available ${formatBytes(availableWorkspaceBytes)}, ${formatBytes(used.workspaceBytes)} already reserved.`;
|
|
1341
|
+
}
|
|
1342
|
+
const availablePids = Math.max(0, capacity.pids - used.pids);
|
|
1343
|
+
if (requested.pids > availablePids) {
|
|
1344
|
+
return `Sandbox PID capacity is insufficient: requested ${requested.pids}, available ${availablePids}, ${used.pids} already reserved.`;
|
|
1345
|
+
}
|
|
1346
|
+
const availableBuildSlots = Math.max(0, capacity.buildSlots - used.buildSlots);
|
|
1347
|
+
return `Sandbox build capacity is insufficient: requested ${requested.buildSlots}, available ${availableBuildSlots}, ${used.buildSlots} already reserved.`;
|
|
1348
|
+
}
|
|
1349
|
+
function formatBytes(bytes) {
|
|
1350
|
+
if (!Number.isFinite(bytes) || bytes <= 0)
|
|
1351
|
+
return "0 B";
|
|
1352
|
+
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
1353
|
+
const power = Math.min(units.length - 1, Math.floor(Math.log(bytes) / Math.log(1024)));
|
|
1354
|
+
const value = bytes / 1024 ** power;
|
|
1355
|
+
return `${value >= 10 || power === 0 ? value.toFixed(0) : value.toFixed(2)} ${units[power]}`;
|
|
1356
|
+
}
|
|
1357
|
+
function sumSandboxResources(resources) {
|
|
1358
|
+
return resources.reduce((total, resource) => ({
|
|
1359
|
+
sandboxSlots: total.sandboxSlots + resource.sandboxSlots,
|
|
1360
|
+
cpuMillis: total.cpuMillis + resource.cpuMillis,
|
|
1361
|
+
workerMemoryLimitBytes: total.workerMemoryLimitBytes + resource.workerMemoryLimitBytes,
|
|
1362
|
+
serviceMemoryLimitBytes: total.serviceMemoryLimitBytes + resource.serviceMemoryLimitBytes,
|
|
1363
|
+
runtimeOverheadBytes: total.runtimeOverheadBytes + resource.runtimeOverheadBytes,
|
|
1364
|
+
memoryBytes: total.memoryBytes + resource.memoryBytes,
|
|
1365
|
+
workspaceBytes: total.workspaceBytes + resource.workspaceBytes,
|
|
1366
|
+
pids: total.pids + resource.pids,
|
|
1367
|
+
buildSlots: total.buildSlots + resource.buildSlots
|
|
1368
|
+
}), zeroSandboxResources());
|
|
1369
|
+
}
|
|
1370
|
+
function zeroSandboxResources() {
|
|
1371
|
+
return {
|
|
1372
|
+
sandboxSlots: 0,
|
|
1373
|
+
cpuMillis: 0,
|
|
1374
|
+
workerMemoryLimitBytes: 0,
|
|
1375
|
+
serviceMemoryLimitBytes: 0,
|
|
1376
|
+
runtimeOverheadBytes: 0,
|
|
1377
|
+
memoryBytes: 0,
|
|
1378
|
+
workspaceBytes: 0,
|
|
1379
|
+
pids: 0,
|
|
1380
|
+
buildSlots: 0,
|
|
1381
|
+
};
|
|
1382
|
+
}
|
|
1383
|
+
function defaultSandboxResources() {
|
|
1384
|
+
return {
|
|
1385
|
+
sandboxSlots: 1,
|
|
1386
|
+
cpuMillis: 2000,
|
|
1387
|
+
workerMemoryLimitBytes: 2 * 1024 ** 3,
|
|
1388
|
+
serviceMemoryLimitBytes: 0,
|
|
1389
|
+
runtimeOverheadBytes: 128 * 1024 ** 2,
|
|
1390
|
+
memoryBytes: 2 * 1024 ** 3 + 128 * 1024 ** 2,
|
|
1391
|
+
workspaceBytes: 10 * 1024 ** 3,
|
|
1392
|
+
pids: 512,
|
|
1393
|
+
buildSlots: 0
|
|
1394
|
+
};
|
|
1395
|
+
}
|
|
1396
|
+
export function resolveDynamicSandboxResources(requested, serviceCount, networkMode, baseRuntimeOverheadBytes = 128 * 1024 ** 2) {
|
|
1397
|
+
const serviceMemoryLimitBytes = Math.max(0, serviceCount) * 1024 ** 3;
|
|
1398
|
+
const runtimeOverheadBytes = Math.max(128 * 1024 ** 2, baseRuntimeOverheadBytes)
|
|
1399
|
+
+ (networkMode === "public_allowlist" ? 256 * 1024 ** 2 : 0);
|
|
1400
|
+
return {
|
|
1401
|
+
...requested,
|
|
1402
|
+
serviceMemoryLimitBytes,
|
|
1403
|
+
runtimeOverheadBytes,
|
|
1404
|
+
memoryBytes: requested.workerMemoryLimitBytes + serviceMemoryLimitBytes + runtimeOverheadBytes,
|
|
1405
|
+
};
|
|
1406
|
+
}
|
|
1407
|
+
export async function resolveSandboxResourcesFromRepo(dispatch, baseRuntimeOverheadBytes = 128 * 1024 ** 2) {
|
|
1408
|
+
const repoPath = dispatch.repo.serverLocalPath;
|
|
1409
|
+
const sandbox = dispatch.sandbox;
|
|
1410
|
+
if (!repoPath || !sandbox) {
|
|
1411
|
+
throw new Error("Sandbox admission needs a mapped repository.");
|
|
1412
|
+
}
|
|
1413
|
+
const selection = selectSandboxConfigRefs(sandbox.executionPolicy, sandbox.sandboxConfigCommitSha, dispatch.repo.defaultBranch);
|
|
1414
|
+
const configCommitSha = await resolveFirstGitCommit(repoPath, selection.refs);
|
|
1415
|
+
if (!configCommitSha) {
|
|
1416
|
+
throw new Error("Sandbox admission could not resolve the immutable sandbox config commit.");
|
|
1417
|
+
}
|
|
1418
|
+
const contractResult = await runSandboxProcess({
|
|
1419
|
+
command: "git",
|
|
1420
|
+
args: ["-C", repoPath, "show", `${configCommitSha}:.forge/forge.sandbox.yml`],
|
|
1421
|
+
timeoutMs: 30_000,
|
|
1422
|
+
maxOutputBytes: 1024 * 1024,
|
|
1423
|
+
});
|
|
1424
|
+
if (contractResult.exitCode !== 0) {
|
|
1425
|
+
throw new Error("The approved immutable sandbox contract is unavailable in the mapped repository.");
|
|
1426
|
+
}
|
|
1427
|
+
const contract = validateSandboxContract(contractResult.stdout);
|
|
1428
|
+
return {
|
|
1429
|
+
resources: resolveDynamicSandboxResources(sandbox.requestedResources, contract.contract.services.length, sandbox.networkPolicy.mode, baseRuntimeOverheadBytes),
|
|
1430
|
+
sandboxConfigCommitSha: configCommitSha,
|
|
1431
|
+
};
|
|
1432
|
+
}
|
|
1433
|
+
async function resolveFirstGitCommit(repoPath, refs) {
|
|
1434
|
+
for (const ref of refs) {
|
|
1435
|
+
const result = await runSandboxProcess({
|
|
1436
|
+
command: "git",
|
|
1437
|
+
args: ["-C", repoPath, "rev-parse", "--verify", `${ref}^{commit}`],
|
|
1438
|
+
timeoutMs: 30_000,
|
|
1439
|
+
maxOutputBytes: 1024 * 1024,
|
|
1440
|
+
});
|
|
1441
|
+
const commitSha = result.stdout.trim().toLowerCase();
|
|
1442
|
+
if (result.exitCode === 0 && /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/u.test(commitSha)) {
|
|
1443
|
+
return commitSha;
|
|
1444
|
+
}
|
|
1445
|
+
}
|
|
1446
|
+
return null;
|
|
1447
|
+
}
|
|
1448
|
+
function parseOfferEnvelope(value) {
|
|
1449
|
+
return parseDispatchEnvelope(value);
|
|
1450
|
+
}
|
|
1451
|
+
function parseStartEnvelope(value) {
|
|
1452
|
+
return parseDispatchEnvelope(value);
|
|
1453
|
+
}
|
|
1454
|
+
function parseDispatchEnvelope(value) {
|
|
1455
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
1456
|
+
return null;
|
|
1457
|
+
const envelope = value;
|
|
1458
|
+
if (envelope.schemaVersion !== 1
|
|
1459
|
+
|| typeof envelope.dispatchAttemptId !== "string"
|
|
1460
|
+
|| typeof envelope.leaseId !== "string"
|
|
1461
|
+
|| typeof envelope.fencingToken !== "string"
|
|
1462
|
+
|| typeof envelope.runnerInstanceId !== "string"
|
|
1463
|
+
|| typeof envelope.expiresAt !== "string"
|
|
1464
|
+
|| !envelope.dispatch)
|
|
1465
|
+
return null;
|
|
1466
|
+
return {
|
|
1467
|
+
dispatchAttemptId: envelope.dispatchAttemptId,
|
|
1468
|
+
leaseId: envelope.leaseId,
|
|
1469
|
+
fencingToken: envelope.fencingToken,
|
|
1470
|
+
runnerInstanceId: envelope.runnerInstanceId,
|
|
1471
|
+
expiresAt: envelope.expiresAt,
|
|
1472
|
+
dispatch: envelope.dispatch
|
|
1473
|
+
};
|
|
1474
|
+
}
|
|
549
1475
|
const maxAttachmentExactSecretLength = 8 * 1024;
|
|
550
1476
|
function exactSecrets(dispatch, config) {
|
|
551
1477
|
return [
|