@loucompanion/forge-bridge 0.1.3-dev.b3bce6ba65a6 → 0.1.3-dev.ba293ccf876b
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +127 -0
- package/dist/bridge/bridge.base.d.ts +8 -1
- package/dist/bridge/bridge.service.d.ts +98 -3
- package/dist/bridge/bridge.service.js +1425 -45
- package/dist/bridge/bridge.types.d.ts +205 -1
- package/dist/bridge/readiness-result-outbox.d.ts +21 -0
- package/dist/bridge/readiness-result-outbox.js +54 -0
- package/dist/bridge/readiness-result-worker.d.ts +2 -0
- package/dist/bridge/readiness-result-worker.js +100 -0
- package/dist/bridge/runner-host-capabilities.d.ts +15 -0
- package/dist/bridge/runner-host-capabilities.js +93 -0
- package/dist/cleanup/cleanup.service.d.ts +14 -2
- package/dist/cleanup/cleanup.service.js +496 -3
- package/dist/cleanup/cleanup.types.d.ts +10 -0
- package/dist/cleanup/index.d.ts +1 -1
- package/dist/cleanup/index.js +1 -1
- package/dist/cleanup/internals.d.ts +2 -2
- package/dist/cleanup/internals.js +1 -1
- package/dist/cli/cli.help.js +47 -1
- package/dist/cli/cli.service.d.ts +45 -2
- package/dist/cli/cli.service.js +377 -27
- package/dist/cli/local-config.d.ts +5 -0
- package/dist/cli/local-config.js +45 -1
- package/dist/runtime/aspnet-runtime-prebuild.d.ts +36 -0
- package/dist/runtime/aspnet-runtime-prebuild.js +340 -0
- package/dist/runtime/attempt-node-runtime-launcher.d.ts +17 -0
- package/dist/runtime/attempt-node-runtime-launcher.js +105 -0
- package/dist/runtime/attempt-path-operation.d.ts +21 -0
- package/dist/runtime/attempt-path-operation.js +94 -0
- package/dist/runtime/attempt-private-directory.d.ts +6 -0
- package/dist/runtime/attempt-private-directory.js +54 -0
- package/dist/runtime/attempt-process-supervisor-registry.d.ts +20 -0
- package/dist/runtime/attempt-process-supervisor-registry.js +64 -0
- package/dist/runtime/attempt-process-supervisor.d.ts +97 -0
- package/dist/runtime/attempt-process-supervisor.js +386 -0
- package/dist/runtime/canonical-source-file.d.ts +3 -0
- package/dist/runtime/canonical-source-file.js +54 -0
- package/dist/runtime/docker-engine-probe.d.ts +83 -0
- package/dist/runtime/docker-engine-probe.js +300 -0
- package/dist/runtime/generated-output-reset.d.ts +10 -0
- package/dist/runtime/generated-output-reset.js +103 -0
- package/dist/runtime/host-diagnostics.d.ts +4 -0
- package/dist/runtime/host-diagnostics.js +95 -0
- package/dist/runtime/host-listener-inspector.d.ts +46 -0
- package/dist/runtime/host-listener-inspector.js +349 -0
- package/dist/runtime/host-port-broker.d.ts +76 -0
- package/dist/runtime/host-port-broker.js +333 -0
- package/dist/runtime/host-runtime-contract.d.ts +56 -0
- package/dist/runtime/host-runtime-contract.js +709 -0
- package/dist/runtime/host-runtime-lease.d.ts +201 -0
- package/dist/runtime/host-runtime-lease.js +2314 -0
- package/dist/runtime/host-runtime-promotion.d.ts +32 -0
- package/dist/runtime/host-runtime-promotion.js +206 -0
- package/dist/runtime/memory-capacity.d.ts +68 -0
- package/dist/runtime/memory-capacity.js +121 -0
- package/dist/runtime/node-host-runtime-adapter.d.ts +54 -0
- package/dist/runtime/node-host-runtime-adapter.js +284 -0
- package/dist/runtime/pinned-directory-process.d.ts +4 -0
- package/dist/runtime/pinned-directory-process.js +20 -0
- package/dist/runtime/pinned-source-directory.d.ts +9 -0
- package/dist/runtime/pinned-source-directory.js +56 -0
- package/dist/runtime/playwright-host-runtime-adapter.d.ts +21 -0
- package/dist/runtime/playwright-host-runtime-adapter.js +268 -0
- package/dist/runtime/prepared-runtime-identity.d.ts +3 -0
- package/dist/runtime/prepared-runtime-identity.js +191 -0
- package/dist/runtime/prepared-runtime-readiness.d.ts +22 -0
- package/dist/runtime/prepared-runtime-readiness.js +191 -0
- package/dist/runtime/runtime-isolation-lease.d.ts +67 -0
- package/dist/runtime/runtime-isolation-lease.js +158 -0
- package/dist/runtime/workspace-resource-estimator.d.ts +17 -0
- package/dist/runtime/workspace-resource-estimator.js +174 -0
- package/dist/sandbox/command-runner.d.ts +4 -0
- package/dist/sandbox/command-runner.js +47 -0
- package/dist/sandbox/contract.d.ts +5 -0
- package/dist/sandbox/contract.js +276 -0
- package/dist/sandbox/event-budget.d.ts +11 -0
- package/dist/sandbox/event-budget.js +27 -0
- package/dist/sandbox/image-policy.d.ts +35 -0
- package/dist/sandbox/image-policy.js +247 -0
- package/dist/sandbox/index.d.ts +5 -0
- package/dist/sandbox/index.js +5 -0
- package/dist/sandbox/lease-watchdog.d.ts +18 -0
- package/dist/sandbox/lease-watchdog.js +67 -0
- package/dist/sandbox/preflight.d.ts +84 -0
- package/dist/sandbox/preflight.js +521 -0
- package/dist/sandbox/process.d.ts +20 -0
- package/dist/sandbox/process.js +209 -0
- package/dist/sandbox/provider-auth/codex.d.ts +26 -0
- package/dist/sandbox/provider-auth/codex.js +212 -0
- package/dist/sandbox/repository-lock.d.ts +5 -0
- package/dist/sandbox/repository-lock.js +121 -0
- package/dist/sandbox/runtime.d.ts +297 -0
- package/dist/sandbox/runtime.js +2338 -0
- package/dist/sandbox/sandbox.types.d.ts +93 -0
- package/dist/sandbox/sandbox.types.js +1 -0
- package/dist/sandbox/smoke.d.ts +76 -0
- package/dist/sandbox/smoke.js +380 -0
- package/dist/sandbox/worker-supervisor.d.ts +13 -0
- package/dist/sandbox/worker-supervisor.js +36 -0
- package/dist/sandbox/worker.d.ts +1 -0
- package/dist/sandbox/worker.js +225 -0
- package/dist/sandbox/workspace.service.d.ts +13 -0
- package/dist/sandbox/workspace.service.js +72 -0
- package/dist/session/managed-attempt-tools.d.ts +21 -0
- package/dist/session/managed-attempt-tools.js +912 -0
- package/dist/session/managed-build-coordinator.d.ts +5 -0
- package/dist/session/managed-build-coordinator.js +105 -0
- package/dist/session/managed-build-receipt.d.ts +21 -0
- package/dist/session/managed-build-receipt.js +335 -0
- package/dist/session/phase-completion.artifacts.d.ts +17 -0
- package/dist/session/phase-completion.artifacts.js +256 -26
- package/dist/session/phase-completion.d.ts +12 -2
- package/dist/session/phase-completion.helpers.d.ts +5 -0
- package/dist/session/phase-completion.helpers.js +51 -10
- package/dist/session/phase-completion.js +14 -3
- package/dist/session/phase-completion.prompts.d.ts +3 -3
- package/dist/session/phase-completion.prompts.js +24 -8
- package/dist/session/phase-completion.runtime.d.ts +13 -1
- package/dist/session/phase-completion.runtime.js +189 -8
- package/dist/session/phase-completion.schemas.js +117 -4
- package/dist/session/phase-completion.validation.d.ts +4 -1
- package/dist/session/phase-completion.validation.js +153 -22
- package/dist/session/pre-publication-validation.d.ts +47 -0
- package/dist/session/pre-publication-validation.js +249 -0
- package/dist/session/provider-cli/ci-repair-readiness.d.ts +16 -0
- package/dist/session/provider-cli/ci-repair-readiness.js +110 -0
- package/dist/session/provider-cli/provider-cli.artifacts.d.ts +3 -2
- package/dist/session/provider-cli/provider-cli.artifacts.js +65 -7
- package/dist/session/provider-cli/provider-cli.base.d.ts +1 -0
- package/dist/session/provider-cli/provider-cli.deterministic.d.ts +2 -0
- package/dist/session/provider-cli/provider-cli.deterministic.js +68 -9
- package/dist/session/provider-cli/provider-cli.process.d.ts +26 -0
- package/dist/session/provider-cli/provider-cli.process.js +474 -23
- package/dist/session/provider-cli/provider-cli.prompts.d.ts +14 -2
- package/dist/session/provider-cli/provider-cli.prompts.js +147 -25
- package/dist/session/provider-cli/provider-cli.qa-sidecars.js +1 -0
- package/dist/session/provider-cli/provider-cli.repair-epoch.d.ts +6 -0
- package/dist/session/provider-cli/provider-cli.repair-epoch.js +11 -0
- package/dist/session/provider-cli/provider-cli.runtime.d.ts +1 -1
- package/dist/session/provider-cli/provider-cli.runtime.js +6 -3
- package/dist/session/provider-cli/provider-cli.service.d.ts +25 -2
- package/dist/session/provider-cli/provider-cli.service.js +636 -84
- package/dist/session/provider-cli/provider-cli.types.d.ts +5 -0
- package/dist/session/provider-cli/provider-output-state.d.ts +2 -0
- package/dist/session/provider-cli/provider-output-state.js +91 -16
- package/dist/session/provider-cli/provider-usage.d.ts +7 -0
- package/dist/session/provider-cli/provider-usage.js +28 -5
- package/dist/session/session-redaction.d.ts +3 -0
- package/dist/session/session-redaction.js +38 -0
- package/dist/session/session.base.d.ts +6 -1
- package/dist/session/session.service.d.ts +8 -2
- package/dist/session/session.service.js +531 -42
- package/dist/session/session.types.d.ts +115 -1
- package/dist/session/session.types.js +194 -3
- package/dist/shared/format/format-bytes.d.ts +1 -0
- package/dist/shared/format/format-bytes.js +9 -0
- package/dist/shared/git/git.js +2 -1
- package/dist/shared/path/path.d.ts +1 -0
- package/dist/shared/path/path.js +21 -1
- package/dist/shared/process/dotnet-toolchain.d.ts +14 -0
- package/dist/shared/process/dotnet-toolchain.js +179 -0
- package/dist/shared/process/host-process-recovery.d.ts +12 -0
- package/dist/shared/process/host-process-recovery.js +78 -0
- package/dist/shared/process/host-process-supervisor.d.ts +59 -0
- package/dist/shared/process/host-process-supervisor.js +330 -0
- package/dist/shared/process/process.d.ts +2 -0
- package/dist/shared/process/process.js +11 -6
- package/dist/shared/provider-capabilities.d.ts +1 -0
- package/dist/shared/provider-capabilities.js +3 -0
- package/dist/shared/redaction/redaction.js +12 -2
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/worktree/artifact-injector.d.ts +3 -2
- package/dist/worktree/artifact-injector.js +43 -4
- package/dist/worktree/dependency-warmer.d.ts +28 -1
- package/dist/worktree/dependency-warmer.js +308 -9
- package/dist/worktree/index.d.ts +2 -1
- package/dist/worktree/index.js +1 -0
- package/dist/worktree/prior-phase-artifact-hydrator.d.ts +2 -0
- package/dist/worktree/prior-phase-artifact-hydrator.js +136 -0
- package/dist/worktree/worktree-paths.d.ts +4 -0
- package/dist/worktree/worktree-paths.js +30 -0
- package/dist/worktree/worktree.base.d.ts +3 -0
- package/dist/worktree/worktree.service.d.ts +14 -2
- package/dist/worktree/worktree.service.js +296 -47
- package/dist/worktree/worktree.types.d.ts +5 -1
- package/package.json +8 -4
- package/runtime/windows-job-supervisor.ps1 +444 -0
- package/runtime/windows-listener-owner.ps1 +33 -0
|
@@ -1,47 +1,214 @@
|
|
|
1
1
|
import { HubConnectionBuilder, HttpTransportType, LogLevel } from "@microsoft/signalr";
|
|
2
2
|
import fs from "node:fs";
|
|
3
3
|
import path from "node:path";
|
|
4
|
+
import { randomUUID } from "node:crypto";
|
|
5
|
+
import os from "node:os";
|
|
4
6
|
import { defaultForgeBridgeConfig } from "../config.js";
|
|
5
|
-
import {
|
|
7
|
+
import { supportsDraftImageAttachmentsFor } from "../shared/provider-capabilities.js";
|
|
8
|
+
import { redactText } from "../shared/redaction/redaction.js";
|
|
6
9
|
import { BridgeSessionService } from "../session/session.service.js";
|
|
10
|
+
import { ProcessBackedProviderCliService, RoutingProviderCliService, StreamingProviderProcessRunner, } from "../session/provider-cli/provider-cli.service.js";
|
|
11
|
+
import { redactRunnerDispatchEvent, redactRunOnceResultPacket } from "../session/session-redaction.js";
|
|
12
|
+
import { HostRuntimeLeaseManager } from "../runtime/host-runtime-lease.js";
|
|
13
|
+
import { buildMemoryAdmissionDiagnostics } from "../runtime/memory-capacity.js";
|
|
14
|
+
import { DockerEngineProbe } from "../runtime/docker-engine-probe.js";
|
|
15
|
+
import { estimateWorkspaceBytesFromRepo } from "../runtime/workspace-resource-estimator.js";
|
|
16
|
+
import { GitWorktreeService } from "../worktree/worktree.service.js";
|
|
17
|
+
import { RepositoryCoordinatorLock } from "../sandbox/repository-lock.js";
|
|
18
|
+
import { AttemptCoordinatedProviderProcessRunner, AttemptProcessSupervisorRegistry, } from "../runtime/attempt-process-supervisor-registry.js";
|
|
7
19
|
import { failureResult, parseRunOnceDispatchPacket } from "../session/session.types.js";
|
|
8
20
|
import { forgeBridgeVersion } from "../version.js";
|
|
21
|
+
import { canRunDockerMaintenance, DockerPreflight } from "../sandbox/preflight.js";
|
|
22
|
+
import { validateSandboxContract } from "../sandbox/contract.js";
|
|
23
|
+
import { runSandboxProcess } from "../sandbox/process.js";
|
|
24
|
+
import { DockerSandboxRuntime, selectSandboxConfigRefs } from "../sandbox/runtime.js";
|
|
25
|
+
import { codexSandboxProviderAuthAdapter, sandboxProviderAuthAdapterFor, sandboxProviderAuthUnavailableMessage } from "../sandbox/provider-auth/codex.js";
|
|
26
|
+
import { formatBytes } from "../shared/format/format-bytes.js";
|
|
27
|
+
import { emptyRunnerHostCapabilitySnapshot, ProcessRunnerHostCapabilityProbe, } from "./runner-host-capabilities.js";
|
|
28
|
+
import { FileReadinessResultOutbox } from "./readiness-result-outbox.js";
|
|
29
|
+
/**
|
|
30
|
+
* Protocol-v2 events are acknowledged one at a time before runOnce returns.
|
|
31
|
+
* Sending the complete transcript again in DispatchResult made long QA and CI
|
|
32
|
+
* sessions exceed SignalR's receive limit, disconnecting the bridge after the
|
|
33
|
+
* provider had already completed. The API combines its acknowledged streamed
|
|
34
|
+
* events with the terminal result, so the callback only needs the outcome and
|
|
35
|
+
* artifacts. Protocol v1 retains the transcript for legacy servers.
|
|
36
|
+
*/
|
|
37
|
+
export function compactTerminalResult(dispatch, result) {
|
|
38
|
+
return dispatch.schemaVersion >= 2 && result.events.length > 0
|
|
39
|
+
? { ...result, events: [] }
|
|
40
|
+
: result;
|
|
41
|
+
}
|
|
9
42
|
export class ForgeBridgeService {
|
|
10
43
|
config;
|
|
11
|
-
sessions;
|
|
12
44
|
connections;
|
|
45
|
+
hostCapabilityProbe;
|
|
13
46
|
connection = null;
|
|
14
47
|
heartbeatTimer = null;
|
|
48
|
+
heartbeatInFlight = false;
|
|
49
|
+
consecutiveHeartbeatFailures = 0;
|
|
50
|
+
reconnectingFromHeartbeatFailure = false;
|
|
51
|
+
leaseWatchdogTimer = null;
|
|
52
|
+
sandboxMaintenanceTimer = null;
|
|
53
|
+
sandboxMaintenanceRunning = false;
|
|
15
54
|
closed = null;
|
|
55
|
+
transportRecovery = null;
|
|
56
|
+
transportRecoveryGeneration = 0;
|
|
16
57
|
stopped = false;
|
|
17
58
|
activeSessions = new Map();
|
|
18
|
-
|
|
59
|
+
reservations = new Map();
|
|
60
|
+
activeReadinessProbes = new Map();
|
|
61
|
+
readinessResultOutbox;
|
|
62
|
+
readinessResultFlush = null;
|
|
63
|
+
preflight;
|
|
64
|
+
dockerEngineProbe;
|
|
65
|
+
sandboxRuntime;
|
|
66
|
+
sandboxResourceResolver;
|
|
67
|
+
sessions;
|
|
68
|
+
sandboxAmbientMemoryCommitmentBytes = null;
|
|
69
|
+
hostCapabilities = emptyRunnerHostCapabilitySnapshot();
|
|
70
|
+
hostCapabilitiesDetected = false;
|
|
71
|
+
constructor(config, sessions = undefined, connections = new SignalRBridgeConnectionFactory(), preflight, sandboxRuntime, sandboxResourceResolver, dockerEngineProbe, hostCapabilityProbe = new ProcessRunnerHostCapabilityProbe(), readinessResultOutbox) {
|
|
19
72
|
this.config = config;
|
|
20
|
-
this.sessions = sessions;
|
|
21
73
|
this.connections = connections;
|
|
74
|
+
this.hostCapabilityProbe = hostCapabilityProbe;
|
|
75
|
+
this.sessions = sessions ?? createDefaultSessionService(config.sandboxStateRoot);
|
|
76
|
+
this.preflight = preflight ?? new DockerPreflight({
|
|
77
|
+
dockerExecutable: config.dockerExecutable,
|
|
78
|
+
dockerContext: config.dockerContext,
|
|
79
|
+
stateRoot: config.sandboxStateRoot,
|
|
80
|
+
runnerInstanceId: config.runnerInstanceId,
|
|
81
|
+
maxConcurrentSandboxes: config.maxConcurrentSandboxes,
|
|
82
|
+
diskHighWatermarkPercent: config.diskHighWatermarkPercent,
|
|
83
|
+
diskLowWatermarkPercent: config.diskLowWatermarkPercent,
|
|
84
|
+
minimumWorkspaceBytes: config.minimumWorkspaceBytes,
|
|
85
|
+
hostMemoryReserveBytes: config.sandboxHostMemoryReserveBytes,
|
|
86
|
+
defaultWorkerMemoryLimitBytes: config.defaultSandboxWorkerMemoryLimitBytes,
|
|
87
|
+
defaultServiceMemoryLimitBytes: config.defaultSandboxServiceMemoryLimitBytes,
|
|
88
|
+
runtimeOverheadBytes: config.sandboxRuntimeOverheadBytes,
|
|
89
|
+
requireRootless: config.requireRootlessDocker
|
|
90
|
+
});
|
|
91
|
+
this.dockerEngineProbe = dockerEngineProbe ?? new DockerEngineProbe({
|
|
92
|
+
dockerExecutable: config.dockerExecutable,
|
|
93
|
+
dockerContext: config.dockerContext,
|
|
94
|
+
recovery: {
|
|
95
|
+
enabled: config.dockerDesktopRecoveryEnabled,
|
|
96
|
+
failureThreshold: config.dockerRecoveryFailureThreshold,
|
|
97
|
+
cooldownMs: config.dockerRecoveryCooldownMs,
|
|
98
|
+
startupTimeoutMs: config.dockerRecoveryStartupTimeoutMs,
|
|
99
|
+
canRecover: () => this.activeSessions.size === 0 && this.reservations.size === 0,
|
|
100
|
+
},
|
|
101
|
+
onStatus: logBridgeStatus,
|
|
102
|
+
});
|
|
103
|
+
this.sandboxRuntime = sandboxRuntime ?? new DockerSandboxRuntime(config, (request, signal) => this.redeemPublishGrant(request, signal));
|
|
104
|
+
this.sandboxResourceResolver = sandboxResourceResolver
|
|
105
|
+
?? (dispatch => resolveSandboxResourcesFromRepo(dispatch, this.config.sandboxRuntimeOverheadBytes));
|
|
106
|
+
this.readinessResultOutbox = readinessResultOutbox ?? new FileReadinessResultOutbox(config.sandboxStateRoot);
|
|
22
107
|
}
|
|
23
108
|
async connectOnce() {
|
|
24
109
|
await this.stopConnection();
|
|
25
110
|
this.stopped = false;
|
|
111
|
+
const hostReap = await this.sessions.reapHostRuntime?.({
|
|
112
|
+
runnerId: this.config.runnerId,
|
|
113
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
114
|
+
});
|
|
115
|
+
if (hostReap && (hostReap.reaped > 0 || hostReap.degraded > 0)) {
|
|
116
|
+
logBridgeStatus(`Managed-host recovery reaped ${hostReap.reaped} stale attempt(s), protected ${hostReap.reusedPidsProtected} reused PID(s), and left ${hostReap.degraded} degraded attempt(s) for reconciliation.`);
|
|
117
|
+
}
|
|
118
|
+
if (this.config.dockerEnabled) {
|
|
119
|
+
const preflight = await this.preflight.refresh();
|
|
120
|
+
if (canReapManagedSandboxes(preflight)) {
|
|
121
|
+
await this.sandboxRuntime.reap({ diskLow: preflight.diskLow });
|
|
122
|
+
await this.preflight.refresh();
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
await this.dockerEngineProbe.refresh(true);
|
|
126
|
+
if (!this.hostCapabilitiesDetected) {
|
|
127
|
+
this.hostCapabilities = await this.hostCapabilityProbe.detect(this.config);
|
|
128
|
+
this.hostCapabilitiesDetected = true;
|
|
129
|
+
}
|
|
26
130
|
const connection = this.connections.create(this.config);
|
|
27
131
|
this.connection = connection;
|
|
132
|
+
this.transportRecovery = null;
|
|
28
133
|
this.closed = new Promise((resolve) => {
|
|
29
|
-
connection.onClose(() => {
|
|
134
|
+
connection.onClose((error) => {
|
|
30
135
|
this.stopHeartbeat();
|
|
136
|
+
this.completeTransportRecovery(error ?? new Error("bridge connection closed"));
|
|
31
137
|
this.abortActiveSessions("bridge connection closed");
|
|
138
|
+
this.abortActiveReadinessProbes("bridge connection closed");
|
|
32
139
|
resolve();
|
|
33
140
|
});
|
|
34
141
|
});
|
|
142
|
+
connection.onReconnecting?.((error) => {
|
|
143
|
+
if (connection !== this.connection || this.stopped)
|
|
144
|
+
return;
|
|
145
|
+
this.stopHeartbeat();
|
|
146
|
+
this.beginTransportRecovery();
|
|
147
|
+
logBridgeStatus(`Forge connection interrupted${error ? `: ${errorMessage(error)}` : ""}. Reconnecting in place; active dispatches remain supervised...`);
|
|
148
|
+
});
|
|
149
|
+
connection.onReconnected?.(async () => {
|
|
150
|
+
if (connection !== this.connection || this.stopped)
|
|
151
|
+
return;
|
|
152
|
+
try {
|
|
153
|
+
await this.registerConnection(connection);
|
|
154
|
+
this.completeTransportRecovery();
|
|
155
|
+
this.startHeartbeat();
|
|
156
|
+
await this.flushReadinessResults().catch(error => {
|
|
157
|
+
logBridgeStatus(`Readiness result replay deferred after reconnect: ${errorMessage(error)}`);
|
|
158
|
+
});
|
|
159
|
+
await this.flushCleanupReports().catch(() => undefined);
|
|
160
|
+
logBridgeStatus("Forge connection restored; active dispatches remain attached to this runner process.");
|
|
161
|
+
}
|
|
162
|
+
catch (error) {
|
|
163
|
+
this.completeTransportRecovery(error);
|
|
164
|
+
logBridgeStatus(`Forge reconnect registration failed: ${errorMessage(error)}. Closing the connection for a clean retry...`);
|
|
165
|
+
await connection.stop().catch(() => undefined);
|
|
166
|
+
}
|
|
167
|
+
});
|
|
35
168
|
connection.onDispatch((dispatch) => this.handleDispatch(dispatch));
|
|
169
|
+
connection.onOffer?.((dispatch) => this.handleOffer(dispatch));
|
|
170
|
+
connection.onStart?.((dispatch) => this.handleStart(dispatch));
|
|
171
|
+
connection.onLeaseUpdate?.((request) => this.handleLeaseUpdate(request));
|
|
36
172
|
connection.onCancel((request) => this.handleCancel(request));
|
|
173
|
+
connection.onReap?.((request) => this.handleReap(request));
|
|
174
|
+
connection.onReadinessProbe?.((request) => this.handleReadinessProbe(request));
|
|
37
175
|
await connection.start();
|
|
38
|
-
await this.
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
minServerVersion: this.config.minServerVersion ?? null,
|
|
42
|
-
capabilities: this.capabilities()
|
|
176
|
+
await this.registerConnection(connection);
|
|
177
|
+
await this.flushReadinessResults().catch(error => {
|
|
178
|
+
logBridgeStatus(`Readiness result replay deferred: ${errorMessage(error)}`);
|
|
43
179
|
});
|
|
180
|
+
await this.flushCleanupReports();
|
|
44
181
|
this.startHeartbeat();
|
|
182
|
+
this.startLeaseWatchdog();
|
|
183
|
+
this.startSandboxMaintenance();
|
|
184
|
+
}
|
|
185
|
+
async registerConnection(connection) {
|
|
186
|
+
let storageRecoveryAttempt = 0;
|
|
187
|
+
while (true) {
|
|
188
|
+
try {
|
|
189
|
+
const ack = await connection.invoke("Hello", {
|
|
190
|
+
version: this.config.version,
|
|
191
|
+
protocolVersion: this.config.protocolVersion,
|
|
192
|
+
minServerVersion: this.config.minServerVersion ?? null,
|
|
193
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
194
|
+
capabilities: this.capabilities()
|
|
195
|
+
});
|
|
196
|
+
if (!ack.accepted) {
|
|
197
|
+
throw new BridgeCallbackRejectedError(ack.code ?? null, ack.message ?? ack.code ?? "Hello callback was rejected.");
|
|
198
|
+
}
|
|
199
|
+
logBridgeStatus(`Registered runner ${this.config.runnerId}, instance ${this.config.runnerInstanceId}, build ${this.config.version}, protocol ${this.config.protocolVersion}.`);
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
catch (error) {
|
|
203
|
+
if (!isProviderStorageUnavailable(error) || storageRecoveryAttempt >= this.config.reconnectBackoffMs.length) {
|
|
204
|
+
throw error;
|
|
205
|
+
}
|
|
206
|
+
const backoff = this.config.reconnectBackoffMs[storageRecoveryAttempt] ?? 5000;
|
|
207
|
+
storageRecoveryAttempt += 1;
|
|
208
|
+
logBridgeStatus(`Hello reached Forge while storage was recovering. Retrying registration ${storageRecoveryAttempt} in ${formatDelay(backoff)}...`);
|
|
209
|
+
await delay(backoff);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
45
212
|
}
|
|
46
213
|
async run(signal) {
|
|
47
214
|
let attempt = 0;
|
|
@@ -72,6 +239,7 @@ export class ForgeBridgeService {
|
|
|
72
239
|
finally {
|
|
73
240
|
if (signal?.aborted) {
|
|
74
241
|
this.abortActiveSessions(String(signal.reason ?? "bridge stopping"));
|
|
242
|
+
this.abortActiveReadinessProbes(String(signal.reason ?? "bridge stopping"));
|
|
75
243
|
}
|
|
76
244
|
await this.stopConnection();
|
|
77
245
|
}
|
|
@@ -79,31 +247,75 @@ export class ForgeBridgeService {
|
|
|
79
247
|
async stop() {
|
|
80
248
|
this.stopped = true;
|
|
81
249
|
this.abortActiveSessions("bridge stopping");
|
|
250
|
+
this.abortActiveReadinessProbes("bridge stopping");
|
|
82
251
|
await this.stopConnection();
|
|
252
|
+
await this.preflight.release();
|
|
83
253
|
}
|
|
84
254
|
async handleDispatch(rawDispatch) {
|
|
85
255
|
const dispatch = this.parseDispatchOrReport(rawDispatch);
|
|
86
256
|
if (!dispatch) {
|
|
87
257
|
return;
|
|
88
258
|
}
|
|
259
|
+
const capabilityFailure = await this.requiredCapabilityFailure(dispatch);
|
|
260
|
+
if (capabilityFailure) {
|
|
261
|
+
await this.sendResult(dispatch, failureResult(capabilityFailure, [{
|
|
262
|
+
role: "runner",
|
|
263
|
+
kind: "error",
|
|
264
|
+
content: capabilityFailure,
|
|
265
|
+
eventType: "docker_engine_unavailable",
|
|
266
|
+
}])).catch(() => undefined);
|
|
267
|
+
return;
|
|
268
|
+
}
|
|
89
269
|
const leaseId = dispatch.lease.id;
|
|
90
270
|
const fencingToken = dispatch.lease.fencingToken;
|
|
91
271
|
const redactionSecrets = exactSecrets(dispatch, this.config);
|
|
92
|
-
|
|
93
|
-
|
|
272
|
+
const existing = this.activeSessions.get(leaseId);
|
|
273
|
+
if (existing) {
|
|
274
|
+
if (existing.fencingToken === fencingToken) {
|
|
275
|
+
return;
|
|
276
|
+
}
|
|
277
|
+
await this.sendResult(dispatch, failureResult("Dispatch fence conflicts with an active bridge session.")).catch(() => undefined);
|
|
94
278
|
return;
|
|
95
279
|
}
|
|
96
280
|
const controller = new AbortController();
|
|
97
|
-
this.activeSessions.set(leaseId, {
|
|
281
|
+
this.activeSessions.set(leaseId, {
|
|
282
|
+
fencingToken,
|
|
283
|
+
dispatchAttemptId: dispatch.sandbox?.dispatchAttemptId ?? null,
|
|
284
|
+
runnerInstanceId: dispatch.sandbox?.runnerInstanceId ?? null,
|
|
285
|
+
leaseExpiresAt: dispatch.lease.expiresAt,
|
|
286
|
+
sandboxed: dispatch.sandbox?.executionMode === "docker",
|
|
287
|
+
resources: dispatch.sandbox?.executionMode === "docker"
|
|
288
|
+
? dispatch.sandbox.requestedResources
|
|
289
|
+
: zeroSandboxResources(),
|
|
290
|
+
controller
|
|
291
|
+
});
|
|
98
292
|
try {
|
|
99
293
|
const normalized = this.normalizeDispatch(dispatch);
|
|
100
|
-
|
|
294
|
+
if (normalized.sandbox) {
|
|
295
|
+
const started = {
|
|
296
|
+
dispatchAttemptId: normalized.sandbox.dispatchAttemptId,
|
|
297
|
+
leaseId,
|
|
298
|
+
fencingToken,
|
|
299
|
+
runnerInstanceId: normalized.sandbox.runnerInstanceId
|
|
300
|
+
};
|
|
301
|
+
await this.invokeAck("DispatchStarted", started);
|
|
302
|
+
}
|
|
303
|
+
const sink = {
|
|
101
304
|
emit: async (event) => {
|
|
305
|
+
if (event.eventType === "provider_cli_started") {
|
|
306
|
+
const active = this.activeSessions.get(leaseId);
|
|
307
|
+
if (active)
|
|
308
|
+
active.resources = { ...active.resources, buildSlots: 0 };
|
|
309
|
+
}
|
|
102
310
|
try {
|
|
103
311
|
await this.invokeAck("DispatchEvent", {
|
|
104
312
|
leaseId,
|
|
105
313
|
fencingToken,
|
|
106
|
-
|
|
314
|
+
...(dispatch.sandbox ? {
|
|
315
|
+
dispatchAttemptId: dispatch.sandbox.dispatchAttemptId,
|
|
316
|
+
runnerInstanceId: dispatch.sandbox.runnerInstanceId
|
|
317
|
+
} : {}),
|
|
318
|
+
event: redactRunnerDispatchEvent({ ...event, eventId: event.eventId ?? randomUUID() }, redactionSecrets)
|
|
107
319
|
});
|
|
108
320
|
}
|
|
109
321
|
catch (error) {
|
|
@@ -114,18 +326,302 @@ export class ForgeBridgeService {
|
|
|
114
326
|
throw error;
|
|
115
327
|
}
|
|
116
328
|
}
|
|
117
|
-
}
|
|
118
|
-
|
|
329
|
+
};
|
|
330
|
+
const result = normalized.sandbox?.executionMode === "docker"
|
|
331
|
+
? await this.sandboxRuntime.runOnce(normalized, sink, controller.signal)
|
|
332
|
+
: await this.sessions.runOnce(normalized, sink, controller.signal);
|
|
333
|
+
if (normalized.sandbox?.executionMode === "docker") {
|
|
334
|
+
await this.flushCleanupReports().catch(() => undefined);
|
|
335
|
+
}
|
|
336
|
+
const redactedResult = redactRunOnceResultPacket(result, redactionSecrets);
|
|
337
|
+
await this.releaseActiveSessionAndRefreshCapacity(leaseId);
|
|
338
|
+
await this.sendResult(dispatch, compactTerminalResult(dispatch, redactedResult));
|
|
119
339
|
}
|
|
120
340
|
catch (error) {
|
|
121
341
|
const summary = controller.signal.aborted
|
|
122
342
|
? `Dispatch cancelled: ${String(controller.signal.reason ?? "cancelled")}`
|
|
123
343
|
: redactText(error instanceof Error ? error.message : String(error), redactionSecrets);
|
|
124
|
-
|
|
344
|
+
const failureEventType = infrastructureFailureEventType(summary);
|
|
345
|
+
const failureEvents = failureEventType ? [{
|
|
346
|
+
role: "runner",
|
|
347
|
+
kind: "error",
|
|
348
|
+
content: summary,
|
|
349
|
+
eventType: failureEventType,
|
|
350
|
+
}] : [];
|
|
351
|
+
await this.releaseActiveSessionAndRefreshCapacity(leaseId);
|
|
352
|
+
await this.sendResult(dispatch, failureResult(summary, failureEvents)).catch(() => undefined);
|
|
125
353
|
}
|
|
126
354
|
finally {
|
|
127
|
-
this.
|
|
355
|
+
await this.releaseActiveSessionAndRefreshCapacity(leaseId);
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
async releaseActiveSessionAndRefreshCapacity(leaseId) {
|
|
359
|
+
if (!this.activeSessions.delete(leaseId))
|
|
360
|
+
return;
|
|
361
|
+
// Publish released host/runtime capacity before the terminal result can
|
|
362
|
+
// enqueue the next pipeline phase. Otherwise the server may reject that
|
|
363
|
+
// handoff from the previous periodic heartbeat's active reservation.
|
|
364
|
+
await this.sendHeartbeat().catch(() => undefined);
|
|
365
|
+
}
|
|
366
|
+
async handleOffer(rawDispatch) {
|
|
367
|
+
const envelope = parseOfferEnvelope(rawDispatch);
|
|
368
|
+
if (!envelope) {
|
|
369
|
+
return;
|
|
370
|
+
}
|
|
371
|
+
const dispatch = this.parseDispatchOrReport(envelope.dispatch);
|
|
372
|
+
if (!dispatch?.sandbox
|
|
373
|
+
|| dispatch.sandbox.dispatchAttemptId !== envelope.dispatchAttemptId
|
|
374
|
+
|| dispatch.lease.id !== envelope.leaseId
|
|
375
|
+
|| dispatch.lease.fencingToken !== envelope.fencingToken)
|
|
376
|
+
return;
|
|
377
|
+
const sandbox = dispatch.sandbox;
|
|
378
|
+
this.expireReservations(Date.now());
|
|
379
|
+
const existing = this.reservations.get(sandbox.dispatchAttemptId);
|
|
380
|
+
if (existing) {
|
|
381
|
+
const sameFence = existing.leaseId === dispatch.lease.id
|
|
382
|
+
&& existing.fencingToken === dispatch.lease.fencingToken
|
|
383
|
+
&& existing.runnerInstanceId === sandbox.runnerInstanceId;
|
|
384
|
+
await this.sendOfferDecision(dispatch, sameFence, sameFence ? null : "stale_fence", sameFence ? null : "Offer fence conflicts with the existing reservation.", sameFence ? existing.reservedUntil : null, sameFence ? existing.resources : null);
|
|
385
|
+
return;
|
|
386
|
+
}
|
|
387
|
+
if (sandbox.runnerInstanceId !== this.config.runnerInstanceId) {
|
|
388
|
+
await this.sendOfferDecision(dispatch, false, "stale_runner_instance", "Offer targets a different runner process instance.");
|
|
389
|
+
return;
|
|
390
|
+
}
|
|
391
|
+
if (Date.parse(envelope.expiresAt) <= Date.now()) {
|
|
392
|
+
await this.sendOfferDecision(dispatch, false, "lease_expired", "Offer lease is already expired.");
|
|
393
|
+
return;
|
|
394
|
+
}
|
|
395
|
+
const preflight = sandbox.executionMode === "docker" && this.config.dockerEnabled
|
|
396
|
+
? await this.preflight.run()
|
|
397
|
+
: this.preflight.current();
|
|
398
|
+
if (sandbox.executionMode === "docker") {
|
|
399
|
+
if (!this.config.dockerEnabled || !preflight?.healthy) {
|
|
400
|
+
await this.sendOfferDecision(dispatch, false, preflight?.unavailableCode ?? "docker_unavailable", preflight?.unavailableReason ?? "Docker sandbox preflight is unavailable.");
|
|
401
|
+
return;
|
|
402
|
+
}
|
|
403
|
+
const networkFailure = sandboxNetworkPolicyFailure(dispatch, this.config);
|
|
404
|
+
if (networkFailure) {
|
|
405
|
+
await this.sendOfferDecision(dispatch, false, "policy_mismatch", networkFailure);
|
|
406
|
+
return;
|
|
407
|
+
}
|
|
408
|
+
const providerNetworkFailure = sandboxProviderNetworkFailure(dispatch, this.config);
|
|
409
|
+
if (providerNetworkFailure) {
|
|
410
|
+
await this.sendOfferDecision(dispatch, false, "provider_network_policy_denied", providerNetworkFailure);
|
|
411
|
+
return;
|
|
412
|
+
}
|
|
413
|
+
const authFailure = sandboxAuthFailure(dispatch, this.config);
|
|
414
|
+
if (authFailure) {
|
|
415
|
+
await this.sendOfferDecision(dispatch, false, "sandbox_auth_unavailable", authFailure);
|
|
416
|
+
return;
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
let requestedResources = sandbox.requestedResources;
|
|
420
|
+
let sandboxConfigCommitSha = sandbox.sandboxConfigCommitSha ?? null;
|
|
421
|
+
const normalizedDispatch = this.normalizeDispatch(dispatch);
|
|
422
|
+
if (sandbox.executionMode === "docker") {
|
|
423
|
+
try {
|
|
424
|
+
const profile = await this.sandboxResourceResolver(normalizedDispatch);
|
|
425
|
+
requestedResources = profile.resources;
|
|
426
|
+
sandboxConfigCommitSha = profile.sandboxConfigCommitSha;
|
|
427
|
+
}
|
|
428
|
+
catch (error) {
|
|
429
|
+
await this.sendOfferDecision(dispatch, false, "sandbox_capacity_profile_unavailable", error instanceof Error ? error.message : "The immutable sandbox contract could not be profiled for admission.");
|
|
430
|
+
return;
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
else if (normalizedDispatch.repo.serverLocalPath) {
|
|
434
|
+
// Host execution uses the same project-derived disk reservation as Docker.
|
|
435
|
+
// The API supplies a placement-time estimate; the runner can safely raise it
|
|
436
|
+
// from its authoritative checkout before admitting the attempt.
|
|
437
|
+
const measuredWorkspaceBytes = await estimateWorkspaceBytesFromRepo(normalizedDispatch.repo.serverLocalPath, { role: normalizedDispatch.execution.role }).catch(() => null);
|
|
438
|
+
if (measuredWorkspaceBytes !== null) {
|
|
439
|
+
requestedResources = {
|
|
440
|
+
...requestedResources,
|
|
441
|
+
workspaceBytes: Math.max(requestedResources.workspaceBytes, measuredWorkspaceBytes),
|
|
442
|
+
};
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
// Resource profiling can await filesystem and Docker work. Admission needs
|
|
446
|
+
// a dynamic host/daemon sample that began after profiling completed, so an
|
|
447
|
+
// older in-flight heartbeat inspection cannot supply the commit decision.
|
|
448
|
+
const admissionPreflight = sandbox.executionMode === "docker"
|
|
449
|
+
? await this.preflight.refreshCapacityForAdmission()
|
|
450
|
+
: preflight;
|
|
451
|
+
// Resource profiling and preflight may await slow filesystem or Docker I/O.
|
|
452
|
+
// Re-read every mutable capacity input only after that work, then decide and
|
|
453
|
+
// commit synchronously. JavaScript cannot interleave another Offer handler
|
|
454
|
+
// inside this non-async method, so the ledger check and insertion are one
|
|
455
|
+
// atomic event-loop turn without holding a lock across external I/O.
|
|
456
|
+
const decision = this.commitOfferReservation(dispatch, requestedResources, admissionPreflight);
|
|
457
|
+
await this.sendOfferDecision(dispatch, decision.accepted, decision.code, decision.message, decision.reservedUntil, decision.resources, sandboxConfigCommitSha);
|
|
458
|
+
}
|
|
459
|
+
commitOfferReservation(dispatch, requestedResources, preparedPreflight) {
|
|
460
|
+
const sandbox = dispatch.sandbox;
|
|
461
|
+
const now = Date.now();
|
|
462
|
+
this.expireReservations(now);
|
|
463
|
+
const existing = this.reservations.get(sandbox.dispatchAttemptId);
|
|
464
|
+
if (existing) {
|
|
465
|
+
const sameFence = existing.leaseId === dispatch.lease.id
|
|
466
|
+
&& existing.fencingToken === dispatch.lease.fencingToken
|
|
467
|
+
&& existing.runnerInstanceId === sandbox.runnerInstanceId;
|
|
468
|
+
return {
|
|
469
|
+
accepted: sameFence,
|
|
470
|
+
code: sameFence ? null : "stale_fence",
|
|
471
|
+
message: sameFence ? null : "Offer fence conflicts with the existing reservation.",
|
|
472
|
+
reservedUntil: sameFence ? existing.reservedUntil : null,
|
|
473
|
+
resources: sameFence ? existing.resources : null,
|
|
474
|
+
};
|
|
475
|
+
}
|
|
476
|
+
if (Date.parse(dispatch.lease.expiresAt) <= now) {
|
|
477
|
+
return rejectedOfferReservation("lease_expired", "Offer lease is already expired.");
|
|
478
|
+
}
|
|
479
|
+
if (sandbox.executionMode === "docker") {
|
|
480
|
+
// A concurrent heartbeat/preflight refresh may have produced a newer
|
|
481
|
+
// conservative dual-domain sample while this offer was being profiled.
|
|
482
|
+
const admissionPreflight = this.preflight.current() ?? preparedPreflight;
|
|
483
|
+
if (!this.config.dockerEnabled || !admissionPreflight?.healthy) {
|
|
484
|
+
return rejectedOfferReservation(admissionPreflight?.unavailableCode ?? "docker_unavailable", admissionPreflight?.unavailableReason ?? "Docker sandbox preflight is unavailable.");
|
|
485
|
+
}
|
|
486
|
+
const runtimeState = this.sandboxRuntime.snapshot();
|
|
487
|
+
const usedResources = this.usedSandboxResources(runtimeState.degraded);
|
|
488
|
+
const activeResources = this.activeSandboxResources(runtimeState.degraded);
|
|
489
|
+
const capacity = this.sandboxResourceCapacity(admissionPreflight, activeResources, usedResources);
|
|
490
|
+
if (!canReserveSandboxResources(usedResources, requestedResources, capacity)) {
|
|
491
|
+
return rejectedOfferReservation("capacity_unavailable", sandboxCapacityFailureMessage(usedResources, requestedResources, capacity, admissionPreflight, this.config));
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
const reservedUntil = new Date(Math.min(Date.parse(dispatch.lease.expiresAt), now + 30_000)).toISOString();
|
|
495
|
+
const reservation = {
|
|
496
|
+
leaseId: dispatch.lease.id,
|
|
497
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
498
|
+
runnerInstanceId: sandbox.runnerInstanceId,
|
|
499
|
+
expiresAt: dispatch.lease.expiresAt,
|
|
500
|
+
reservedUntil,
|
|
501
|
+
sandboxed: sandbox.executionMode === "docker",
|
|
502
|
+
resources: requestedResources,
|
|
503
|
+
};
|
|
504
|
+
this.reservations.set(sandbox.dispatchAttemptId, reservation);
|
|
505
|
+
return {
|
|
506
|
+
accepted: true,
|
|
507
|
+
code: null,
|
|
508
|
+
message: null,
|
|
509
|
+
reservedUntil,
|
|
510
|
+
resources: requestedResources,
|
|
511
|
+
};
|
|
512
|
+
}
|
|
513
|
+
async handleStart(rawDispatch) {
|
|
514
|
+
const envelope = parseStartEnvelope(rawDispatch);
|
|
515
|
+
if (!envelope) {
|
|
516
|
+
return;
|
|
517
|
+
}
|
|
518
|
+
let dispatch = this.parseDispatchOrReport(envelope.dispatch);
|
|
519
|
+
if (!dispatch?.sandbox
|
|
520
|
+
|| dispatch.sandbox.dispatchAttemptId !== envelope.dispatchAttemptId
|
|
521
|
+
|| dispatch.lease.id !== envelope.leaseId
|
|
522
|
+
|| dispatch.lease.fencingToken !== envelope.fencingToken)
|
|
523
|
+
return;
|
|
524
|
+
const reservation = this.reservations.get(dispatch.sandbox.dispatchAttemptId);
|
|
525
|
+
if (!reservation
|
|
526
|
+
|| reservation.leaseId !== dispatch.lease.id
|
|
527
|
+
|| reservation.fencingToken !== dispatch.lease.fencingToken
|
|
528
|
+
|| reservation.runnerInstanceId !== dispatch.sandbox.runnerInstanceId
|
|
529
|
+
|| Date.parse(reservation.reservedUntil) <= Date.now()) {
|
|
530
|
+
await this.sendResult(dispatch, failureResult("Start rejected because its reservation is missing or stale.")).catch(() => undefined);
|
|
531
|
+
return;
|
|
532
|
+
}
|
|
533
|
+
if (dispatch.apiKeySecretGrant) {
|
|
534
|
+
this.reservations.delete(dispatch.sandbox.dispatchAttemptId);
|
|
535
|
+
await this.sendResult(dispatch, failureResult("Start rejected because protocol-v2 packets may not contain a plaintext provider grant.")).catch(() => undefined);
|
|
536
|
+
return;
|
|
537
|
+
}
|
|
538
|
+
if (dispatch.apiKeySecretGrantHandle) {
|
|
539
|
+
const response = await this.redeemSecretGrant(dispatch);
|
|
540
|
+
const grant = response.grant;
|
|
541
|
+
if (!response.accepted
|
|
542
|
+
|| !grant
|
|
543
|
+
|| grant.grantId !== dispatch.apiKeySecretGrantHandle.grantId
|
|
544
|
+
|| grant.provider !== dispatch.apiKeySecretGrantHandle.provider
|
|
545
|
+
|| Date.parse(grant.expiresAt) <= Date.now()) {
|
|
546
|
+
this.reservations.delete(dispatch.sandbox.dispatchAttemptId);
|
|
547
|
+
await this.sendResult(dispatch, failureResult(response.message ?? "Secret grant redemption was rejected.")).catch(() => undefined);
|
|
548
|
+
return;
|
|
549
|
+
}
|
|
550
|
+
dispatch = { ...dispatch, apiKeySecretGrant: grant, apiKeySecretGrantHandle: null };
|
|
128
551
|
}
|
|
552
|
+
else if (dispatch.execution.requiresApiKeySecretGrant) {
|
|
553
|
+
this.reservations.delete(dispatch.sandbox.dispatchAttemptId);
|
|
554
|
+
await this.sendResult(dispatch, failureResult("Start rejected because the provider secret grant handle is missing.")).catch(() => undefined);
|
|
555
|
+
return;
|
|
556
|
+
}
|
|
557
|
+
dispatch = {
|
|
558
|
+
...dispatch,
|
|
559
|
+
sandbox: { ...dispatch.sandbox, requestedResources: reservation.resources },
|
|
560
|
+
};
|
|
561
|
+
this.reservations.delete(envelope.dispatchAttemptId);
|
|
562
|
+
await this.handleDispatch(dispatch);
|
|
563
|
+
}
|
|
564
|
+
async redeemSecretGrant(dispatch) {
|
|
565
|
+
const sandbox = dispatch.sandbox;
|
|
566
|
+
const handle = dispatch.apiKeySecretGrantHandle;
|
|
567
|
+
const connection = this.connection;
|
|
568
|
+
if (!connection)
|
|
569
|
+
return { accepted: false, code: "bridge.disconnected", message: "Bridge is disconnected.", grant: null };
|
|
570
|
+
return await connection.invoke("RedeemSecretGrant", {
|
|
571
|
+
dispatchAttemptId: sandbox.dispatchAttemptId,
|
|
572
|
+
leaseId: dispatch.lease.id,
|
|
573
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
574
|
+
runnerInstanceId: sandbox.runnerInstanceId,
|
|
575
|
+
grantId: handle.grantId
|
|
576
|
+
});
|
|
577
|
+
}
|
|
578
|
+
async redeemPublishGrant(request, signal) {
|
|
579
|
+
if (signal?.aborted)
|
|
580
|
+
throw new Error(String(signal.reason ?? "publication cancelled"));
|
|
581
|
+
const connection = this.connection;
|
|
582
|
+
if (!connection)
|
|
583
|
+
throw new Error("sandbox_output_publish_forbidden: bridge is disconnected.");
|
|
584
|
+
const response = await connection.invoke("RedeemPublishGrant", request);
|
|
585
|
+
if (!response.accepted || !response.credential) {
|
|
586
|
+
throw new Error(`sandbox_output_publish_forbidden: ${response.message ?? response.code ?? "publication grant redemption was rejected"}`);
|
|
587
|
+
}
|
|
588
|
+
return response.credential;
|
|
589
|
+
}
|
|
590
|
+
async handleLeaseUpdate(request) {
|
|
591
|
+
if (request.runnerInstanceId !== this.config.runnerInstanceId) {
|
|
592
|
+
return;
|
|
593
|
+
}
|
|
594
|
+
const reservation = this.reservations.get(request.dispatchAttemptId);
|
|
595
|
+
if (reservation && reservation.leaseId === request.leaseId && reservation.fencingToken === request.fencingToken) {
|
|
596
|
+
reservation.expiresAt = request.expiresAt;
|
|
597
|
+
}
|
|
598
|
+
const active = this.activeSessions.get(request.leaseId);
|
|
599
|
+
if (active && active.fencingToken === request.fencingToken && active.dispatchAttemptId === request.dispatchAttemptId) {
|
|
600
|
+
active.leaseExpiresAt = request.expiresAt;
|
|
601
|
+
if (active.sandboxed) {
|
|
602
|
+
await this.sandboxRuntime.updateLease(request.dispatchAttemptId, request.fencingToken, request.expiresAt);
|
|
603
|
+
}
|
|
604
|
+
else {
|
|
605
|
+
await this.sessions.updateLease?.(request.dispatchAttemptId, request.fencingToken, request.expiresAt);
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
async sendOfferDecision(dispatch, accepted, code = null, message = null, reservedUntil = null, reservedResources = null, sandboxConfigCommitSha = null) {
|
|
610
|
+
if (!dispatch.sandbox)
|
|
611
|
+
return;
|
|
612
|
+
const payload = {
|
|
613
|
+
dispatchAttemptId: dispatch.sandbox.dispatchAttemptId,
|
|
614
|
+
leaseId: dispatch.lease.id,
|
|
615
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
616
|
+
runnerInstanceId: dispatch.sandbox.runnerInstanceId,
|
|
617
|
+
accepted,
|
|
618
|
+
code,
|
|
619
|
+
message,
|
|
620
|
+
reservedUntil: accepted ? reservedUntil ?? dispatch.lease.expiresAt : null,
|
|
621
|
+
reservedResources: accepted ? reservedResources : null,
|
|
622
|
+
sandboxConfigCommitSha: accepted ? sandboxConfigCommitSha : null,
|
|
623
|
+
};
|
|
624
|
+
await this.invokeAck("OfferDecision", payload);
|
|
129
625
|
}
|
|
130
626
|
parseDispatchOrReport(rawDispatch) {
|
|
131
627
|
try {
|
|
@@ -137,7 +633,7 @@ export class ForgeBridgeService {
|
|
|
137
633
|
return null;
|
|
138
634
|
}
|
|
139
635
|
const summary = invalidDispatchSummary(error);
|
|
140
|
-
void this.
|
|
636
|
+
void this.sendLegacyResult(identifiers.leaseId, identifiers.fencingToken, failureResult(summary, [
|
|
141
637
|
{
|
|
142
638
|
role: "runner",
|
|
143
639
|
kind: "error",
|
|
@@ -155,12 +651,135 @@ export class ForgeBridgeService {
|
|
|
155
651
|
}
|
|
156
652
|
active.controller.abort(request.reason ?? "cancelled by server");
|
|
157
653
|
}
|
|
654
|
+
async handleReap(request) {
|
|
655
|
+
if (request.runnerId !== this.config.runnerId
|
|
656
|
+
|| request.runnerInstanceId !== this.config.runnerInstanceId)
|
|
657
|
+
return;
|
|
658
|
+
await this.runSandboxMaintenance();
|
|
659
|
+
await this.sendHeartbeat();
|
|
660
|
+
}
|
|
661
|
+
async handleReadinessProbe(rawRequest) {
|
|
662
|
+
const request = parseReadinessProbe(rawRequest);
|
|
663
|
+
if (!request || request.runnerInstanceId !== this.config.runnerInstanceId)
|
|
664
|
+
return;
|
|
665
|
+
const existing = this.activeReadinessProbes.get(request.requestId);
|
|
666
|
+
if (existing)
|
|
667
|
+
return;
|
|
668
|
+
const controller = new AbortController();
|
|
669
|
+
this.activeReadinessProbes.set(request.requestId, controller);
|
|
670
|
+
const deadlineMs = Date.parse(request.expiresAt) - Date.now();
|
|
671
|
+
const timer = setTimeout(() => controller.abort("readiness probe expired"), Math.max(0, deadlineMs));
|
|
672
|
+
timer.unref?.();
|
|
673
|
+
let result;
|
|
674
|
+
try {
|
|
675
|
+
const accepted = {
|
|
676
|
+
schemaVersion: 1,
|
|
677
|
+
requestId: request.requestId,
|
|
678
|
+
fencingToken: request.fencingToken,
|
|
679
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
680
|
+
acceptedAt: new Date().toISOString(),
|
|
681
|
+
};
|
|
682
|
+
await this.invokeAck("ReadinessAccepted", accepted);
|
|
683
|
+
if (!this.sessions.probeHostRuntime) {
|
|
684
|
+
throw new Error("host_runtime_probe_unavailable: this bridge does not implement runtime readiness probes.");
|
|
685
|
+
}
|
|
686
|
+
const probe = await this.sessions.probeHostRuntime({
|
|
687
|
+
...request,
|
|
688
|
+
runnerId: this.config.runnerId,
|
|
689
|
+
repoRoot: this.config.defaultRepoRoot,
|
|
690
|
+
cleanupWorktrees: this.config.cleanupWorktrees,
|
|
691
|
+
}, controller.signal);
|
|
692
|
+
result = {
|
|
693
|
+
schemaVersion: 1,
|
|
694
|
+
requestId: request.requestId,
|
|
695
|
+
fencingToken: request.fencingToken,
|
|
696
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
697
|
+
...probe,
|
|
698
|
+
};
|
|
699
|
+
}
|
|
700
|
+
catch (error) {
|
|
701
|
+
result = {
|
|
702
|
+
schemaVersion: 1,
|
|
703
|
+
requestId: request.requestId,
|
|
704
|
+
fencingToken: request.fencingToken,
|
|
705
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
706
|
+
status: "failed",
|
|
707
|
+
reasonCode: "host_runtime_probe_failed",
|
|
708
|
+
reasonMessage: errorMessage(error).slice(0, 1000),
|
|
709
|
+
checkedAt: new Date().toISOString(),
|
|
710
|
+
};
|
|
711
|
+
}
|
|
712
|
+
finally {
|
|
713
|
+
clearTimeout(timer);
|
|
714
|
+
this.activeReadinessProbes.delete(request.requestId);
|
|
715
|
+
}
|
|
716
|
+
try {
|
|
717
|
+
await this.readinessResultOutbox.persist(result, request.expiresAt);
|
|
718
|
+
}
|
|
719
|
+
catch (error) {
|
|
720
|
+
if (errorMessage(error).includes("runtime_readiness_result_conflict:")) {
|
|
721
|
+
logBridgeStatus(`Readiness result ${request.requestId} rejected conflicting reuse of a durable request identity.`);
|
|
722
|
+
await this.sendHeartbeat().catch(() => undefined);
|
|
723
|
+
return;
|
|
724
|
+
}
|
|
725
|
+
logBridgeStatus(`Readiness result ${request.requestId} could not be persisted; attempting direct delivery: ${errorMessage(error)}`);
|
|
726
|
+
await this.invokeAck("ReadinessResult", result).catch(deliveryError => {
|
|
727
|
+
logBridgeStatus(`Readiness result ${request.requestId} could not be persisted or delivered: ${errorMessage(deliveryError)}`);
|
|
728
|
+
});
|
|
729
|
+
await this.sendHeartbeat().catch(() => undefined);
|
|
730
|
+
return;
|
|
731
|
+
}
|
|
732
|
+
try {
|
|
733
|
+
await this.flushReadinessResults();
|
|
734
|
+
}
|
|
735
|
+
catch (error) {
|
|
736
|
+
logBridgeStatus(`Readiness result ${request.requestId} remains queued for replay: ${errorMessage(error)}`);
|
|
737
|
+
}
|
|
738
|
+
await this.sendHeartbeat().catch(() => undefined);
|
|
739
|
+
}
|
|
740
|
+
async flushReadinessResults() {
|
|
741
|
+
if (this.readinessResultFlush)
|
|
742
|
+
return await this.readinessResultFlush;
|
|
743
|
+
const flush = this.flushReadinessResultsCore();
|
|
744
|
+
this.readinessResultFlush = flush;
|
|
745
|
+
try {
|
|
746
|
+
await flush;
|
|
747
|
+
}
|
|
748
|
+
finally {
|
|
749
|
+
if (this.readinessResultFlush === flush)
|
|
750
|
+
this.readinessResultFlush = null;
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
async flushReadinessResultsCore() {
|
|
754
|
+
for (const pending of await this.readinessResultOutbox.pending()) {
|
|
755
|
+
if (Date.parse(pending.expiresAt) <= Date.now()) {
|
|
756
|
+
await this.readinessResultOutbox.acknowledge(pending.result.requestId);
|
|
757
|
+
continue;
|
|
758
|
+
}
|
|
759
|
+
try {
|
|
760
|
+
await this.invokeAck("ReadinessResult", pending.result);
|
|
761
|
+
await this.readinessResultOutbox.acknowledge(pending.result.requestId);
|
|
762
|
+
}
|
|
763
|
+
catch (error) {
|
|
764
|
+
if (isReadinessNoLongerPending(error)) {
|
|
765
|
+
await this.readinessResultOutbox.acknowledge(pending.result.requestId);
|
|
766
|
+
continue;
|
|
767
|
+
}
|
|
768
|
+
throw error;
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
}
|
|
158
772
|
abortActiveSessions(reason) {
|
|
159
773
|
for (const active of this.activeSessions.values()) {
|
|
160
774
|
active.controller.abort(reason);
|
|
161
775
|
}
|
|
162
776
|
this.activeSessions.clear();
|
|
163
777
|
}
|
|
778
|
+
abortActiveReadinessProbes(reason) {
|
|
779
|
+
for (const controller of this.activeReadinessProbes.values())
|
|
780
|
+
controller.abort(reason);
|
|
781
|
+
this.activeReadinessProbes.clear();
|
|
782
|
+
}
|
|
164
783
|
normalizeDispatch(dispatch) {
|
|
165
784
|
const mapping = findRepoMapping(dispatch, repoMappingsFor(this.config));
|
|
166
785
|
const localPath = mapping?.localPath?.trim();
|
|
@@ -185,12 +804,20 @@ export class ForgeBridgeService {
|
|
|
185
804
|
: dispatch.qaBrowser
|
|
186
805
|
};
|
|
187
806
|
}
|
|
188
|
-
async sendResult(
|
|
807
|
+
async sendResult(dispatch, result) {
|
|
189
808
|
const payload = {
|
|
190
|
-
leaseId,
|
|
191
|
-
fencingToken,
|
|
809
|
+
leaseId: dispatch.lease.id,
|
|
810
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
811
|
+
dispatchAttemptId: dispatch.sandbox?.dispatchAttemptId ?? null,
|
|
812
|
+
runnerInstanceId: dispatch.sandbox?.runnerInstanceId ?? null,
|
|
192
813
|
result
|
|
193
814
|
};
|
|
815
|
+
await this.sendResultPayload(payload);
|
|
816
|
+
}
|
|
817
|
+
async sendLegacyResult(leaseId, fencingToken, result) {
|
|
818
|
+
await this.sendResultPayload({ leaseId, fencingToken, result });
|
|
819
|
+
}
|
|
820
|
+
async sendResultPayload(payload) {
|
|
194
821
|
try {
|
|
195
822
|
await this.invokeAck("DispatchResult", payload);
|
|
196
823
|
}
|
|
@@ -203,19 +830,143 @@ export class ForgeBridgeService {
|
|
|
203
830
|
}
|
|
204
831
|
startHeartbeat() {
|
|
205
832
|
this.stopHeartbeat();
|
|
833
|
+
this.consecutiveHeartbeatFailures = 0;
|
|
834
|
+
this.reconnectingFromHeartbeatFailure = false;
|
|
206
835
|
const intervalMs = positiveFiniteNumber(this.config.heartbeatIntervalMs, 30_000);
|
|
207
836
|
this.heartbeatTimer = setInterval(() => {
|
|
208
|
-
this.
|
|
837
|
+
void this.heartbeatTick();
|
|
209
838
|
}, intervalMs);
|
|
210
839
|
this.heartbeatTimer.unref?.();
|
|
211
840
|
}
|
|
841
|
+
async heartbeatTick() {
|
|
842
|
+
if (this.heartbeatInFlight || this.reconnectingFromHeartbeatFailure)
|
|
843
|
+
return;
|
|
844
|
+
const connection = this.connection;
|
|
845
|
+
if (!connection)
|
|
846
|
+
return;
|
|
847
|
+
this.heartbeatInFlight = true;
|
|
848
|
+
try {
|
|
849
|
+
await this.flushReadinessResults();
|
|
850
|
+
await this.sendHeartbeat();
|
|
851
|
+
this.consecutiveHeartbeatFailures = 0;
|
|
852
|
+
}
|
|
853
|
+
catch (error) {
|
|
854
|
+
if (connection !== this.connection || this.stopped)
|
|
855
|
+
return;
|
|
856
|
+
this.consecutiveHeartbeatFailures += 1;
|
|
857
|
+
logBridgeStatus(`Heartbeat ${this.consecutiveHeartbeatFailures} failed: ${errorMessage(error)}.`);
|
|
858
|
+
if (this.consecutiveHeartbeatFailures < 2)
|
|
859
|
+
return;
|
|
860
|
+
this.reconnectingFromHeartbeatFailure = true;
|
|
861
|
+
this.stopHeartbeat();
|
|
862
|
+
logBridgeStatus("Heartbeat failed twice. Closing the stale connection so the Bridge can reconnect...");
|
|
863
|
+
await connection.stop().catch(() => undefined);
|
|
864
|
+
}
|
|
865
|
+
finally {
|
|
866
|
+
this.heartbeatInFlight = false;
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
startLeaseWatchdog() {
|
|
870
|
+
this.stopLeaseWatchdog();
|
|
871
|
+
this.leaseWatchdogTimer = setInterval(() => this.expireLeases(), 1000);
|
|
872
|
+
this.leaseWatchdogTimer.unref?.();
|
|
873
|
+
}
|
|
874
|
+
stopLeaseWatchdog() {
|
|
875
|
+
if (this.leaseWatchdogTimer) {
|
|
876
|
+
clearInterval(this.leaseWatchdogTimer);
|
|
877
|
+
this.leaseWatchdogTimer = null;
|
|
878
|
+
}
|
|
879
|
+
}
|
|
880
|
+
expireLeases() {
|
|
881
|
+
const now = Date.now();
|
|
882
|
+
this.expireReservations(now);
|
|
883
|
+
for (const active of this.activeSessions.values()) {
|
|
884
|
+
if (Date.parse(active.leaseExpiresAt) <= now && !active.controller.signal.aborted) {
|
|
885
|
+
active.controller.abort("dispatch lease expired");
|
|
886
|
+
}
|
|
887
|
+
}
|
|
888
|
+
}
|
|
889
|
+
expireReservations(now) {
|
|
890
|
+
for (const [attemptId, reservation] of this.reservations) {
|
|
891
|
+
if (Date.parse(reservation.expiresAt) <= now || Date.parse(reservation.reservedUntil) <= now) {
|
|
892
|
+
this.reservations.delete(attemptId);
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
}
|
|
212
896
|
stopHeartbeat() {
|
|
213
897
|
if (this.heartbeatTimer) {
|
|
214
898
|
clearInterval(this.heartbeatTimer);
|
|
215
899
|
this.heartbeatTimer = null;
|
|
216
900
|
}
|
|
217
901
|
}
|
|
902
|
+
startSandboxMaintenance() {
|
|
903
|
+
this.stopSandboxMaintenance();
|
|
904
|
+
this.sandboxMaintenanceTimer = setInterval(() => {
|
|
905
|
+
this.runSandboxMaintenance().catch(() => undefined);
|
|
906
|
+
}, 60_000);
|
|
907
|
+
this.sandboxMaintenanceTimer.unref?.();
|
|
908
|
+
}
|
|
909
|
+
stopSandboxMaintenance() {
|
|
910
|
+
if (this.sandboxMaintenanceTimer) {
|
|
911
|
+
clearInterval(this.sandboxMaintenanceTimer);
|
|
912
|
+
this.sandboxMaintenanceTimer = null;
|
|
913
|
+
}
|
|
914
|
+
}
|
|
915
|
+
async runSandboxMaintenance() {
|
|
916
|
+
if (this.sandboxMaintenanceRunning)
|
|
917
|
+
return;
|
|
918
|
+
this.sandboxMaintenanceRunning = true;
|
|
919
|
+
try {
|
|
920
|
+
const hostReap = await this.sessions.reapHostRuntime?.({
|
|
921
|
+
runnerId: this.config.runnerId,
|
|
922
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
923
|
+
});
|
|
924
|
+
if (hostReap && (hostReap.reaped > 0 || hostReap.degraded > 0)) {
|
|
925
|
+
logBridgeStatus(`Managed-host maintenance reaped ${hostReap.reaped} stale attempt(s) and left ${hostReap.degraded} degraded attempt(s) for reconciliation.`);
|
|
926
|
+
}
|
|
927
|
+
if (this.config.dockerEnabled) {
|
|
928
|
+
const preflight = await this.preflight.refresh();
|
|
929
|
+
if (canReapManagedSandboxes(preflight)) {
|
|
930
|
+
const attemptSnapshot = await this.readAttemptSnapshot();
|
|
931
|
+
await this.sandboxRuntime.reap({ diskLow: preflight.diskLow, attemptSnapshot });
|
|
932
|
+
await this.flushCleanupReports();
|
|
933
|
+
}
|
|
934
|
+
}
|
|
935
|
+
}
|
|
936
|
+
finally {
|
|
937
|
+
this.sandboxMaintenanceRunning = false;
|
|
938
|
+
}
|
|
939
|
+
}
|
|
940
|
+
async readAttemptSnapshot() {
|
|
941
|
+
const connection = this.connection;
|
|
942
|
+
if (!connection)
|
|
943
|
+
throw new Error("Bridge is not connected.");
|
|
944
|
+
const response = await connection.invoke("AttemptSnapshot", {
|
|
945
|
+
runnerInstanceId: this.config.runnerInstanceId
|
|
946
|
+
});
|
|
947
|
+
if (!response.accepted) {
|
|
948
|
+
throw new BridgeCallbackRejectedError(response.code ?? null, response.message ?? response.code ?? "Attempt snapshot was rejected.");
|
|
949
|
+
}
|
|
950
|
+
return response.attempts ?? [];
|
|
951
|
+
}
|
|
952
|
+
async flushCleanupReports() {
|
|
953
|
+
if (!this.sandboxRuntime.pendingCleanupReports || !this.sandboxRuntime.acknowledgeCleanupReport)
|
|
954
|
+
return;
|
|
955
|
+
for (const report of await this.sandboxRuntime.pendingCleanupReports()) {
|
|
956
|
+
await this.invokeAck("CleanupReport", {
|
|
957
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
958
|
+
report
|
|
959
|
+
});
|
|
960
|
+
await this.sandboxRuntime.acknowledgeCleanupReport(report.reportId);
|
|
961
|
+
}
|
|
962
|
+
}
|
|
218
963
|
async sendHeartbeat() {
|
|
964
|
+
// Heartbeats are the recovery path for capacity-blocked work. Refresh the
|
|
965
|
+
// dynamic sample before advertising it, while reusing the static Docker
|
|
966
|
+
// capability snapshot established at startup or by an offer.
|
|
967
|
+
if (this.config.dockerEnabled)
|
|
968
|
+
await this.preflight.refreshCapacity();
|
|
969
|
+
await this.dockerEngineProbe.refresh();
|
|
219
970
|
await this.invokeAck("Heartbeat", {
|
|
220
971
|
status: "online",
|
|
221
972
|
capabilities: this.capabilities(),
|
|
@@ -224,8 +975,26 @@ export class ForgeBridgeService {
|
|
|
224
975
|
});
|
|
225
976
|
}
|
|
226
977
|
capabilities() {
|
|
978
|
+
const preflight = this.preflight.current();
|
|
979
|
+
const dockerEngine = this.dockerEngineProbe.current();
|
|
980
|
+
const runtimeState = this.sandboxRuntime.snapshot();
|
|
981
|
+
const reserved = [...this.reservations.values()].filter(reservation => reservation.sandboxed).length;
|
|
982
|
+
const used = this.usedSandboxResources(runtimeState.degraded);
|
|
983
|
+
const activeResources = this.activeSandboxResources(runtimeState.degraded);
|
|
984
|
+
const capacity = this.sandboxResourceCapacity(preflight, activeResources, used);
|
|
985
|
+
const ambientMemoryCommitmentBytes = this.sandboxAmbientMemoryCommitmentBytes ?? 0;
|
|
986
|
+
const memoryAdmission = preflight ? buildMemoryAdmissionDiagnostics({
|
|
987
|
+
sampledAt: preflight.sampledAt,
|
|
988
|
+
totalMemoryBytes: preflight.totalMemoryBytes,
|
|
989
|
+
rawAvailableMemoryBytes: preflight.rawAvailableMemoryBytes,
|
|
990
|
+
stabilizedAvailableMemoryBytes: preflight.availableMemoryBytes,
|
|
991
|
+
controlPlaneMemoryBytes: process.memoryUsage().rss,
|
|
992
|
+
reserveMemoryBytes: preflight.hostReserveBytes,
|
|
993
|
+
committedMemoryBytes: ambientMemoryCommitmentBytes + used.memoryBytes,
|
|
994
|
+
pendingCommitmentMemoryBytes: Math.max(0, used.memoryBytes - activeResources.memoryBytes),
|
|
995
|
+
}) : null;
|
|
227
996
|
return {
|
|
228
|
-
schemaVersion:
|
|
997
|
+
schemaVersion: 5,
|
|
229
998
|
activeSessionCount: this.activeSessions.size,
|
|
230
999
|
cliProviders: this.config.cliProviders,
|
|
231
1000
|
providerStatuses: this.config.providerStatuses,
|
|
@@ -233,6 +1002,53 @@ export class ForgeBridgeService {
|
|
|
233
1002
|
repoMappings: repoMappingsFor(this.config),
|
|
234
1003
|
os: this.config.os,
|
|
235
1004
|
sandboxProfiles: this.config.sandboxProfiles,
|
|
1005
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
1006
|
+
trustDomainId: this.config.trustDomainId,
|
|
1007
|
+
executionFacilities: dockerEngine?.available ? ["docker_engine"] : [],
|
|
1008
|
+
dockerEngine,
|
|
1009
|
+
toolchains: {
|
|
1010
|
+
...this.hostCapabilities.toolchains,
|
|
1011
|
+
docker: {
|
|
1012
|
+
available: dockerEngine?.available === true,
|
|
1013
|
+
executable: this.config.dockerExecutable,
|
|
1014
|
+
version: dockerEngine?.runtimeVersion ?? null,
|
|
1015
|
+
},
|
|
1016
|
+
},
|
|
1017
|
+
playwrightProjects: this.hostCapabilities.playwrightProjects,
|
|
1018
|
+
browserLaunch: this.hostCapabilities.browserLaunch,
|
|
1019
|
+
providerConnectivity: this.hostCapabilities.providerConnectivity,
|
|
1020
|
+
maxConcurrentSandboxes: capacity.sandboxSlots,
|
|
1021
|
+
reservedSandboxCount: reserved,
|
|
1022
|
+
activeSandboxCount: runtimeState.active,
|
|
1023
|
+
startingSandboxCount: runtimeState.starting,
|
|
1024
|
+
degradedReservationCount: runtimeState.degraded,
|
|
1025
|
+
sandboxRuntimes: preflight?.healthy ? ["docker"] : [],
|
|
1026
|
+
sandboxRuntimeVersion: preflight?.runtimeVersion ?? null,
|
|
1027
|
+
sandboxFeatures: preflight?.features ?? null,
|
|
1028
|
+
allowsOpenPublicSandbox: this.config.allowOpenPublicSandbox,
|
|
1029
|
+
sandboxPublicationModes: [this.config.sandboxPublicationMode ?? "shared_filesystem"],
|
|
1030
|
+
sandboxAuthModes: sandboxAuthModesFor(this.config),
|
|
1031
|
+
availableMemoryBytes: memoryAdmission?.availableForAdmissionBytes ?? null,
|
|
1032
|
+
availableCpuMillis: preflight ? Math.max(0, capacity.cpuMillis - used.cpuMillis) : null,
|
|
1033
|
+
availableWorkspaceBytes: preflight ? Math.max(0, capacity.workspaceBytes - used.workspaceBytes) : null,
|
|
1034
|
+
diskLow: preflight?.diskLow ?? null,
|
|
1035
|
+
preflightGeneration: preflight?.generation ?? 0,
|
|
1036
|
+
sandboxUnavailableCode: this.config.dockerEnabled ? preflight?.unavailableCode ?? null : "docker_disabled",
|
|
1037
|
+
sandboxUnavailableReason: this.config.dockerEnabled ? preflight?.unavailableReason ?? null : "Docker sandboxing is disabled by runner configuration.",
|
|
1038
|
+
sandboxMemoryCapacity: preflight ? {
|
|
1039
|
+
defaultWorkerMemoryLimitBytes: this.config.defaultSandboxWorkerMemoryLimitBytes,
|
|
1040
|
+
defaultServiceMemoryLimitBytes: this.config.defaultSandboxServiceMemoryLimitBytes,
|
|
1041
|
+
runtimeOverheadBytes: this.config.sandboxRuntimeOverheadBytes,
|
|
1042
|
+
hostReserveBytes: preflight.hostReserveBytes,
|
|
1043
|
+
activeReservationBytes: used.memoryBytes,
|
|
1044
|
+
daemonTotalBytes: preflight.totalMemoryBytes,
|
|
1045
|
+
daemonAvailableBytes: preflight.rawDaemonAvailableMemoryBytes ?? 0,
|
|
1046
|
+
hostAvailableBytes: preflight.rawHostAvailableMemoryBytes,
|
|
1047
|
+
conservativeAvailableBytes: preflight.rawAvailableMemoryBytes,
|
|
1048
|
+
availableForAdmissionBytes: memoryAdmission.availableForAdmissionBytes,
|
|
1049
|
+
memoryAdmission: memoryAdmission,
|
|
1050
|
+
} : null,
|
|
1051
|
+
hostRuntime: this.sessions.runtimeCapacity?.() ?? null,
|
|
236
1052
|
supportsApiSecretGrant: this.config.supportsApiSecretGrant,
|
|
237
1053
|
supportsManagedWorktrees: this.config.supportsManagedWorktrees,
|
|
238
1054
|
supportsManagedForgeArtifacts: this.config.supportsManagedForgeArtifacts,
|
|
@@ -242,25 +1058,143 @@ export class ForgeBridgeService {
|
|
|
242
1058
|
supportsDraftImageAttachments: this.config.supportsDraftImageAttachments
|
|
243
1059
|
};
|
|
244
1060
|
}
|
|
1061
|
+
async requiredCapabilityFailure(dispatch) {
|
|
1062
|
+
const requiredCapabilities = dispatch.qaScope?.required_capabilities ?? [];
|
|
1063
|
+
if (!requiredCapabilities.includes("docker_engine"))
|
|
1064
|
+
return null;
|
|
1065
|
+
const executionMode = dispatch.sandbox?.executionMode;
|
|
1066
|
+
if (executionMode === "docker") {
|
|
1067
|
+
return dockerEngineRequirementFailure(requiredCapabilities, executionMode, null);
|
|
1068
|
+
}
|
|
1069
|
+
return dockerEngineRequirementFailure(requiredCapabilities, executionMode, await this.dockerEngineProbe.refresh(true));
|
|
1070
|
+
}
|
|
1071
|
+
usedSandboxResources(degradedReservations) {
|
|
1072
|
+
const resources = [
|
|
1073
|
+
...[...this.reservations.values()]
|
|
1074
|
+
.filter(reservation => reservation.sandboxed)
|
|
1075
|
+
.map(reservation => reservation.resources),
|
|
1076
|
+
...[...this.activeSessions.values()]
|
|
1077
|
+
.filter(session => session.sandboxed)
|
|
1078
|
+
.map(session => session.resources),
|
|
1079
|
+
...Array.from({ length: degradedReservations }, () => defaultSandboxResources())
|
|
1080
|
+
];
|
|
1081
|
+
return sumSandboxResources(resources);
|
|
1082
|
+
}
|
|
1083
|
+
activeSandboxResources(degradedReservations) {
|
|
1084
|
+
return sumSandboxResources([
|
|
1085
|
+
...[...this.activeSessions.values()]
|
|
1086
|
+
.filter(session => session.sandboxed)
|
|
1087
|
+
.map(session => session.resources),
|
|
1088
|
+
...Array.from({ length: degradedReservations }, () => defaultSandboxResources())
|
|
1089
|
+
]);
|
|
1090
|
+
}
|
|
1091
|
+
sandboxResourceCapacity(preflight, activeResources, usedResources) {
|
|
1092
|
+
if (!preflight)
|
|
1093
|
+
return zeroSandboxResources();
|
|
1094
|
+
if (usedResources.memoryBytes === 0 || this.sandboxAmbientMemoryCommitmentBytes === null) {
|
|
1095
|
+
// Freeze non-Forge daemon use while reservations exist. If the bridge
|
|
1096
|
+
// reconnects with managed containers already present, the initial value
|
|
1097
|
+
// is deliberately conservative until their ledger drains.
|
|
1098
|
+
this.sandboxAmbientMemoryCommitmentBytes = Math.max(0, preflight.totalMemoryBytes - preflight.rawAvailableMemoryBytes);
|
|
1099
|
+
}
|
|
1100
|
+
return {
|
|
1101
|
+
// Host free memory and disk already reflect active containers. Add their
|
|
1102
|
+
// commitments back to form a stable total capacity, then subtract them
|
|
1103
|
+
// exactly once in usedSandboxResources.
|
|
1104
|
+
sandboxSlots: Math.min(this.config.maxConcurrentSandboxes, preflight.maxConcurrentSandboxes + activeResources.sandboxSlots),
|
|
1105
|
+
cpuMillis: preflight.availableCpuMillis,
|
|
1106
|
+
workerMemoryLimitBytes: 0,
|
|
1107
|
+
serviceMemoryLimitBytes: 0,
|
|
1108
|
+
runtimeOverheadBytes: 0,
|
|
1109
|
+
memoryBytes: sandboxMemoryCommitmentCapacity(preflight, activeResources.memoryBytes, this.sandboxAmbientMemoryCommitmentBytes),
|
|
1110
|
+
workspaceBytes: Math.max(0, preflight.availableWorkspaceBytes
|
|
1111
|
+
- this.config.minimumWorkspaceBytes
|
|
1112
|
+
+ activeResources.workspaceBytes),
|
|
1113
|
+
pids: this.config.sandboxPidCapacity,
|
|
1114
|
+
buildSlots: this.config.maxConcurrentSandboxBuilds
|
|
1115
|
+
};
|
|
1116
|
+
}
|
|
245
1117
|
async invokeAck(methodName, payload) {
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
1118
|
+
let storageRecoveryAttempt = 0;
|
|
1119
|
+
while (true) {
|
|
1120
|
+
const connection = this.connection;
|
|
1121
|
+
if (!connection) {
|
|
1122
|
+
throw new Error("Bridge is not connected.");
|
|
1123
|
+
}
|
|
1124
|
+
await this.waitForTransportRecovery(connection, this.transportRecovery);
|
|
1125
|
+
const recoveryGeneration = this.transportRecoveryGeneration;
|
|
1126
|
+
try {
|
|
1127
|
+
const ack = await waitForCallbackAck(connection.invoke(methodName, payload), methodName, positiveFiniteNumber(this.config.callbackAckTimeoutMs ?? 30_000, 30_000));
|
|
1128
|
+
if (!ack.accepted) {
|
|
1129
|
+
throw new BridgeCallbackRejectedError(ack.code ?? null, ack.message ?? ack.code ?? `${methodName} callback was rejected.`);
|
|
1130
|
+
}
|
|
1131
|
+
return ack;
|
|
1132
|
+
}
|
|
1133
|
+
catch (error) {
|
|
1134
|
+
if (connection !== this.connection || this.stopped)
|
|
1135
|
+
throw error;
|
|
1136
|
+
const recovery = this.transportRecovery;
|
|
1137
|
+
if (recovery && recovery.generation > recoveryGeneration) {
|
|
1138
|
+
await this.waitForTransportRecovery(connection, recovery);
|
|
1139
|
+
continue;
|
|
1140
|
+
}
|
|
1141
|
+
if (isProviderStorageUnavailable(error) && storageRecoveryAttempt < this.config.reconnectBackoffMs.length) {
|
|
1142
|
+
const backoff = this.config.reconnectBackoffMs[storageRecoveryAttempt] ?? 5000;
|
|
1143
|
+
storageRecoveryAttempt += 1;
|
|
1144
|
+
logBridgeStatus(`${methodName} reached Forge while storage was recovering. Retrying callback ${storageRecoveryAttempt} in ${formatDelay(backoff)}...`);
|
|
1145
|
+
await delay(backoff);
|
|
1146
|
+
continue;
|
|
1147
|
+
}
|
|
1148
|
+
throw error;
|
|
1149
|
+
}
|
|
249
1150
|
}
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
1151
|
+
}
|
|
1152
|
+
beginTransportRecovery() {
|
|
1153
|
+
if (this.transportRecovery && !this.transportRecovery.completed)
|
|
1154
|
+
return;
|
|
1155
|
+
let resolve = () => undefined;
|
|
1156
|
+
const promise = new Promise((complete) => { resolve = complete; });
|
|
1157
|
+
this.transportRecoveryGeneration += 1;
|
|
1158
|
+
this.transportRecovery = {
|
|
1159
|
+
generation: this.transportRecoveryGeneration,
|
|
1160
|
+
promise,
|
|
1161
|
+
resolve,
|
|
1162
|
+
completed: false,
|
|
1163
|
+
error: null,
|
|
1164
|
+
};
|
|
1165
|
+
}
|
|
1166
|
+
completeTransportRecovery(error = null) {
|
|
1167
|
+
const recovery = this.transportRecovery;
|
|
1168
|
+
if (!recovery || recovery.completed)
|
|
1169
|
+
return;
|
|
1170
|
+
recovery.completed = true;
|
|
1171
|
+
recovery.error = error;
|
|
1172
|
+
recovery.resolve();
|
|
1173
|
+
}
|
|
1174
|
+
async waitForTransportRecovery(connection, recovery) {
|
|
1175
|
+
if (!recovery)
|
|
1176
|
+
return;
|
|
1177
|
+
if (recovery.completed) {
|
|
1178
|
+
if (recovery.error)
|
|
1179
|
+
throw recovery.error;
|
|
1180
|
+
return;
|
|
253
1181
|
}
|
|
254
|
-
|
|
1182
|
+
await recovery.promise;
|
|
1183
|
+
if (connection !== this.connection)
|
|
1184
|
+
throw new Error("Bridge connection changed while recovering.");
|
|
1185
|
+
if (recovery.error)
|
|
1186
|
+
throw recovery.error;
|
|
255
1187
|
}
|
|
256
1188
|
async waitForClose(signal) {
|
|
257
1189
|
if (!this.closed) {
|
|
258
1190
|
return;
|
|
259
1191
|
}
|
|
260
|
-
await
|
|
1192
|
+
await waitForPromiseOrAbort(this.closed, signal);
|
|
261
1193
|
}
|
|
262
1194
|
async stopConnection() {
|
|
263
1195
|
this.stopHeartbeat();
|
|
1196
|
+
this.stopLeaseWatchdog();
|
|
1197
|
+
this.stopSandboxMaintenance();
|
|
264
1198
|
if (this.connection) {
|
|
265
1199
|
await this.connection.stop().catch(() => undefined);
|
|
266
1200
|
this.connection = null;
|
|
@@ -281,6 +1215,21 @@ function isDispatchNoLongerPending(error) {
|
|
|
281
1215
|
}
|
|
282
1216
|
return error instanceof Error && error.message === "Dispatch is no longer pending.";
|
|
283
1217
|
}
|
|
1218
|
+
function isReadinessNoLongerPending(error) {
|
|
1219
|
+
return error instanceof BridgeCallbackRejectedError
|
|
1220
|
+
&& (error.code === "bridge.stale_readiness_probe" || error.code === "bridge.stale_runner_instance");
|
|
1221
|
+
}
|
|
1222
|
+
function isProviderStorageUnavailable(error) {
|
|
1223
|
+
const message = errorMessage(error).toLowerCase();
|
|
1224
|
+
return message.includes("provider_storage_unavailable")
|
|
1225
|
+
|| (message.includes("storage is recovering") && message.includes("503"));
|
|
1226
|
+
}
|
|
1227
|
+
function createDefaultSessionService(sandboxStateRoot) {
|
|
1228
|
+
const rawProcessRunner = new StreamingProviderProcessRunner();
|
|
1229
|
+
const processSupervisors = new AttemptProcessSupervisorRegistry();
|
|
1230
|
+
const coordinatedProcessRunner = new AttemptCoordinatedProviderProcessRunner(processSupervisors, rawProcessRunner);
|
|
1231
|
+
return new BridgeSessionService(new GitWorktreeService(undefined, new RepositoryCoordinatorLock(sandboxStateRoot)), new RoutingProviderCliService(new ProcessBackedProviderCliService(coordinatedProcessRunner)), new HostRuntimeLeaseManager(sandboxStateRoot, undefined, undefined, processSupervisors, rawProcessRunner));
|
|
1232
|
+
}
|
|
284
1233
|
export class SignalRBridgeConnectionFactory {
|
|
285
1234
|
create(config) {
|
|
286
1235
|
return new SignalRBridgeConnection(config);
|
|
@@ -303,8 +1252,12 @@ export class SignalRBridgeConnection {
|
|
|
303
1252
|
Authorization: `Bearer ${config.deviceToken}`
|
|
304
1253
|
}
|
|
305
1254
|
})
|
|
1255
|
+
.withAutomaticReconnect([...config.reconnectBackoffMs])
|
|
306
1256
|
.configureLogging(LogLevel.Warning)
|
|
307
1257
|
.build();
|
|
1258
|
+
this.connection.on("Superseded", () => {
|
|
1259
|
+
void this.connection.stop().catch(() => undefined);
|
|
1260
|
+
});
|
|
308
1261
|
}
|
|
309
1262
|
start() {
|
|
310
1263
|
return this.connection.start();
|
|
@@ -315,16 +1268,49 @@ export class SignalRBridgeConnection {
|
|
|
315
1268
|
onClose(handler) {
|
|
316
1269
|
this.connection.onclose(handler);
|
|
317
1270
|
}
|
|
1271
|
+
onReconnecting(handler) {
|
|
1272
|
+
this.connection.onreconnecting(handler);
|
|
1273
|
+
}
|
|
1274
|
+
onReconnected(handler) {
|
|
1275
|
+
this.connection.onreconnected((connectionId) => {
|
|
1276
|
+
void Promise.resolve(handler(connectionId)).catch(() => undefined);
|
|
1277
|
+
});
|
|
1278
|
+
}
|
|
318
1279
|
onDispatch(handler) {
|
|
319
1280
|
this.connection.on("Dispatch", (raw) => {
|
|
320
1281
|
void handler(raw).catch(() => undefined);
|
|
321
1282
|
});
|
|
322
1283
|
}
|
|
1284
|
+
onOffer(handler) {
|
|
1285
|
+
this.connection.on("Offer", (raw) => {
|
|
1286
|
+
void handler(raw).catch(() => undefined);
|
|
1287
|
+
});
|
|
1288
|
+
}
|
|
1289
|
+
onStart(handler) {
|
|
1290
|
+
this.connection.on("Start", (raw) => {
|
|
1291
|
+
void handler(raw).catch(() => undefined);
|
|
1292
|
+
});
|
|
1293
|
+
}
|
|
1294
|
+
onLeaseUpdate(handler) {
|
|
1295
|
+
this.connection.on("LeaseUpdate", (request) => {
|
|
1296
|
+
void handler(request).catch(() => undefined);
|
|
1297
|
+
});
|
|
1298
|
+
}
|
|
323
1299
|
onCancel(handler) {
|
|
324
1300
|
this.connection.on("Cancel", (request) => {
|
|
325
1301
|
void handler(request).catch(() => undefined);
|
|
326
1302
|
});
|
|
327
1303
|
}
|
|
1304
|
+
onReap(handler) {
|
|
1305
|
+
this.connection.on("Reap", (request) => {
|
|
1306
|
+
void handler(request).catch(() => undefined);
|
|
1307
|
+
});
|
|
1308
|
+
}
|
|
1309
|
+
onReadinessProbe(handler) {
|
|
1310
|
+
this.connection.on("ReadinessProbe", (request) => {
|
|
1311
|
+
void handler(request).catch(() => undefined);
|
|
1312
|
+
});
|
|
1313
|
+
}
|
|
328
1314
|
invoke(methodName, payload) {
|
|
329
1315
|
return this.connection.invoke(methodName, payload);
|
|
330
1316
|
}
|
|
@@ -332,9 +1318,41 @@ export class SignalRBridgeConnection {
|
|
|
332
1318
|
export function defaultBridgeServiceConfig(overrides = {}) {
|
|
333
1319
|
const defaults = defaultForgeBridgeConfig();
|
|
334
1320
|
const cliProviders = splitCsv(process.env.FORGE_CLI_PROVIDERS ?? "codex");
|
|
1321
|
+
const configuredSandboxHostMemoryReserveBytes = optionalPositiveFiniteNumber(process.env.FORGE_SANDBOX_HOST_MEMORY_RESERVE_BYTES);
|
|
1322
|
+
const sandboxPublicationMode = process.env.FORGE_SANDBOX_PUBLICATION_MODE === "bridge_push" ? "bridge_push" : "shared_filesystem";
|
|
335
1323
|
const config = {
|
|
336
1324
|
serverUrl: process.env.FORGE_URL ?? "http://localhost:5208",
|
|
337
1325
|
runnerId: process.env.FORGE_RUNNER_ID ?? "",
|
|
1326
|
+
runnerInstanceId: process.env.FORGE_RUNNER_INSTANCE_ID ?? randomUUID(),
|
|
1327
|
+
trustDomainId: process.env.FORGE_TRUST_DOMAIN_ID ?? "default",
|
|
1328
|
+
maxConcurrentSandboxes: positiveFiniteNumber(Number(process.env.FORGE_MAX_CONCURRENT_SANDBOXES ?? 3), 3),
|
|
1329
|
+
maxConcurrentSandboxBuilds: positiveFiniteNumber(Number(process.env.FORGE_MAX_CONCURRENT_SANDBOX_BUILDS ?? 1), 1),
|
|
1330
|
+
sandboxPidCapacity: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_PID_CAPACITY ?? 2048), 2048),
|
|
1331
|
+
dockerEnabled: process.env.FORGE_DOCKER_SANDBOX !== "false",
|
|
1332
|
+
allowOpenPublicSandbox: process.env.FORGE_ALLOW_OPEN_PUBLIC_SANDBOX === "true",
|
|
1333
|
+
sandboxPublicationMode,
|
|
1334
|
+
dockerExecutable: process.env.FORGE_DOCKER_EXECUTABLE ?? "docker",
|
|
1335
|
+
dockerContext: process.env.FORGE_DOCKER_CONTEXT ?? "",
|
|
1336
|
+
dockerDesktopRecoveryEnabled: process.env.FORGE_DOCKER_DESKTOP_AUTO_RECOVERY === "true",
|
|
1337
|
+
dockerRecoveryFailureThreshold: positiveFiniteNumber(Number(process.env.FORGE_DOCKER_RECOVERY_FAILURE_THRESHOLD ?? 3), 3),
|
|
1338
|
+
dockerRecoveryCooldownMs: positiveFiniteNumber(Number(process.env.FORGE_DOCKER_RECOVERY_COOLDOWN_MS ?? 10 * 60_000), 10 * 60_000),
|
|
1339
|
+
dockerRecoveryStartupTimeoutMs: positiveFiniteNumber(Number(process.env.FORGE_DOCKER_RECOVERY_STARTUP_TIMEOUT_MS ?? 2 * 60_000), 2 * 60_000),
|
|
1340
|
+
sandboxStateRoot: process.env.FORGE_SANDBOX_STATE_ROOT ?? path.join(os.homedir(), ".forge", "bridge", "sandboxes"),
|
|
1341
|
+
requireRootlessDocker: process.env.FORGE_REQUIRE_ROOTLESS_DOCKER === "true" || process.platform === "linux",
|
|
1342
|
+
diskHighWatermarkPercent: positiveFiniteNumber(Number(process.env.FORGE_DISK_HIGH_WATERMARK_PERCENT ?? 85), 85),
|
|
1343
|
+
diskLowWatermarkPercent: positiveFiniteNumber(Number(process.env.FORGE_DISK_LOW_WATERMARK_PERCENT ?? 75), 75),
|
|
1344
|
+
minimumWorkspaceBytes: positiveFiniteNumber(Number(process.env.FORGE_MINIMUM_WORKSPACE_BYTES ?? 10 * 1024 ** 3), 10 * 1024 ** 3),
|
|
1345
|
+
// Undefined means preflight derives the bounded reserve from the actual
|
|
1346
|
+
// Docker/host capacity domain after sampling it.
|
|
1347
|
+
sandboxHostMemoryReserveBytes: configuredSandboxHostMemoryReserveBytes,
|
|
1348
|
+
defaultSandboxWorkerMemoryLimitBytes: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_DEFAULT_WORKER_MEMORY_LIMIT_BYTES ?? 2 * 1024 ** 3), 2 * 1024 ** 3),
|
|
1349
|
+
defaultSandboxServiceMemoryLimitBytes: nonNegativeFiniteNumber(Number(process.env.FORGE_SANDBOX_DEFAULT_SERVICE_MEMORY_LIMIT_BYTES ?? 0), 0),
|
|
1350
|
+
sandboxRuntimeOverheadBytes: Math.max(128 * 1024 ** 2, nonNegativeFiniteNumber(Number(process.env.FORGE_SANDBOX_RUNTIME_OVERHEAD_BYTES ?? 128 * 1024 ** 2), 128 * 1024 ** 2)),
|
|
1351
|
+
sandboxMemorySampleIntervalMs: Math.min(30_000, Math.max(250, positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_MEMORY_SAMPLE_INTERVAL_MS ?? 1000), 1000))),
|
|
1352
|
+
imageRetentionCount: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_IMAGE_RETENTION_COUNT ?? 3), 3),
|
|
1353
|
+
imageRetentionTtlMs: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_IMAGE_RETENTION_TTL_MS ?? 7 * 24 * 60 * 60 * 1000), 7 * 24 * 60 * 60 * 1000),
|
|
1354
|
+
auditRetentionCount: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_AUDIT_RETENTION_COUNT ?? 200), 200),
|
|
1355
|
+
auditRetentionTtlMs: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_AUDIT_RETENTION_TTL_MS ?? 30 * 24 * 60 * 60 * 1000), 30 * 24 * 60 * 60 * 1000),
|
|
338
1356
|
deviceToken: process.env.FORGE_DEVICE_TOKEN ?? "",
|
|
339
1357
|
deviceName: process.env.FORGE_DEVICE_NAME ?? "",
|
|
340
1358
|
defaultRepoRoot: process.env.FORGE_REPO_ROOT ?? defaults.defaultRepoRoot,
|
|
@@ -351,18 +1369,22 @@ export function defaultBridgeServiceConfig(overrides = {}) {
|
|
|
351
1369
|
supportsRunnerAuthoredPhaseArtifacts: true,
|
|
352
1370
|
supportsDraftRunOnce: true,
|
|
353
1371
|
supportsDraftAttachments: true,
|
|
354
|
-
supportsDraftImageAttachments:
|
|
1372
|
+
supportsDraftImageAttachments: supportsDraftImageAttachmentsFor(cliProviders),
|
|
355
1373
|
version: process.env.FORGE_BRIDGE_VERSION ?? forgeBridgeVersion,
|
|
356
|
-
protocolVersion:
|
|
1374
|
+
protocolVersion: 3,
|
|
357
1375
|
minServerVersion: null,
|
|
358
1376
|
heartbeatIntervalMs: positiveFiniteNumber(Number(process.env.FORGE_HEARTBEAT_INTERVAL_MS ?? 30_000), 30_000),
|
|
1377
|
+
callbackAckTimeoutMs: positiveFiniteNumber(Number(process.env.FORGE_CALLBACK_ACK_TIMEOUT_MS ?? 30_000), 30_000),
|
|
359
1378
|
reconnectBackoffMs: [1000, 2000, 5000, 10000, 30000],
|
|
360
1379
|
...overrides
|
|
361
1380
|
};
|
|
362
1381
|
return {
|
|
363
1382
|
...config,
|
|
1383
|
+
providerStatuses: overrides.providerStatuses ?? providerStatusesFor(config.cliProviders),
|
|
364
1384
|
sandboxProfiles: config.sandboxProfiles ?? defaultSandboxProfiles(config.cliProviders),
|
|
365
|
-
|
|
1385
|
+
supportsDraftImageAttachments: (overrides.supportsDraftImageAttachments ?? true) && supportsDraftImageAttachmentsFor(config.cliProviders),
|
|
1386
|
+
heartbeatIntervalMs: positiveFiniteNumber(config.heartbeatIntervalMs, 30_000),
|
|
1387
|
+
callbackAckTimeoutMs: positiveFiniteNumber(config.callbackAckTimeoutMs ?? 30_000, 30_000),
|
|
366
1388
|
};
|
|
367
1389
|
}
|
|
368
1390
|
export function resolveBridgeRunnerHubUrl(serverUrl) {
|
|
@@ -377,9 +1399,42 @@ function normalizeServerUrl(serverUrl) {
|
|
|
377
1399
|
function positiveFiniteNumber(value, fallback) {
|
|
378
1400
|
return Number.isFinite(value) && value > 0 ? value : fallback;
|
|
379
1401
|
}
|
|
1402
|
+
function optionalPositiveFiniteNumber(value) {
|
|
1403
|
+
if (value === undefined || value.trim().length === 0)
|
|
1404
|
+
return undefined;
|
|
1405
|
+
const parsed = Number(value);
|
|
1406
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
|
|
1407
|
+
}
|
|
1408
|
+
function nonNegativeFiniteNumber(value, fallback) {
|
|
1409
|
+
return Number.isFinite(value) && value >= 0 ? value : fallback;
|
|
1410
|
+
}
|
|
380
1411
|
function providerStatusesFor(providers) {
|
|
381
1412
|
return Object.fromEntries(providers.map((provider) => [provider, { signedIn: true, version: "", models: configuredModelsFor(provider) }]));
|
|
382
1413
|
}
|
|
1414
|
+
function sandboxAuthModesFor(config) {
|
|
1415
|
+
return Object.fromEntries(config.cliProviders.map((provider) => {
|
|
1416
|
+
const modes = ["api_key"];
|
|
1417
|
+
if (sandboxBridgeCliAuthAvailable(config, provider)) {
|
|
1418
|
+
modes.push("bridge_cli");
|
|
1419
|
+
}
|
|
1420
|
+
return [provider, modes];
|
|
1421
|
+
}));
|
|
1422
|
+
}
|
|
1423
|
+
function sandboxBridgeCliAuthAvailable(config, provider) {
|
|
1424
|
+
if (provider.trim().toLowerCase() !== "codex") {
|
|
1425
|
+
return false;
|
|
1426
|
+
}
|
|
1427
|
+
const status = providerStatus(config, provider);
|
|
1428
|
+
if (status?.signedIn === false) {
|
|
1429
|
+
return false;
|
|
1430
|
+
}
|
|
1431
|
+
return codexSandboxProviderAuthAdapter.doctor().available;
|
|
1432
|
+
}
|
|
1433
|
+
function providerStatus(config, provider) {
|
|
1434
|
+
const normalized = provider.trim().toLowerCase();
|
|
1435
|
+
return Object.entries(config.providerStatuses)
|
|
1436
|
+
.find(([key]) => key.trim().toLowerCase() === normalized)?.[1];
|
|
1437
|
+
}
|
|
383
1438
|
function defaultSandboxProfiles(providers, platform = process.platform) {
|
|
384
1439
|
if (!providers.some((provider) => provider.toLowerCase() === "codex")) {
|
|
385
1440
|
return ["none"];
|
|
@@ -474,10 +1529,11 @@ function normalizeQaAppLaunchTarget(target) {
|
|
|
474
1529
|
command: cleanString(target.command),
|
|
475
1530
|
working_directory: cleanString(target.working_directory),
|
|
476
1531
|
ready_url: cleanString(target.ready_url),
|
|
1532
|
+
managed_runtime_port: cleanString(target.managed_runtime_port),
|
|
477
1533
|
routes,
|
|
478
1534
|
notes: cleanString(target.notes)
|
|
479
1535
|
};
|
|
480
|
-
return normalized.url || normalized.command || normalized.ready_url ||
|
|
1536
|
+
return normalized.url || normalized.command || normalized.ready_url || normalized.managed_runtime_port
|
|
481
1537
|
? normalized
|
|
482
1538
|
: null;
|
|
483
1539
|
}
|
|
@@ -519,29 +1575,353 @@ function formatDelay(ms) {
|
|
|
519
1575
|
}
|
|
520
1576
|
return `${ms}ms`;
|
|
521
1577
|
}
|
|
1578
|
+
async function waitForCallbackAck(callback, methodName, timeoutMs) {
|
|
1579
|
+
let timeout = null;
|
|
1580
|
+
const expired = new Promise((_resolve, reject) => {
|
|
1581
|
+
timeout = setTimeout(() => {
|
|
1582
|
+
reject(new Error(`bridge_callback_timeout: ${methodName} callback was not acknowledged within ${formatDelay(timeoutMs)}.`));
|
|
1583
|
+
}, timeoutMs);
|
|
1584
|
+
});
|
|
1585
|
+
try {
|
|
1586
|
+
return await Promise.race([callback, expired]);
|
|
1587
|
+
}
|
|
1588
|
+
finally {
|
|
1589
|
+
if (timeout)
|
|
1590
|
+
clearTimeout(timeout);
|
|
1591
|
+
}
|
|
1592
|
+
}
|
|
522
1593
|
function logBridgeStatus(message) {
|
|
523
1594
|
process.stderr.write(`[${new Date().toISOString()}] ${message}\n`);
|
|
524
1595
|
}
|
|
525
|
-
function delay(ms, signal) {
|
|
1596
|
+
export function delay(ms, signal) {
|
|
526
1597
|
if (signal?.aborted) {
|
|
527
1598
|
return Promise.resolve();
|
|
528
1599
|
}
|
|
529
1600
|
return new Promise((resolve) => {
|
|
530
|
-
const
|
|
531
|
-
signal?.addEventListener("abort", () => {
|
|
1601
|
+
const complete = () => {
|
|
532
1602
|
clearTimeout(timeout);
|
|
1603
|
+
signal?.removeEventListener("abort", complete);
|
|
533
1604
|
resolve();
|
|
534
|
-
}
|
|
1605
|
+
};
|
|
1606
|
+
const timeout = setTimeout(complete, ms);
|
|
1607
|
+
signal?.addEventListener("abort", complete, { once: true });
|
|
1608
|
+
if (signal?.aborted)
|
|
1609
|
+
complete();
|
|
535
1610
|
});
|
|
536
1611
|
}
|
|
537
|
-
function
|
|
1612
|
+
export async function waitForPromiseOrAbort(promise, signal) {
|
|
538
1613
|
if (!signal) {
|
|
539
|
-
|
|
1614
|
+
await promise;
|
|
1615
|
+
return;
|
|
540
1616
|
}
|
|
541
1617
|
if (signal.aborted) {
|
|
542
|
-
return
|
|
1618
|
+
return;
|
|
1619
|
+
}
|
|
1620
|
+
let resolveAbort;
|
|
1621
|
+
const abort = new Promise(resolve => { resolveAbort = resolve; });
|
|
1622
|
+
signal.addEventListener("abort", resolveAbort, { once: true });
|
|
1623
|
+
try {
|
|
1624
|
+
await Promise.race([promise, abort]);
|
|
1625
|
+
}
|
|
1626
|
+
finally {
|
|
1627
|
+
signal.removeEventListener("abort", resolveAbort);
|
|
1628
|
+
}
|
|
1629
|
+
}
|
|
1630
|
+
export function canReapManagedSandboxes(preflight) {
|
|
1631
|
+
return canRunDockerMaintenance(preflight);
|
|
1632
|
+
}
|
|
1633
|
+
function sandboxNetworkPolicyFailure(dispatch, config) {
|
|
1634
|
+
const sandbox = dispatch.sandbox;
|
|
1635
|
+
if (!sandbox)
|
|
1636
|
+
return "Protocol-v2 sandbox context is missing.";
|
|
1637
|
+
if (sandbox.networkPolicy.mode === "open_public"
|
|
1638
|
+
&& (sandbox.executionPolicy === "sandbox_required" || !config.allowOpenPublicSandbox)) {
|
|
1639
|
+
return "open_public sandbox networking is not permitted by this runner policy.";
|
|
1640
|
+
}
|
|
1641
|
+
if (sandbox.networkPolicy.mode === "public_allowlist"
|
|
1642
|
+
&& sandbox.networkPolicy.allowedHosts.length > 0
|
|
1643
|
+
&& process.env.FORGE_SANDBOX_EGRESS_ENFORCED !== "true") {
|
|
1644
|
+
return "public_allowlist requires an operator-enforced Forge egress proxy or firewall.";
|
|
1645
|
+
}
|
|
1646
|
+
return null;
|
|
1647
|
+
}
|
|
1648
|
+
export function infrastructureFailureEventType(summary) {
|
|
1649
|
+
const validationFailure = /^(prepublication_(?:validation_failed|capability_unavailable|contract_invalid|execution_scope_invalid)):/u.exec(summary)?.[1];
|
|
1650
|
+
if (validationFailure)
|
|
1651
|
+
return validationFailure;
|
|
1652
|
+
const sandboxFailure = /^(sandbox_[a-z0-9_.-]+|image_[a-z0-9_.-]+|contract_[a-z0-9_.-]+|teardown_[a-z0-9_.-]+):/u.exec(summary)?.[1];
|
|
1653
|
+
if (sandboxFailure)
|
|
1654
|
+
return sandboxFailure;
|
|
1655
|
+
if (summary.startsWith("worktree_prepare_failed:"))
|
|
1656
|
+
return "worktree_prepare_failed";
|
|
1657
|
+
if (summary.startsWith("provider_network_unavailable:"))
|
|
1658
|
+
return "provider_network_unavailable";
|
|
1659
|
+
if (summary.startsWith("provider_start_unavailable:"))
|
|
1660
|
+
return "provider_start_unavailable";
|
|
1661
|
+
if (summary.startsWith("host_port_collision:"))
|
|
1662
|
+
return "host_port_collision";
|
|
1663
|
+
if (summary.startsWith("host_runtime_process_exited:"))
|
|
1664
|
+
return "host_runtime_process_exited";
|
|
1665
|
+
if (summary.startsWith("host_runtime_listener_timeout:"))
|
|
1666
|
+
return "host_runtime_listener_timeout";
|
|
1667
|
+
if (summary.startsWith("host_runtime_http_readiness_timeout:"))
|
|
1668
|
+
return "host_runtime_http_readiness_timeout";
|
|
1669
|
+
if (summary.startsWith("docker_engine_unavailable:"))
|
|
1670
|
+
return "docker_engine_unavailable";
|
|
1671
|
+
if (summary.startsWith("attempt_process_supervisor_start_failed:"))
|
|
1672
|
+
return "attempt_process_supervisor_start_failed";
|
|
1673
|
+
if (summary.startsWith("attempt_process_supervisor_start_timeout:"))
|
|
1674
|
+
return "attempt_process_supervisor_start_timeout";
|
|
1675
|
+
if (summary.startsWith("bridge_callback_timeout:"))
|
|
1676
|
+
return "bridge_callback_timeout";
|
|
1677
|
+
return null;
|
|
1678
|
+
}
|
|
1679
|
+
export function dockerEngineRequirementFailure(requiredCapabilities, executionMode, capability) {
|
|
1680
|
+
if (!requiredCapabilities.includes("docker_engine"))
|
|
1681
|
+
return null;
|
|
1682
|
+
if (executionMode === "docker") {
|
|
1683
|
+
return "docker_engine_unavailable: docker_engine is not exposed inside Forge agent sandboxes; use a managed host runner with Docker-daemon access.";
|
|
1684
|
+
}
|
|
1685
|
+
return capability?.available
|
|
1686
|
+
? null
|
|
1687
|
+
: `docker_engine_unavailable: ${capability?.unavailableReason ?? "Docker daemon is unreachable from the bridge process."}`;
|
|
1688
|
+
}
|
|
1689
|
+
function parseReadinessProbe(value) {
|
|
1690
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
1691
|
+
return null;
|
|
1692
|
+
const request = value;
|
|
1693
|
+
const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
|
|
1694
|
+
const bounded = (candidate, maximum) => typeof candidate === "string" && candidate.trim().length > 0 && candidate.length <= maximum;
|
|
1695
|
+
if (request.schemaVersion !== 1
|
|
1696
|
+
|| !uuid.test(request.requestId ?? "")
|
|
1697
|
+
|| !uuid.test(request.fencingToken ?? "")
|
|
1698
|
+
|| !uuid.test(request.userId ?? "")
|
|
1699
|
+
|| !uuid.test(request.projectId ?? "")
|
|
1700
|
+
|| !uuid.test(request.repoId ?? "")
|
|
1701
|
+
|| !uuid.test(request.runnerInstanceId ?? "")
|
|
1702
|
+
|| !Number.isInteger(request.attempt) || (request.attempt ?? 0) < 1
|
|
1703
|
+
|| !bounded(request.repoName, 160)
|
|
1704
|
+
|| typeof request.remoteUrl !== "string" || request.remoteUrl.length > 2048
|
|
1705
|
+
|| !bounded(request.defaultBranch, 512)
|
|
1706
|
+
|| typeof request.cloneOnDemand !== "boolean"
|
|
1707
|
+
|| !bounded(request.runnerRepoLocalPath, 4096)
|
|
1708
|
+
|| !bounded(request.revision, 512)
|
|
1709
|
+
|| !Number.isFinite(Date.parse(request.expiresAt ?? ""))
|
|
1710
|
+
|| Date.parse(request.expiresAt ?? "") <= Date.now()) {
|
|
1711
|
+
return null;
|
|
1712
|
+
}
|
|
1713
|
+
return request;
|
|
1714
|
+
}
|
|
1715
|
+
function sandboxProviderNetworkFailure(dispatch, config) {
|
|
1716
|
+
const sandbox = dispatch.sandbox;
|
|
1717
|
+
if (!sandbox || sandbox.networkPolicy.mode !== "offline")
|
|
1718
|
+
return null;
|
|
1719
|
+
const provider = dispatch.execution.provider.trim().toLowerCase();
|
|
1720
|
+
const isNetworkProvider = config.cliProviders.some(candidate => candidate.trim().toLowerCase() === provider);
|
|
1721
|
+
if (!isNetworkProvider)
|
|
1722
|
+
return null;
|
|
1723
|
+
return `Provider ${dispatch.execution.provider} requires outbound network access, but this Docker attempt uses the offline network policy.`;
|
|
1724
|
+
}
|
|
1725
|
+
function sandboxAuthFailure(dispatch, config) {
|
|
1726
|
+
const unsupported = sandboxProviderAuthUnavailableMessage(dispatch.execution.provider, dispatch.execution.authKind);
|
|
1727
|
+
if (unsupported) {
|
|
1728
|
+
return unsupported;
|
|
1729
|
+
}
|
|
1730
|
+
if (dispatch.execution.authKind.trim().toLowerCase() !== "bridge_cli") {
|
|
1731
|
+
return null;
|
|
1732
|
+
}
|
|
1733
|
+
const status = providerStatus(config, dispatch.execution.provider);
|
|
1734
|
+
if (status?.signedIn === false) {
|
|
1735
|
+
return `${dispatch.execution.provider} Local CLI is not signed in on this runner.`;
|
|
1736
|
+
}
|
|
1737
|
+
const adapter = sandboxProviderAuthAdapterFor(dispatch.execution.provider, dispatch.execution.authKind);
|
|
1738
|
+
if (!adapter) {
|
|
1739
|
+
return `${dispatch.execution.provider} Local CLI auth is not portable into Docker yet.`;
|
|
543
1740
|
}
|
|
544
|
-
|
|
1741
|
+
const doctor = adapter.doctor();
|
|
1742
|
+
return doctor.available ? null : doctor.message;
|
|
1743
|
+
}
|
|
1744
|
+
function rejectedOfferReservation(code, message) {
|
|
1745
|
+
return {
|
|
1746
|
+
accepted: false,
|
|
1747
|
+
code,
|
|
1748
|
+
message,
|
|
1749
|
+
reservedUntil: null,
|
|
1750
|
+
resources: null,
|
|
1751
|
+
};
|
|
1752
|
+
}
|
|
1753
|
+
export function canReserveSandboxResources(used, requested, capacity) {
|
|
1754
|
+
return used.sandboxSlots + requested.sandboxSlots <= capacity.sandboxSlots
|
|
1755
|
+
&& used.cpuMillis + requested.cpuMillis <= capacity.cpuMillis
|
|
1756
|
+
&& used.memoryBytes + requested.memoryBytes <= capacity.memoryBytes
|
|
1757
|
+
&& used.workspaceBytes + requested.workspaceBytes <= capacity.workspaceBytes
|
|
1758
|
+
&& used.pids + requested.pids <= capacity.pids
|
|
1759
|
+
&& used.buildSlots + requested.buildSlots <= capacity.buildSlots;
|
|
1760
|
+
}
|
|
1761
|
+
export function sandboxMemoryCommitmentCapacity(preflight, activeMemoryBytes, ambientMemoryCommitmentBytes = 0) {
|
|
1762
|
+
const ambientCapacity = preflight.availableMemoryBytes
|
|
1763
|
+
- preflight.hostReserveBytes
|
|
1764
|
+
+ activeMemoryBytes;
|
|
1765
|
+
const commitmentCapacity = preflight.totalMemoryBytes
|
|
1766
|
+
- preflight.hostReserveBytes
|
|
1767
|
+
- Math.max(0, ambientMemoryCommitmentBytes);
|
|
1768
|
+
return Math.max(0, Math.min(ambientCapacity, commitmentCapacity));
|
|
1769
|
+
}
|
|
1770
|
+
export function sandboxCapacityFailureMessage(used, requested, capacity, preflight, config) {
|
|
1771
|
+
const availableSlots = Math.max(0, capacity.sandboxSlots - used.sandboxSlots);
|
|
1772
|
+
if (requested.sandboxSlots > availableSlots) {
|
|
1773
|
+
return `Sandbox slot capacity is insufficient: requested ${requested.sandboxSlots}, available ${availableSlots}, ${used.sandboxSlots} already reserved.`;
|
|
1774
|
+
}
|
|
1775
|
+
const availableMemoryBytes = Math.max(0, capacity.memoryBytes - used.memoryBytes);
|
|
1776
|
+
if (requested.memoryBytes > availableMemoryBytes) {
|
|
1777
|
+
return [
|
|
1778
|
+
`Sandbox memory capacity is insufficient: requested ${formatBytes(requested.memoryBytes)} total`,
|
|
1779
|
+
`(worker ${formatBytes(requested.workerMemoryLimitBytes)}, services ${formatBytes(requested.serviceMemoryLimitBytes)}, runtime overhead ${formatBytes(requested.runtimeOverheadBytes)}),`,
|
|
1780
|
+
`but ${formatBytes(availableMemoryBytes)} is available for admission.`,
|
|
1781
|
+
`Docker reports ${formatBytes(preflight?.availableMemoryBytes ?? 0)} available of ${formatBytes(preflight?.totalMemoryBytes ?? 0)};`,
|
|
1782
|
+
`Forge reserves ${formatBytes(preflight?.hostReserveBytes ?? config.sandboxHostMemoryReserveBytes ?? 0)} for the host and tracks ${formatBytes(used.memoryBytes)} in active or reserved attempts.`,
|
|
1783
|
+
].join(" ");
|
|
1784
|
+
}
|
|
1785
|
+
const availableCpuMillis = Math.max(0, capacity.cpuMillis - used.cpuMillis);
|
|
1786
|
+
if (requested.cpuMillis > availableCpuMillis) {
|
|
1787
|
+
return `Sandbox CPU capacity is insufficient: requested ${requested.cpuMillis} millis, available ${availableCpuMillis} millis, ${used.cpuMillis} millis already reserved.`;
|
|
1788
|
+
}
|
|
1789
|
+
const availableWorkspaceBytes = Math.max(0, capacity.workspaceBytes - used.workspaceBytes);
|
|
1790
|
+
if (requested.workspaceBytes > availableWorkspaceBytes) {
|
|
1791
|
+
return `Sandbox workspace capacity is insufficient: requested ${formatBytes(requested.workspaceBytes)}, available ${formatBytes(availableWorkspaceBytes)}, ${formatBytes(used.workspaceBytes)} already reserved.`;
|
|
1792
|
+
}
|
|
1793
|
+
const availablePids = Math.max(0, capacity.pids - used.pids);
|
|
1794
|
+
if (requested.pids > availablePids) {
|
|
1795
|
+
return `Sandbox PID capacity is insufficient: requested ${requested.pids}, available ${availablePids}, ${used.pids} already reserved.`;
|
|
1796
|
+
}
|
|
1797
|
+
const availableBuildSlots = Math.max(0, capacity.buildSlots - used.buildSlots);
|
|
1798
|
+
return `Sandbox build capacity is insufficient: requested ${requested.buildSlots}, available ${availableBuildSlots}, ${used.buildSlots} already reserved.`;
|
|
1799
|
+
}
|
|
1800
|
+
function sumSandboxResources(resources) {
|
|
1801
|
+
return resources.reduce((total, resource) => ({
|
|
1802
|
+
sandboxSlots: total.sandboxSlots + resource.sandboxSlots,
|
|
1803
|
+
cpuMillis: total.cpuMillis + resource.cpuMillis,
|
|
1804
|
+
workerMemoryLimitBytes: total.workerMemoryLimitBytes + resource.workerMemoryLimitBytes,
|
|
1805
|
+
serviceMemoryLimitBytes: total.serviceMemoryLimitBytes + resource.serviceMemoryLimitBytes,
|
|
1806
|
+
runtimeOverheadBytes: total.runtimeOverheadBytes + resource.runtimeOverheadBytes,
|
|
1807
|
+
memoryBytes: total.memoryBytes + resource.memoryBytes,
|
|
1808
|
+
workspaceBytes: total.workspaceBytes + resource.workspaceBytes,
|
|
1809
|
+
pids: total.pids + resource.pids,
|
|
1810
|
+
buildSlots: total.buildSlots + resource.buildSlots
|
|
1811
|
+
}), zeroSandboxResources());
|
|
1812
|
+
}
|
|
1813
|
+
function zeroSandboxResources() {
|
|
1814
|
+
return {
|
|
1815
|
+
sandboxSlots: 0,
|
|
1816
|
+
cpuMillis: 0,
|
|
1817
|
+
workerMemoryLimitBytes: 0,
|
|
1818
|
+
serviceMemoryLimitBytes: 0,
|
|
1819
|
+
runtimeOverheadBytes: 0,
|
|
1820
|
+
memoryBytes: 0,
|
|
1821
|
+
workspaceBytes: 0,
|
|
1822
|
+
pids: 0,
|
|
1823
|
+
buildSlots: 0,
|
|
1824
|
+
};
|
|
1825
|
+
}
|
|
1826
|
+
function defaultSandboxResources() {
|
|
1827
|
+
return {
|
|
1828
|
+
sandboxSlots: 1,
|
|
1829
|
+
cpuMillis: 2000,
|
|
1830
|
+
workerMemoryLimitBytes: 2 * 1024 ** 3,
|
|
1831
|
+
serviceMemoryLimitBytes: 0,
|
|
1832
|
+
runtimeOverheadBytes: 128 * 1024 ** 2,
|
|
1833
|
+
memoryBytes: 2 * 1024 ** 3 + 128 * 1024 ** 2,
|
|
1834
|
+
workspaceBytes: 10 * 1024 ** 3,
|
|
1835
|
+
pids: 512,
|
|
1836
|
+
buildSlots: 0
|
|
1837
|
+
};
|
|
1838
|
+
}
|
|
1839
|
+
export function resolveDynamicSandboxResources(requested, serviceCount, networkMode, baseRuntimeOverheadBytes = 128 * 1024 ** 2) {
|
|
1840
|
+
const serviceMemoryLimitBytes = Math.max(0, serviceCount) * 1024 ** 3;
|
|
1841
|
+
const runtimeOverheadBytes = Math.max(128 * 1024 ** 2, baseRuntimeOverheadBytes)
|
|
1842
|
+
+ (networkMode === "public_allowlist" ? 256 * 1024 ** 2 : 0);
|
|
1843
|
+
return {
|
|
1844
|
+
...requested,
|
|
1845
|
+
serviceMemoryLimitBytes,
|
|
1846
|
+
runtimeOverheadBytes,
|
|
1847
|
+
memoryBytes: requested.workerMemoryLimitBytes + serviceMemoryLimitBytes + runtimeOverheadBytes,
|
|
1848
|
+
};
|
|
1849
|
+
}
|
|
1850
|
+
export async function resolveSandboxResourcesFromRepo(dispatch, baseRuntimeOverheadBytes = 128 * 1024 ** 2) {
|
|
1851
|
+
const repoPath = dispatch.repo.serverLocalPath;
|
|
1852
|
+
const sandbox = dispatch.sandbox;
|
|
1853
|
+
if (!repoPath || !sandbox) {
|
|
1854
|
+
throw new Error("Sandbox admission needs a mapped repository.");
|
|
1855
|
+
}
|
|
1856
|
+
const selection = selectSandboxConfigRefs(sandbox.executionPolicy, sandbox.sandboxConfigCommitSha, dispatch.repo.defaultBranch);
|
|
1857
|
+
const configCommitSha = await resolveFirstGitCommit(repoPath, selection.refs);
|
|
1858
|
+
if (!configCommitSha) {
|
|
1859
|
+
throw new Error("Sandbox admission could not resolve the immutable sandbox config commit.");
|
|
1860
|
+
}
|
|
1861
|
+
const contractResult = await runSandboxProcess({
|
|
1862
|
+
command: "git",
|
|
1863
|
+
args: ["-C", repoPath, "show", `${configCommitSha}:.forge/forge.sandbox.yml`],
|
|
1864
|
+
timeoutMs: 30_000,
|
|
1865
|
+
maxOutputBytes: 1024 * 1024,
|
|
1866
|
+
});
|
|
1867
|
+
if (contractResult.exitCode !== 0) {
|
|
1868
|
+
throw new Error("The approved immutable sandbox contract is unavailable in the mapped repository.");
|
|
1869
|
+
}
|
|
1870
|
+
const contract = validateSandboxContract(contractResult.stdout);
|
|
1871
|
+
const workspaceBytes = await estimateWorkspaceBytesFromRepo(repoPath, {
|
|
1872
|
+
role: dispatch.execution.role,
|
|
1873
|
+
serviceCount: contract.contract.services.length,
|
|
1874
|
+
browser: contract.contract.browser,
|
|
1875
|
+
});
|
|
1876
|
+
return {
|
|
1877
|
+
resources: resolveDynamicSandboxResources({
|
|
1878
|
+
...sandbox.requestedResources,
|
|
1879
|
+
workspaceBytes: Math.max(sandbox.requestedResources.workspaceBytes, workspaceBytes),
|
|
1880
|
+
}, contract.contract.services.length, sandbox.networkPolicy.mode, baseRuntimeOverheadBytes),
|
|
1881
|
+
sandboxConfigCommitSha: configCommitSha,
|
|
1882
|
+
};
|
|
1883
|
+
}
|
|
1884
|
+
async function resolveFirstGitCommit(repoPath, refs) {
|
|
1885
|
+
for (const ref of refs) {
|
|
1886
|
+
const result = await runSandboxProcess({
|
|
1887
|
+
command: "git",
|
|
1888
|
+
args: ["-C", repoPath, "rev-parse", "--verify", `${ref}^{commit}`],
|
|
1889
|
+
timeoutMs: 30_000,
|
|
1890
|
+
maxOutputBytes: 1024 * 1024,
|
|
1891
|
+
});
|
|
1892
|
+
const commitSha = result.stdout.trim().toLowerCase();
|
|
1893
|
+
if (result.exitCode === 0 && /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/u.test(commitSha)) {
|
|
1894
|
+
return commitSha;
|
|
1895
|
+
}
|
|
1896
|
+
}
|
|
1897
|
+
return null;
|
|
1898
|
+
}
|
|
1899
|
+
function parseOfferEnvelope(value) {
|
|
1900
|
+
return parseDispatchEnvelope(value);
|
|
1901
|
+
}
|
|
1902
|
+
function parseStartEnvelope(value) {
|
|
1903
|
+
return parseDispatchEnvelope(value);
|
|
1904
|
+
}
|
|
1905
|
+
function parseDispatchEnvelope(value) {
|
|
1906
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
1907
|
+
return null;
|
|
1908
|
+
const envelope = value;
|
|
1909
|
+
if (envelope.schemaVersion !== 1
|
|
1910
|
+
|| typeof envelope.dispatchAttemptId !== "string"
|
|
1911
|
+
|| typeof envelope.leaseId !== "string"
|
|
1912
|
+
|| typeof envelope.fencingToken !== "string"
|
|
1913
|
+
|| typeof envelope.runnerInstanceId !== "string"
|
|
1914
|
+
|| typeof envelope.expiresAt !== "string"
|
|
1915
|
+
|| !envelope.dispatch)
|
|
1916
|
+
return null;
|
|
1917
|
+
return {
|
|
1918
|
+
dispatchAttemptId: envelope.dispatchAttemptId,
|
|
1919
|
+
leaseId: envelope.leaseId,
|
|
1920
|
+
fencingToken: envelope.fencingToken,
|
|
1921
|
+
runnerInstanceId: envelope.runnerInstanceId,
|
|
1922
|
+
expiresAt: envelope.expiresAt,
|
|
1923
|
+
dispatch: envelope.dispatch
|
|
1924
|
+
};
|
|
545
1925
|
}
|
|
546
1926
|
const maxAttachmentExactSecretLength = 8 * 1024;
|
|
547
1927
|
function exactSecrets(dispatch, config) {
|