@loucompanion/forge-bridge 0.1.3-dev.9bc5014a2270 → 0.1.3-dev.a58d7cd1092f
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +127 -0
- package/dist/bridge/bridge.base.d.ts +8 -1
- package/dist/bridge/bridge.service.d.ts +93 -3
- package/dist/bridge/bridge.service.js +1358 -47
- package/dist/bridge/bridge.types.d.ts +202 -1
- package/dist/bridge/runner-host-capabilities.d.ts +15 -0
- package/dist/bridge/runner-host-capabilities.js +93 -0
- package/dist/cleanup/cleanup.service.d.ts +14 -2
- package/dist/cleanup/cleanup.service.js +496 -3
- package/dist/cleanup/cleanup.types.d.ts +10 -0
- package/dist/cleanup/index.d.ts +1 -1
- package/dist/cleanup/index.js +1 -1
- package/dist/cleanup/internals.d.ts +2 -2
- package/dist/cleanup/internals.js +1 -1
- package/dist/cli/cli.help.js +47 -1
- package/dist/cli/cli.service.d.ts +45 -2
- package/dist/cli/cli.service.js +378 -26
- package/dist/cli/local-config.d.ts +7 -0
- package/dist/cli/local-config.js +51 -3
- package/dist/runtime/aspnet-runtime-prebuild.d.ts +36 -0
- package/dist/runtime/aspnet-runtime-prebuild.js +340 -0
- package/dist/runtime/attempt-node-runtime-launcher.d.ts +17 -0
- package/dist/runtime/attempt-node-runtime-launcher.js +105 -0
- package/dist/runtime/attempt-process-supervisor-registry.d.ts +20 -0
- package/dist/runtime/attempt-process-supervisor-registry.js +64 -0
- package/dist/runtime/attempt-process-supervisor.d.ts +97 -0
- package/dist/runtime/attempt-process-supervisor.js +386 -0
- package/dist/runtime/docker-engine-probe.d.ts +83 -0
- package/dist/runtime/docker-engine-probe.js +300 -0
- package/dist/runtime/generated-output-reset.d.ts +10 -0
- package/dist/runtime/generated-output-reset.js +103 -0
- package/dist/runtime/host-listener-inspector.d.ts +46 -0
- package/dist/runtime/host-listener-inspector.js +349 -0
- package/dist/runtime/host-port-broker.d.ts +76 -0
- package/dist/runtime/host-port-broker.js +333 -0
- package/dist/runtime/host-runtime-contract.d.ts +47 -0
- package/dist/runtime/host-runtime-contract.js +700 -0
- package/dist/runtime/host-runtime-lease.d.ts +197 -0
- package/dist/runtime/host-runtime-lease.js +2203 -0
- package/dist/runtime/host-runtime-promotion.d.ts +23 -0
- package/dist/runtime/host-runtime-promotion.js +159 -0
- package/dist/runtime/memory-capacity.d.ts +68 -0
- package/dist/runtime/memory-capacity.js +121 -0
- package/dist/runtime/node-host-runtime-adapter.d.ts +54 -0
- package/dist/runtime/node-host-runtime-adapter.js +284 -0
- package/dist/runtime/playwright-host-runtime-adapter.d.ts +21 -0
- package/dist/runtime/playwright-host-runtime-adapter.js +231 -0
- package/dist/runtime/runtime-isolation-lease.d.ts +67 -0
- package/dist/runtime/runtime-isolation-lease.js +158 -0
- package/dist/runtime/workspace-resource-estimator.d.ts +17 -0
- package/dist/runtime/workspace-resource-estimator.js +174 -0
- package/dist/sandbox/command-runner.d.ts +4 -0
- package/dist/sandbox/command-runner.js +47 -0
- package/dist/sandbox/contract.d.ts +5 -0
- package/dist/sandbox/contract.js +276 -0
- package/dist/sandbox/event-budget.d.ts +11 -0
- package/dist/sandbox/event-budget.js +27 -0
- package/dist/sandbox/image-policy.d.ts +35 -0
- package/dist/sandbox/image-policy.js +247 -0
- package/dist/sandbox/index.d.ts +5 -0
- package/dist/sandbox/index.js +5 -0
- package/dist/sandbox/lease-watchdog.d.ts +18 -0
- package/dist/sandbox/lease-watchdog.js +67 -0
- package/dist/sandbox/preflight.d.ts +84 -0
- package/dist/sandbox/preflight.js +521 -0
- package/dist/sandbox/process.d.ts +20 -0
- package/dist/sandbox/process.js +209 -0
- package/dist/sandbox/provider-auth/codex.d.ts +26 -0
- package/dist/sandbox/provider-auth/codex.js +212 -0
- package/dist/sandbox/repository-lock.d.ts +5 -0
- package/dist/sandbox/repository-lock.js +121 -0
- package/dist/sandbox/runtime.d.ts +297 -0
- package/dist/sandbox/runtime.js +2338 -0
- package/dist/sandbox/sandbox.types.d.ts +93 -0
- package/dist/sandbox/sandbox.types.js +1 -0
- package/dist/sandbox/smoke.d.ts +76 -0
- package/dist/sandbox/smoke.js +380 -0
- package/dist/sandbox/worker-supervisor.d.ts +13 -0
- package/dist/sandbox/worker-supervisor.js +36 -0
- package/dist/sandbox/worker.d.ts +1 -0
- package/dist/sandbox/worker.js +225 -0
- package/dist/sandbox/workspace.service.d.ts +13 -0
- package/dist/sandbox/workspace.service.js +72 -0
- package/dist/session/managed-attempt-tools.d.ts +21 -0
- package/dist/session/managed-attempt-tools.js +912 -0
- package/dist/session/managed-build-coordinator.d.ts +5 -0
- package/dist/session/managed-build-coordinator.js +105 -0
- package/dist/session/managed-build-receipt.d.ts +21 -0
- package/dist/session/managed-build-receipt.js +335 -0
- package/dist/session/phase-completion.artifacts.d.ts +17 -0
- package/dist/session/phase-completion.artifacts.js +256 -26
- package/dist/session/phase-completion.d.ts +12 -2
- package/dist/session/phase-completion.helpers.d.ts +5 -0
- package/dist/session/phase-completion.helpers.js +51 -10
- package/dist/session/phase-completion.js +14 -3
- package/dist/session/phase-completion.prompts.d.ts +3 -3
- package/dist/session/phase-completion.prompts.js +24 -8
- package/dist/session/phase-completion.runtime.d.ts +13 -1
- package/dist/session/phase-completion.runtime.js +189 -8
- package/dist/session/phase-completion.schemas.js +117 -4
- package/dist/session/phase-completion.validation.d.ts +4 -1
- package/dist/session/phase-completion.validation.js +153 -22
- package/dist/session/pre-publication-validation.d.ts +46 -0
- package/dist/session/pre-publication-validation.js +208 -0
- package/dist/session/provider-cli/ci-repair-readiness.d.ts +16 -0
- package/dist/session/provider-cli/ci-repair-readiness.js +110 -0
- package/dist/session/provider-cli/provider-cli.artifacts.d.ts +3 -2
- package/dist/session/provider-cli/provider-cli.artifacts.js +65 -7
- package/dist/session/provider-cli/provider-cli.base.d.ts +1 -0
- package/dist/session/provider-cli/provider-cli.deterministic.d.ts +2 -0
- package/dist/session/provider-cli/provider-cli.deterministic.js +66 -9
- package/dist/session/provider-cli/provider-cli.process.d.ts +26 -0
- package/dist/session/provider-cli/provider-cli.process.js +474 -23
- package/dist/session/provider-cli/provider-cli.prompts.d.ts +16 -2
- package/dist/session/provider-cli/provider-cli.prompts.js +224 -24
- package/dist/session/provider-cli/provider-cli.qa-sidecars.js +1 -0
- package/dist/session/provider-cli/provider-cli.runtime.d.ts +1 -1
- package/dist/session/provider-cli/provider-cli.runtime.js +6 -3
- package/dist/session/provider-cli/provider-cli.service.d.ts +26 -3
- package/dist/session/provider-cli/provider-cli.service.js +646 -90
- package/dist/session/provider-cli/provider-cli.types.d.ts +5 -0
- package/dist/session/provider-cli/provider-output-state.d.ts +2 -0
- package/dist/session/provider-cli/provider-output-state.js +91 -16
- package/dist/session/provider-cli/provider-usage.d.ts +7 -0
- package/dist/session/provider-cli/provider-usage.js +28 -5
- package/dist/session/session-redaction.d.ts +3 -0
- package/dist/session/session-redaction.js +38 -0
- package/dist/session/session.base.d.ts +6 -1
- package/dist/session/session.service.d.ts +8 -2
- package/dist/session/session.service.js +530 -42
- package/dist/session/session.types.d.ts +124 -1
- package/dist/session/session.types.js +231 -4
- package/dist/shared/format/format-bytes.d.ts +1 -0
- package/dist/shared/format/format-bytes.js +9 -0
- package/dist/shared/path/path.d.ts +1 -0
- package/dist/shared/path/path.js +21 -1
- package/dist/shared/process/dotnet-toolchain.d.ts +14 -0
- package/dist/shared/process/dotnet-toolchain.js +179 -0
- package/dist/shared/process/host-process-recovery.d.ts +12 -0
- package/dist/shared/process/host-process-recovery.js +78 -0
- package/dist/shared/process/host-process-supervisor.d.ts +59 -0
- package/dist/shared/process/host-process-supervisor.js +330 -0
- package/dist/shared/process/process.d.ts +1 -0
- package/dist/shared/process/process.js +1 -0
- package/dist/shared/process/windows-launch.d.ts +1 -1
- package/dist/shared/process/windows-launch.js +6 -3
- package/dist/shared/provider-capabilities.d.ts +1 -0
- package/dist/shared/provider-capabilities.js +3 -0
- package/dist/shared/redaction/redaction.js +16 -4
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/worktree/artifact-injector.d.ts +3 -2
- package/dist/worktree/artifact-injector.js +43 -4
- package/dist/worktree/dependency-warmer.d.ts +36 -2
- package/dist/worktree/dependency-warmer.js +347 -11
- package/dist/worktree/index.d.ts +2 -1
- package/dist/worktree/index.js +1 -0
- package/dist/worktree/prior-phase-artifact-hydrator.d.ts +2 -0
- package/dist/worktree/prior-phase-artifact-hydrator.js +136 -0
- package/dist/worktree/worktree-paths.d.ts +4 -0
- package/dist/worktree/worktree-paths.js +30 -0
- package/dist/worktree/worktree.base.d.ts +3 -0
- package/dist/worktree/worktree.service.d.ts +13 -2
- package/dist/worktree/worktree.service.js +238 -47
- package/dist/worktree/worktree.types.d.ts +5 -1
- package/package.json +8 -4
- package/runtime/windows-job-supervisor.ps1 +444 -0
- package/runtime/windows-listener-owner.ps1 +33 -0
|
@@ -1,47 +1,203 @@
|
|
|
1
1
|
import { HubConnectionBuilder, HttpTransportType, LogLevel } from "@microsoft/signalr";
|
|
2
2
|
import fs from "node:fs";
|
|
3
3
|
import path from "node:path";
|
|
4
|
+
import { randomUUID } from "node:crypto";
|
|
5
|
+
import os from "node:os";
|
|
4
6
|
import { defaultForgeBridgeConfig } from "../config.js";
|
|
5
|
-
import {
|
|
7
|
+
import { supportsDraftImageAttachmentsFor } from "../shared/provider-capabilities.js";
|
|
8
|
+
import { redactText } from "../shared/redaction/redaction.js";
|
|
6
9
|
import { BridgeSessionService } from "../session/session.service.js";
|
|
10
|
+
import { ProcessBackedProviderCliService, RoutingProviderCliService, StreamingProviderProcessRunner, } from "../session/provider-cli/provider-cli.service.js";
|
|
11
|
+
import { redactRunnerDispatchEvent, redactRunOnceResultPacket } from "../session/session-redaction.js";
|
|
12
|
+
import { HostRuntimeLeaseManager } from "../runtime/host-runtime-lease.js";
|
|
13
|
+
import { buildMemoryAdmissionDiagnostics } from "../runtime/memory-capacity.js";
|
|
14
|
+
import { DockerEngineProbe } from "../runtime/docker-engine-probe.js";
|
|
15
|
+
import { estimateWorkspaceBytesFromRepo } from "../runtime/workspace-resource-estimator.js";
|
|
16
|
+
import { GitWorktreeService } from "../worktree/worktree.service.js";
|
|
17
|
+
import { RepositoryCoordinatorLock } from "../sandbox/repository-lock.js";
|
|
18
|
+
import { AttemptCoordinatedProviderProcessRunner, AttemptProcessSupervisorRegistry, } from "../runtime/attempt-process-supervisor-registry.js";
|
|
7
19
|
import { failureResult, parseRunOnceDispatchPacket } from "../session/session.types.js";
|
|
8
20
|
import { forgeBridgeVersion } from "../version.js";
|
|
21
|
+
import { canRunDockerMaintenance, DockerPreflight } from "../sandbox/preflight.js";
|
|
22
|
+
import { validateSandboxContract } from "../sandbox/contract.js";
|
|
23
|
+
import { runSandboxProcess } from "../sandbox/process.js";
|
|
24
|
+
import { DockerSandboxRuntime, selectSandboxConfigRefs } from "../sandbox/runtime.js";
|
|
25
|
+
import { codexSandboxProviderAuthAdapter, sandboxProviderAuthAdapterFor, sandboxProviderAuthUnavailableMessage } from "../sandbox/provider-auth/codex.js";
|
|
26
|
+
import { formatBytes } from "../shared/format/format-bytes.js";
|
|
27
|
+
import { emptyRunnerHostCapabilitySnapshot, ProcessRunnerHostCapabilityProbe, } from "./runner-host-capabilities.js";
|
|
28
|
+
/**
|
|
29
|
+
* Protocol-v2 events are acknowledged one at a time before runOnce returns.
|
|
30
|
+
* Sending the complete transcript again in DispatchResult made long QA and CI
|
|
31
|
+
* sessions exceed SignalR's receive limit, disconnecting the bridge after the
|
|
32
|
+
* provider had already completed. The API combines its acknowledged streamed
|
|
33
|
+
* events with the terminal result, so the callback only needs the outcome and
|
|
34
|
+
* artifacts. Protocol v1 retains the transcript for legacy servers.
|
|
35
|
+
*/
|
|
36
|
+
export function compactTerminalResult(dispatch, result) {
|
|
37
|
+
return dispatch.schemaVersion >= 2 && result.events.length > 0
|
|
38
|
+
? { ...result, events: [] }
|
|
39
|
+
: result;
|
|
40
|
+
}
|
|
9
41
|
export class ForgeBridgeService {
|
|
10
42
|
config;
|
|
11
|
-
sessions;
|
|
12
43
|
connections;
|
|
44
|
+
hostCapabilityProbe;
|
|
13
45
|
connection = null;
|
|
14
46
|
heartbeatTimer = null;
|
|
47
|
+
heartbeatInFlight = false;
|
|
48
|
+
consecutiveHeartbeatFailures = 0;
|
|
49
|
+
reconnectingFromHeartbeatFailure = false;
|
|
50
|
+
leaseWatchdogTimer = null;
|
|
51
|
+
sandboxMaintenanceTimer = null;
|
|
52
|
+
sandboxMaintenanceRunning = false;
|
|
15
53
|
closed = null;
|
|
54
|
+
transportRecovery = null;
|
|
55
|
+
transportRecoveryGeneration = 0;
|
|
16
56
|
stopped = false;
|
|
17
57
|
activeSessions = new Map();
|
|
18
|
-
|
|
58
|
+
reservations = new Map();
|
|
59
|
+
activeReadinessProbes = new Map();
|
|
60
|
+
preflight;
|
|
61
|
+
dockerEngineProbe;
|
|
62
|
+
sandboxRuntime;
|
|
63
|
+
sandboxResourceResolver;
|
|
64
|
+
sessions;
|
|
65
|
+
sandboxAmbientMemoryCommitmentBytes = null;
|
|
66
|
+
hostCapabilities = emptyRunnerHostCapabilitySnapshot();
|
|
67
|
+
hostCapabilitiesDetected = false;
|
|
68
|
+
constructor(config, sessions = undefined, connections = new SignalRBridgeConnectionFactory(), preflight, sandboxRuntime, sandboxResourceResolver, dockerEngineProbe, hostCapabilityProbe = new ProcessRunnerHostCapabilityProbe()) {
|
|
19
69
|
this.config = config;
|
|
20
|
-
this.sessions = sessions;
|
|
21
70
|
this.connections = connections;
|
|
71
|
+
this.hostCapabilityProbe = hostCapabilityProbe;
|
|
72
|
+
this.sessions = sessions ?? createDefaultSessionService(config.sandboxStateRoot);
|
|
73
|
+
this.preflight = preflight ?? new DockerPreflight({
|
|
74
|
+
dockerExecutable: config.dockerExecutable,
|
|
75
|
+
dockerContext: config.dockerContext,
|
|
76
|
+
stateRoot: config.sandboxStateRoot,
|
|
77
|
+
runnerInstanceId: config.runnerInstanceId,
|
|
78
|
+
maxConcurrentSandboxes: config.maxConcurrentSandboxes,
|
|
79
|
+
diskHighWatermarkPercent: config.diskHighWatermarkPercent,
|
|
80
|
+
diskLowWatermarkPercent: config.diskLowWatermarkPercent,
|
|
81
|
+
minimumWorkspaceBytes: config.minimumWorkspaceBytes,
|
|
82
|
+
hostMemoryReserveBytes: config.sandboxHostMemoryReserveBytes,
|
|
83
|
+
defaultWorkerMemoryLimitBytes: config.defaultSandboxWorkerMemoryLimitBytes,
|
|
84
|
+
defaultServiceMemoryLimitBytes: config.defaultSandboxServiceMemoryLimitBytes,
|
|
85
|
+
runtimeOverheadBytes: config.sandboxRuntimeOverheadBytes,
|
|
86
|
+
requireRootless: config.requireRootlessDocker
|
|
87
|
+
});
|
|
88
|
+
this.dockerEngineProbe = dockerEngineProbe ?? new DockerEngineProbe({
|
|
89
|
+
dockerExecutable: config.dockerExecutable,
|
|
90
|
+
dockerContext: config.dockerContext,
|
|
91
|
+
recovery: {
|
|
92
|
+
enabled: config.dockerDesktopRecoveryEnabled,
|
|
93
|
+
failureThreshold: config.dockerRecoveryFailureThreshold,
|
|
94
|
+
cooldownMs: config.dockerRecoveryCooldownMs,
|
|
95
|
+
startupTimeoutMs: config.dockerRecoveryStartupTimeoutMs,
|
|
96
|
+
canRecover: () => this.activeSessions.size === 0 && this.reservations.size === 0,
|
|
97
|
+
},
|
|
98
|
+
onStatus: logBridgeStatus,
|
|
99
|
+
});
|
|
100
|
+
this.sandboxRuntime = sandboxRuntime ?? new DockerSandboxRuntime(config, (request, signal) => this.redeemPublishGrant(request, signal));
|
|
101
|
+
this.sandboxResourceResolver = sandboxResourceResolver
|
|
102
|
+
?? (dispatch => resolveSandboxResourcesFromRepo(dispatch, this.config.sandboxRuntimeOverheadBytes));
|
|
22
103
|
}
|
|
23
104
|
async connectOnce() {
|
|
24
105
|
await this.stopConnection();
|
|
25
106
|
this.stopped = false;
|
|
107
|
+
const hostReap = await this.sessions.reapHostRuntime?.({
|
|
108
|
+
runnerId: this.config.runnerId,
|
|
109
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
110
|
+
});
|
|
111
|
+
if (hostReap && (hostReap.reaped > 0 || hostReap.degraded > 0)) {
|
|
112
|
+
logBridgeStatus(`Managed-host recovery reaped ${hostReap.reaped} stale attempt(s), protected ${hostReap.reusedPidsProtected} reused PID(s), and left ${hostReap.degraded} degraded attempt(s) for reconciliation.`);
|
|
113
|
+
}
|
|
114
|
+
if (this.config.dockerEnabled) {
|
|
115
|
+
const preflight = await this.preflight.refresh();
|
|
116
|
+
if (canReapManagedSandboxes(preflight)) {
|
|
117
|
+
await this.sandboxRuntime.reap({ diskLow: preflight.diskLow });
|
|
118
|
+
await this.preflight.refresh();
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
await this.dockerEngineProbe.refresh(true);
|
|
122
|
+
if (!this.hostCapabilitiesDetected) {
|
|
123
|
+
this.hostCapabilities = await this.hostCapabilityProbe.detect(this.config);
|
|
124
|
+
this.hostCapabilitiesDetected = true;
|
|
125
|
+
}
|
|
26
126
|
const connection = this.connections.create(this.config);
|
|
27
127
|
this.connection = connection;
|
|
128
|
+
this.transportRecovery = null;
|
|
28
129
|
this.closed = new Promise((resolve) => {
|
|
29
|
-
connection.onClose(() => {
|
|
130
|
+
connection.onClose((error) => {
|
|
30
131
|
this.stopHeartbeat();
|
|
132
|
+
this.completeTransportRecovery(error ?? new Error("bridge connection closed"));
|
|
31
133
|
this.abortActiveSessions("bridge connection closed");
|
|
134
|
+
this.abortActiveReadinessProbes("bridge connection closed");
|
|
32
135
|
resolve();
|
|
33
136
|
});
|
|
34
137
|
});
|
|
138
|
+
connection.onReconnecting?.((error) => {
|
|
139
|
+
if (connection !== this.connection || this.stopped)
|
|
140
|
+
return;
|
|
141
|
+
this.stopHeartbeat();
|
|
142
|
+
this.beginTransportRecovery();
|
|
143
|
+
logBridgeStatus(`Forge connection interrupted${error ? `: ${errorMessage(error)}` : ""}. Reconnecting in place; active dispatches remain supervised...`);
|
|
144
|
+
});
|
|
145
|
+
connection.onReconnected?.(async () => {
|
|
146
|
+
if (connection !== this.connection || this.stopped)
|
|
147
|
+
return;
|
|
148
|
+
try {
|
|
149
|
+
await this.registerConnection(connection);
|
|
150
|
+
this.completeTransportRecovery();
|
|
151
|
+
this.startHeartbeat();
|
|
152
|
+
await this.flushCleanupReports().catch(() => undefined);
|
|
153
|
+
logBridgeStatus("Forge connection restored; active dispatches remain attached to this runner process.");
|
|
154
|
+
}
|
|
155
|
+
catch (error) {
|
|
156
|
+
this.completeTransportRecovery(error);
|
|
157
|
+
logBridgeStatus(`Forge reconnect registration failed: ${errorMessage(error)}. Closing the connection for a clean retry...`);
|
|
158
|
+
await connection.stop().catch(() => undefined);
|
|
159
|
+
}
|
|
160
|
+
});
|
|
35
161
|
connection.onDispatch((dispatch) => this.handleDispatch(dispatch));
|
|
162
|
+
connection.onOffer?.((dispatch) => this.handleOffer(dispatch));
|
|
163
|
+
connection.onStart?.((dispatch) => this.handleStart(dispatch));
|
|
164
|
+
connection.onLeaseUpdate?.((request) => this.handleLeaseUpdate(request));
|
|
36
165
|
connection.onCancel((request) => this.handleCancel(request));
|
|
166
|
+
connection.onReap?.((request) => this.handleReap(request));
|
|
167
|
+
connection.onReadinessProbe?.((request) => this.handleReadinessProbe(request));
|
|
37
168
|
await connection.start();
|
|
38
|
-
await this.
|
|
39
|
-
|
|
40
|
-
protocolVersion: this.config.protocolVersion,
|
|
41
|
-
minServerVersion: this.config.minServerVersion ?? null,
|
|
42
|
-
capabilities: this.capabilities()
|
|
43
|
-
});
|
|
169
|
+
await this.registerConnection(connection);
|
|
170
|
+
await this.flushCleanupReports();
|
|
44
171
|
this.startHeartbeat();
|
|
172
|
+
this.startLeaseWatchdog();
|
|
173
|
+
this.startSandboxMaintenance();
|
|
174
|
+
}
|
|
175
|
+
async registerConnection(connection) {
|
|
176
|
+
let storageRecoveryAttempt = 0;
|
|
177
|
+
while (true) {
|
|
178
|
+
try {
|
|
179
|
+
const ack = await connection.invoke("Hello", {
|
|
180
|
+
version: this.config.version,
|
|
181
|
+
protocolVersion: this.config.protocolVersion,
|
|
182
|
+
minServerVersion: this.config.minServerVersion ?? null,
|
|
183
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
184
|
+
capabilities: this.capabilities()
|
|
185
|
+
});
|
|
186
|
+
if (!ack.accepted) {
|
|
187
|
+
throw new BridgeCallbackRejectedError(ack.code ?? null, ack.message ?? ack.code ?? "Hello callback was rejected.");
|
|
188
|
+
}
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
catch (error) {
|
|
192
|
+
if (!isProviderStorageUnavailable(error) || storageRecoveryAttempt >= this.config.reconnectBackoffMs.length) {
|
|
193
|
+
throw error;
|
|
194
|
+
}
|
|
195
|
+
const backoff = this.config.reconnectBackoffMs[storageRecoveryAttempt] ?? 5000;
|
|
196
|
+
storageRecoveryAttempt += 1;
|
|
197
|
+
logBridgeStatus(`Hello reached Forge while storage was recovering. Retrying registration ${storageRecoveryAttempt} in ${formatDelay(backoff)}...`);
|
|
198
|
+
await delay(backoff);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
45
201
|
}
|
|
46
202
|
async run(signal) {
|
|
47
203
|
let attempt = 0;
|
|
@@ -72,6 +228,7 @@ export class ForgeBridgeService {
|
|
|
72
228
|
finally {
|
|
73
229
|
if (signal?.aborted) {
|
|
74
230
|
this.abortActiveSessions(String(signal.reason ?? "bridge stopping"));
|
|
231
|
+
this.abortActiveReadinessProbes(String(signal.reason ?? "bridge stopping"));
|
|
75
232
|
}
|
|
76
233
|
await this.stopConnection();
|
|
77
234
|
}
|
|
@@ -79,31 +236,75 @@ export class ForgeBridgeService {
|
|
|
79
236
|
async stop() {
|
|
80
237
|
this.stopped = true;
|
|
81
238
|
this.abortActiveSessions("bridge stopping");
|
|
239
|
+
this.abortActiveReadinessProbes("bridge stopping");
|
|
82
240
|
await this.stopConnection();
|
|
241
|
+
await this.preflight.release();
|
|
83
242
|
}
|
|
84
243
|
async handleDispatch(rawDispatch) {
|
|
85
244
|
const dispatch = this.parseDispatchOrReport(rawDispatch);
|
|
86
245
|
if (!dispatch) {
|
|
87
246
|
return;
|
|
88
247
|
}
|
|
248
|
+
const capabilityFailure = await this.requiredCapabilityFailure(dispatch);
|
|
249
|
+
if (capabilityFailure) {
|
|
250
|
+
await this.sendResult(dispatch, failureResult(capabilityFailure, [{
|
|
251
|
+
role: "runner",
|
|
252
|
+
kind: "error",
|
|
253
|
+
content: capabilityFailure,
|
|
254
|
+
eventType: "docker_engine_unavailable",
|
|
255
|
+
}])).catch(() => undefined);
|
|
256
|
+
return;
|
|
257
|
+
}
|
|
89
258
|
const leaseId = dispatch.lease.id;
|
|
90
259
|
const fencingToken = dispatch.lease.fencingToken;
|
|
91
260
|
const redactionSecrets = exactSecrets(dispatch, this.config);
|
|
92
|
-
|
|
93
|
-
|
|
261
|
+
const existing = this.activeSessions.get(leaseId);
|
|
262
|
+
if (existing) {
|
|
263
|
+
if (existing.fencingToken === fencingToken) {
|
|
264
|
+
return;
|
|
265
|
+
}
|
|
266
|
+
await this.sendResult(dispatch, failureResult("Dispatch fence conflicts with an active bridge session.")).catch(() => undefined);
|
|
94
267
|
return;
|
|
95
268
|
}
|
|
96
269
|
const controller = new AbortController();
|
|
97
|
-
this.activeSessions.set(leaseId, {
|
|
270
|
+
this.activeSessions.set(leaseId, {
|
|
271
|
+
fencingToken,
|
|
272
|
+
dispatchAttemptId: dispatch.sandbox?.dispatchAttemptId ?? null,
|
|
273
|
+
runnerInstanceId: dispatch.sandbox?.runnerInstanceId ?? null,
|
|
274
|
+
leaseExpiresAt: dispatch.lease.expiresAt,
|
|
275
|
+
sandboxed: dispatch.sandbox?.executionMode === "docker",
|
|
276
|
+
resources: dispatch.sandbox?.executionMode === "docker"
|
|
277
|
+
? dispatch.sandbox.requestedResources
|
|
278
|
+
: zeroSandboxResources(),
|
|
279
|
+
controller
|
|
280
|
+
});
|
|
98
281
|
try {
|
|
99
282
|
const normalized = this.normalizeDispatch(dispatch);
|
|
100
|
-
|
|
283
|
+
if (normalized.sandbox) {
|
|
284
|
+
const started = {
|
|
285
|
+
dispatchAttemptId: normalized.sandbox.dispatchAttemptId,
|
|
286
|
+
leaseId,
|
|
287
|
+
fencingToken,
|
|
288
|
+
runnerInstanceId: normalized.sandbox.runnerInstanceId
|
|
289
|
+
};
|
|
290
|
+
await this.invokeAck("DispatchStarted", started);
|
|
291
|
+
}
|
|
292
|
+
const sink = {
|
|
101
293
|
emit: async (event) => {
|
|
294
|
+
if (event.eventType === "provider_cli_started") {
|
|
295
|
+
const active = this.activeSessions.get(leaseId);
|
|
296
|
+
if (active)
|
|
297
|
+
active.resources = { ...active.resources, buildSlots: 0 };
|
|
298
|
+
}
|
|
102
299
|
try {
|
|
103
300
|
await this.invokeAck("DispatchEvent", {
|
|
104
301
|
leaseId,
|
|
105
302
|
fencingToken,
|
|
106
|
-
|
|
303
|
+
...(dispatch.sandbox ? {
|
|
304
|
+
dispatchAttemptId: dispatch.sandbox.dispatchAttemptId,
|
|
305
|
+
runnerInstanceId: dispatch.sandbox.runnerInstanceId
|
|
306
|
+
} : {}),
|
|
307
|
+
event: redactRunnerDispatchEvent(event, redactionSecrets)
|
|
107
308
|
});
|
|
108
309
|
}
|
|
109
310
|
catch (error) {
|
|
@@ -114,18 +315,302 @@ export class ForgeBridgeService {
|
|
|
114
315
|
throw error;
|
|
115
316
|
}
|
|
116
317
|
}
|
|
117
|
-
}
|
|
118
|
-
|
|
318
|
+
};
|
|
319
|
+
const result = normalized.sandbox?.executionMode === "docker"
|
|
320
|
+
? await this.sandboxRuntime.runOnce(normalized, sink, controller.signal)
|
|
321
|
+
: await this.sessions.runOnce(normalized, sink, controller.signal);
|
|
322
|
+
if (normalized.sandbox?.executionMode === "docker") {
|
|
323
|
+
await this.flushCleanupReports().catch(() => undefined);
|
|
324
|
+
}
|
|
325
|
+
const redactedResult = redactRunOnceResultPacket(result, redactionSecrets);
|
|
326
|
+
await this.releaseActiveSessionAndRefreshCapacity(leaseId);
|
|
327
|
+
await this.sendResult(dispatch, compactTerminalResult(dispatch, redactedResult));
|
|
119
328
|
}
|
|
120
329
|
catch (error) {
|
|
121
330
|
const summary = controller.signal.aborted
|
|
122
331
|
? `Dispatch cancelled: ${String(controller.signal.reason ?? "cancelled")}`
|
|
123
332
|
: redactText(error instanceof Error ? error.message : String(error), redactionSecrets);
|
|
124
|
-
|
|
333
|
+
const failureEventType = infrastructureFailureEventType(summary);
|
|
334
|
+
const failureEvents = failureEventType ? [{
|
|
335
|
+
role: "runner",
|
|
336
|
+
kind: "error",
|
|
337
|
+
content: summary,
|
|
338
|
+
eventType: failureEventType,
|
|
339
|
+
}] : [];
|
|
340
|
+
await this.releaseActiveSessionAndRefreshCapacity(leaseId);
|
|
341
|
+
await this.sendResult(dispatch, failureResult(summary, failureEvents)).catch(() => undefined);
|
|
125
342
|
}
|
|
126
343
|
finally {
|
|
127
|
-
this.
|
|
344
|
+
await this.releaseActiveSessionAndRefreshCapacity(leaseId);
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
async releaseActiveSessionAndRefreshCapacity(leaseId) {
|
|
348
|
+
if (!this.activeSessions.delete(leaseId))
|
|
349
|
+
return;
|
|
350
|
+
// Publish released host/runtime capacity before the terminal result can
|
|
351
|
+
// enqueue the next pipeline phase. Otherwise the server may reject that
|
|
352
|
+
// handoff from the previous periodic heartbeat's active reservation.
|
|
353
|
+
await this.sendHeartbeat().catch(() => undefined);
|
|
354
|
+
}
|
|
355
|
+
async handleOffer(rawDispatch) {
|
|
356
|
+
const envelope = parseOfferEnvelope(rawDispatch);
|
|
357
|
+
if (!envelope) {
|
|
358
|
+
return;
|
|
359
|
+
}
|
|
360
|
+
const dispatch = this.parseDispatchOrReport(envelope.dispatch);
|
|
361
|
+
if (!dispatch?.sandbox
|
|
362
|
+
|| dispatch.sandbox.dispatchAttemptId !== envelope.dispatchAttemptId
|
|
363
|
+
|| dispatch.lease.id !== envelope.leaseId
|
|
364
|
+
|| dispatch.lease.fencingToken !== envelope.fencingToken)
|
|
365
|
+
return;
|
|
366
|
+
const sandbox = dispatch.sandbox;
|
|
367
|
+
this.expireReservations(Date.now());
|
|
368
|
+
const existing = this.reservations.get(sandbox.dispatchAttemptId);
|
|
369
|
+
if (existing) {
|
|
370
|
+
const sameFence = existing.leaseId === dispatch.lease.id
|
|
371
|
+
&& existing.fencingToken === dispatch.lease.fencingToken
|
|
372
|
+
&& existing.runnerInstanceId === sandbox.runnerInstanceId;
|
|
373
|
+
await this.sendOfferDecision(dispatch, sameFence, sameFence ? null : "stale_fence", sameFence ? null : "Offer fence conflicts with the existing reservation.", sameFence ? existing.reservedUntil : null, sameFence ? existing.resources : null);
|
|
374
|
+
return;
|
|
375
|
+
}
|
|
376
|
+
if (sandbox.runnerInstanceId !== this.config.runnerInstanceId) {
|
|
377
|
+
await this.sendOfferDecision(dispatch, false, "stale_runner_instance", "Offer targets a different runner process instance.");
|
|
378
|
+
return;
|
|
379
|
+
}
|
|
380
|
+
if (Date.parse(envelope.expiresAt) <= Date.now()) {
|
|
381
|
+
await this.sendOfferDecision(dispatch, false, "lease_expired", "Offer lease is already expired.");
|
|
382
|
+
return;
|
|
383
|
+
}
|
|
384
|
+
const preflight = sandbox.executionMode === "docker" && this.config.dockerEnabled
|
|
385
|
+
? await this.preflight.run()
|
|
386
|
+
: this.preflight.current();
|
|
387
|
+
if (sandbox.executionMode === "docker") {
|
|
388
|
+
if (!this.config.dockerEnabled || !preflight?.healthy) {
|
|
389
|
+
await this.sendOfferDecision(dispatch, false, preflight?.unavailableCode ?? "docker_unavailable", preflight?.unavailableReason ?? "Docker sandbox preflight is unavailable.");
|
|
390
|
+
return;
|
|
391
|
+
}
|
|
392
|
+
const networkFailure = sandboxNetworkPolicyFailure(dispatch, this.config);
|
|
393
|
+
if (networkFailure) {
|
|
394
|
+
await this.sendOfferDecision(dispatch, false, "policy_mismatch", networkFailure);
|
|
395
|
+
return;
|
|
396
|
+
}
|
|
397
|
+
const providerNetworkFailure = sandboxProviderNetworkFailure(dispatch, this.config);
|
|
398
|
+
if (providerNetworkFailure) {
|
|
399
|
+
await this.sendOfferDecision(dispatch, false, "provider_network_policy_denied", providerNetworkFailure);
|
|
400
|
+
return;
|
|
401
|
+
}
|
|
402
|
+
const authFailure = sandboxAuthFailure(dispatch, this.config);
|
|
403
|
+
if (authFailure) {
|
|
404
|
+
await this.sendOfferDecision(dispatch, false, "sandbox_auth_unavailable", authFailure);
|
|
405
|
+
return;
|
|
406
|
+
}
|
|
128
407
|
}
|
|
408
|
+
let requestedResources = sandbox.requestedResources;
|
|
409
|
+
let sandboxConfigCommitSha = sandbox.sandboxConfigCommitSha ?? null;
|
|
410
|
+
const normalizedDispatch = this.normalizeDispatch(dispatch);
|
|
411
|
+
if (sandbox.executionMode === "docker") {
|
|
412
|
+
try {
|
|
413
|
+
const profile = await this.sandboxResourceResolver(normalizedDispatch);
|
|
414
|
+
requestedResources = profile.resources;
|
|
415
|
+
sandboxConfigCommitSha = profile.sandboxConfigCommitSha;
|
|
416
|
+
}
|
|
417
|
+
catch (error) {
|
|
418
|
+
await this.sendOfferDecision(dispatch, false, "sandbox_capacity_profile_unavailable", error instanceof Error ? error.message : "The immutable sandbox contract could not be profiled for admission.");
|
|
419
|
+
return;
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
else if (normalizedDispatch.repo.serverLocalPath) {
|
|
423
|
+
// Host execution uses the same project-derived disk reservation as Docker.
|
|
424
|
+
// The API supplies a placement-time estimate; the runner can safely raise it
|
|
425
|
+
// from its authoritative checkout before admitting the attempt.
|
|
426
|
+
const measuredWorkspaceBytes = await estimateWorkspaceBytesFromRepo(normalizedDispatch.repo.serverLocalPath, { role: normalizedDispatch.execution.role }).catch(() => null);
|
|
427
|
+
if (measuredWorkspaceBytes !== null) {
|
|
428
|
+
requestedResources = {
|
|
429
|
+
...requestedResources,
|
|
430
|
+
workspaceBytes: Math.max(requestedResources.workspaceBytes, measuredWorkspaceBytes),
|
|
431
|
+
};
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
// Resource profiling can await filesystem and Docker work. Admission needs
|
|
435
|
+
// a dynamic host/daemon sample that began after profiling completed, so an
|
|
436
|
+
// older in-flight heartbeat inspection cannot supply the commit decision.
|
|
437
|
+
const admissionPreflight = sandbox.executionMode === "docker"
|
|
438
|
+
? await this.preflight.refreshCapacityForAdmission()
|
|
439
|
+
: preflight;
|
|
440
|
+
// Resource profiling and preflight may await slow filesystem or Docker I/O.
|
|
441
|
+
// Re-read every mutable capacity input only after that work, then decide and
|
|
442
|
+
// commit synchronously. JavaScript cannot interleave another Offer handler
|
|
443
|
+
// inside this non-async method, so the ledger check and insertion are one
|
|
444
|
+
// atomic event-loop turn without holding a lock across external I/O.
|
|
445
|
+
const decision = this.commitOfferReservation(dispatch, requestedResources, admissionPreflight);
|
|
446
|
+
await this.sendOfferDecision(dispatch, decision.accepted, decision.code, decision.message, decision.reservedUntil, decision.resources, sandboxConfigCommitSha);
|
|
447
|
+
}
|
|
448
|
+
commitOfferReservation(dispatch, requestedResources, preparedPreflight) {
|
|
449
|
+
const sandbox = dispatch.sandbox;
|
|
450
|
+
const now = Date.now();
|
|
451
|
+
this.expireReservations(now);
|
|
452
|
+
const existing = this.reservations.get(sandbox.dispatchAttemptId);
|
|
453
|
+
if (existing) {
|
|
454
|
+
const sameFence = existing.leaseId === dispatch.lease.id
|
|
455
|
+
&& existing.fencingToken === dispatch.lease.fencingToken
|
|
456
|
+
&& existing.runnerInstanceId === sandbox.runnerInstanceId;
|
|
457
|
+
return {
|
|
458
|
+
accepted: sameFence,
|
|
459
|
+
code: sameFence ? null : "stale_fence",
|
|
460
|
+
message: sameFence ? null : "Offer fence conflicts with the existing reservation.",
|
|
461
|
+
reservedUntil: sameFence ? existing.reservedUntil : null,
|
|
462
|
+
resources: sameFence ? existing.resources : null,
|
|
463
|
+
};
|
|
464
|
+
}
|
|
465
|
+
if (Date.parse(dispatch.lease.expiresAt) <= now) {
|
|
466
|
+
return rejectedOfferReservation("lease_expired", "Offer lease is already expired.");
|
|
467
|
+
}
|
|
468
|
+
if (sandbox.executionMode === "docker") {
|
|
469
|
+
// A concurrent heartbeat/preflight refresh may have produced a newer
|
|
470
|
+
// conservative dual-domain sample while this offer was being profiled.
|
|
471
|
+
const admissionPreflight = this.preflight.current() ?? preparedPreflight;
|
|
472
|
+
if (!this.config.dockerEnabled || !admissionPreflight?.healthy) {
|
|
473
|
+
return rejectedOfferReservation(admissionPreflight?.unavailableCode ?? "docker_unavailable", admissionPreflight?.unavailableReason ?? "Docker sandbox preflight is unavailable.");
|
|
474
|
+
}
|
|
475
|
+
const runtimeState = this.sandboxRuntime.snapshot();
|
|
476
|
+
const usedResources = this.usedSandboxResources(runtimeState.degraded);
|
|
477
|
+
const activeResources = this.activeSandboxResources(runtimeState.degraded);
|
|
478
|
+
const capacity = this.sandboxResourceCapacity(admissionPreflight, activeResources, usedResources);
|
|
479
|
+
if (!canReserveSandboxResources(usedResources, requestedResources, capacity)) {
|
|
480
|
+
return rejectedOfferReservation("capacity_unavailable", sandboxCapacityFailureMessage(usedResources, requestedResources, capacity, admissionPreflight, this.config));
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
const reservedUntil = new Date(Math.min(Date.parse(dispatch.lease.expiresAt), now + 30_000)).toISOString();
|
|
484
|
+
const reservation = {
|
|
485
|
+
leaseId: dispatch.lease.id,
|
|
486
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
487
|
+
runnerInstanceId: sandbox.runnerInstanceId,
|
|
488
|
+
expiresAt: dispatch.lease.expiresAt,
|
|
489
|
+
reservedUntil,
|
|
490
|
+
sandboxed: sandbox.executionMode === "docker",
|
|
491
|
+
resources: requestedResources,
|
|
492
|
+
};
|
|
493
|
+
this.reservations.set(sandbox.dispatchAttemptId, reservation);
|
|
494
|
+
return {
|
|
495
|
+
accepted: true,
|
|
496
|
+
code: null,
|
|
497
|
+
message: null,
|
|
498
|
+
reservedUntil,
|
|
499
|
+
resources: requestedResources,
|
|
500
|
+
};
|
|
501
|
+
}
|
|
502
|
+
async handleStart(rawDispatch) {
|
|
503
|
+
const envelope = parseStartEnvelope(rawDispatch);
|
|
504
|
+
if (!envelope) {
|
|
505
|
+
return;
|
|
506
|
+
}
|
|
507
|
+
let dispatch = this.parseDispatchOrReport(envelope.dispatch);
|
|
508
|
+
if (!dispatch?.sandbox
|
|
509
|
+
|| dispatch.sandbox.dispatchAttemptId !== envelope.dispatchAttemptId
|
|
510
|
+
|| dispatch.lease.id !== envelope.leaseId
|
|
511
|
+
|| dispatch.lease.fencingToken !== envelope.fencingToken)
|
|
512
|
+
return;
|
|
513
|
+
const reservation = this.reservations.get(dispatch.sandbox.dispatchAttemptId);
|
|
514
|
+
if (!reservation
|
|
515
|
+
|| reservation.leaseId !== dispatch.lease.id
|
|
516
|
+
|| reservation.fencingToken !== dispatch.lease.fencingToken
|
|
517
|
+
|| reservation.runnerInstanceId !== dispatch.sandbox.runnerInstanceId
|
|
518
|
+
|| Date.parse(reservation.reservedUntil) <= Date.now()) {
|
|
519
|
+
await this.sendResult(dispatch, failureResult("Start rejected because its reservation is missing or stale.")).catch(() => undefined);
|
|
520
|
+
return;
|
|
521
|
+
}
|
|
522
|
+
if (dispatch.apiKeySecretGrant) {
|
|
523
|
+
this.reservations.delete(dispatch.sandbox.dispatchAttemptId);
|
|
524
|
+
await this.sendResult(dispatch, failureResult("Start rejected because protocol-v2 packets may not contain a plaintext provider grant.")).catch(() => undefined);
|
|
525
|
+
return;
|
|
526
|
+
}
|
|
527
|
+
if (dispatch.apiKeySecretGrantHandle) {
|
|
528
|
+
const response = await this.redeemSecretGrant(dispatch);
|
|
529
|
+
const grant = response.grant;
|
|
530
|
+
if (!response.accepted
|
|
531
|
+
|| !grant
|
|
532
|
+
|| grant.grantId !== dispatch.apiKeySecretGrantHandle.grantId
|
|
533
|
+
|| grant.provider !== dispatch.apiKeySecretGrantHandle.provider
|
|
534
|
+
|| Date.parse(grant.expiresAt) <= Date.now()) {
|
|
535
|
+
this.reservations.delete(dispatch.sandbox.dispatchAttemptId);
|
|
536
|
+
await this.sendResult(dispatch, failureResult(response.message ?? "Secret grant redemption was rejected.")).catch(() => undefined);
|
|
537
|
+
return;
|
|
538
|
+
}
|
|
539
|
+
dispatch = { ...dispatch, apiKeySecretGrant: grant, apiKeySecretGrantHandle: null };
|
|
540
|
+
}
|
|
541
|
+
else if (dispatch.execution.requiresApiKeySecretGrant) {
|
|
542
|
+
this.reservations.delete(dispatch.sandbox.dispatchAttemptId);
|
|
543
|
+
await this.sendResult(dispatch, failureResult("Start rejected because the provider secret grant handle is missing.")).catch(() => undefined);
|
|
544
|
+
return;
|
|
545
|
+
}
|
|
546
|
+
dispatch = {
|
|
547
|
+
...dispatch,
|
|
548
|
+
sandbox: { ...dispatch.sandbox, requestedResources: reservation.resources },
|
|
549
|
+
};
|
|
550
|
+
this.reservations.delete(envelope.dispatchAttemptId);
|
|
551
|
+
await this.handleDispatch(dispatch);
|
|
552
|
+
}
|
|
553
|
+
async redeemSecretGrant(dispatch) {
|
|
554
|
+
const sandbox = dispatch.sandbox;
|
|
555
|
+
const handle = dispatch.apiKeySecretGrantHandle;
|
|
556
|
+
const connection = this.connection;
|
|
557
|
+
if (!connection)
|
|
558
|
+
return { accepted: false, code: "bridge.disconnected", message: "Bridge is disconnected.", grant: null };
|
|
559
|
+
return await connection.invoke("RedeemSecretGrant", {
|
|
560
|
+
dispatchAttemptId: sandbox.dispatchAttemptId,
|
|
561
|
+
leaseId: dispatch.lease.id,
|
|
562
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
563
|
+
runnerInstanceId: sandbox.runnerInstanceId,
|
|
564
|
+
grantId: handle.grantId
|
|
565
|
+
});
|
|
566
|
+
}
|
|
567
|
+
async redeemPublishGrant(request, signal) {
|
|
568
|
+
if (signal?.aborted)
|
|
569
|
+
throw new Error(String(signal.reason ?? "publication cancelled"));
|
|
570
|
+
const connection = this.connection;
|
|
571
|
+
if (!connection)
|
|
572
|
+
throw new Error("sandbox_output_publish_forbidden: bridge is disconnected.");
|
|
573
|
+
const response = await connection.invoke("RedeemPublishGrant", request);
|
|
574
|
+
if (!response.accepted || !response.credential) {
|
|
575
|
+
throw new Error(`sandbox_output_publish_forbidden: ${response.message ?? response.code ?? "publication grant redemption was rejected"}`);
|
|
576
|
+
}
|
|
577
|
+
return response.credential;
|
|
578
|
+
}
|
|
579
|
+
async handleLeaseUpdate(request) {
|
|
580
|
+
if (request.runnerInstanceId !== this.config.runnerInstanceId) {
|
|
581
|
+
return;
|
|
582
|
+
}
|
|
583
|
+
const reservation = this.reservations.get(request.dispatchAttemptId);
|
|
584
|
+
if (reservation && reservation.leaseId === request.leaseId && reservation.fencingToken === request.fencingToken) {
|
|
585
|
+
reservation.expiresAt = request.expiresAt;
|
|
586
|
+
}
|
|
587
|
+
const active = this.activeSessions.get(request.leaseId);
|
|
588
|
+
if (active && active.fencingToken === request.fencingToken && active.dispatchAttemptId === request.dispatchAttemptId) {
|
|
589
|
+
active.leaseExpiresAt = request.expiresAt;
|
|
590
|
+
if (active.sandboxed) {
|
|
591
|
+
await this.sandboxRuntime.updateLease(request.dispatchAttemptId, request.fencingToken, request.expiresAt);
|
|
592
|
+
}
|
|
593
|
+
else {
|
|
594
|
+
await this.sessions.updateLease?.(request.dispatchAttemptId, request.fencingToken, request.expiresAt);
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
async sendOfferDecision(dispatch, accepted, code = null, message = null, reservedUntil = null, reservedResources = null, sandboxConfigCommitSha = null) {
|
|
599
|
+
if (!dispatch.sandbox)
|
|
600
|
+
return;
|
|
601
|
+
const payload = {
|
|
602
|
+
dispatchAttemptId: dispatch.sandbox.dispatchAttemptId,
|
|
603
|
+
leaseId: dispatch.lease.id,
|
|
604
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
605
|
+
runnerInstanceId: dispatch.sandbox.runnerInstanceId,
|
|
606
|
+
accepted,
|
|
607
|
+
code,
|
|
608
|
+
message,
|
|
609
|
+
reservedUntil: accepted ? reservedUntil ?? dispatch.lease.expiresAt : null,
|
|
610
|
+
reservedResources: accepted ? reservedResources : null,
|
|
611
|
+
sandboxConfigCommitSha: accepted ? sandboxConfigCommitSha : null,
|
|
612
|
+
};
|
|
613
|
+
await this.invokeAck("OfferDecision", payload);
|
|
129
614
|
}
|
|
130
615
|
parseDispatchOrReport(rawDispatch) {
|
|
131
616
|
try {
|
|
@@ -137,7 +622,7 @@ export class ForgeBridgeService {
|
|
|
137
622
|
return null;
|
|
138
623
|
}
|
|
139
624
|
const summary = invalidDispatchSummary(error);
|
|
140
|
-
void this.
|
|
625
|
+
void this.sendLegacyResult(identifiers.leaseId, identifiers.fencingToken, failureResult(summary, [
|
|
141
626
|
{
|
|
142
627
|
role: "runner",
|
|
143
628
|
kind: "error",
|
|
@@ -155,12 +640,74 @@ export class ForgeBridgeService {
|
|
|
155
640
|
}
|
|
156
641
|
active.controller.abort(request.reason ?? "cancelled by server");
|
|
157
642
|
}
|
|
643
|
+
async handleReap(request) {
|
|
644
|
+
if (request.runnerId !== this.config.runnerId
|
|
645
|
+
|| request.runnerInstanceId !== this.config.runnerInstanceId)
|
|
646
|
+
return;
|
|
647
|
+
await this.runSandboxMaintenance();
|
|
648
|
+
await this.sendHeartbeat();
|
|
649
|
+
}
|
|
650
|
+
async handleReadinessProbe(rawRequest) {
|
|
651
|
+
const request = parseReadinessProbe(rawRequest);
|
|
652
|
+
if (!request || request.runnerInstanceId !== this.config.runnerInstanceId)
|
|
653
|
+
return;
|
|
654
|
+
const existing = this.activeReadinessProbes.get(request.requestId);
|
|
655
|
+
if (existing)
|
|
656
|
+
return;
|
|
657
|
+
const controller = new AbortController();
|
|
658
|
+
this.activeReadinessProbes.set(request.requestId, controller);
|
|
659
|
+
const deadlineMs = Date.parse(request.expiresAt) - Date.now();
|
|
660
|
+
const timer = setTimeout(() => controller.abort("readiness probe expired"), Math.max(0, deadlineMs));
|
|
661
|
+
timer.unref?.();
|
|
662
|
+
let result;
|
|
663
|
+
try {
|
|
664
|
+
if (!this.sessions.probeHostRuntime) {
|
|
665
|
+
throw new Error("host_runtime_probe_unavailable: this bridge does not implement runtime readiness probes.");
|
|
666
|
+
}
|
|
667
|
+
const probe = await this.sessions.probeHostRuntime({
|
|
668
|
+
...request,
|
|
669
|
+
runnerId: this.config.runnerId,
|
|
670
|
+
repoRoot: this.config.defaultRepoRoot,
|
|
671
|
+
cleanupWorktrees: this.config.cleanupWorktrees,
|
|
672
|
+
}, controller.signal);
|
|
673
|
+
result = {
|
|
674
|
+
schemaVersion: 1,
|
|
675
|
+
requestId: request.requestId,
|
|
676
|
+
fencingToken: request.fencingToken,
|
|
677
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
678
|
+
...probe,
|
|
679
|
+
};
|
|
680
|
+
}
|
|
681
|
+
catch (error) {
|
|
682
|
+
result = {
|
|
683
|
+
schemaVersion: 1,
|
|
684
|
+
requestId: request.requestId,
|
|
685
|
+
fencingToken: request.fencingToken,
|
|
686
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
687
|
+
status: "failed",
|
|
688
|
+
reasonCode: "host_runtime_probe_failed",
|
|
689
|
+
reasonMessage: errorMessage(error).slice(0, 1000),
|
|
690
|
+
checkedAt: new Date().toISOString(),
|
|
691
|
+
};
|
|
692
|
+
}
|
|
693
|
+
finally {
|
|
694
|
+
clearTimeout(timer);
|
|
695
|
+
this.activeReadinessProbes.delete(request.requestId);
|
|
696
|
+
}
|
|
697
|
+
await this.invokeAck("ReadinessResult", result).catch(() => undefined);
|
|
698
|
+
await this.sendHeartbeat().catch(() => undefined);
|
|
699
|
+
}
|
|
158
700
|
abortActiveSessions(reason) {
|
|
159
701
|
for (const active of this.activeSessions.values()) {
|
|
160
702
|
active.controller.abort(reason);
|
|
161
703
|
}
|
|
162
704
|
this.activeSessions.clear();
|
|
163
705
|
}
|
|
706
|
+
abortActiveReadinessProbes(reason) {
|
|
707
|
+
for (const controller of this.activeReadinessProbes.values())
|
|
708
|
+
controller.abort(reason);
|
|
709
|
+
this.activeReadinessProbes.clear();
|
|
710
|
+
}
|
|
164
711
|
normalizeDispatch(dispatch) {
|
|
165
712
|
const mapping = findRepoMapping(dispatch, repoMappingsFor(this.config));
|
|
166
713
|
const localPath = mapping?.localPath?.trim();
|
|
@@ -185,12 +732,20 @@ export class ForgeBridgeService {
|
|
|
185
732
|
: dispatch.qaBrowser
|
|
186
733
|
};
|
|
187
734
|
}
|
|
188
|
-
async sendResult(
|
|
735
|
+
async sendResult(dispatch, result) {
|
|
189
736
|
const payload = {
|
|
190
|
-
leaseId,
|
|
191
|
-
fencingToken,
|
|
737
|
+
leaseId: dispatch.lease.id,
|
|
738
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
739
|
+
dispatchAttemptId: dispatch.sandbox?.dispatchAttemptId ?? null,
|
|
740
|
+
runnerInstanceId: dispatch.sandbox?.runnerInstanceId ?? null,
|
|
192
741
|
result
|
|
193
742
|
};
|
|
743
|
+
await this.sendResultPayload(payload);
|
|
744
|
+
}
|
|
745
|
+
async sendLegacyResult(leaseId, fencingToken, result) {
|
|
746
|
+
await this.sendResultPayload({ leaseId, fencingToken, result });
|
|
747
|
+
}
|
|
748
|
+
async sendResultPayload(payload) {
|
|
194
749
|
try {
|
|
195
750
|
await this.invokeAck("DispatchResult", payload);
|
|
196
751
|
}
|
|
@@ -203,19 +758,142 @@ export class ForgeBridgeService {
|
|
|
203
758
|
}
|
|
204
759
|
startHeartbeat() {
|
|
205
760
|
this.stopHeartbeat();
|
|
761
|
+
this.consecutiveHeartbeatFailures = 0;
|
|
762
|
+
this.reconnectingFromHeartbeatFailure = false;
|
|
206
763
|
const intervalMs = positiveFiniteNumber(this.config.heartbeatIntervalMs, 30_000);
|
|
207
764
|
this.heartbeatTimer = setInterval(() => {
|
|
208
|
-
this.
|
|
765
|
+
void this.heartbeatTick();
|
|
209
766
|
}, intervalMs);
|
|
210
767
|
this.heartbeatTimer.unref?.();
|
|
211
768
|
}
|
|
769
|
+
async heartbeatTick() {
|
|
770
|
+
if (this.heartbeatInFlight || this.reconnectingFromHeartbeatFailure)
|
|
771
|
+
return;
|
|
772
|
+
const connection = this.connection;
|
|
773
|
+
if (!connection)
|
|
774
|
+
return;
|
|
775
|
+
this.heartbeatInFlight = true;
|
|
776
|
+
try {
|
|
777
|
+
await this.sendHeartbeat();
|
|
778
|
+
this.consecutiveHeartbeatFailures = 0;
|
|
779
|
+
}
|
|
780
|
+
catch (error) {
|
|
781
|
+
if (connection !== this.connection || this.stopped)
|
|
782
|
+
return;
|
|
783
|
+
this.consecutiveHeartbeatFailures += 1;
|
|
784
|
+
logBridgeStatus(`Heartbeat ${this.consecutiveHeartbeatFailures} failed: ${errorMessage(error)}.`);
|
|
785
|
+
if (this.consecutiveHeartbeatFailures < 2)
|
|
786
|
+
return;
|
|
787
|
+
this.reconnectingFromHeartbeatFailure = true;
|
|
788
|
+
this.stopHeartbeat();
|
|
789
|
+
logBridgeStatus("Heartbeat failed twice. Closing the stale connection so the Bridge can reconnect...");
|
|
790
|
+
await connection.stop().catch(() => undefined);
|
|
791
|
+
}
|
|
792
|
+
finally {
|
|
793
|
+
this.heartbeatInFlight = false;
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
startLeaseWatchdog() {
|
|
797
|
+
this.stopLeaseWatchdog();
|
|
798
|
+
this.leaseWatchdogTimer = setInterval(() => this.expireLeases(), 1000);
|
|
799
|
+
this.leaseWatchdogTimer.unref?.();
|
|
800
|
+
}
|
|
801
|
+
stopLeaseWatchdog() {
|
|
802
|
+
if (this.leaseWatchdogTimer) {
|
|
803
|
+
clearInterval(this.leaseWatchdogTimer);
|
|
804
|
+
this.leaseWatchdogTimer = null;
|
|
805
|
+
}
|
|
806
|
+
}
|
|
807
|
+
expireLeases() {
|
|
808
|
+
const now = Date.now();
|
|
809
|
+
this.expireReservations(now);
|
|
810
|
+
for (const active of this.activeSessions.values()) {
|
|
811
|
+
if (Date.parse(active.leaseExpiresAt) <= now && !active.controller.signal.aborted) {
|
|
812
|
+
active.controller.abort("dispatch lease expired");
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
expireReservations(now) {
|
|
817
|
+
for (const [attemptId, reservation] of this.reservations) {
|
|
818
|
+
if (Date.parse(reservation.expiresAt) <= now || Date.parse(reservation.reservedUntil) <= now) {
|
|
819
|
+
this.reservations.delete(attemptId);
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
}
|
|
212
823
|
stopHeartbeat() {
|
|
213
824
|
if (this.heartbeatTimer) {
|
|
214
825
|
clearInterval(this.heartbeatTimer);
|
|
215
826
|
this.heartbeatTimer = null;
|
|
216
827
|
}
|
|
217
828
|
}
|
|
829
|
+
startSandboxMaintenance() {
|
|
830
|
+
this.stopSandboxMaintenance();
|
|
831
|
+
this.sandboxMaintenanceTimer = setInterval(() => {
|
|
832
|
+
this.runSandboxMaintenance().catch(() => undefined);
|
|
833
|
+
}, 60_000);
|
|
834
|
+
this.sandboxMaintenanceTimer.unref?.();
|
|
835
|
+
}
|
|
836
|
+
stopSandboxMaintenance() {
|
|
837
|
+
if (this.sandboxMaintenanceTimer) {
|
|
838
|
+
clearInterval(this.sandboxMaintenanceTimer);
|
|
839
|
+
this.sandboxMaintenanceTimer = null;
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
async runSandboxMaintenance() {
|
|
843
|
+
if (this.sandboxMaintenanceRunning)
|
|
844
|
+
return;
|
|
845
|
+
this.sandboxMaintenanceRunning = true;
|
|
846
|
+
try {
|
|
847
|
+
const hostReap = await this.sessions.reapHostRuntime?.({
|
|
848
|
+
runnerId: this.config.runnerId,
|
|
849
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
850
|
+
});
|
|
851
|
+
if (hostReap && (hostReap.reaped > 0 || hostReap.degraded > 0)) {
|
|
852
|
+
logBridgeStatus(`Managed-host maintenance reaped ${hostReap.reaped} stale attempt(s) and left ${hostReap.degraded} degraded attempt(s) for reconciliation.`);
|
|
853
|
+
}
|
|
854
|
+
if (this.config.dockerEnabled) {
|
|
855
|
+
const preflight = await this.preflight.refresh();
|
|
856
|
+
if (canReapManagedSandboxes(preflight)) {
|
|
857
|
+
const attemptSnapshot = await this.readAttemptSnapshot();
|
|
858
|
+
await this.sandboxRuntime.reap({ diskLow: preflight.diskLow, attemptSnapshot });
|
|
859
|
+
await this.flushCleanupReports();
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
finally {
|
|
864
|
+
this.sandboxMaintenanceRunning = false;
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
async readAttemptSnapshot() {
|
|
868
|
+
const connection = this.connection;
|
|
869
|
+
if (!connection)
|
|
870
|
+
throw new Error("Bridge is not connected.");
|
|
871
|
+
const response = await connection.invoke("AttemptSnapshot", {
|
|
872
|
+
runnerInstanceId: this.config.runnerInstanceId
|
|
873
|
+
});
|
|
874
|
+
if (!response.accepted) {
|
|
875
|
+
throw new BridgeCallbackRejectedError(response.code ?? null, response.message ?? response.code ?? "Attempt snapshot was rejected.");
|
|
876
|
+
}
|
|
877
|
+
return response.attempts ?? [];
|
|
878
|
+
}
|
|
879
|
+
async flushCleanupReports() {
|
|
880
|
+
if (!this.sandboxRuntime.pendingCleanupReports || !this.sandboxRuntime.acknowledgeCleanupReport)
|
|
881
|
+
return;
|
|
882
|
+
for (const report of await this.sandboxRuntime.pendingCleanupReports()) {
|
|
883
|
+
await this.invokeAck("CleanupReport", {
|
|
884
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
885
|
+
report
|
|
886
|
+
});
|
|
887
|
+
await this.sandboxRuntime.acknowledgeCleanupReport(report.reportId);
|
|
888
|
+
}
|
|
889
|
+
}
|
|
218
890
|
async sendHeartbeat() {
|
|
891
|
+
// Heartbeats are the recovery path for capacity-blocked work. Refresh the
|
|
892
|
+
// dynamic sample before advertising it, while reusing the static Docker
|
|
893
|
+
// capability snapshot established at startup or by an offer.
|
|
894
|
+
if (this.config.dockerEnabled)
|
|
895
|
+
await this.preflight.refreshCapacity();
|
|
896
|
+
await this.dockerEngineProbe.refresh();
|
|
219
897
|
await this.invokeAck("Heartbeat", {
|
|
220
898
|
status: "online",
|
|
221
899
|
capabilities: this.capabilities(),
|
|
@@ -224,8 +902,26 @@ export class ForgeBridgeService {
|
|
|
224
902
|
});
|
|
225
903
|
}
|
|
226
904
|
capabilities() {
|
|
905
|
+
const preflight = this.preflight.current();
|
|
906
|
+
const dockerEngine = this.dockerEngineProbe.current();
|
|
907
|
+
const runtimeState = this.sandboxRuntime.snapshot();
|
|
908
|
+
const reserved = [...this.reservations.values()].filter(reservation => reservation.sandboxed).length;
|
|
909
|
+
const used = this.usedSandboxResources(runtimeState.degraded);
|
|
910
|
+
const activeResources = this.activeSandboxResources(runtimeState.degraded);
|
|
911
|
+
const capacity = this.sandboxResourceCapacity(preflight, activeResources, used);
|
|
912
|
+
const ambientMemoryCommitmentBytes = this.sandboxAmbientMemoryCommitmentBytes ?? 0;
|
|
913
|
+
const memoryAdmission = preflight ? buildMemoryAdmissionDiagnostics({
|
|
914
|
+
sampledAt: preflight.sampledAt,
|
|
915
|
+
totalMemoryBytes: preflight.totalMemoryBytes,
|
|
916
|
+
rawAvailableMemoryBytes: preflight.rawAvailableMemoryBytes,
|
|
917
|
+
stabilizedAvailableMemoryBytes: preflight.availableMemoryBytes,
|
|
918
|
+
controlPlaneMemoryBytes: process.memoryUsage().rss,
|
|
919
|
+
reserveMemoryBytes: preflight.hostReserveBytes,
|
|
920
|
+
committedMemoryBytes: ambientMemoryCommitmentBytes + used.memoryBytes,
|
|
921
|
+
pendingCommitmentMemoryBytes: Math.max(0, used.memoryBytes - activeResources.memoryBytes),
|
|
922
|
+
}) : null;
|
|
227
923
|
return {
|
|
228
|
-
schemaVersion:
|
|
924
|
+
schemaVersion: 5,
|
|
229
925
|
activeSessionCount: this.activeSessions.size,
|
|
230
926
|
cliProviders: this.config.cliProviders,
|
|
231
927
|
providerStatuses: this.config.providerStatuses,
|
|
@@ -233,32 +929,199 @@ export class ForgeBridgeService {
|
|
|
233
929
|
repoMappings: repoMappingsFor(this.config),
|
|
234
930
|
os: this.config.os,
|
|
235
931
|
sandboxProfiles: this.config.sandboxProfiles,
|
|
932
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
933
|
+
trustDomainId: this.config.trustDomainId,
|
|
934
|
+
executionFacilities: dockerEngine?.available ? ["docker_engine"] : [],
|
|
935
|
+
dockerEngine,
|
|
936
|
+
toolchains: {
|
|
937
|
+
...this.hostCapabilities.toolchains,
|
|
938
|
+
docker: {
|
|
939
|
+
available: dockerEngine?.available === true,
|
|
940
|
+
executable: this.config.dockerExecutable,
|
|
941
|
+
version: dockerEngine?.runtimeVersion ?? null,
|
|
942
|
+
},
|
|
943
|
+
},
|
|
944
|
+
playwrightProjects: this.hostCapabilities.playwrightProjects,
|
|
945
|
+
browserLaunch: this.hostCapabilities.browserLaunch,
|
|
946
|
+
providerConnectivity: this.hostCapabilities.providerConnectivity,
|
|
947
|
+
maxConcurrentSandboxes: capacity.sandboxSlots,
|
|
948
|
+
reservedSandboxCount: reserved,
|
|
949
|
+
activeSandboxCount: runtimeState.active,
|
|
950
|
+
startingSandboxCount: runtimeState.starting,
|
|
951
|
+
degradedReservationCount: runtimeState.degraded,
|
|
952
|
+
sandboxRuntimes: preflight?.healthy ? ["docker"] : [],
|
|
953
|
+
sandboxRuntimeVersion: preflight?.runtimeVersion ?? null,
|
|
954
|
+
sandboxFeatures: preflight?.features ?? null,
|
|
955
|
+
allowsOpenPublicSandbox: this.config.allowOpenPublicSandbox,
|
|
956
|
+
sandboxPublicationModes: [this.config.sandboxPublicationMode ?? "shared_filesystem"],
|
|
957
|
+
sandboxAuthModes: sandboxAuthModesFor(this.config),
|
|
958
|
+
availableMemoryBytes: memoryAdmission?.availableForAdmissionBytes ?? null,
|
|
959
|
+
availableCpuMillis: preflight ? Math.max(0, capacity.cpuMillis - used.cpuMillis) : null,
|
|
960
|
+
availableWorkspaceBytes: preflight ? Math.max(0, capacity.workspaceBytes - used.workspaceBytes) : null,
|
|
961
|
+
diskLow: preflight?.diskLow ?? null,
|
|
962
|
+
preflightGeneration: preflight?.generation ?? 0,
|
|
963
|
+
sandboxUnavailableCode: this.config.dockerEnabled ? preflight?.unavailableCode ?? null : "docker_disabled",
|
|
964
|
+
sandboxUnavailableReason: this.config.dockerEnabled ? preflight?.unavailableReason ?? null : "Docker sandboxing is disabled by runner configuration.",
|
|
965
|
+
sandboxMemoryCapacity: preflight ? {
|
|
966
|
+
defaultWorkerMemoryLimitBytes: this.config.defaultSandboxWorkerMemoryLimitBytes,
|
|
967
|
+
defaultServiceMemoryLimitBytes: this.config.defaultSandboxServiceMemoryLimitBytes,
|
|
968
|
+
runtimeOverheadBytes: this.config.sandboxRuntimeOverheadBytes,
|
|
969
|
+
hostReserveBytes: preflight.hostReserveBytes,
|
|
970
|
+
activeReservationBytes: used.memoryBytes,
|
|
971
|
+
daemonTotalBytes: preflight.totalMemoryBytes,
|
|
972
|
+
daemonAvailableBytes: preflight.rawDaemonAvailableMemoryBytes ?? 0,
|
|
973
|
+
hostAvailableBytes: preflight.rawHostAvailableMemoryBytes,
|
|
974
|
+
conservativeAvailableBytes: preflight.rawAvailableMemoryBytes,
|
|
975
|
+
availableForAdmissionBytes: memoryAdmission.availableForAdmissionBytes,
|
|
976
|
+
memoryAdmission: memoryAdmission,
|
|
977
|
+
} : null,
|
|
978
|
+
hostRuntime: this.sessions.runtimeCapacity?.() ?? null,
|
|
236
979
|
supportsApiSecretGrant: this.config.supportsApiSecretGrant,
|
|
237
980
|
supportsManagedWorktrees: this.config.supportsManagedWorktrees,
|
|
238
981
|
supportsManagedForgeArtifacts: this.config.supportsManagedForgeArtifacts,
|
|
239
982
|
supportsRunnerAuthoredPhaseArtifacts: this.config.supportsRunnerAuthoredPhaseArtifacts,
|
|
240
|
-
supportsDraftRunOnce: this.config.supportsDraftRunOnce
|
|
983
|
+
supportsDraftRunOnce: this.config.supportsDraftRunOnce,
|
|
984
|
+
supportsDraftAttachments: this.config.supportsDraftAttachments,
|
|
985
|
+
supportsDraftImageAttachments: this.config.supportsDraftImageAttachments
|
|
986
|
+
};
|
|
987
|
+
}
|
|
988
|
+
async requiredCapabilityFailure(dispatch) {
|
|
989
|
+
const requiredCapabilities = dispatch.qaScope?.required_capabilities ?? [];
|
|
990
|
+
if (!requiredCapabilities.includes("docker_engine"))
|
|
991
|
+
return null;
|
|
992
|
+
const executionMode = dispatch.sandbox?.executionMode;
|
|
993
|
+
if (executionMode === "docker") {
|
|
994
|
+
return dockerEngineRequirementFailure(requiredCapabilities, executionMode, null);
|
|
995
|
+
}
|
|
996
|
+
return dockerEngineRequirementFailure(requiredCapabilities, executionMode, await this.dockerEngineProbe.refresh(true));
|
|
997
|
+
}
|
|
998
|
+
usedSandboxResources(degradedReservations) {
|
|
999
|
+
const resources = [
|
|
1000
|
+
...[...this.reservations.values()]
|
|
1001
|
+
.filter(reservation => reservation.sandboxed)
|
|
1002
|
+
.map(reservation => reservation.resources),
|
|
1003
|
+
...[...this.activeSessions.values()]
|
|
1004
|
+
.filter(session => session.sandboxed)
|
|
1005
|
+
.map(session => session.resources),
|
|
1006
|
+
...Array.from({ length: degradedReservations }, () => defaultSandboxResources())
|
|
1007
|
+
];
|
|
1008
|
+
return sumSandboxResources(resources);
|
|
1009
|
+
}
|
|
1010
|
+
activeSandboxResources(degradedReservations) {
|
|
1011
|
+
return sumSandboxResources([
|
|
1012
|
+
...[...this.activeSessions.values()]
|
|
1013
|
+
.filter(session => session.sandboxed)
|
|
1014
|
+
.map(session => session.resources),
|
|
1015
|
+
...Array.from({ length: degradedReservations }, () => defaultSandboxResources())
|
|
1016
|
+
]);
|
|
1017
|
+
}
|
|
1018
|
+
sandboxResourceCapacity(preflight, activeResources, usedResources) {
|
|
1019
|
+
if (!preflight)
|
|
1020
|
+
return zeroSandboxResources();
|
|
1021
|
+
if (usedResources.memoryBytes === 0 || this.sandboxAmbientMemoryCommitmentBytes === null) {
|
|
1022
|
+
// Freeze non-Forge daemon use while reservations exist. If the bridge
|
|
1023
|
+
// reconnects with managed containers already present, the initial value
|
|
1024
|
+
// is deliberately conservative until their ledger drains.
|
|
1025
|
+
this.sandboxAmbientMemoryCommitmentBytes = Math.max(0, preflight.totalMemoryBytes - preflight.rawAvailableMemoryBytes);
|
|
1026
|
+
}
|
|
1027
|
+
return {
|
|
1028
|
+
// Host free memory and disk already reflect active containers. Add their
|
|
1029
|
+
// commitments back to form a stable total capacity, then subtract them
|
|
1030
|
+
// exactly once in usedSandboxResources.
|
|
1031
|
+
sandboxSlots: Math.min(this.config.maxConcurrentSandboxes, preflight.maxConcurrentSandboxes + activeResources.sandboxSlots),
|
|
1032
|
+
cpuMillis: preflight.availableCpuMillis,
|
|
1033
|
+
workerMemoryLimitBytes: 0,
|
|
1034
|
+
serviceMemoryLimitBytes: 0,
|
|
1035
|
+
runtimeOverheadBytes: 0,
|
|
1036
|
+
memoryBytes: sandboxMemoryCommitmentCapacity(preflight, activeResources.memoryBytes, this.sandboxAmbientMemoryCommitmentBytes),
|
|
1037
|
+
workspaceBytes: Math.max(0, preflight.availableWorkspaceBytes
|
|
1038
|
+
- this.config.minimumWorkspaceBytes
|
|
1039
|
+
+ activeResources.workspaceBytes),
|
|
1040
|
+
pids: this.config.sandboxPidCapacity,
|
|
1041
|
+
buildSlots: this.config.maxConcurrentSandboxBuilds
|
|
241
1042
|
};
|
|
242
1043
|
}
|
|
243
1044
|
async invokeAck(methodName, payload) {
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
1045
|
+
let storageRecoveryAttempt = 0;
|
|
1046
|
+
while (true) {
|
|
1047
|
+
const connection = this.connection;
|
|
1048
|
+
if (!connection) {
|
|
1049
|
+
throw new Error("Bridge is not connected.");
|
|
1050
|
+
}
|
|
1051
|
+
await this.waitForTransportRecovery(connection, this.transportRecovery);
|
|
1052
|
+
const recoveryGeneration = this.transportRecoveryGeneration;
|
|
1053
|
+
try {
|
|
1054
|
+
const ack = await waitForCallbackAck(connection.invoke(methodName, payload), methodName, positiveFiniteNumber(this.config.callbackAckTimeoutMs ?? 30_000, 30_000));
|
|
1055
|
+
if (!ack.accepted) {
|
|
1056
|
+
throw new BridgeCallbackRejectedError(ack.code ?? null, ack.message ?? ack.code ?? `${methodName} callback was rejected.`);
|
|
1057
|
+
}
|
|
1058
|
+
return ack;
|
|
1059
|
+
}
|
|
1060
|
+
catch (error) {
|
|
1061
|
+
if (connection !== this.connection || this.stopped)
|
|
1062
|
+
throw error;
|
|
1063
|
+
const recovery = this.transportRecovery;
|
|
1064
|
+
if (recovery && recovery.generation > recoveryGeneration) {
|
|
1065
|
+
await this.waitForTransportRecovery(connection, recovery);
|
|
1066
|
+
continue;
|
|
1067
|
+
}
|
|
1068
|
+
if (isProviderStorageUnavailable(error) && storageRecoveryAttempt < this.config.reconnectBackoffMs.length) {
|
|
1069
|
+
const backoff = this.config.reconnectBackoffMs[storageRecoveryAttempt] ?? 5000;
|
|
1070
|
+
storageRecoveryAttempt += 1;
|
|
1071
|
+
logBridgeStatus(`${methodName} reached Forge while storage was recovering. Retrying callback ${storageRecoveryAttempt} in ${formatDelay(backoff)}...`);
|
|
1072
|
+
await delay(backoff);
|
|
1073
|
+
continue;
|
|
1074
|
+
}
|
|
1075
|
+
throw error;
|
|
1076
|
+
}
|
|
247
1077
|
}
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
1078
|
+
}
|
|
1079
|
+
beginTransportRecovery() {
|
|
1080
|
+
if (this.transportRecovery && !this.transportRecovery.completed)
|
|
1081
|
+
return;
|
|
1082
|
+
let resolve = () => undefined;
|
|
1083
|
+
const promise = new Promise((complete) => { resolve = complete; });
|
|
1084
|
+
this.transportRecoveryGeneration += 1;
|
|
1085
|
+
this.transportRecovery = {
|
|
1086
|
+
generation: this.transportRecoveryGeneration,
|
|
1087
|
+
promise,
|
|
1088
|
+
resolve,
|
|
1089
|
+
completed: false,
|
|
1090
|
+
error: null,
|
|
1091
|
+
};
|
|
1092
|
+
}
|
|
1093
|
+
completeTransportRecovery(error = null) {
|
|
1094
|
+
const recovery = this.transportRecovery;
|
|
1095
|
+
if (!recovery || recovery.completed)
|
|
1096
|
+
return;
|
|
1097
|
+
recovery.completed = true;
|
|
1098
|
+
recovery.error = error;
|
|
1099
|
+
recovery.resolve();
|
|
1100
|
+
}
|
|
1101
|
+
async waitForTransportRecovery(connection, recovery) {
|
|
1102
|
+
if (!recovery)
|
|
1103
|
+
return;
|
|
1104
|
+
if (recovery.completed) {
|
|
1105
|
+
if (recovery.error)
|
|
1106
|
+
throw recovery.error;
|
|
1107
|
+
return;
|
|
251
1108
|
}
|
|
252
|
-
|
|
1109
|
+
await recovery.promise;
|
|
1110
|
+
if (connection !== this.connection)
|
|
1111
|
+
throw new Error("Bridge connection changed while recovering.");
|
|
1112
|
+
if (recovery.error)
|
|
1113
|
+
throw recovery.error;
|
|
253
1114
|
}
|
|
254
1115
|
async waitForClose(signal) {
|
|
255
1116
|
if (!this.closed) {
|
|
256
1117
|
return;
|
|
257
1118
|
}
|
|
258
|
-
await
|
|
1119
|
+
await waitForPromiseOrAbort(this.closed, signal);
|
|
259
1120
|
}
|
|
260
1121
|
async stopConnection() {
|
|
261
1122
|
this.stopHeartbeat();
|
|
1123
|
+
this.stopLeaseWatchdog();
|
|
1124
|
+
this.stopSandboxMaintenance();
|
|
262
1125
|
if (this.connection) {
|
|
263
1126
|
await this.connection.stop().catch(() => undefined);
|
|
264
1127
|
this.connection = null;
|
|
@@ -279,6 +1142,17 @@ function isDispatchNoLongerPending(error) {
|
|
|
279
1142
|
}
|
|
280
1143
|
return error instanceof Error && error.message === "Dispatch is no longer pending.";
|
|
281
1144
|
}
|
|
1145
|
+
function isProviderStorageUnavailable(error) {
|
|
1146
|
+
const message = errorMessage(error).toLowerCase();
|
|
1147
|
+
return message.includes("provider_storage_unavailable")
|
|
1148
|
+
|| (message.includes("storage is recovering") && message.includes("503"));
|
|
1149
|
+
}
|
|
1150
|
+
function createDefaultSessionService(sandboxStateRoot) {
|
|
1151
|
+
const rawProcessRunner = new StreamingProviderProcessRunner();
|
|
1152
|
+
const processSupervisors = new AttemptProcessSupervisorRegistry();
|
|
1153
|
+
const coordinatedProcessRunner = new AttemptCoordinatedProviderProcessRunner(processSupervisors, rawProcessRunner);
|
|
1154
|
+
return new BridgeSessionService(new GitWorktreeService(undefined, new RepositoryCoordinatorLock(sandboxStateRoot)), new RoutingProviderCliService(new ProcessBackedProviderCliService(coordinatedProcessRunner)), new HostRuntimeLeaseManager(sandboxStateRoot, undefined, undefined, processSupervisors, rawProcessRunner));
|
|
1155
|
+
}
|
|
282
1156
|
export class SignalRBridgeConnectionFactory {
|
|
283
1157
|
create(config) {
|
|
284
1158
|
return new SignalRBridgeConnection(config);
|
|
@@ -301,8 +1175,12 @@ export class SignalRBridgeConnection {
|
|
|
301
1175
|
Authorization: `Bearer ${config.deviceToken}`
|
|
302
1176
|
}
|
|
303
1177
|
})
|
|
1178
|
+
.withAutomaticReconnect([...config.reconnectBackoffMs])
|
|
304
1179
|
.configureLogging(LogLevel.Warning)
|
|
305
1180
|
.build();
|
|
1181
|
+
this.connection.on("Superseded", () => {
|
|
1182
|
+
void this.connection.stop().catch(() => undefined);
|
|
1183
|
+
});
|
|
306
1184
|
}
|
|
307
1185
|
start() {
|
|
308
1186
|
return this.connection.start();
|
|
@@ -313,16 +1191,49 @@ export class SignalRBridgeConnection {
|
|
|
313
1191
|
onClose(handler) {
|
|
314
1192
|
this.connection.onclose(handler);
|
|
315
1193
|
}
|
|
1194
|
+
onReconnecting(handler) {
|
|
1195
|
+
this.connection.onreconnecting(handler);
|
|
1196
|
+
}
|
|
1197
|
+
onReconnected(handler) {
|
|
1198
|
+
this.connection.onreconnected((connectionId) => {
|
|
1199
|
+
void Promise.resolve(handler(connectionId)).catch(() => undefined);
|
|
1200
|
+
});
|
|
1201
|
+
}
|
|
316
1202
|
onDispatch(handler) {
|
|
317
1203
|
this.connection.on("Dispatch", (raw) => {
|
|
318
1204
|
void handler(raw).catch(() => undefined);
|
|
319
1205
|
});
|
|
320
1206
|
}
|
|
1207
|
+
onOffer(handler) {
|
|
1208
|
+
this.connection.on("Offer", (raw) => {
|
|
1209
|
+
void handler(raw).catch(() => undefined);
|
|
1210
|
+
});
|
|
1211
|
+
}
|
|
1212
|
+
onStart(handler) {
|
|
1213
|
+
this.connection.on("Start", (raw) => {
|
|
1214
|
+
void handler(raw).catch(() => undefined);
|
|
1215
|
+
});
|
|
1216
|
+
}
|
|
1217
|
+
onLeaseUpdate(handler) {
|
|
1218
|
+
this.connection.on("LeaseUpdate", (request) => {
|
|
1219
|
+
void handler(request).catch(() => undefined);
|
|
1220
|
+
});
|
|
1221
|
+
}
|
|
321
1222
|
onCancel(handler) {
|
|
322
1223
|
this.connection.on("Cancel", (request) => {
|
|
323
1224
|
void handler(request).catch(() => undefined);
|
|
324
1225
|
});
|
|
325
1226
|
}
|
|
1227
|
+
onReap(handler) {
|
|
1228
|
+
this.connection.on("Reap", (request) => {
|
|
1229
|
+
void handler(request).catch(() => undefined);
|
|
1230
|
+
});
|
|
1231
|
+
}
|
|
1232
|
+
onReadinessProbe(handler) {
|
|
1233
|
+
this.connection.on("ReadinessProbe", (request) => {
|
|
1234
|
+
void handler(request).catch(() => undefined);
|
|
1235
|
+
});
|
|
1236
|
+
}
|
|
326
1237
|
invoke(methodName, payload) {
|
|
327
1238
|
return this.connection.invoke(methodName, payload);
|
|
328
1239
|
}
|
|
@@ -330,9 +1241,41 @@ export class SignalRBridgeConnection {
|
|
|
330
1241
|
export function defaultBridgeServiceConfig(overrides = {}) {
|
|
331
1242
|
const defaults = defaultForgeBridgeConfig();
|
|
332
1243
|
const cliProviders = splitCsv(process.env.FORGE_CLI_PROVIDERS ?? "codex");
|
|
1244
|
+
const configuredSandboxHostMemoryReserveBytes = optionalPositiveFiniteNumber(process.env.FORGE_SANDBOX_HOST_MEMORY_RESERVE_BYTES);
|
|
1245
|
+
const sandboxPublicationMode = process.env.FORGE_SANDBOX_PUBLICATION_MODE === "bridge_push" ? "bridge_push" : "shared_filesystem";
|
|
333
1246
|
const config = {
|
|
334
1247
|
serverUrl: process.env.FORGE_URL ?? "http://localhost:5208",
|
|
335
1248
|
runnerId: process.env.FORGE_RUNNER_ID ?? "",
|
|
1249
|
+
runnerInstanceId: process.env.FORGE_RUNNER_INSTANCE_ID ?? randomUUID(),
|
|
1250
|
+
trustDomainId: process.env.FORGE_TRUST_DOMAIN_ID ?? "default",
|
|
1251
|
+
maxConcurrentSandboxes: positiveFiniteNumber(Number(process.env.FORGE_MAX_CONCURRENT_SANDBOXES ?? 3), 3),
|
|
1252
|
+
maxConcurrentSandboxBuilds: positiveFiniteNumber(Number(process.env.FORGE_MAX_CONCURRENT_SANDBOX_BUILDS ?? 1), 1),
|
|
1253
|
+
sandboxPidCapacity: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_PID_CAPACITY ?? 2048), 2048),
|
|
1254
|
+
dockerEnabled: process.env.FORGE_DOCKER_SANDBOX !== "false",
|
|
1255
|
+
allowOpenPublicSandbox: process.env.FORGE_ALLOW_OPEN_PUBLIC_SANDBOX === "true",
|
|
1256
|
+
sandboxPublicationMode,
|
|
1257
|
+
dockerExecutable: process.env.FORGE_DOCKER_EXECUTABLE ?? "docker",
|
|
1258
|
+
dockerContext: process.env.FORGE_DOCKER_CONTEXT ?? "",
|
|
1259
|
+
dockerDesktopRecoveryEnabled: process.env.FORGE_DOCKER_DESKTOP_AUTO_RECOVERY === "true",
|
|
1260
|
+
dockerRecoveryFailureThreshold: positiveFiniteNumber(Number(process.env.FORGE_DOCKER_RECOVERY_FAILURE_THRESHOLD ?? 3), 3),
|
|
1261
|
+
dockerRecoveryCooldownMs: positiveFiniteNumber(Number(process.env.FORGE_DOCKER_RECOVERY_COOLDOWN_MS ?? 10 * 60_000), 10 * 60_000),
|
|
1262
|
+
dockerRecoveryStartupTimeoutMs: positiveFiniteNumber(Number(process.env.FORGE_DOCKER_RECOVERY_STARTUP_TIMEOUT_MS ?? 2 * 60_000), 2 * 60_000),
|
|
1263
|
+
sandboxStateRoot: process.env.FORGE_SANDBOX_STATE_ROOT ?? path.join(os.homedir(), ".forge", "bridge", "sandboxes"),
|
|
1264
|
+
requireRootlessDocker: process.env.FORGE_REQUIRE_ROOTLESS_DOCKER === "true" || process.platform === "linux",
|
|
1265
|
+
diskHighWatermarkPercent: positiveFiniteNumber(Number(process.env.FORGE_DISK_HIGH_WATERMARK_PERCENT ?? 85), 85),
|
|
1266
|
+
diskLowWatermarkPercent: positiveFiniteNumber(Number(process.env.FORGE_DISK_LOW_WATERMARK_PERCENT ?? 75), 75),
|
|
1267
|
+
minimumWorkspaceBytes: positiveFiniteNumber(Number(process.env.FORGE_MINIMUM_WORKSPACE_BYTES ?? 10 * 1024 ** 3), 10 * 1024 ** 3),
|
|
1268
|
+
// Undefined means preflight derives the bounded reserve from the actual
|
|
1269
|
+
// Docker/host capacity domain after sampling it.
|
|
1270
|
+
sandboxHostMemoryReserveBytes: configuredSandboxHostMemoryReserveBytes,
|
|
1271
|
+
defaultSandboxWorkerMemoryLimitBytes: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_DEFAULT_WORKER_MEMORY_LIMIT_BYTES ?? 2 * 1024 ** 3), 2 * 1024 ** 3),
|
|
1272
|
+
defaultSandboxServiceMemoryLimitBytes: nonNegativeFiniteNumber(Number(process.env.FORGE_SANDBOX_DEFAULT_SERVICE_MEMORY_LIMIT_BYTES ?? 0), 0),
|
|
1273
|
+
sandboxRuntimeOverheadBytes: Math.max(128 * 1024 ** 2, nonNegativeFiniteNumber(Number(process.env.FORGE_SANDBOX_RUNTIME_OVERHEAD_BYTES ?? 128 * 1024 ** 2), 128 * 1024 ** 2)),
|
|
1274
|
+
sandboxMemorySampleIntervalMs: Math.min(30_000, Math.max(250, positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_MEMORY_SAMPLE_INTERVAL_MS ?? 1000), 1000))),
|
|
1275
|
+
imageRetentionCount: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_IMAGE_RETENTION_COUNT ?? 3), 3),
|
|
1276
|
+
imageRetentionTtlMs: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_IMAGE_RETENTION_TTL_MS ?? 7 * 24 * 60 * 60 * 1000), 7 * 24 * 60 * 60 * 1000),
|
|
1277
|
+
auditRetentionCount: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_AUDIT_RETENTION_COUNT ?? 200), 200),
|
|
1278
|
+
auditRetentionTtlMs: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_AUDIT_RETENTION_TTL_MS ?? 30 * 24 * 60 * 60 * 1000), 30 * 24 * 60 * 60 * 1000),
|
|
336
1279
|
deviceToken: process.env.FORGE_DEVICE_TOKEN ?? "",
|
|
337
1280
|
deviceName: process.env.FORGE_DEVICE_NAME ?? "",
|
|
338
1281
|
defaultRepoRoot: process.env.FORGE_REPO_ROOT ?? defaults.defaultRepoRoot,
|
|
@@ -348,17 +1291,23 @@ export function defaultBridgeServiceConfig(overrides = {}) {
|
|
|
348
1291
|
supportsManagedForgeArtifacts: true,
|
|
349
1292
|
supportsRunnerAuthoredPhaseArtifacts: true,
|
|
350
1293
|
supportsDraftRunOnce: true,
|
|
1294
|
+
supportsDraftAttachments: true,
|
|
1295
|
+
supportsDraftImageAttachments: supportsDraftImageAttachmentsFor(cliProviders),
|
|
351
1296
|
version: process.env.FORGE_BRIDGE_VERSION ?? forgeBridgeVersion,
|
|
352
|
-
protocolVersion:
|
|
1297
|
+
protocolVersion: 2,
|
|
353
1298
|
minServerVersion: null,
|
|
354
1299
|
heartbeatIntervalMs: positiveFiniteNumber(Number(process.env.FORGE_HEARTBEAT_INTERVAL_MS ?? 30_000), 30_000),
|
|
1300
|
+
callbackAckTimeoutMs: positiveFiniteNumber(Number(process.env.FORGE_CALLBACK_ACK_TIMEOUT_MS ?? 30_000), 30_000),
|
|
355
1301
|
reconnectBackoffMs: [1000, 2000, 5000, 10000, 30000],
|
|
356
1302
|
...overrides
|
|
357
1303
|
};
|
|
358
1304
|
return {
|
|
359
1305
|
...config,
|
|
1306
|
+
providerStatuses: overrides.providerStatuses ?? providerStatusesFor(config.cliProviders),
|
|
360
1307
|
sandboxProfiles: config.sandboxProfiles ?? defaultSandboxProfiles(config.cliProviders),
|
|
361
|
-
|
|
1308
|
+
supportsDraftImageAttachments: (overrides.supportsDraftImageAttachments ?? true) && supportsDraftImageAttachmentsFor(config.cliProviders),
|
|
1309
|
+
heartbeatIntervalMs: positiveFiniteNumber(config.heartbeatIntervalMs, 30_000),
|
|
1310
|
+
callbackAckTimeoutMs: positiveFiniteNumber(config.callbackAckTimeoutMs ?? 30_000, 30_000),
|
|
362
1311
|
};
|
|
363
1312
|
}
|
|
364
1313
|
export function resolveBridgeRunnerHubUrl(serverUrl) {
|
|
@@ -373,9 +1322,42 @@ function normalizeServerUrl(serverUrl) {
|
|
|
373
1322
|
function positiveFiniteNumber(value, fallback) {
|
|
374
1323
|
return Number.isFinite(value) && value > 0 ? value : fallback;
|
|
375
1324
|
}
|
|
1325
|
+
function optionalPositiveFiniteNumber(value) {
|
|
1326
|
+
if (value === undefined || value.trim().length === 0)
|
|
1327
|
+
return undefined;
|
|
1328
|
+
const parsed = Number(value);
|
|
1329
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
|
|
1330
|
+
}
|
|
1331
|
+
function nonNegativeFiniteNumber(value, fallback) {
|
|
1332
|
+
return Number.isFinite(value) && value >= 0 ? value : fallback;
|
|
1333
|
+
}
|
|
376
1334
|
function providerStatusesFor(providers) {
|
|
377
1335
|
return Object.fromEntries(providers.map((provider) => [provider, { signedIn: true, version: "", models: configuredModelsFor(provider) }]));
|
|
378
1336
|
}
|
|
1337
|
+
function sandboxAuthModesFor(config) {
|
|
1338
|
+
return Object.fromEntries(config.cliProviders.map((provider) => {
|
|
1339
|
+
const modes = ["api_key"];
|
|
1340
|
+
if (sandboxBridgeCliAuthAvailable(config, provider)) {
|
|
1341
|
+
modes.push("bridge_cli");
|
|
1342
|
+
}
|
|
1343
|
+
return [provider, modes];
|
|
1344
|
+
}));
|
|
1345
|
+
}
|
|
1346
|
+
function sandboxBridgeCliAuthAvailable(config, provider) {
|
|
1347
|
+
if (provider.trim().toLowerCase() !== "codex") {
|
|
1348
|
+
return false;
|
|
1349
|
+
}
|
|
1350
|
+
const status = providerStatus(config, provider);
|
|
1351
|
+
if (status?.signedIn === false) {
|
|
1352
|
+
return false;
|
|
1353
|
+
}
|
|
1354
|
+
return codexSandboxProviderAuthAdapter.doctor().available;
|
|
1355
|
+
}
|
|
1356
|
+
function providerStatus(config, provider) {
|
|
1357
|
+
const normalized = provider.trim().toLowerCase();
|
|
1358
|
+
return Object.entries(config.providerStatuses)
|
|
1359
|
+
.find(([key]) => key.trim().toLowerCase() === normalized)?.[1];
|
|
1360
|
+
}
|
|
379
1361
|
function defaultSandboxProfiles(providers, platform = process.platform) {
|
|
380
1362
|
if (!providers.some((provider) => provider.toLowerCase() === "codex")) {
|
|
381
1363
|
return ["none"];
|
|
@@ -470,10 +1452,11 @@ function normalizeQaAppLaunchTarget(target) {
|
|
|
470
1452
|
command: cleanString(target.command),
|
|
471
1453
|
working_directory: cleanString(target.working_directory),
|
|
472
1454
|
ready_url: cleanString(target.ready_url),
|
|
1455
|
+
managed_runtime_port: cleanString(target.managed_runtime_port),
|
|
473
1456
|
routes,
|
|
474
1457
|
notes: cleanString(target.notes)
|
|
475
1458
|
};
|
|
476
|
-
return normalized.url || normalized.command || normalized.ready_url ||
|
|
1459
|
+
return normalized.url || normalized.command || normalized.ready_url || normalized.managed_runtime_port
|
|
477
1460
|
? normalized
|
|
478
1461
|
: null;
|
|
479
1462
|
}
|
|
@@ -515,33 +1498,361 @@ function formatDelay(ms) {
|
|
|
515
1498
|
}
|
|
516
1499
|
return `${ms}ms`;
|
|
517
1500
|
}
|
|
1501
|
+
async function waitForCallbackAck(callback, methodName, timeoutMs) {
|
|
1502
|
+
let timeout = null;
|
|
1503
|
+
const expired = new Promise((_resolve, reject) => {
|
|
1504
|
+
timeout = setTimeout(() => {
|
|
1505
|
+
reject(new Error(`bridge_callback_timeout: ${methodName} callback was not acknowledged within ${formatDelay(timeoutMs)}.`));
|
|
1506
|
+
}, timeoutMs);
|
|
1507
|
+
});
|
|
1508
|
+
try {
|
|
1509
|
+
return await Promise.race([callback, expired]);
|
|
1510
|
+
}
|
|
1511
|
+
finally {
|
|
1512
|
+
if (timeout)
|
|
1513
|
+
clearTimeout(timeout);
|
|
1514
|
+
}
|
|
1515
|
+
}
|
|
518
1516
|
function logBridgeStatus(message) {
|
|
519
1517
|
process.stderr.write(`[${new Date().toISOString()}] ${message}\n`);
|
|
520
1518
|
}
|
|
521
|
-
function delay(ms, signal) {
|
|
1519
|
+
export function delay(ms, signal) {
|
|
522
1520
|
if (signal?.aborted) {
|
|
523
1521
|
return Promise.resolve();
|
|
524
1522
|
}
|
|
525
1523
|
return new Promise((resolve) => {
|
|
526
|
-
const
|
|
527
|
-
signal?.addEventListener("abort", () => {
|
|
1524
|
+
const complete = () => {
|
|
528
1525
|
clearTimeout(timeout);
|
|
1526
|
+
signal?.removeEventListener("abort", complete);
|
|
529
1527
|
resolve();
|
|
530
|
-
}
|
|
1528
|
+
};
|
|
1529
|
+
const timeout = setTimeout(complete, ms);
|
|
1530
|
+
signal?.addEventListener("abort", complete, { once: true });
|
|
1531
|
+
if (signal?.aborted)
|
|
1532
|
+
complete();
|
|
531
1533
|
});
|
|
532
1534
|
}
|
|
533
|
-
function
|
|
1535
|
+
export async function waitForPromiseOrAbort(promise, signal) {
|
|
534
1536
|
if (!signal) {
|
|
535
|
-
|
|
1537
|
+
await promise;
|
|
1538
|
+
return;
|
|
536
1539
|
}
|
|
537
1540
|
if (signal.aborted) {
|
|
538
|
-
return
|
|
1541
|
+
return;
|
|
1542
|
+
}
|
|
1543
|
+
let resolveAbort;
|
|
1544
|
+
const abort = new Promise(resolve => { resolveAbort = resolve; });
|
|
1545
|
+
signal.addEventListener("abort", resolveAbort, { once: true });
|
|
1546
|
+
try {
|
|
1547
|
+
await Promise.race([promise, abort]);
|
|
1548
|
+
}
|
|
1549
|
+
finally {
|
|
1550
|
+
signal.removeEventListener("abort", resolveAbort);
|
|
1551
|
+
}
|
|
1552
|
+
}
|
|
1553
|
+
export function canReapManagedSandboxes(preflight) {
|
|
1554
|
+
return canRunDockerMaintenance(preflight);
|
|
1555
|
+
}
|
|
1556
|
+
function sandboxNetworkPolicyFailure(dispatch, config) {
|
|
1557
|
+
const sandbox = dispatch.sandbox;
|
|
1558
|
+
if (!sandbox)
|
|
1559
|
+
return "Protocol-v2 sandbox context is missing.";
|
|
1560
|
+
if (sandbox.networkPolicy.mode === "open_public"
|
|
1561
|
+
&& (sandbox.executionPolicy === "sandbox_required" || !config.allowOpenPublicSandbox)) {
|
|
1562
|
+
return "open_public sandbox networking is not permitted by this runner policy.";
|
|
1563
|
+
}
|
|
1564
|
+
if (sandbox.networkPolicy.mode === "public_allowlist"
|
|
1565
|
+
&& sandbox.networkPolicy.allowedHosts.length > 0
|
|
1566
|
+
&& process.env.FORGE_SANDBOX_EGRESS_ENFORCED !== "true") {
|
|
1567
|
+
return "public_allowlist requires an operator-enforced Forge egress proxy or firewall.";
|
|
1568
|
+
}
|
|
1569
|
+
return null;
|
|
1570
|
+
}
|
|
1571
|
+
export function infrastructureFailureEventType(summary) {
|
|
1572
|
+
const sandboxFailure = /^(sandbox_[a-z0-9_.-]+|image_[a-z0-9_.-]+|contract_[a-z0-9_.-]+|teardown_[a-z0-9_.-]+):/u.exec(summary)?.[1];
|
|
1573
|
+
if (sandboxFailure)
|
|
1574
|
+
return sandboxFailure;
|
|
1575
|
+
if (summary.startsWith("worktree_prepare_failed:"))
|
|
1576
|
+
return "worktree_prepare_failed";
|
|
1577
|
+
if (summary.startsWith("provider_network_unavailable:"))
|
|
1578
|
+
return "provider_network_unavailable";
|
|
1579
|
+
if (summary.startsWith("provider_start_unavailable:"))
|
|
1580
|
+
return "provider_start_unavailable";
|
|
1581
|
+
if (summary.startsWith("host_port_collision:"))
|
|
1582
|
+
return "host_port_collision";
|
|
1583
|
+
if (summary.startsWith("host_runtime_process_exited:"))
|
|
1584
|
+
return "host_runtime_process_exited";
|
|
1585
|
+
if (summary.startsWith("host_runtime_listener_timeout:"))
|
|
1586
|
+
return "host_runtime_listener_timeout";
|
|
1587
|
+
if (summary.startsWith("host_runtime_http_readiness_timeout:"))
|
|
1588
|
+
return "host_runtime_http_readiness_timeout";
|
|
1589
|
+
if (summary.startsWith("docker_engine_unavailable:"))
|
|
1590
|
+
return "docker_engine_unavailable";
|
|
1591
|
+
if (summary.startsWith("attempt_process_supervisor_start_failed:"))
|
|
1592
|
+
return "attempt_process_supervisor_start_failed";
|
|
1593
|
+
if (summary.startsWith("attempt_process_supervisor_start_timeout:"))
|
|
1594
|
+
return "attempt_process_supervisor_start_timeout";
|
|
1595
|
+
if (summary.startsWith("bridge_callback_timeout:"))
|
|
1596
|
+
return "bridge_callback_timeout";
|
|
1597
|
+
return null;
|
|
1598
|
+
}
|
|
1599
|
+
export function dockerEngineRequirementFailure(requiredCapabilities, executionMode, capability) {
|
|
1600
|
+
if (!requiredCapabilities.includes("docker_engine"))
|
|
1601
|
+
return null;
|
|
1602
|
+
if (executionMode === "docker") {
|
|
1603
|
+
return "docker_engine_unavailable: docker_engine is not exposed inside Forge agent sandboxes; use a managed host runner with Docker-daemon access.";
|
|
1604
|
+
}
|
|
1605
|
+
return capability?.available
|
|
1606
|
+
? null
|
|
1607
|
+
: `docker_engine_unavailable: ${capability?.unavailableReason ?? "Docker daemon is unreachable from the bridge process."}`;
|
|
1608
|
+
}
|
|
1609
|
+
function parseReadinessProbe(value) {
|
|
1610
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
1611
|
+
return null;
|
|
1612
|
+
const request = value;
|
|
1613
|
+
const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
|
|
1614
|
+
const bounded = (candidate, maximum) => typeof candidate === "string" && candidate.trim().length > 0 && candidate.length <= maximum;
|
|
1615
|
+
if (request.schemaVersion !== 1
|
|
1616
|
+
|| !uuid.test(request.requestId ?? "")
|
|
1617
|
+
|| !uuid.test(request.fencingToken ?? "")
|
|
1618
|
+
|| !uuid.test(request.userId ?? "")
|
|
1619
|
+
|| !uuid.test(request.projectId ?? "")
|
|
1620
|
+
|| !uuid.test(request.repoId ?? "")
|
|
1621
|
+
|| !uuid.test(request.runnerInstanceId ?? "")
|
|
1622
|
+
|| !Number.isInteger(request.attempt) || (request.attempt ?? 0) < 1
|
|
1623
|
+
|| !bounded(request.repoName, 160)
|
|
1624
|
+
|| typeof request.remoteUrl !== "string" || request.remoteUrl.length > 2048
|
|
1625
|
+
|| !bounded(request.defaultBranch, 512)
|
|
1626
|
+
|| typeof request.cloneOnDemand !== "boolean"
|
|
1627
|
+
|| !bounded(request.runnerRepoLocalPath, 4096)
|
|
1628
|
+
|| !bounded(request.revision, 512)
|
|
1629
|
+
|| !Number.isFinite(Date.parse(request.expiresAt ?? ""))
|
|
1630
|
+
|| Date.parse(request.expiresAt ?? "") <= Date.now()) {
|
|
1631
|
+
return null;
|
|
539
1632
|
}
|
|
540
|
-
return
|
|
1633
|
+
return request;
|
|
1634
|
+
}
|
|
1635
|
+
function sandboxProviderNetworkFailure(dispatch, config) {
|
|
1636
|
+
const sandbox = dispatch.sandbox;
|
|
1637
|
+
if (!sandbox || sandbox.networkPolicy.mode !== "offline")
|
|
1638
|
+
return null;
|
|
1639
|
+
const provider = dispatch.execution.provider.trim().toLowerCase();
|
|
1640
|
+
const isNetworkProvider = config.cliProviders.some(candidate => candidate.trim().toLowerCase() === provider);
|
|
1641
|
+
if (!isNetworkProvider)
|
|
1642
|
+
return null;
|
|
1643
|
+
return `Provider ${dispatch.execution.provider} requires outbound network access, but this Docker attempt uses the offline network policy.`;
|
|
1644
|
+
}
|
|
1645
|
+
function sandboxAuthFailure(dispatch, config) {
|
|
1646
|
+
const unsupported = sandboxProviderAuthUnavailableMessage(dispatch.execution.provider, dispatch.execution.authKind);
|
|
1647
|
+
if (unsupported) {
|
|
1648
|
+
return unsupported;
|
|
1649
|
+
}
|
|
1650
|
+
if (dispatch.execution.authKind.trim().toLowerCase() !== "bridge_cli") {
|
|
1651
|
+
return null;
|
|
1652
|
+
}
|
|
1653
|
+
const status = providerStatus(config, dispatch.execution.provider);
|
|
1654
|
+
if (status?.signedIn === false) {
|
|
1655
|
+
return `${dispatch.execution.provider} Local CLI is not signed in on this runner.`;
|
|
1656
|
+
}
|
|
1657
|
+
const adapter = sandboxProviderAuthAdapterFor(dispatch.execution.provider, dispatch.execution.authKind);
|
|
1658
|
+
if (!adapter) {
|
|
1659
|
+
return `${dispatch.execution.provider} Local CLI auth is not portable into Docker yet.`;
|
|
1660
|
+
}
|
|
1661
|
+
const doctor = adapter.doctor();
|
|
1662
|
+
return doctor.available ? null : doctor.message;
|
|
1663
|
+
}
|
|
1664
|
+
function rejectedOfferReservation(code, message) {
|
|
1665
|
+
return {
|
|
1666
|
+
accepted: false,
|
|
1667
|
+
code,
|
|
1668
|
+
message,
|
|
1669
|
+
reservedUntil: null,
|
|
1670
|
+
resources: null,
|
|
1671
|
+
};
|
|
1672
|
+
}
|
|
1673
|
+
export function canReserveSandboxResources(used, requested, capacity) {
|
|
1674
|
+
return used.sandboxSlots + requested.sandboxSlots <= capacity.sandboxSlots
|
|
1675
|
+
&& used.cpuMillis + requested.cpuMillis <= capacity.cpuMillis
|
|
1676
|
+
&& used.memoryBytes + requested.memoryBytes <= capacity.memoryBytes
|
|
1677
|
+
&& used.workspaceBytes + requested.workspaceBytes <= capacity.workspaceBytes
|
|
1678
|
+
&& used.pids + requested.pids <= capacity.pids
|
|
1679
|
+
&& used.buildSlots + requested.buildSlots <= capacity.buildSlots;
|
|
541
1680
|
}
|
|
1681
|
+
export function sandboxMemoryCommitmentCapacity(preflight, activeMemoryBytes, ambientMemoryCommitmentBytes = 0) {
|
|
1682
|
+
const ambientCapacity = preflight.availableMemoryBytes
|
|
1683
|
+
- preflight.hostReserveBytes
|
|
1684
|
+
+ activeMemoryBytes;
|
|
1685
|
+
const commitmentCapacity = preflight.totalMemoryBytes
|
|
1686
|
+
- preflight.hostReserveBytes
|
|
1687
|
+
- Math.max(0, ambientMemoryCommitmentBytes);
|
|
1688
|
+
return Math.max(0, Math.min(ambientCapacity, commitmentCapacity));
|
|
1689
|
+
}
|
|
1690
|
+
export function sandboxCapacityFailureMessage(used, requested, capacity, preflight, config) {
|
|
1691
|
+
const availableSlots = Math.max(0, capacity.sandboxSlots - used.sandboxSlots);
|
|
1692
|
+
if (requested.sandboxSlots > availableSlots) {
|
|
1693
|
+
return `Sandbox slot capacity is insufficient: requested ${requested.sandboxSlots}, available ${availableSlots}, ${used.sandboxSlots} already reserved.`;
|
|
1694
|
+
}
|
|
1695
|
+
const availableMemoryBytes = Math.max(0, capacity.memoryBytes - used.memoryBytes);
|
|
1696
|
+
if (requested.memoryBytes > availableMemoryBytes) {
|
|
1697
|
+
return [
|
|
1698
|
+
`Sandbox memory capacity is insufficient: requested ${formatBytes(requested.memoryBytes)} total`,
|
|
1699
|
+
`(worker ${formatBytes(requested.workerMemoryLimitBytes)}, services ${formatBytes(requested.serviceMemoryLimitBytes)}, runtime overhead ${formatBytes(requested.runtimeOverheadBytes)}),`,
|
|
1700
|
+
`but ${formatBytes(availableMemoryBytes)} is available for admission.`,
|
|
1701
|
+
`Docker reports ${formatBytes(preflight?.availableMemoryBytes ?? 0)} available of ${formatBytes(preflight?.totalMemoryBytes ?? 0)};`,
|
|
1702
|
+
`Forge reserves ${formatBytes(preflight?.hostReserveBytes ?? config.sandboxHostMemoryReserveBytes ?? 0)} for the host and tracks ${formatBytes(used.memoryBytes)} in active or reserved attempts.`,
|
|
1703
|
+
].join(" ");
|
|
1704
|
+
}
|
|
1705
|
+
const availableCpuMillis = Math.max(0, capacity.cpuMillis - used.cpuMillis);
|
|
1706
|
+
if (requested.cpuMillis > availableCpuMillis) {
|
|
1707
|
+
return `Sandbox CPU capacity is insufficient: requested ${requested.cpuMillis} millis, available ${availableCpuMillis} millis, ${used.cpuMillis} millis already reserved.`;
|
|
1708
|
+
}
|
|
1709
|
+
const availableWorkspaceBytes = Math.max(0, capacity.workspaceBytes - used.workspaceBytes);
|
|
1710
|
+
if (requested.workspaceBytes > availableWorkspaceBytes) {
|
|
1711
|
+
return `Sandbox workspace capacity is insufficient: requested ${formatBytes(requested.workspaceBytes)}, available ${formatBytes(availableWorkspaceBytes)}, ${formatBytes(used.workspaceBytes)} already reserved.`;
|
|
1712
|
+
}
|
|
1713
|
+
const availablePids = Math.max(0, capacity.pids - used.pids);
|
|
1714
|
+
if (requested.pids > availablePids) {
|
|
1715
|
+
return `Sandbox PID capacity is insufficient: requested ${requested.pids}, available ${availablePids}, ${used.pids} already reserved.`;
|
|
1716
|
+
}
|
|
1717
|
+
const availableBuildSlots = Math.max(0, capacity.buildSlots - used.buildSlots);
|
|
1718
|
+
return `Sandbox build capacity is insufficient: requested ${requested.buildSlots}, available ${availableBuildSlots}, ${used.buildSlots} already reserved.`;
|
|
1719
|
+
}
|
|
1720
|
+
function sumSandboxResources(resources) {
|
|
1721
|
+
return resources.reduce((total, resource) => ({
|
|
1722
|
+
sandboxSlots: total.sandboxSlots + resource.sandboxSlots,
|
|
1723
|
+
cpuMillis: total.cpuMillis + resource.cpuMillis,
|
|
1724
|
+
workerMemoryLimitBytes: total.workerMemoryLimitBytes + resource.workerMemoryLimitBytes,
|
|
1725
|
+
serviceMemoryLimitBytes: total.serviceMemoryLimitBytes + resource.serviceMemoryLimitBytes,
|
|
1726
|
+
runtimeOverheadBytes: total.runtimeOverheadBytes + resource.runtimeOverheadBytes,
|
|
1727
|
+
memoryBytes: total.memoryBytes + resource.memoryBytes,
|
|
1728
|
+
workspaceBytes: total.workspaceBytes + resource.workspaceBytes,
|
|
1729
|
+
pids: total.pids + resource.pids,
|
|
1730
|
+
buildSlots: total.buildSlots + resource.buildSlots
|
|
1731
|
+
}), zeroSandboxResources());
|
|
1732
|
+
}
|
|
1733
|
+
function zeroSandboxResources() {
|
|
1734
|
+
return {
|
|
1735
|
+
sandboxSlots: 0,
|
|
1736
|
+
cpuMillis: 0,
|
|
1737
|
+
workerMemoryLimitBytes: 0,
|
|
1738
|
+
serviceMemoryLimitBytes: 0,
|
|
1739
|
+
runtimeOverheadBytes: 0,
|
|
1740
|
+
memoryBytes: 0,
|
|
1741
|
+
workspaceBytes: 0,
|
|
1742
|
+
pids: 0,
|
|
1743
|
+
buildSlots: 0,
|
|
1744
|
+
};
|
|
1745
|
+
}
|
|
1746
|
+
function defaultSandboxResources() {
|
|
1747
|
+
return {
|
|
1748
|
+
sandboxSlots: 1,
|
|
1749
|
+
cpuMillis: 2000,
|
|
1750
|
+
workerMemoryLimitBytes: 2 * 1024 ** 3,
|
|
1751
|
+
serviceMemoryLimitBytes: 0,
|
|
1752
|
+
runtimeOverheadBytes: 128 * 1024 ** 2,
|
|
1753
|
+
memoryBytes: 2 * 1024 ** 3 + 128 * 1024 ** 2,
|
|
1754
|
+
workspaceBytes: 10 * 1024 ** 3,
|
|
1755
|
+
pids: 512,
|
|
1756
|
+
buildSlots: 0
|
|
1757
|
+
};
|
|
1758
|
+
}
|
|
1759
|
+
export function resolveDynamicSandboxResources(requested, serviceCount, networkMode, baseRuntimeOverheadBytes = 128 * 1024 ** 2) {
|
|
1760
|
+
const serviceMemoryLimitBytes = Math.max(0, serviceCount) * 1024 ** 3;
|
|
1761
|
+
const runtimeOverheadBytes = Math.max(128 * 1024 ** 2, baseRuntimeOverheadBytes)
|
|
1762
|
+
+ (networkMode === "public_allowlist" ? 256 * 1024 ** 2 : 0);
|
|
1763
|
+
return {
|
|
1764
|
+
...requested,
|
|
1765
|
+
serviceMemoryLimitBytes,
|
|
1766
|
+
runtimeOverheadBytes,
|
|
1767
|
+
memoryBytes: requested.workerMemoryLimitBytes + serviceMemoryLimitBytes + runtimeOverheadBytes,
|
|
1768
|
+
};
|
|
1769
|
+
}
|
|
1770
|
+
export async function resolveSandboxResourcesFromRepo(dispatch, baseRuntimeOverheadBytes = 128 * 1024 ** 2) {
|
|
1771
|
+
const repoPath = dispatch.repo.serverLocalPath;
|
|
1772
|
+
const sandbox = dispatch.sandbox;
|
|
1773
|
+
if (!repoPath || !sandbox) {
|
|
1774
|
+
throw new Error("Sandbox admission needs a mapped repository.");
|
|
1775
|
+
}
|
|
1776
|
+
const selection = selectSandboxConfigRefs(sandbox.executionPolicy, sandbox.sandboxConfigCommitSha, dispatch.repo.defaultBranch);
|
|
1777
|
+
const configCommitSha = await resolveFirstGitCommit(repoPath, selection.refs);
|
|
1778
|
+
if (!configCommitSha) {
|
|
1779
|
+
throw new Error("Sandbox admission could not resolve the immutable sandbox config commit.");
|
|
1780
|
+
}
|
|
1781
|
+
const contractResult = await runSandboxProcess({
|
|
1782
|
+
command: "git",
|
|
1783
|
+
args: ["-C", repoPath, "show", `${configCommitSha}:.forge/forge.sandbox.yml`],
|
|
1784
|
+
timeoutMs: 30_000,
|
|
1785
|
+
maxOutputBytes: 1024 * 1024,
|
|
1786
|
+
});
|
|
1787
|
+
if (contractResult.exitCode !== 0) {
|
|
1788
|
+
throw new Error("The approved immutable sandbox contract is unavailable in the mapped repository.");
|
|
1789
|
+
}
|
|
1790
|
+
const contract = validateSandboxContract(contractResult.stdout);
|
|
1791
|
+
const workspaceBytes = await estimateWorkspaceBytesFromRepo(repoPath, {
|
|
1792
|
+
role: dispatch.execution.role,
|
|
1793
|
+
serviceCount: contract.contract.services.length,
|
|
1794
|
+
browser: contract.contract.browser,
|
|
1795
|
+
});
|
|
1796
|
+
return {
|
|
1797
|
+
resources: resolveDynamicSandboxResources({
|
|
1798
|
+
...sandbox.requestedResources,
|
|
1799
|
+
workspaceBytes: Math.max(sandbox.requestedResources.workspaceBytes, workspaceBytes),
|
|
1800
|
+
}, contract.contract.services.length, sandbox.networkPolicy.mode, baseRuntimeOverheadBytes),
|
|
1801
|
+
sandboxConfigCommitSha: configCommitSha,
|
|
1802
|
+
};
|
|
1803
|
+
}
|
|
1804
|
+
async function resolveFirstGitCommit(repoPath, refs) {
|
|
1805
|
+
for (const ref of refs) {
|
|
1806
|
+
const result = await runSandboxProcess({
|
|
1807
|
+
command: "git",
|
|
1808
|
+
args: ["-C", repoPath, "rev-parse", "--verify", `${ref}^{commit}`],
|
|
1809
|
+
timeoutMs: 30_000,
|
|
1810
|
+
maxOutputBytes: 1024 * 1024,
|
|
1811
|
+
});
|
|
1812
|
+
const commitSha = result.stdout.trim().toLowerCase();
|
|
1813
|
+
if (result.exitCode === 0 && /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/u.test(commitSha)) {
|
|
1814
|
+
return commitSha;
|
|
1815
|
+
}
|
|
1816
|
+
}
|
|
1817
|
+
return null;
|
|
1818
|
+
}
|
|
1819
|
+
function parseOfferEnvelope(value) {
|
|
1820
|
+
return parseDispatchEnvelope(value);
|
|
1821
|
+
}
|
|
1822
|
+
function parseStartEnvelope(value) {
|
|
1823
|
+
return parseDispatchEnvelope(value);
|
|
1824
|
+
}
|
|
1825
|
+
function parseDispatchEnvelope(value) {
|
|
1826
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
1827
|
+
return null;
|
|
1828
|
+
const envelope = value;
|
|
1829
|
+
if (envelope.schemaVersion !== 1
|
|
1830
|
+
|| typeof envelope.dispatchAttemptId !== "string"
|
|
1831
|
+
|| typeof envelope.leaseId !== "string"
|
|
1832
|
+
|| typeof envelope.fencingToken !== "string"
|
|
1833
|
+
|| typeof envelope.runnerInstanceId !== "string"
|
|
1834
|
+
|| typeof envelope.expiresAt !== "string"
|
|
1835
|
+
|| !envelope.dispatch)
|
|
1836
|
+
return null;
|
|
1837
|
+
return {
|
|
1838
|
+
dispatchAttemptId: envelope.dispatchAttemptId,
|
|
1839
|
+
leaseId: envelope.leaseId,
|
|
1840
|
+
fencingToken: envelope.fencingToken,
|
|
1841
|
+
runnerInstanceId: envelope.runnerInstanceId,
|
|
1842
|
+
expiresAt: envelope.expiresAt,
|
|
1843
|
+
dispatch: envelope.dispatch
|
|
1844
|
+
};
|
|
1845
|
+
}
|
|
1846
|
+
const maxAttachmentExactSecretLength = 8 * 1024;
|
|
542
1847
|
function exactSecrets(dispatch, config) {
|
|
543
1848
|
return [
|
|
544
1849
|
dispatch.apiKeySecretGrant?.secret,
|
|
545
|
-
config.deviceToken
|
|
1850
|
+
config.deviceToken,
|
|
1851
|
+
...boundedAttachmentExactSecrets(dispatch)
|
|
546
1852
|
].filter((secret) => typeof secret === "string" && secret.length > 0);
|
|
547
1853
|
}
|
|
1854
|
+
function boundedAttachmentExactSecrets(dispatch) {
|
|
1855
|
+
return (dispatch.draft?.attachments ?? [])
|
|
1856
|
+
.flatMap((attachment) => [attachment.textContent, attachment.contentBase64])
|
|
1857
|
+
.filter((secret) => typeof secret === "string" && secret.length > 0 && secret.length <= maxAttachmentExactSecretLength);
|
|
1858
|
+
}
|