@loucompanion/forge-bridge 0.1.3-dev.fbe15023cdfe → 0.1.4-dev.bfa11ebe98c6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +88 -2
- package/dist/bridge/bridge-chat.types.d.ts +111 -0
- package/dist/bridge/bridge-chat.types.js +401 -0
- package/dist/bridge/bridge.base.d.ts +4 -1
- package/dist/bridge/bridge.service.d.ts +72 -3
- package/dist/bridge/bridge.service.js +1562 -152
- package/dist/bridge/bridge.types.d.ts +46 -1
- package/dist/bridge/dispatch-result-outbox.d.ts +25 -0
- package/dist/bridge/dispatch-result-outbox.js +62 -0
- package/dist/bridge/dispatch-result-worker.d.ts +2 -0
- package/dist/bridge/dispatch-result-worker.js +122 -0
- package/dist/bridge/internals.d.ts +2 -0
- package/dist/bridge/internals.js +1 -0
- package/dist/bridge/readiness-result-outbox.d.ts +21 -0
- package/dist/bridge/readiness-result-outbox.js +54 -0
- package/dist/bridge/readiness-result-worker.d.ts +2 -0
- package/dist/bridge/readiness-result-worker.js +121 -0
- package/dist/bridge/runner-host-capabilities.d.ts +15 -0
- package/dist/bridge/runner-host-capabilities.js +93 -0
- package/dist/cleanup/cleanup.service.d.ts +1 -0
- package/dist/cleanup/cleanup.service.js +70 -9
- package/dist/cli/cli.help.js +2 -2
- package/dist/cli/cli.service.d.ts +14 -6
- package/dist/cli/cli.service.js +75 -16
- package/dist/cli/local-config.d.ts +1 -0
- package/dist/cli/local-config.js +4 -0
- package/dist/runtime/aspnet-runtime-prebuild.d.ts +36 -0
- package/dist/runtime/aspnet-runtime-prebuild.js +341 -0
- package/dist/runtime/attempt-node-runtime-launcher.d.ts +2 -1
- package/dist/runtime/attempt-node-runtime-launcher.js +57 -18
- package/dist/runtime/attempt-path-operation.d.ts +21 -0
- package/dist/runtime/attempt-path-operation.js +94 -0
- package/dist/runtime/attempt-private-directory.d.ts +6 -0
- package/dist/runtime/attempt-private-directory.js +54 -0
- package/dist/runtime/attempt-process-supervisor.d.ts +11 -0
- package/dist/runtime/attempt-process-supervisor.js +67 -20
- package/dist/runtime/canonical-source-file.d.ts +3 -0
- package/dist/runtime/canonical-source-file.js +54 -0
- package/dist/runtime/docker-engine-probe.d.ts +90 -0
- package/dist/runtime/docker-engine-probe.js +333 -0
- package/dist/runtime/generated-output-reset.d.ts +10 -0
- package/dist/runtime/generated-output-reset.js +62 -8
- package/dist/runtime/host-database-capability-boundary.d.ts +16 -0
- package/dist/runtime/host-database-capability-boundary.js +62 -0
- package/dist/runtime/host-diagnostics.d.ts +11 -0
- package/dist/runtime/host-diagnostics.js +191 -0
- package/dist/runtime/host-execution-root.d.ts +4 -0
- package/dist/runtime/host-execution-root.js +39 -0
- package/dist/runtime/host-listener-inspector.d.ts +8 -1
- package/dist/runtime/host-listener-inspector.js +121 -60
- package/dist/runtime/host-runtime-contract.d.ts +12 -1
- package/dist/runtime/host-runtime-contract.js +107 -53
- package/dist/runtime/host-runtime-lease.d.ts +90 -8
- package/dist/runtime/host-runtime-lease.js +1097 -108
- package/dist/runtime/host-runtime-promotion.d.ts +12 -3
- package/dist/runtime/host-runtime-promotion.js +59 -12
- package/dist/runtime/memory-capacity.d.ts +70 -0
- package/dist/runtime/memory-capacity.js +123 -0
- package/dist/runtime/native-test-postgres-fallback.d.ts +25 -0
- package/dist/runtime/native-test-postgres-fallback.js +91 -0
- package/dist/runtime/node-host-runtime-adapter.d.ts +3 -0
- package/dist/runtime/node-host-runtime-adapter.js +28 -11
- package/dist/runtime/pinned-directory-process.d.ts +9 -0
- package/dist/runtime/pinned-directory-process.js +61 -0
- package/dist/runtime/pinned-source-directory.d.ts +17 -0
- package/dist/runtime/pinned-source-directory.js +94 -0
- package/dist/runtime/playwright-host-runtime-adapter.d.ts +1 -0
- package/dist/runtime/playwright-host-runtime-adapter.js +53 -5
- package/dist/runtime/prepared-runtime-identity.d.ts +6 -0
- package/dist/runtime/prepared-runtime-identity.js +218 -0
- package/dist/runtime/prepared-runtime-readiness.d.ts +30 -0
- package/dist/runtime/prepared-runtime-readiness.js +285 -0
- package/dist/runtime/runtime-isolation-lease.js +4 -1
- package/dist/runtime/workspace-resource-estimator.d.ts +17 -0
- package/dist/runtime/workspace-resource-estimator.js +174 -0
- package/dist/sandbox/preflight.d.ts +31 -1
- package/dist/sandbox/preflight.js +237 -51
- package/dist/sandbox/repository-lock.d.ts +2 -1
- package/dist/sandbox/repository-lock.js +17 -8
- package/dist/sandbox/runtime.d.ts +6 -1
- package/dist/sandbox/runtime.js +132 -103
- package/dist/sandbox/sandbox.types.d.ts +7 -0
- package/dist/sandbox/workspace.service.js +2 -0
- package/dist/session/managed-attempt-tools.d.ts +17 -1
- package/dist/session/managed-attempt-tools.js +481 -31
- package/dist/session/managed-build-coordinator.d.ts +5 -0
- package/dist/session/managed-build-coordinator.js +105 -0
- package/dist/session/managed-build-receipt.d.ts +21 -0
- package/dist/session/managed-build-receipt.js +337 -0
- package/dist/session/managed-verification-timeouts.d.ts +8 -0
- package/dist/session/managed-verification-timeouts.js +17 -0
- package/dist/session/phase-completion.artifacts.d.ts +17 -0
- package/dist/session/phase-completion.artifacts.js +224 -22
- package/dist/session/phase-completion.d.ts +11 -1
- package/dist/session/phase-completion.helpers.d.ts +5 -0
- package/dist/session/phase-completion.helpers.js +30 -7
- package/dist/session/phase-completion.js +14 -3
- package/dist/session/phase-completion.prompts.js +17 -18
- package/dist/session/phase-completion.runtime.d.ts +3 -1
- package/dist/session/phase-completion.runtime.js +118 -16
- package/dist/session/phase-completion.schemas.js +49 -2
- package/dist/session/phase-completion.validation.d.ts +1 -0
- package/dist/session/phase-completion.validation.js +111 -21
- package/dist/session/pre-publication-validation.d.ts +75 -0
- package/dist/session/pre-publication-validation.js +657 -0
- package/dist/session/provider-cli/bridge-cli.conformance-fixtures.d.ts +72 -0
- package/dist/session/provider-cli/bridge-cli.conformance-fixtures.js +462 -0
- package/dist/session/provider-cli/ci-repair-readiness.d.ts +8 -0
- package/dist/session/provider-cli/ci-repair-readiness.js +23 -3
- package/dist/session/provider-cli/provider-cli.artifacts.d.ts +4 -2
- package/dist/session/provider-cli/provider-cli.artifacts.js +47 -13
- package/dist/session/provider-cli/provider-cli.deterministic.js +34 -2
- package/dist/session/provider-cli/provider-cli.process.d.ts +51 -0
- package/dist/session/provider-cli/provider-cli.process.js +422 -22
- package/dist/session/provider-cli/provider-cli.prompts.d.ts +5 -0
- package/dist/session/provider-cli/provider-cli.prompts.js +136 -16
- package/dist/session/provider-cli/provider-cli.repair-epoch.d.ts +6 -0
- package/dist/session/provider-cli/provider-cli.repair-epoch.js +11 -0
- package/dist/session/provider-cli/provider-cli.runtime.d.ts +8 -1
- package/dist/session/provider-cli/provider-cli.runtime.js +40 -3
- package/dist/session/provider-cli/provider-cli.service.d.ts +30 -5
- package/dist/session/provider-cli/provider-cli.service.js +771 -68
- package/dist/session/provider-cli/provider-cli.types.d.ts +2 -0
- package/dist/session/provider-cli/provider-output-state.d.ts +10 -0
- package/dist/session/provider-cli/provider-output-state.js +361 -17
- package/dist/session/provider-cli/provider-usage.d.ts +7 -0
- package/dist/session/provider-cli/provider-usage.js +28 -5
- package/dist/session/session-redaction.js +434 -9
- package/dist/session/session.service.js +145 -23
- package/dist/session/session.types.d.ts +147 -2
- package/dist/session/session.types.js +587 -7
- package/dist/session/unreferenced-exports.d.ts +12 -0
- package/dist/session/unreferenced-exports.js +59 -0
- package/dist/shared/format/format-bytes.d.ts +1 -0
- package/dist/shared/format/format-bytes.js +9 -0
- package/dist/shared/git/git.js +2 -1
- package/dist/shared/path/path.d.ts +1 -0
- package/dist/shared/path/path.js +18 -0
- package/dist/shared/process/dotnet-toolchain.d.ts +17 -0
- package/dist/shared/process/dotnet-toolchain.js +307 -0
- package/dist/shared/process/host-process-supervisor.d.ts +2 -0
- package/dist/shared/process/host-process-supervisor.js +57 -9
- package/dist/shared/process/process.d.ts +2 -0
- package/dist/shared/process/process.js +12 -7
- package/dist/shared/provider-capabilities.d.ts +3 -0
- package/dist/shared/provider-capabilities.js +85 -0
- package/dist/shared/redaction/redaction.d.ts +1 -0
- package/dist/shared/redaction/redaction.js +39 -3
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/worktree/dependency-warmer.d.ts +41 -9
- package/dist/worktree/dependency-warmer.js +497 -22
- package/dist/worktree/index.d.ts +1 -1
- package/dist/worktree/story-qa-reference-hydrator.d.ts +4 -0
- package/dist/worktree/story-qa-reference-hydrator.js +253 -0
- package/dist/worktree/worktree.service.d.ts +7 -0
- package/dist/worktree/worktree.service.js +253 -21
- package/dist/worktree/worktree.types.d.ts +7 -1
- package/package.json +7 -5
- package/runtime/windows-job-supervisor.ps1 +380 -14
|
@@ -4,25 +4,54 @@ import path from "node:path";
|
|
|
4
4
|
import { randomUUID } from "node:crypto";
|
|
5
5
|
import os from "node:os";
|
|
6
6
|
import { defaultForgeBridgeConfig } from "../config.js";
|
|
7
|
-
import { supportsDraftImageAttachmentsFor } from "../shared/provider-capabilities.js";
|
|
7
|
+
import { bridgeChatCapabilityFor, supportsDraftImageAttachmentsFor } from "../shared/provider-capabilities.js";
|
|
8
8
|
import { redactText } from "../shared/redaction/redaction.js";
|
|
9
9
|
import { BridgeSessionService } from "../session/session.service.js";
|
|
10
10
|
import { ProcessBackedProviderCliService, RoutingProviderCliService, StreamingProviderProcessRunner, } from "../session/provider-cli/provider-cli.service.js";
|
|
11
|
+
import { codexSandboxModeForPlatform, formatProviderCliCommandForCheckpoint } from "../session/provider-cli/provider-cli.prompts.js";
|
|
12
|
+
import { clip, createProviderOutputState } from "../session/provider-cli/provider-output-state.js";
|
|
11
13
|
import { redactRunnerDispatchEvent, redactRunOnceResultPacket } from "../session/session-redaction.js";
|
|
12
|
-
import {
|
|
14
|
+
import { PrePublicationValidator } from "../session/pre-publication-validation.js";
|
|
15
|
+
import { HostRuntimeLeaseManager, } from "../runtime/host-runtime-lease.js";
|
|
16
|
+
import { assertHostDatabaseCapabilityBoundary, establishHostDatabaseCapabilityBoundary, } from "../runtime/host-database-capability-boundary.js";
|
|
17
|
+
import { defaultHostExecutionRoot } from "../runtime/host-execution-root.js";
|
|
18
|
+
import { buildMemoryAdmissionDiagnostics } from "../runtime/memory-capacity.js";
|
|
19
|
+
import { DockerEngineProbe } from "../runtime/docker-engine-probe.js";
|
|
20
|
+
import { readNativeTestPostgresFallback } from "../runtime/native-test-postgres-fallback.js";
|
|
21
|
+
import { estimateWorkspaceBytesFromRepo } from "../runtime/workspace-resource-estimator.js";
|
|
13
22
|
import { GitWorktreeService } from "../worktree/worktree.service.js";
|
|
14
23
|
import { RepositoryCoordinatorLock } from "../sandbox/repository-lock.js";
|
|
15
24
|
import { AttemptCoordinatedProviderProcessRunner, AttemptProcessSupervisorRegistry, } from "../runtime/attempt-process-supervisor-registry.js";
|
|
16
|
-
import { failureResult, parseRunOnceDispatchPacket } from "../session/session.types.js";
|
|
25
|
+
import { failureFactsForCode, failureFactsFromError, failureResult, parseRunOnceDispatchPacket } from "../session/session.types.js";
|
|
17
26
|
import { forgeBridgeVersion } from "../version.js";
|
|
18
27
|
import { canRunDockerMaintenance, DockerPreflight } from "../sandbox/preflight.js";
|
|
19
28
|
import { validateSandboxContract } from "../sandbox/contract.js";
|
|
20
29
|
import { runSandboxProcess } from "../sandbox/process.js";
|
|
21
30
|
import { DockerSandboxRuntime, selectSandboxConfigRefs } from "../sandbox/runtime.js";
|
|
22
31
|
import { codexSandboxProviderAuthAdapter, sandboxProviderAuthAdapterFor, sandboxProviderAuthUnavailableMessage } from "../sandbox/provider-auth/codex.js";
|
|
32
|
+
import { assertBridgeChatCallbackPayload, bridgeChatMaxProviderThreadIdChars, bridgeChatProtocolVersion, isTerminalChatTurnCallbackKind, parseChatTurnDispatchPacket, } from "./bridge-chat.types.js";
|
|
33
|
+
import { formatBytes } from "../shared/format/format-bytes.js";
|
|
34
|
+
import { emptyRunnerHostCapabilitySnapshot, ProcessRunnerHostCapabilityProbe, } from "./runner-host-capabilities.js";
|
|
35
|
+
import { FileReadinessResultOutbox } from "./readiness-result-outbox.js";
|
|
36
|
+
import { FileDispatchResultOutbox } from "./dispatch-result-outbox.js";
|
|
37
|
+
/**
|
|
38
|
+
* Protocol-v2 events are acknowledged one at a time before runOnce returns.
|
|
39
|
+
* Sending the complete transcript again in DispatchResult made long QA and CI
|
|
40
|
+
* sessions exceed SignalR's receive limit, disconnecting the bridge after the
|
|
41
|
+
* provider had already completed. The API combines its acknowledged streamed
|
|
42
|
+
* events with the terminal result, so the callback only needs the outcome and
|
|
43
|
+
* artifacts. Protocol v1 retains the transcript for legacy servers.
|
|
44
|
+
*/
|
|
45
|
+
export function compactTerminalResult(dispatch, result) {
|
|
46
|
+
return dispatch.schemaVersion >= 2 && result.events.length > 0
|
|
47
|
+
? { ...result, events: [] }
|
|
48
|
+
: result;
|
|
49
|
+
}
|
|
23
50
|
export class ForgeBridgeService {
|
|
24
51
|
config;
|
|
25
52
|
connections;
|
|
53
|
+
hostCapabilityProbe;
|
|
54
|
+
chatProcessRunner;
|
|
26
55
|
connection = null;
|
|
27
56
|
heartbeatTimer = null;
|
|
28
57
|
heartbeatInFlight = false;
|
|
@@ -32,18 +61,40 @@ export class ForgeBridgeService {
|
|
|
32
61
|
sandboxMaintenanceTimer = null;
|
|
33
62
|
sandboxMaintenanceRunning = false;
|
|
34
63
|
closed = null;
|
|
64
|
+
transportRecovery = null;
|
|
65
|
+
transportRecoveryGeneration = 0;
|
|
35
66
|
stopped = false;
|
|
36
67
|
activeSessions = new Map();
|
|
37
68
|
reservations = new Map();
|
|
38
69
|
activeReadinessProbes = new Map();
|
|
70
|
+
readinessResultOutbox;
|
|
71
|
+
readinessResultFlush = null;
|
|
72
|
+
dispatchResultOutbox;
|
|
73
|
+
dispatchResultFlush = null;
|
|
74
|
+
dispatchResultGeneration = 0;
|
|
75
|
+
dispatchResultFlushedGeneration = -1;
|
|
76
|
+
dispatchResultHandoffs = new Map();
|
|
39
77
|
preflight;
|
|
78
|
+
dockerEngineProbe;
|
|
40
79
|
sandboxRuntime;
|
|
41
80
|
sandboxResourceResolver;
|
|
42
81
|
sessions;
|
|
43
|
-
|
|
82
|
+
enforceHostDatabaseCapabilityBoundary;
|
|
83
|
+
hostCapabilities = emptyRunnerHostCapabilitySnapshot();
|
|
84
|
+
// Reaping deletes the stale ledger, so its stopped-process proof is held here
|
|
85
|
+
// until a Hello carrying it is accepted. A bridge restart forgets it, which
|
|
86
|
+
// leaves those attempts on Forge's conservative unknown-outcome path.
|
|
87
|
+
recoveredHostAttempts = new Map();
|
|
88
|
+
hostCapabilitiesDetected = false;
|
|
89
|
+
constructor(config, sessions = undefined, connections = new SignalRBridgeConnectionFactory(), preflight, sandboxRuntime, sandboxResourceResolver, dockerEngineProbe, hostCapabilityProbe = new ProcessRunnerHostCapabilityProbe(), readinessResultOutbox, chatProcessRunner = new StreamingProviderProcessRunner(), dispatchResultOutbox) {
|
|
44
90
|
this.config = config;
|
|
45
91
|
this.connections = connections;
|
|
46
|
-
this.
|
|
92
|
+
this.hostCapabilityProbe = hostCapabilityProbe;
|
|
93
|
+
this.chatProcessRunner = chatProcessRunner;
|
|
94
|
+
this.enforceHostDatabaseCapabilityBoundary = sessions === undefined;
|
|
95
|
+
if (this.enforceHostDatabaseCapabilityBoundary) {
|
|
96
|
+
establishHostDatabaseCapabilityBoundary();
|
|
97
|
+
}
|
|
47
98
|
this.preflight = preflight ?? new DockerPreflight({
|
|
48
99
|
dockerExecutable: config.dockerExecutable,
|
|
49
100
|
dockerContext: config.dockerContext,
|
|
@@ -59,17 +110,36 @@ export class ForgeBridgeService {
|
|
|
59
110
|
runtimeOverheadBytes: config.sandboxRuntimeOverheadBytes,
|
|
60
111
|
requireRootless: config.requireRootlessDocker
|
|
61
112
|
});
|
|
113
|
+
this.dockerEngineProbe = dockerEngineProbe ?? new DockerEngineProbe({
|
|
114
|
+
dockerExecutable: config.dockerExecutable,
|
|
115
|
+
dockerContext: config.dockerContext,
|
|
116
|
+
recovery: {
|
|
117
|
+
enabled: config.dockerDesktopRecoveryEnabled,
|
|
118
|
+
failureThreshold: config.dockerRecoveryFailureThreshold,
|
|
119
|
+
cooldownMs: config.dockerRecoveryCooldownMs,
|
|
120
|
+
startupTimeoutMs: config.dockerRecoveryStartupTimeoutMs,
|
|
121
|
+
canRecover: () => this.activeSessions.size === 0 && this.reservations.size === 0,
|
|
122
|
+
},
|
|
123
|
+
onStatus: logBridgeStatus,
|
|
124
|
+
});
|
|
125
|
+
this.sessions = sessions ?? createDefaultSessionService(config.sandboxStateRoot, config.sandboxExecutionRoot, this.dockerEngineProbe);
|
|
62
126
|
this.sandboxRuntime = sandboxRuntime ?? new DockerSandboxRuntime(config, (request, signal) => this.redeemPublishGrant(request, signal));
|
|
63
127
|
this.sandboxResourceResolver = sandboxResourceResolver
|
|
64
128
|
?? (dispatch => resolveSandboxResourcesFromRepo(dispatch, this.config.sandboxRuntimeOverheadBytes));
|
|
129
|
+
this.readinessResultOutbox = readinessResultOutbox ?? new FileReadinessResultOutbox(config.sandboxStateRoot);
|
|
130
|
+
this.dispatchResultOutbox = dispatchResultOutbox ?? new FileDispatchResultOutbox(config.sandboxStateRoot);
|
|
65
131
|
}
|
|
66
132
|
async connectOnce() {
|
|
67
133
|
await this.stopConnection();
|
|
68
134
|
this.stopped = false;
|
|
135
|
+
if (this.enforceHostDatabaseCapabilityBoundary) {
|
|
136
|
+
assertHostDatabaseCapabilityBoundary();
|
|
137
|
+
}
|
|
69
138
|
const hostReap = await this.sessions.reapHostRuntime?.({
|
|
70
139
|
runnerId: this.config.runnerId,
|
|
71
140
|
runnerInstanceId: this.config.runnerInstanceId,
|
|
72
141
|
});
|
|
142
|
+
this.rememberRecoveredHostAttempts(hostReap);
|
|
73
143
|
if (hostReap && (hostReap.reaped > 0 || hostReap.degraded > 0)) {
|
|
74
144
|
logBridgeStatus(`Managed-host recovery reaped ${hostReap.reaped} stale attempt(s), protected ${hostReap.reusedPidsProtected} reused PID(s), and left ${hostReap.degraded} degraded attempt(s) for reconciliation.`);
|
|
75
145
|
}
|
|
@@ -80,17 +150,54 @@ export class ForgeBridgeService {
|
|
|
80
150
|
await this.preflight.refresh();
|
|
81
151
|
}
|
|
82
152
|
}
|
|
153
|
+
await this.dockerEngineProbe.refresh(true);
|
|
154
|
+
if (!this.hostCapabilitiesDetected) {
|
|
155
|
+
this.hostCapabilities = await this.hostCapabilityProbe.detect(this.config);
|
|
156
|
+
this.hostCapabilitiesDetected = true;
|
|
157
|
+
}
|
|
83
158
|
const connection = this.connections.create(this.config);
|
|
84
159
|
this.connection = connection;
|
|
160
|
+
this.transportRecovery = null;
|
|
85
161
|
this.closed = new Promise((resolve) => {
|
|
86
|
-
connection.onClose(() => {
|
|
162
|
+
connection.onClose((error) => {
|
|
87
163
|
this.stopHeartbeat();
|
|
164
|
+
this.completeTransportRecovery(error ?? new Error("bridge connection closed"));
|
|
88
165
|
this.abortActiveSessions("bridge connection closed");
|
|
89
166
|
this.abortActiveReadinessProbes("bridge connection closed");
|
|
90
167
|
resolve();
|
|
91
168
|
});
|
|
92
169
|
});
|
|
170
|
+
connection.onReconnecting?.((error) => {
|
|
171
|
+
if (connection !== this.connection || this.stopped)
|
|
172
|
+
return;
|
|
173
|
+
this.stopHeartbeat();
|
|
174
|
+
this.beginTransportRecovery();
|
|
175
|
+
logBridgeStatus(`Forge connection interrupted${error ? `: ${errorMessage(error)}` : ""}. Reconnecting in place; active dispatches remain supervised...`);
|
|
176
|
+
});
|
|
177
|
+
connection.onReconnected?.(async () => {
|
|
178
|
+
if (connection !== this.connection || this.stopped)
|
|
179
|
+
return;
|
|
180
|
+
try {
|
|
181
|
+
await this.registerConnection(connection);
|
|
182
|
+
this.completeTransportRecovery();
|
|
183
|
+
this.startHeartbeat();
|
|
184
|
+
await this.flushDispatchResults(true).catch(error => {
|
|
185
|
+
logBridgeStatus(`Dispatch result replay deferred after reconnect: ${errorMessage(error)}`);
|
|
186
|
+
});
|
|
187
|
+
await this.flushReadinessResults().catch(error => {
|
|
188
|
+
logBridgeStatus(`Readiness result replay deferred after reconnect: ${errorMessage(error)}`);
|
|
189
|
+
});
|
|
190
|
+
await this.flushCleanupReports().catch(() => undefined);
|
|
191
|
+
logBridgeStatus("Forge connection restored; active dispatches remain attached to this runner process.");
|
|
192
|
+
}
|
|
193
|
+
catch (error) {
|
|
194
|
+
this.completeTransportRecovery(error);
|
|
195
|
+
logBridgeStatus(`Forge reconnect registration failed: ${errorMessage(error)}. Closing the connection for a clean retry...`);
|
|
196
|
+
await connection.stop().catch(() => undefined);
|
|
197
|
+
}
|
|
198
|
+
});
|
|
93
199
|
connection.onDispatch((dispatch) => this.handleDispatch(dispatch));
|
|
200
|
+
connection.onChatTurn?.((dispatch) => this.handleChatTurn(dispatch));
|
|
94
201
|
connection.onOffer?.((dispatch) => this.handleOffer(dispatch));
|
|
95
202
|
connection.onStart?.((dispatch) => this.handleStart(dispatch));
|
|
96
203
|
connection.onLeaseUpdate?.((request) => this.handleLeaseUpdate(request));
|
|
@@ -98,18 +205,67 @@ export class ForgeBridgeService {
|
|
|
98
205
|
connection.onReap?.((request) => this.handleReap(request));
|
|
99
206
|
connection.onReadinessProbe?.((request) => this.handleReadinessProbe(request));
|
|
100
207
|
await connection.start();
|
|
101
|
-
await this.
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
208
|
+
await this.registerConnection(connection);
|
|
209
|
+
await this.flushDispatchResults(true).catch(error => {
|
|
210
|
+
logBridgeStatus(`Dispatch result replay deferred: ${errorMessage(error)}`);
|
|
211
|
+
});
|
|
212
|
+
await this.flushReadinessResults().catch(error => {
|
|
213
|
+
logBridgeStatus(`Readiness result replay deferred: ${errorMessage(error)}`);
|
|
107
214
|
});
|
|
108
215
|
await this.flushCleanupReports();
|
|
109
216
|
this.startHeartbeat();
|
|
110
217
|
this.startLeaseWatchdog();
|
|
111
218
|
this.startSandboxMaintenance();
|
|
112
219
|
}
|
|
220
|
+
rememberRecoveredHostAttempts(report) {
|
|
221
|
+
for (const attempt of report?.recoveredAttempts ?? []) {
|
|
222
|
+
if (this.recoveredHostAttempts.size >= maxRecoveredHostAttempts)
|
|
223
|
+
break;
|
|
224
|
+
this.recoveredHostAttempts.set(attempt.dispatchAttemptId, attempt);
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
async recoveredHostAttemptsForHello() {
|
|
228
|
+
if (this.recoveredHostAttempts.size === 0)
|
|
229
|
+
return [];
|
|
230
|
+
// A durable terminal result outranks the stopped-process proof: it replays
|
|
231
|
+
// after Hello and Forge accepts the provider's real outcome instead.
|
|
232
|
+
const pendingResults = new Set((await this.dispatchResultOutbox.pending())
|
|
233
|
+
.map(entry => entry.payload.dispatchAttemptId)
|
|
234
|
+
.filter((id) => typeof id === "string"));
|
|
235
|
+
return [...this.recoveredHostAttempts.values()]
|
|
236
|
+
.filter(attempt => !pendingResults.has(attempt.dispatchAttemptId));
|
|
237
|
+
}
|
|
238
|
+
async registerConnection(connection) {
|
|
239
|
+
let storageRecoveryAttempt = 0;
|
|
240
|
+
while (true) {
|
|
241
|
+
try {
|
|
242
|
+
const recoveredHostAttempts = await this.recoveredHostAttemptsForHello();
|
|
243
|
+
const ack = await connection.invoke("Hello", {
|
|
244
|
+
version: this.config.version,
|
|
245
|
+
protocolVersion: this.config.protocolVersion,
|
|
246
|
+
minServerVersion: this.config.minServerVersion ?? null,
|
|
247
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
248
|
+
capabilities: this.capabilities(),
|
|
249
|
+
recoveredHostAttempts,
|
|
250
|
+
});
|
|
251
|
+
if (!ack.accepted) {
|
|
252
|
+
throw new BridgeCallbackRejectedError(ack.code ?? null, ack.message ?? ack.code ?? "Hello callback was rejected.");
|
|
253
|
+
}
|
|
254
|
+
this.recoveredHostAttempts.clear();
|
|
255
|
+
logBridgeStatus(`Registered runner ${this.config.runnerId}, instance ${this.config.runnerInstanceId}, build ${this.config.version}, protocol ${this.config.protocolVersion}.`);
|
|
256
|
+
return;
|
|
257
|
+
}
|
|
258
|
+
catch (error) {
|
|
259
|
+
if (!isProviderStorageUnavailable(error) || storageRecoveryAttempt >= this.config.reconnectBackoffMs.length) {
|
|
260
|
+
throw error;
|
|
261
|
+
}
|
|
262
|
+
const backoff = this.config.reconnectBackoffMs[storageRecoveryAttempt] ?? 5000;
|
|
263
|
+
storageRecoveryAttempt += 1;
|
|
264
|
+
logBridgeStatus(`Hello reached Forge while storage was recovering. Retrying registration ${storageRecoveryAttempt} in ${formatDelay(backoff)}...`);
|
|
265
|
+
await delay(backoff);
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
}
|
|
113
269
|
async run(signal) {
|
|
114
270
|
let attempt = 0;
|
|
115
271
|
try {
|
|
@@ -151,24 +307,351 @@ export class ForgeBridgeService {
|
|
|
151
307
|
await this.stopConnection();
|
|
152
308
|
await this.preflight.release();
|
|
153
309
|
}
|
|
154
|
-
async
|
|
155
|
-
const
|
|
310
|
+
async handleChatTurn(rawDispatch) {
|
|
311
|
+
const packet = this.parseChatTurnOrReport(rawDispatch);
|
|
312
|
+
if (!packet) {
|
|
313
|
+
return;
|
|
314
|
+
}
|
|
315
|
+
if (packet.runnerInstanceId !== this.config.runnerInstanceId) {
|
|
316
|
+
logBridgeStatus(`Rejected Bridge Chat turn ${packet.turnId}: stale runner instance.`);
|
|
317
|
+
return;
|
|
318
|
+
}
|
|
319
|
+
const capabilityFailure = this.requiredChatCapabilityFailure(packet);
|
|
320
|
+
if (capabilityFailure) {
|
|
321
|
+
await this.sendStandaloneChatTerminal(packet, "failed", {
|
|
322
|
+
code: "bridge_chat.runner_unavailable",
|
|
323
|
+
message: capabilityFailure,
|
|
324
|
+
}).catch(() => undefined);
|
|
325
|
+
return;
|
|
326
|
+
}
|
|
327
|
+
const existing = this.activeSessions.get(packet.leaseId);
|
|
328
|
+
if (existing) {
|
|
329
|
+
if (existing.kind === "chat"
|
|
330
|
+
&& existing.fencingToken === packet.fencingToken
|
|
331
|
+
&& existing.chatRunId === packet.chatRunId
|
|
332
|
+
&& existing.conversationId === packet.conversationId
|
|
333
|
+
&& existing.turnId === packet.turnId) {
|
|
334
|
+
return;
|
|
335
|
+
}
|
|
336
|
+
await this.sendStandaloneChatTerminal(packet, "failed", {
|
|
337
|
+
code: "bridge_chat.stale_fencing_token",
|
|
338
|
+
message: "Bridge Chat turn fence conflicts with an active bridge session.",
|
|
339
|
+
}).catch(() => undefined);
|
|
340
|
+
return;
|
|
341
|
+
}
|
|
342
|
+
const controller = new AbortController();
|
|
343
|
+
const active = {
|
|
344
|
+
leaseId: packet.leaseId,
|
|
345
|
+
kind: "chat",
|
|
346
|
+
fencingToken: packet.fencingToken,
|
|
347
|
+
dispatchAttemptId: null,
|
|
348
|
+
runnerInstanceId: packet.runnerInstanceId,
|
|
349
|
+
leaseExpiresAt: chatTurnLeaseExpiresAt(packet),
|
|
350
|
+
sandboxed: false,
|
|
351
|
+
resources: zeroSandboxResources(),
|
|
352
|
+
controller,
|
|
353
|
+
chatRunId: packet.chatRunId,
|
|
354
|
+
conversationId: packet.conversationId,
|
|
355
|
+
turnId: packet.turnId,
|
|
356
|
+
providerThreadId: packet.providerThreadId ?? null,
|
|
357
|
+
chatSequence: 0,
|
|
358
|
+
terminalCallbackAttempted: false,
|
|
359
|
+
};
|
|
360
|
+
this.activeSessions.set(packet.leaseId, active);
|
|
361
|
+
let workspacePath = null;
|
|
362
|
+
let cleanupAttempted = false;
|
|
363
|
+
let cleanupOutcome = null;
|
|
364
|
+
let cleanupReported = false;
|
|
365
|
+
const cleanup = async () => {
|
|
366
|
+
if (!workspacePath || cleanupAttempted)
|
|
367
|
+
return null;
|
|
368
|
+
cleanupAttempted = true;
|
|
369
|
+
cleanupOutcome = await cleanupChatTurnWorkspace(workspacePath, chatRedactionSecrets(packet));
|
|
370
|
+
return cleanupOutcome;
|
|
371
|
+
};
|
|
372
|
+
const reportCleanup = async (outcome) => {
|
|
373
|
+
if (!outcome || cleanupReported)
|
|
374
|
+
return;
|
|
375
|
+
cleanupReported = true;
|
|
376
|
+
await this.sendChatCallback(active, "output_delta", outcome).catch(() => undefined);
|
|
377
|
+
};
|
|
378
|
+
try {
|
|
379
|
+
workspacePath = await this.createChatTurnWorkspace(packet);
|
|
380
|
+
await materializeChatTurnMetadata(packet, workspacePath);
|
|
381
|
+
const run = await this.runChatProviderProcess(packet, active, workspacePath);
|
|
382
|
+
cleanupOutcome = await cleanup();
|
|
383
|
+
await reportCleanup(cleanupOutcome);
|
|
384
|
+
if (controller.signal.aborted) {
|
|
385
|
+
await this.sendChatAbortTerminal(packet, active, cleanupOutcome);
|
|
386
|
+
return;
|
|
387
|
+
}
|
|
388
|
+
if (cleanupOutcome?.status === "failed") {
|
|
389
|
+
throw new Error(`chat_turn_cleanup_failed: ${cleanupOutcome.message}`);
|
|
390
|
+
}
|
|
391
|
+
if (run.result.exitCode !== 0) {
|
|
392
|
+
throw chatProviderExitFailure(run.command, run.result, chatRedactionSecrets(packet));
|
|
393
|
+
}
|
|
394
|
+
await this.sendChatCallback(active, "output_completed", {
|
|
395
|
+
code: "output_completed",
|
|
396
|
+
text: clip(redactText(run.output.finalMessage ?? "", chatRedactionSecrets(packet)), maxChatTerminalTextChars),
|
|
397
|
+
provider: packet.providerKind,
|
|
398
|
+
model: packet.model,
|
|
399
|
+
cleanup: cleanupOutcome,
|
|
400
|
+
});
|
|
401
|
+
}
|
|
402
|
+
catch (error) {
|
|
403
|
+
cleanupOutcome = cleanupOutcome ?? await cleanup();
|
|
404
|
+
if (active.terminalCallbackAttempted) {
|
|
405
|
+
return;
|
|
406
|
+
}
|
|
407
|
+
await reportCleanup(cleanupOutcome);
|
|
408
|
+
const payload = controller.signal.aborted
|
|
409
|
+
? chatAbortPayload(controller.signal.reason, cleanupOutcome, packet)
|
|
410
|
+
: {
|
|
411
|
+
code: "provider_failed",
|
|
412
|
+
message: clip(redactText(errorMessage(error), chatRedactionSecrets(packet)), 2000),
|
|
413
|
+
cleanup: cleanupOutcome,
|
|
414
|
+
};
|
|
415
|
+
const kind = controller.signal.aborted ? chatAbortKind(controller.signal.reason) : "failed";
|
|
416
|
+
await this.sendChatCallback(active, kind, payload).catch(callbackError => {
|
|
417
|
+
logBridgeStatus(`Bridge Chat terminal callback for turn ${packet.turnId} was not accepted: ${errorMessage(callbackError)}`);
|
|
418
|
+
});
|
|
419
|
+
}
|
|
420
|
+
finally {
|
|
421
|
+
await this.releaseActiveSessionAndRefreshCapacity(packet.leaseId);
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
async createChatTurnWorkspace(packet) {
|
|
425
|
+
const root = path.join(this.config.sandboxStateRoot, "chat-turns");
|
|
426
|
+
await fs.promises.mkdir(root, { recursive: true });
|
|
427
|
+
return await fs.promises.mkdtemp(path.join(root, `${safePathComponent(packet.turnId)}-`));
|
|
428
|
+
}
|
|
429
|
+
async runChatProviderProcess(packet, active, cwd) {
|
|
430
|
+
const output = createProviderOutputState(packet.providerKind);
|
|
431
|
+
const plan = buildChatProviderCommand(packet, cwd);
|
|
432
|
+
const redactionSecrets = chatRedactionSecrets(packet);
|
|
433
|
+
let started = false;
|
|
434
|
+
const result = await this.chatProcessRunner.run({
|
|
435
|
+
command: plan.command.command,
|
|
436
|
+
args: plan.command.args,
|
|
437
|
+
cwd,
|
|
438
|
+
env: {
|
|
439
|
+
FORGE_BRIDGE_CHAT_RUN_ID: packet.chatRunId,
|
|
440
|
+
FORGE_BRIDGE_CHAT_TURN_ID: packet.turnId,
|
|
441
|
+
},
|
|
442
|
+
stdin: plan.stdin,
|
|
443
|
+
}, {
|
|
444
|
+
onStarted: async () => {
|
|
445
|
+
started = true;
|
|
446
|
+
await this.sendChatCallback(active, "turn_started", {
|
|
447
|
+
status: "running",
|
|
448
|
+
provider: packet.providerKind,
|
|
449
|
+
model: packet.model,
|
|
450
|
+
command: redactText(formatProviderCliCommandForCheckpoint(plan.command), redactionSecrets),
|
|
451
|
+
});
|
|
452
|
+
},
|
|
453
|
+
onStdoutLine: async (line) => {
|
|
454
|
+
for (const event of output.consume(line)) {
|
|
455
|
+
await this.sendChatProviderEvent(packet, active, event, output.providerThreadId);
|
|
456
|
+
}
|
|
457
|
+
},
|
|
458
|
+
}, active.controller.signal);
|
|
459
|
+
if (started && !active.controller.signal.aborted) {
|
|
460
|
+
await this.sendChatCallback(active, "output_delta", {
|
|
461
|
+
eventType: "provider_cli_completed",
|
|
462
|
+
role: "runner",
|
|
463
|
+
kind: "checkpoint",
|
|
464
|
+
content: "Bridge Chat provider process exited.",
|
|
465
|
+
}).catch(() => undefined);
|
|
466
|
+
}
|
|
467
|
+
return { command: plan.command, output, result };
|
|
468
|
+
}
|
|
469
|
+
async sendChatProviderEvent(packet, active, event, providerThreadId) {
|
|
470
|
+
if (providerThreadId) {
|
|
471
|
+
active.providerThreadId = providerThreadId;
|
|
472
|
+
}
|
|
473
|
+
else if (event.eventType === "provider_thread_started") {
|
|
474
|
+
active.providerThreadId = providerThreadIdFromEvent(event) ?? active.providerThreadId;
|
|
475
|
+
}
|
|
476
|
+
if (event.eventType === "provider_tool_heartbeat") {
|
|
477
|
+
return;
|
|
478
|
+
}
|
|
479
|
+
const redactionSecrets = chatRedactionSecrets(packet);
|
|
480
|
+
const payload = {
|
|
481
|
+
role: event.role,
|
|
482
|
+
kind: event.kind,
|
|
483
|
+
eventType: event.eventType ?? "provider_event",
|
|
484
|
+
content: clip(redactText(event.content, redactionSecrets), maxChatEventContentChars),
|
|
485
|
+
data: event.data ? clip(redactText(event.data, redactionSecrets), maxChatEventContentChars) : undefined,
|
|
486
|
+
};
|
|
487
|
+
await this.sendChatCallback(active, "output_delta", payload);
|
|
488
|
+
}
|
|
489
|
+
async sendChatAbortTerminal(packet, active, cleanupOutcome) {
|
|
490
|
+
await this.sendChatCallback(active, chatAbortKind(active.controller.signal.reason), chatAbortPayload(active.controller.signal.reason, cleanupOutcome, packet));
|
|
491
|
+
}
|
|
492
|
+
async sendStandaloneChatTerminal(packet, kind, payload) {
|
|
493
|
+
const active = {
|
|
494
|
+
leaseId: packet.leaseId,
|
|
495
|
+
kind: "chat",
|
|
496
|
+
fencingToken: packet.fencingToken,
|
|
497
|
+
dispatchAttemptId: null,
|
|
498
|
+
runnerInstanceId: packet.runnerInstanceId,
|
|
499
|
+
leaseExpiresAt: new Date(Date.now() + defaultChatTurnLeaseMs).toISOString(),
|
|
500
|
+
sandboxed: false,
|
|
501
|
+
resources: zeroSandboxResources(),
|
|
502
|
+
controller: new AbortController(),
|
|
503
|
+
chatRunId: packet.chatRunId,
|
|
504
|
+
conversationId: packet.conversationId,
|
|
505
|
+
turnId: packet.turnId,
|
|
506
|
+
providerThreadId: packet.providerThreadId ?? null,
|
|
507
|
+
chatSequence: 0,
|
|
508
|
+
terminalCallbackAttempted: false,
|
|
509
|
+
};
|
|
510
|
+
await this.sendChatCallback(active, kind, payload);
|
|
511
|
+
}
|
|
512
|
+
async sendChatCallback(active, kind, payload = null) {
|
|
513
|
+
if (active.kind !== "chat" || !active.chatRunId || !active.conversationId || !active.turnId) {
|
|
514
|
+
throw new Error("Bridge Chat callback requires an active chat turn.");
|
|
515
|
+
}
|
|
516
|
+
const terminal = isTerminalChatTurnCallbackKind(kind);
|
|
517
|
+
if (terminal && active.terminalCallbackAttempted) {
|
|
518
|
+
return;
|
|
519
|
+
}
|
|
520
|
+
assertBridgeChatCallbackPayload(payload);
|
|
521
|
+
const sequence = (active.chatSequence ?? 0) + 1;
|
|
522
|
+
const envelope = {
|
|
523
|
+
callbackId: randomUUID(),
|
|
524
|
+
chatRunId: active.chatRunId,
|
|
525
|
+
conversationId: active.conversationId,
|
|
526
|
+
turnId: active.turnId,
|
|
527
|
+
leaseId: active.leaseId,
|
|
528
|
+
fencingToken: active.fencingToken,
|
|
529
|
+
runnerId: this.config.runnerId,
|
|
530
|
+
runnerInstanceId: active.runnerInstanceId ?? this.config.runnerInstanceId,
|
|
531
|
+
sequence,
|
|
532
|
+
eventType: kind,
|
|
533
|
+
redactionStatus: "redacted",
|
|
534
|
+
providerThreadId: active.providerThreadId ?? null,
|
|
535
|
+
payload,
|
|
536
|
+
occurredAt: new Date().toISOString(),
|
|
537
|
+
protocolVersion: bridgeChatProtocolVersion,
|
|
538
|
+
};
|
|
539
|
+
if (terminal) {
|
|
540
|
+
active.terminalCallbackAttempted = true;
|
|
541
|
+
}
|
|
542
|
+
await this.invokeChatCallbackWithRetry(envelope);
|
|
543
|
+
active.chatSequence = sequence;
|
|
544
|
+
}
|
|
545
|
+
async invokeChatCallbackWithRetry(envelope) {
|
|
546
|
+
const maxAttempts = 3;
|
|
547
|
+
for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
|
|
548
|
+
try {
|
|
549
|
+
await this.invokeAck("ChatTurnCallback", envelope);
|
|
550
|
+
return;
|
|
551
|
+
}
|
|
552
|
+
catch (error) {
|
|
553
|
+
if (error instanceof BridgeCallbackRejectedError || attempt === maxAttempts - 1) {
|
|
554
|
+
throw error;
|
|
555
|
+
}
|
|
556
|
+
await delay(200 * (attempt + 1));
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
requiredChatCapabilityFailure(packet) {
|
|
561
|
+
const capability = this.bridgeChatCapability();
|
|
562
|
+
if (capability.availability === "unavailable" || !capability.supportsTextStreaming) {
|
|
563
|
+
return capability.unavailableReason ?? "Bridge Chat text capability is unavailable.";
|
|
564
|
+
}
|
|
565
|
+
if (capability.supportedProviders.length > 0
|
|
566
|
+
&& !containsIgnoreCase(capability.supportedProviders, packet.providerKind)) {
|
|
567
|
+
return `Bridge Chat provider '${packet.providerKind}' is not advertised by this bridge.`;
|
|
568
|
+
}
|
|
569
|
+
if (capability.supportedModels.length > 0
|
|
570
|
+
&& !containsIgnoreCase(capability.supportedModels, packet.model)) {
|
|
571
|
+
return `Bridge Chat model '${packet.model}' is not advertised by this bridge.`;
|
|
572
|
+
}
|
|
573
|
+
if (packet.requestedCapabilities.includes("cancel") && !capability.supportsCancel) {
|
|
574
|
+
return "Bridge Chat cancellation is not advertised by this bridge.";
|
|
575
|
+
}
|
|
576
|
+
if (packet.requestedCapabilities.includes("resume") && !capability.supportsResume) {
|
|
577
|
+
return "Bridge Chat resume is not advertised by this bridge.";
|
|
578
|
+
}
|
|
579
|
+
if (packet.requestedCapabilities.includes("thread_continuity")
|
|
580
|
+
&& !capability.supportsProviderThreadContinuity) {
|
|
581
|
+
return "Bridge Chat provider thread continuity is not advertised by this bridge.";
|
|
582
|
+
}
|
|
583
|
+
if (packet.requestedCapabilities.includes("attachment_handoffs") && !capability.supportsAttachments) {
|
|
584
|
+
return "Bridge Chat attachment handoffs are not advertised by this bridge.";
|
|
585
|
+
}
|
|
586
|
+
if (packet.requestedCapabilities.includes("action_broker_grants") && !capability.supportsActionBrokerGrants) {
|
|
587
|
+
return "Bridge Chat action-broker grants are not advertised by this bridge.";
|
|
588
|
+
}
|
|
589
|
+
return null;
|
|
590
|
+
}
|
|
591
|
+
bridgeChatCapability() {
|
|
592
|
+
return bridgeChatCapabilityFor(this.config.cliProviders, this.config.providerStatuses, this.config.bridgeChat);
|
|
593
|
+
}
|
|
594
|
+
async handleDispatch(rawDispatch, reservationHandoff) {
|
|
595
|
+
const dispatch = await this.parseDispatchOrReport(rawDispatch);
|
|
156
596
|
if (!dispatch) {
|
|
597
|
+
this.releaseReservedDispatchHandoff(reservationHandoff);
|
|
598
|
+
return;
|
|
599
|
+
}
|
|
600
|
+
if (this.enforceHostDatabaseCapabilityBoundary) {
|
|
601
|
+
try {
|
|
602
|
+
assertHostDatabaseCapabilityBoundary();
|
|
603
|
+
}
|
|
604
|
+
catch (error) {
|
|
605
|
+
const message = errorMessage(error);
|
|
606
|
+
await this.sendResult(dispatch, failureResult(message, [{
|
|
607
|
+
role: "runner",
|
|
608
|
+
kind: "error",
|
|
609
|
+
content: message,
|
|
610
|
+
eventType: "host_database_capability_forbidden",
|
|
611
|
+
}], failureFactsFromError(error, { source: "bridge" }))).catch(() => undefined);
|
|
612
|
+
this.releaseReservedDispatchHandoff(reservationHandoff);
|
|
613
|
+
return;
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
let capabilityFailure;
|
|
617
|
+
try {
|
|
618
|
+
capabilityFailure = await this.requiredCapabilityFailure(dispatch);
|
|
619
|
+
}
|
|
620
|
+
catch (error) {
|
|
621
|
+
this.releaseReservedDispatchHandoff(reservationHandoff);
|
|
622
|
+
throw error;
|
|
623
|
+
}
|
|
624
|
+
if (capabilityFailure) {
|
|
625
|
+
await this.sendResult(dispatch, failureResult(capabilityFailure, [{
|
|
626
|
+
role: "runner",
|
|
627
|
+
kind: "error",
|
|
628
|
+
content: capabilityFailure,
|
|
629
|
+
eventType: "docker_engine_unavailable",
|
|
630
|
+
}], failureFactsForCode("docker_engine_unavailable"))).catch(() => undefined);
|
|
631
|
+
this.releaseReservedDispatchHandoff(reservationHandoff);
|
|
157
632
|
return;
|
|
158
633
|
}
|
|
159
634
|
const leaseId = dispatch.lease.id;
|
|
160
635
|
const fencingToken = dispatch.lease.fencingToken;
|
|
161
636
|
const redactionSecrets = exactSecrets(dispatch, this.config);
|
|
637
|
+
if (reservationHandoff && !this.reservedDispatchHandoffIsCurrent(reservationHandoff, dispatch)) {
|
|
638
|
+
await this.sendResult(dispatch, failureResult("Start rejected because its reservation is missing or stale.", [], failureFactsForCode("bridge.reservation_stale"))).catch(() => undefined);
|
|
639
|
+
return;
|
|
640
|
+
}
|
|
162
641
|
const existing = this.activeSessions.get(leaseId);
|
|
163
642
|
if (existing) {
|
|
164
643
|
if (existing.fencingToken === fencingToken) {
|
|
644
|
+
this.releaseReservedDispatchHandoff(reservationHandoff);
|
|
165
645
|
return;
|
|
166
646
|
}
|
|
167
|
-
await this.sendResult(dispatch, failureResult("Dispatch fence conflicts with an active bridge session.")).catch(() => undefined);
|
|
647
|
+
await this.sendResult(dispatch, failureResult("Dispatch fence conflicts with an active bridge session.", [], failureFactsForCode("bridge.dispatch_fence_conflict"))).catch(() => undefined);
|
|
648
|
+
this.releaseReservedDispatchHandoff(reservationHandoff);
|
|
168
649
|
return;
|
|
169
650
|
}
|
|
170
|
-
const controller = new AbortController();
|
|
651
|
+
const controller = reservationHandoff?.controller ?? new AbortController();
|
|
171
652
|
this.activeSessions.set(leaseId, {
|
|
653
|
+
leaseId,
|
|
654
|
+
kind: "dispatch",
|
|
172
655
|
fencingToken,
|
|
173
656
|
dispatchAttemptId: dispatch.sandbox?.dispatchAttemptId ?? null,
|
|
174
657
|
runnerInstanceId: dispatch.sandbox?.runnerInstanceId ?? null,
|
|
@@ -179,6 +662,18 @@ export class ForgeBridgeService {
|
|
|
179
662
|
: zeroSandboxResources(),
|
|
180
663
|
controller
|
|
181
664
|
});
|
|
665
|
+
if (reservationHandoff) {
|
|
666
|
+
// Promotion is synchronous with active-session registration. A qualified
|
|
667
|
+
// pre-start Cancel therefore sees either the cancellable reservation or
|
|
668
|
+
// the fully active owner; there is no cancellation-blind handoff gap.
|
|
669
|
+
this.releaseReservedDispatchHandoff(reservationHandoff);
|
|
670
|
+
}
|
|
671
|
+
// Server-requested runtime readiness is background maintenance. A probe may
|
|
672
|
+
// have passed admission during the narrow handoff between prepared-runtime
|
|
673
|
+
// verification and the foreground attempt's final admission. Preempt it as
|
|
674
|
+
// soon as real agent work arrives so the probe cannot consume the memory
|
|
675
|
+
// headroom that the foreground attempt is waiting to acquire.
|
|
676
|
+
this.deferActiveReadinessProbes();
|
|
182
677
|
try {
|
|
183
678
|
const normalized = this.normalizeDispatch(dispatch);
|
|
184
679
|
if (normalized.sandbox) {
|
|
@@ -205,7 +700,10 @@ export class ForgeBridgeService {
|
|
|
205
700
|
dispatchAttemptId: dispatch.sandbox.dispatchAttemptId,
|
|
206
701
|
runnerInstanceId: dispatch.sandbox.runnerInstanceId
|
|
207
702
|
} : {}),
|
|
208
|
-
event: redactRunnerDispatchEvent(event, redactionSecrets)
|
|
703
|
+
event: redactRunnerDispatchEvent({ ...event, eventId: event.eventId ?? randomUUID() }, redactionSecrets)
|
|
704
|
+
}, {
|
|
705
|
+
replayCallbackTimeout: true,
|
|
706
|
+
signal: controller.signal,
|
|
209
707
|
});
|
|
210
708
|
}
|
|
211
709
|
catch (error) {
|
|
@@ -223,23 +721,86 @@ export class ForgeBridgeService {
|
|
|
223
721
|
if (normalized.sandbox?.executionMode === "docker") {
|
|
224
722
|
await this.flushCleanupReports().catch(() => undefined);
|
|
225
723
|
}
|
|
226
|
-
|
|
724
|
+
const redactedResult = redactRunOnceResultPacket(result, redactionSecrets);
|
|
725
|
+
await this.sendResult(dispatch, compactTerminalResult(dispatch, redactedResult));
|
|
227
726
|
}
|
|
228
727
|
catch (error) {
|
|
229
728
|
const summary = controller.signal.aborted
|
|
230
729
|
? `Dispatch cancelled: ${String(controller.signal.reason ?? "cancelled")}`
|
|
231
730
|
: redactText(error instanceof Error ? error.message : String(error), redactionSecrets);
|
|
232
|
-
const
|
|
731
|
+
const failureFacts = failureFactsFromError(error, {
|
|
732
|
+
cancelled: controller.signal.aborted,
|
|
733
|
+
source: "bridge",
|
|
734
|
+
});
|
|
735
|
+
const failureEventType = failureFacts.category === "unknown" ? null : failureFacts.code;
|
|
233
736
|
const failureEvents = failureEventType ? [{
|
|
234
737
|
role: "runner",
|
|
235
738
|
kind: "error",
|
|
236
739
|
content: summary,
|
|
237
740
|
eventType: failureEventType,
|
|
238
741
|
}] : [];
|
|
239
|
-
await this.sendResult(dispatch, failureResult(summary, failureEvents)).catch(() => undefined);
|
|
742
|
+
await this.sendResult(dispatch, failureResult(summary, failureEvents, failureFacts)).catch(() => undefined);
|
|
240
743
|
}
|
|
241
744
|
finally {
|
|
242
|
-
this.
|
|
745
|
+
await this.releaseActiveSessionAndRefreshCapacity(leaseId);
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
reservedDispatchHandoffIsCurrent(handoff, dispatch) {
|
|
749
|
+
const sandbox = dispatch.sandbox;
|
|
750
|
+
const reservation = handoff.reservation;
|
|
751
|
+
return sandbox !== null
|
|
752
|
+
&& sandbox !== undefined
|
|
753
|
+
&& sandbox.dispatchAttemptId === handoff.dispatchAttemptId
|
|
754
|
+
&& this.reservations.get(handoff.dispatchAttemptId) === reservation
|
|
755
|
+
&& reservation.startClaim === handoff
|
|
756
|
+
&& !handoff.controller.signal.aborted
|
|
757
|
+
&& reservation.leaseId === dispatch.lease.id
|
|
758
|
+
&& reservation.fencingToken === dispatch.lease.fencingToken
|
|
759
|
+
&& reservation.runnerInstanceId === sandbox.runnerInstanceId
|
|
760
|
+
&& Date.parse(reservation.reservedUntil) > Date.now();
|
|
761
|
+
}
|
|
762
|
+
releaseReservedDispatchHandoff(handoff) {
|
|
763
|
+
if (!handoff
|
|
764
|
+
|| this.reservations.get(handoff.dispatchAttemptId) !== handoff.reservation
|
|
765
|
+
|| handoff.reservation.startClaim !== handoff)
|
|
766
|
+
return;
|
|
767
|
+
this.reservations.delete(handoff.dispatchAttemptId);
|
|
768
|
+
}
|
|
769
|
+
async releaseActiveSessionAndRefreshCapacity(leaseId, expectedDispatch) {
|
|
770
|
+
if (expectedDispatch) {
|
|
771
|
+
const active = this.activeSessions.get(leaseId);
|
|
772
|
+
// Admission failures may arrive after a newer attempt starts on the same
|
|
773
|
+
// lease. A terminal callback can release only the dispatch it identifies.
|
|
774
|
+
if (!active || active.kind !== "dispatch"
|
|
775
|
+
|| active.fencingToken !== expectedDispatch.lease.fencingToken
|
|
776
|
+
|| active.dispatchAttemptId !== (expectedDispatch.sandbox?.dispatchAttemptId ?? null)
|
|
777
|
+
|| active.runnerInstanceId !== (expectedDispatch.sandbox?.runnerInstanceId ?? null))
|
|
778
|
+
return;
|
|
779
|
+
}
|
|
780
|
+
if (!this.activeSessions.delete(leaseId))
|
|
781
|
+
return;
|
|
782
|
+
// Publish released host/runtime capacity before the terminal result can
|
|
783
|
+
// enqueue the next pipeline phase. Otherwise the server may reject that
|
|
784
|
+
// handoff from the previous periodic heartbeat's active reservation.
|
|
785
|
+
// Publish the updated reservation counts from cached samples only. Probes
|
|
786
|
+
// belong to periodic/admission heartbeats and must not outlive this bounded
|
|
787
|
+
// terminal handoff. Storage recovery must not block durable result delivery.
|
|
788
|
+
const controller = new AbortController();
|
|
789
|
+
let timer;
|
|
790
|
+
try {
|
|
791
|
+
await Promise.race([
|
|
792
|
+
this.sendHeartbeatSnapshot(controller.signal).catch(() => undefined),
|
|
793
|
+
new Promise(resolve => {
|
|
794
|
+
timer = setTimeout(() => {
|
|
795
|
+
controller.abort("terminal capacity refresh deadline");
|
|
796
|
+
resolve();
|
|
797
|
+
}, Math.min(5000, positiveFiniteNumber(this.config.callbackAckTimeoutMs ?? 30_000, 30_000)));
|
|
798
|
+
}),
|
|
799
|
+
]);
|
|
800
|
+
}
|
|
801
|
+
finally {
|
|
802
|
+
if (timer)
|
|
803
|
+
clearTimeout(timer);
|
|
243
804
|
}
|
|
244
805
|
}
|
|
245
806
|
async handleOffer(rawDispatch) {
|
|
@@ -247,20 +808,19 @@ export class ForgeBridgeService {
|
|
|
247
808
|
if (!envelope) {
|
|
248
809
|
return;
|
|
249
810
|
}
|
|
250
|
-
const dispatch = this.parseDispatchOrReport(envelope.dispatch);
|
|
811
|
+
const dispatch = await this.parseDispatchOrReport(envelope.dispatch);
|
|
251
812
|
if (!dispatch?.sandbox
|
|
252
813
|
|| dispatch.sandbox.dispatchAttemptId !== envelope.dispatchAttemptId
|
|
253
814
|
|| dispatch.lease.id !== envelope.leaseId
|
|
254
815
|
|| dispatch.lease.fencingToken !== envelope.fencingToken)
|
|
255
816
|
return;
|
|
256
817
|
const sandbox = dispatch.sandbox;
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
this.reservations.delete(sandbox.dispatchAttemptId);
|
|
260
|
-
existing = undefined;
|
|
261
|
-
}
|
|
818
|
+
this.expireReservations(Date.now());
|
|
819
|
+
const existing = this.reservations.get(sandbox.dispatchAttemptId);
|
|
262
820
|
if (existing) {
|
|
263
|
-
const sameFence = existing.
|
|
821
|
+
const sameFence = existing.leaseId === dispatch.lease.id
|
|
822
|
+
&& existing.fencingToken === dispatch.lease.fencingToken
|
|
823
|
+
&& existing.runnerInstanceId === sandbox.runnerInstanceId;
|
|
264
824
|
await this.sendOfferDecision(dispatch, sameFence, sameFence ? null : "stale_fence", sameFence ? null : "Offer fence conflicts with the existing reservation.", sameFence ? existing.reservedUntil : null, sameFence ? existing.resources : null);
|
|
265
825
|
return;
|
|
266
826
|
}
|
|
@@ -287,7 +847,7 @@ export class ForgeBridgeService {
|
|
|
287
847
|
}
|
|
288
848
|
const providerNetworkFailure = sandboxProviderNetworkFailure(dispatch, this.config);
|
|
289
849
|
if (providerNetworkFailure) {
|
|
290
|
-
await this.sendOfferDecision(dispatch, false, "
|
|
850
|
+
await this.sendOfferDecision(dispatch, false, "provider_network_policy_denied", providerNetworkFailure);
|
|
291
851
|
return;
|
|
292
852
|
}
|
|
293
853
|
const authFailure = sandboxAuthFailure(dispatch, this.config);
|
|
@@ -296,15 +856,12 @@ export class ForgeBridgeService {
|
|
|
296
856
|
return;
|
|
297
857
|
}
|
|
298
858
|
}
|
|
299
|
-
const runtimeState = this.sandboxRuntime.snapshot();
|
|
300
|
-
const usedResources = this.usedSandboxResources(runtimeState.degraded);
|
|
301
|
-
const activeResources = this.activeSandboxResources(runtimeState.degraded);
|
|
302
|
-
const capacity = this.sandboxResourceCapacity(preflight, activeResources);
|
|
303
859
|
let requestedResources = sandbox.requestedResources;
|
|
304
860
|
let sandboxConfigCommitSha = sandbox.sandboxConfigCommitSha ?? null;
|
|
861
|
+
const normalizedDispatch = this.normalizeDispatch(dispatch);
|
|
305
862
|
if (sandbox.executionMode === "docker") {
|
|
306
863
|
try {
|
|
307
|
-
const profile = await this.sandboxResourceResolver(
|
|
864
|
+
const profile = await this.sandboxResourceResolver(normalizedDispatch);
|
|
308
865
|
requestedResources = profile.resources;
|
|
309
866
|
sandboxConfigCommitSha = profile.sandboxConfigCommitSha;
|
|
310
867
|
}
|
|
@@ -313,73 +870,166 @@ export class ForgeBridgeService {
|
|
|
313
870
|
return;
|
|
314
871
|
}
|
|
315
872
|
}
|
|
316
|
-
if (
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
873
|
+
else if (normalizedDispatch.repo.serverLocalPath) {
|
|
874
|
+
// Host execution uses the same project-derived disk reservation as Docker.
|
|
875
|
+
// The API supplies a placement-time estimate; the runner can safely raise it
|
|
876
|
+
// from its authoritative checkout before admitting the attempt.
|
|
877
|
+
const measuredWorkspaceBytes = await estimateWorkspaceBytesFromRepo(normalizedDispatch.repo.serverLocalPath, { role: normalizedDispatch.execution.role }).catch(() => null);
|
|
878
|
+
if (measuredWorkspaceBytes !== null) {
|
|
879
|
+
requestedResources = {
|
|
880
|
+
...requestedResources,
|
|
881
|
+
workspaceBytes: Math.max(requestedResources.workspaceBytes, measuredWorkspaceBytes),
|
|
882
|
+
};
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
// Resource profiling can await filesystem and Docker work. Admission needs
|
|
886
|
+
// a dynamic host/daemon sample that began after profiling completed, so an
|
|
887
|
+
// older in-flight heartbeat inspection cannot supply the commit decision.
|
|
888
|
+
const admissionPreflight = sandbox.executionMode === "docker"
|
|
889
|
+
? await this.preflight.refreshCapacityForAdmission()
|
|
890
|
+
: preflight;
|
|
891
|
+
// Resource profiling and preflight may await slow filesystem or Docker I/O.
|
|
892
|
+
// Re-read every mutable capacity input only after that work, then decide and
|
|
893
|
+
// commit synchronously. JavaScript cannot interleave another Offer handler
|
|
894
|
+
// inside this non-async method, so the ledger check and insertion are one
|
|
895
|
+
// atomic event-loop turn without holding a lock across external I/O.
|
|
896
|
+
const decision = this.commitOfferReservation(dispatch, requestedResources, admissionPreflight);
|
|
897
|
+
await this.sendOfferDecision(dispatch, decision.accepted, decision.code, decision.message, decision.reservedUntil, decision.resources, sandboxConfigCommitSha);
|
|
898
|
+
}
|
|
899
|
+
commitOfferReservation(dispatch, requestedResources, preparedPreflight) {
|
|
900
|
+
const sandbox = dispatch.sandbox;
|
|
901
|
+
const now = Date.now();
|
|
902
|
+
this.expireReservations(now);
|
|
903
|
+
const existing = this.reservations.get(sandbox.dispatchAttemptId);
|
|
904
|
+
if (existing) {
|
|
905
|
+
const sameFence = existing.leaseId === dispatch.lease.id
|
|
906
|
+
&& existing.fencingToken === dispatch.lease.fencingToken
|
|
907
|
+
&& existing.runnerInstanceId === sandbox.runnerInstanceId;
|
|
908
|
+
return {
|
|
909
|
+
accepted: sameFence,
|
|
910
|
+
code: sameFence ? null : "stale_fence",
|
|
911
|
+
message: sameFence ? null : "Offer fence conflicts with the existing reservation.",
|
|
912
|
+
reservedUntil: sameFence ? existing.reservedUntil : null,
|
|
913
|
+
resources: sameFence ? existing.resources : null,
|
|
914
|
+
};
|
|
915
|
+
}
|
|
916
|
+
if (Date.parse(dispatch.lease.expiresAt) <= now) {
|
|
917
|
+
return rejectedOfferReservation("lease_expired", "Offer lease is already expired.");
|
|
320
918
|
}
|
|
321
|
-
|
|
322
|
-
|
|
919
|
+
if (sandbox.executionMode === "docker") {
|
|
920
|
+
// A concurrent heartbeat/preflight refresh may have produced a newer
|
|
921
|
+
// conservative dual-domain sample while this offer was being profiled.
|
|
922
|
+
const admissionPreflight = this.preflight.current() ?? preparedPreflight;
|
|
923
|
+
if (!this.config.dockerEnabled || !admissionPreflight?.healthy) {
|
|
924
|
+
return rejectedOfferReservation(admissionPreflight?.unavailableCode ?? "docker_unavailable", admissionPreflight?.unavailableReason ?? "Docker sandbox preflight is unavailable.");
|
|
925
|
+
}
|
|
926
|
+
const runtimeState = this.sandboxRuntime.snapshot();
|
|
927
|
+
const usedResources = this.usedSandboxResources(runtimeState.degraded);
|
|
928
|
+
const activeResources = this.activeSandboxResources(runtimeState.degraded);
|
|
929
|
+
const capacity = this.sandboxResourceCapacity(admissionPreflight, activeResources);
|
|
930
|
+
if (!canReserveSandboxResources(usedResources, requestedResources, capacity)) {
|
|
931
|
+
return rejectedOfferReservation("capacity_unavailable", sandboxCapacityFailureMessage(usedResources, requestedResources, capacity, admissionPreflight, this.config));
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
const reservedUntil = new Date(Math.min(Date.parse(dispatch.lease.expiresAt), now + 30_000)).toISOString();
|
|
935
|
+
const reservation = {
|
|
323
936
|
leaseId: dispatch.lease.id,
|
|
324
937
|
fencingToken: dispatch.lease.fencingToken,
|
|
325
938
|
runnerInstanceId: sandbox.runnerInstanceId,
|
|
326
939
|
expiresAt: dispatch.lease.expiresAt,
|
|
327
940
|
reservedUntil,
|
|
328
941
|
sandboxed: sandbox.executionMode === "docker",
|
|
329
|
-
resources: requestedResources
|
|
330
|
-
}
|
|
331
|
-
|
|
942
|
+
resources: requestedResources,
|
|
943
|
+
};
|
|
944
|
+
this.reservations.set(sandbox.dispatchAttemptId, reservation);
|
|
945
|
+
return {
|
|
946
|
+
accepted: true,
|
|
947
|
+
code: null,
|
|
948
|
+
message: null,
|
|
949
|
+
reservedUntil,
|
|
950
|
+
resources: requestedResources,
|
|
951
|
+
};
|
|
332
952
|
}
|
|
333
953
|
async handleStart(rawDispatch) {
|
|
334
954
|
const envelope = parseStartEnvelope(rawDispatch);
|
|
335
955
|
if (!envelope) {
|
|
336
956
|
return;
|
|
337
957
|
}
|
|
338
|
-
let dispatch = this.parseDispatchOrReport(envelope.dispatch);
|
|
958
|
+
let dispatch = await this.parseDispatchOrReport(envelope.dispatch);
|
|
339
959
|
if (!dispatch?.sandbox
|
|
340
960
|
|| dispatch.sandbox.dispatchAttemptId !== envelope.dispatchAttemptId
|
|
341
961
|
|| dispatch.lease.id !== envelope.leaseId
|
|
342
962
|
|| dispatch.lease.fencingToken !== envelope.fencingToken)
|
|
343
963
|
return;
|
|
964
|
+
const active = this.activeSessions.get(dispatch.lease.id);
|
|
965
|
+
if (active?.kind === "dispatch"
|
|
966
|
+
&& active.fencingToken === dispatch.lease.fencingToken
|
|
967
|
+
&& active.dispatchAttemptId === dispatch.sandbox.dispatchAttemptId
|
|
968
|
+
&& active.runnerInstanceId === dispatch.sandbox.runnerInstanceId) {
|
|
969
|
+
return;
|
|
970
|
+
}
|
|
344
971
|
const reservation = this.reservations.get(dispatch.sandbox.dispatchAttemptId);
|
|
345
972
|
if (!reservation
|
|
346
973
|
|| reservation.leaseId !== dispatch.lease.id
|
|
347
974
|
|| reservation.fencingToken !== dispatch.lease.fencingToken
|
|
348
975
|
|| reservation.runnerInstanceId !== dispatch.sandbox.runnerInstanceId
|
|
349
976
|
|| Date.parse(reservation.reservedUntil) <= Date.now()) {
|
|
350
|
-
await this.sendResult(dispatch, failureResult("Start rejected because its reservation is missing or stale.")).catch(() => undefined);
|
|
977
|
+
await this.sendResult(dispatch, failureResult("Start rejected because its reservation is missing or stale.", [], failureFactsForCode("bridge.reservation_stale"))).catch(() => undefined);
|
|
351
978
|
return;
|
|
352
979
|
}
|
|
980
|
+
// Claim the exact reservation before redemption or any other post-validation
|
|
981
|
+
// await. A duplicate Start for the same reservation is an idempotent replay:
|
|
982
|
+
// it must neither redeem a one-time grant nor disturb the original owner.
|
|
983
|
+
if (reservation.startClaim)
|
|
984
|
+
return;
|
|
985
|
+
const startClaim = {
|
|
986
|
+
dispatchAttemptId: envelope.dispatchAttemptId,
|
|
987
|
+
reservation,
|
|
988
|
+
controller: new AbortController(),
|
|
989
|
+
};
|
|
990
|
+
reservation.startClaim = startClaim;
|
|
353
991
|
if (dispatch.apiKeySecretGrant) {
|
|
354
|
-
this.
|
|
355
|
-
await this.sendResult(dispatch, failureResult("Start rejected because protocol-v2 packets may not contain a plaintext provider grant.")).catch(() => undefined);
|
|
992
|
+
this.releaseReservedDispatchHandoff(startClaim);
|
|
993
|
+
await this.sendResult(dispatch, failureResult("Start rejected because protocol-v2 packets may not contain a plaintext provider grant.", [], failureFactsForCode("contract_plaintext_secret_rejected"))).catch(() => undefined);
|
|
356
994
|
return;
|
|
357
995
|
}
|
|
358
996
|
if (dispatch.apiKeySecretGrantHandle) {
|
|
359
|
-
|
|
997
|
+
let response;
|
|
998
|
+
try {
|
|
999
|
+
response = await this.redeemSecretGrant(dispatch);
|
|
1000
|
+
}
|
|
1001
|
+
catch (error) {
|
|
1002
|
+
this.releaseReservedDispatchHandoff(startClaim);
|
|
1003
|
+
throw error;
|
|
1004
|
+
}
|
|
1005
|
+
// Redemption may yield while a qualified Cancel releases this reservation.
|
|
1006
|
+
// Never launch from it or remove a successor reservation after that wait.
|
|
1007
|
+
if (!this.reservedDispatchHandoffIsCurrent(startClaim, dispatch)) {
|
|
1008
|
+
await this.sendResult(dispatch, failureResult("Start rejected because its reservation is missing or stale.", [], failureFactsForCode("bridge.reservation_stale"))).catch(() => undefined);
|
|
1009
|
+
return;
|
|
1010
|
+
}
|
|
360
1011
|
const grant = response.grant;
|
|
361
1012
|
if (!response.accepted
|
|
362
1013
|
|| !grant
|
|
363
1014
|
|| grant.grantId !== dispatch.apiKeySecretGrantHandle.grantId
|
|
364
1015
|
|| grant.provider !== dispatch.apiKeySecretGrantHandle.provider
|
|
365
1016
|
|| Date.parse(grant.expiresAt) <= Date.now()) {
|
|
366
|
-
this.
|
|
367
|
-
await this.sendResult(dispatch, failureResult(response.message ?? "Secret grant redemption was rejected.")).catch(() => undefined);
|
|
1017
|
+
this.releaseReservedDispatchHandoff(startClaim);
|
|
1018
|
+
await this.sendResult(dispatch, failureResult(response.message ?? "Secret grant redemption was rejected.", [], failureFactsForCode("bridge.api_key_secret_grant_denied"))).catch(() => undefined);
|
|
368
1019
|
return;
|
|
369
1020
|
}
|
|
370
1021
|
dispatch = { ...dispatch, apiKeySecretGrant: grant, apiKeySecretGrantHandle: null };
|
|
371
1022
|
}
|
|
372
1023
|
else if (dispatch.execution.requiresApiKeySecretGrant) {
|
|
373
|
-
this.
|
|
374
|
-
await this.sendResult(dispatch, failureResult("Start rejected because the provider secret grant handle is missing.")).catch(() => undefined);
|
|
1024
|
+
this.releaseReservedDispatchHandoff(startClaim);
|
|
1025
|
+
await this.sendResult(dispatch, failureResult("Start rejected because the provider secret grant handle is missing.", [], failureFactsForCode("bridge.api_key_secret_grant_missing"))).catch(() => undefined);
|
|
375
1026
|
return;
|
|
376
1027
|
}
|
|
377
1028
|
dispatch = {
|
|
378
1029
|
...dispatch,
|
|
379
1030
|
sandbox: { ...dispatch.sandbox, requestedResources: reservation.resources },
|
|
380
1031
|
};
|
|
381
|
-
this.
|
|
382
|
-
await this.handleDispatch(dispatch);
|
|
1032
|
+
await this.handleDispatch(dispatch, startClaim);
|
|
383
1033
|
}
|
|
384
1034
|
async redeemSecretGrant(dispatch) {
|
|
385
1035
|
const sandbox = dispatch.sandbox;
|
|
@@ -443,7 +1093,7 @@ export class ForgeBridgeService {
|
|
|
443
1093
|
};
|
|
444
1094
|
await this.invokeAck("OfferDecision", payload);
|
|
445
1095
|
}
|
|
446
|
-
parseDispatchOrReport(rawDispatch) {
|
|
1096
|
+
async parseDispatchOrReport(rawDispatch) {
|
|
447
1097
|
try {
|
|
448
1098
|
return parseRunOnceDispatchPacket(rawDispatch);
|
|
449
1099
|
}
|
|
@@ -453,24 +1103,78 @@ export class ForgeBridgeService {
|
|
|
453
1103
|
return null;
|
|
454
1104
|
}
|
|
455
1105
|
const summary = invalidDispatchSummary(error);
|
|
456
|
-
|
|
1106
|
+
await this.sendLegacyResult(identifiers.leaseId, identifiers.fencingToken, failureResult(summary, [
|
|
457
1107
|
{
|
|
458
1108
|
role: "runner",
|
|
459
1109
|
kind: "error",
|
|
460
1110
|
content: summary,
|
|
461
1111
|
eventType: "invalid_dispatch"
|
|
462
1112
|
}
|
|
463
|
-
])).catch(() => undefined);
|
|
1113
|
+
], failureFactsForCode("invalid_dispatch"))).catch(() => undefined);
|
|
1114
|
+
return null;
|
|
1115
|
+
}
|
|
1116
|
+
}
|
|
1117
|
+
parseChatTurnOrReport(rawDispatch) {
|
|
1118
|
+
try {
|
|
1119
|
+
return parseChatTurnDispatchPacket(rawDispatch, this.bridgeChatCapability());
|
|
1120
|
+
}
|
|
1121
|
+
catch (error) {
|
|
1122
|
+
logBridgeStatus(invalidChatTurnSummary(error));
|
|
464
1123
|
return null;
|
|
465
1124
|
}
|
|
466
1125
|
}
|
|
467
1126
|
async handleCancel(request) {
|
|
1127
|
+
if (request.dispatchAttemptId !== undefined && request.dispatchAttemptId !== null) {
|
|
1128
|
+
// Qualified cancellation is reservation-only. In particular, a delayed
|
|
1129
|
+
// pre-start Cancel must not abort an active session if Start won the race.
|
|
1130
|
+
const reservation = this.reservations.get(request.dispatchAttemptId);
|
|
1131
|
+
if (reservation
|
|
1132
|
+
&& request.runnerId === this.config.runnerId
|
|
1133
|
+
&& request.runnerInstanceId === this.config.runnerInstanceId
|
|
1134
|
+
&& reservation.runnerInstanceId === request.runnerInstanceId
|
|
1135
|
+
&& reservation.leaseId === request.leaseId
|
|
1136
|
+
&& reservation.fencingToken === request.fencingToken) {
|
|
1137
|
+
reservation.startClaim?.controller.abort(request.reason ?? "cancelled by server");
|
|
1138
|
+
this.reservations.delete(request.dispatchAttemptId);
|
|
1139
|
+
}
|
|
1140
|
+
return;
|
|
1141
|
+
}
|
|
468
1142
|
const active = this.activeSessions.get(request.leaseId);
|
|
469
1143
|
if (!active || active.fencingToken !== request.fencingToken) {
|
|
470
1144
|
return;
|
|
471
1145
|
}
|
|
1146
|
+
if (request.runnerId && request.runnerId !== this.config.runnerId) {
|
|
1147
|
+
return;
|
|
1148
|
+
}
|
|
1149
|
+
if (request.runnerInstanceId && request.runnerInstanceId !== (active.runnerInstanceId ?? this.config.runnerInstanceId)) {
|
|
1150
|
+
return;
|
|
1151
|
+
}
|
|
1152
|
+
if (active.kind === "chat" && !this.chatCancelMatchesActiveTurn(active, request)) {
|
|
1153
|
+
return;
|
|
1154
|
+
}
|
|
472
1155
|
active.controller.abort(request.reason ?? "cancelled by server");
|
|
473
1156
|
}
|
|
1157
|
+
chatCancelMatchesActiveTurn(active, request) {
|
|
1158
|
+
if (request.protocolVersion && request.protocolVersion !== bridgeChatProtocolVersion) {
|
|
1159
|
+
return false;
|
|
1160
|
+
}
|
|
1161
|
+
try {
|
|
1162
|
+
assertBridgeChatCallbackPayload(request.payload, "cancel payload");
|
|
1163
|
+
}
|
|
1164
|
+
catch {
|
|
1165
|
+
return false;
|
|
1166
|
+
}
|
|
1167
|
+
if (request.chatRunId && request.chatRunId !== active.chatRunId)
|
|
1168
|
+
return false;
|
|
1169
|
+
if (request.conversationId && request.conversationId !== active.conversationId)
|
|
1170
|
+
return false;
|
|
1171
|
+
if (request.turnId && request.turnId !== active.turnId)
|
|
1172
|
+
return false;
|
|
1173
|
+
if (request.providerThreadId && active.providerThreadId && request.providerThreadId !== active.providerThreadId) {
|
|
1174
|
+
return false;
|
|
1175
|
+
}
|
|
1176
|
+
return true;
|
|
1177
|
+
}
|
|
474
1178
|
async handleReap(request) {
|
|
475
1179
|
if (request.runnerId !== this.config.runnerId
|
|
476
1180
|
|| request.runnerInstanceId !== this.config.runnerInstanceId)
|
|
@@ -492,32 +1196,43 @@ export class ForgeBridgeService {
|
|
|
492
1196
|
timer.unref?.();
|
|
493
1197
|
let result;
|
|
494
1198
|
try {
|
|
495
|
-
|
|
496
|
-
throw new Error("host_runtime_probe_unavailable: this bridge does not implement runtime readiness probes.");
|
|
497
|
-
}
|
|
498
|
-
const probe = await this.sessions.probeHostRuntime({
|
|
499
|
-
...request,
|
|
500
|
-
runnerId: this.config.runnerId,
|
|
501
|
-
repoRoot: this.config.defaultRepoRoot,
|
|
502
|
-
cleanupWorktrees: this.config.cleanupWorktrees,
|
|
503
|
-
}, controller.signal);
|
|
504
|
-
result = {
|
|
1199
|
+
const accepted = {
|
|
505
1200
|
schemaVersion: 1,
|
|
506
1201
|
requestId: request.requestId,
|
|
507
1202
|
fencingToken: request.fencingToken,
|
|
508
1203
|
runnerInstanceId: this.config.runnerInstanceId,
|
|
509
|
-
|
|
1204
|
+
acceptedAt: new Date().toISOString(),
|
|
510
1205
|
};
|
|
1206
|
+
await this.invokeAck("ReadinessAccepted", accepted);
|
|
1207
|
+
if (this.activeSessions.size > 0) {
|
|
1208
|
+
result = deferredReadinessResult(request, this.config.runnerInstanceId);
|
|
1209
|
+
}
|
|
1210
|
+
else if (!this.sessions.probeHostRuntime) {
|
|
1211
|
+
throw new Error("host_runtime_probe_unavailable: this bridge does not implement runtime readiness probes.");
|
|
1212
|
+
}
|
|
1213
|
+
else {
|
|
1214
|
+
const probe = await this.sessions.probeHostRuntime(this.normalizeReadinessProbe(request), controller.signal);
|
|
1215
|
+
result = readinessDeferralReason(controller.signal)
|
|
1216
|
+
? deferredReadinessResult(request, this.config.runnerInstanceId)
|
|
1217
|
+
: {
|
|
1218
|
+
schemaVersion: 1,
|
|
1219
|
+
requestId: request.requestId,
|
|
1220
|
+
fencingToken: request.fencingToken,
|
|
1221
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
1222
|
+
...probe,
|
|
1223
|
+
};
|
|
1224
|
+
}
|
|
511
1225
|
}
|
|
512
1226
|
catch (error) {
|
|
1227
|
+
const deferredReason = readinessDeferralReason(controller.signal);
|
|
513
1228
|
result = {
|
|
514
1229
|
schemaVersion: 1,
|
|
515
1230
|
requestId: request.requestId,
|
|
516
1231
|
fencingToken: request.fencingToken,
|
|
517
1232
|
runnerInstanceId: this.config.runnerInstanceId,
|
|
518
1233
|
status: "failed",
|
|
519
|
-
reasonCode: "host_runtime_probe_failed",
|
|
520
|
-
reasonMessage: errorMessage(error).slice(0, 1000),
|
|
1234
|
+
reasonCode: deferredReason ? "host_runtime_probe_deferred" : "host_runtime_probe_failed",
|
|
1235
|
+
reasonMessage: (deferredReason ?? errorMessage(error)).slice(0, 1000),
|
|
521
1236
|
checkedAt: new Date().toISOString(),
|
|
522
1237
|
};
|
|
523
1238
|
}
|
|
@@ -525,9 +1240,69 @@ export class ForgeBridgeService {
|
|
|
525
1240
|
clearTimeout(timer);
|
|
526
1241
|
this.activeReadinessProbes.delete(request.requestId);
|
|
527
1242
|
}
|
|
528
|
-
|
|
1243
|
+
try {
|
|
1244
|
+
await this.readinessResultOutbox.persist(result, request.expiresAt);
|
|
1245
|
+
}
|
|
1246
|
+
catch (error) {
|
|
1247
|
+
if (errorMessage(error).includes("runtime_readiness_result_conflict:")) {
|
|
1248
|
+
logBridgeStatus(`Readiness result ${request.requestId} rejected conflicting reuse of a durable request identity.`);
|
|
1249
|
+
await this.sendHeartbeat().catch(() => undefined);
|
|
1250
|
+
return;
|
|
1251
|
+
}
|
|
1252
|
+
logBridgeStatus(`Readiness result ${request.requestId} could not be persisted; attempting direct delivery: ${errorMessage(error)}`);
|
|
1253
|
+
await this.invokeAck("ReadinessResult", result).catch(deliveryError => {
|
|
1254
|
+
logBridgeStatus(`Readiness result ${request.requestId} could not be persisted or delivered: ${errorMessage(deliveryError)}`);
|
|
1255
|
+
});
|
|
1256
|
+
await this.sendHeartbeat().catch(() => undefined);
|
|
1257
|
+
return;
|
|
1258
|
+
}
|
|
1259
|
+
try {
|
|
1260
|
+
await this.flushReadinessResults();
|
|
1261
|
+
}
|
|
1262
|
+
catch (error) {
|
|
1263
|
+
logBridgeStatus(`Readiness result ${request.requestId} remains queued for replay: ${errorMessage(error)}`);
|
|
1264
|
+
}
|
|
529
1265
|
await this.sendHeartbeat().catch(() => undefined);
|
|
530
1266
|
}
|
|
1267
|
+
deferActiveReadinessProbes() {
|
|
1268
|
+
for (const controller of this.activeReadinessProbes.values()) {
|
|
1269
|
+
if (!controller.signal.aborted) {
|
|
1270
|
+
controller.abort("host_runtime_probe_deferred: foreground agent work took priority over background runtime readiness.");
|
|
1271
|
+
}
|
|
1272
|
+
}
|
|
1273
|
+
}
|
|
1274
|
+
async flushReadinessResults() {
|
|
1275
|
+
if (this.readinessResultFlush)
|
|
1276
|
+
return await this.readinessResultFlush;
|
|
1277
|
+
const flush = this.flushReadinessResultsCore();
|
|
1278
|
+
this.readinessResultFlush = flush;
|
|
1279
|
+
try {
|
|
1280
|
+
await flush;
|
|
1281
|
+
}
|
|
1282
|
+
finally {
|
|
1283
|
+
if (this.readinessResultFlush === flush)
|
|
1284
|
+
this.readinessResultFlush = null;
|
|
1285
|
+
}
|
|
1286
|
+
}
|
|
1287
|
+
async flushReadinessResultsCore() {
|
|
1288
|
+
for (const pending of await this.readinessResultOutbox.pending()) {
|
|
1289
|
+
if (Date.parse(pending.expiresAt) <= Date.now()) {
|
|
1290
|
+
await this.readinessResultOutbox.acknowledge(pending.result.requestId);
|
|
1291
|
+
continue;
|
|
1292
|
+
}
|
|
1293
|
+
try {
|
|
1294
|
+
await this.invokeAck("ReadinessResult", pending.result);
|
|
1295
|
+
await this.readinessResultOutbox.acknowledge(pending.result.requestId);
|
|
1296
|
+
}
|
|
1297
|
+
catch (error) {
|
|
1298
|
+
if (isReadinessNoLongerPending(error)) {
|
|
1299
|
+
await this.readinessResultOutbox.acknowledge(pending.result.requestId);
|
|
1300
|
+
continue;
|
|
1301
|
+
}
|
|
1302
|
+
throw error;
|
|
1303
|
+
}
|
|
1304
|
+
}
|
|
1305
|
+
}
|
|
531
1306
|
abortActiveSessions(reason) {
|
|
532
1307
|
for (const active of this.activeSessions.values()) {
|
|
533
1308
|
active.controller.abort(reason);
|
|
@@ -539,8 +1314,42 @@ export class ForgeBridgeService {
|
|
|
539
1314
|
controller.abort(reason);
|
|
540
1315
|
this.activeReadinessProbes.clear();
|
|
541
1316
|
}
|
|
1317
|
+
normalizeReadinessProbe(request) {
|
|
1318
|
+
const mapping = findRepoMapping({
|
|
1319
|
+
id: request.repoId,
|
|
1320
|
+
name: request.repoName,
|
|
1321
|
+
remoteUrl: request.remoteUrl,
|
|
1322
|
+
}, repoMappingsFor(this.config));
|
|
1323
|
+
const configuredPath = mapping?.localPath?.trim();
|
|
1324
|
+
if (!configuredPath) {
|
|
1325
|
+
throw new Error("bridge_repo_mapping_missing: readiness requires a matching local repository mapping.");
|
|
1326
|
+
}
|
|
1327
|
+
// Local configuration, not a path supplied by the server, grants checkout
|
|
1328
|
+
// authority. Equality also excludes a different child/sibling of this repo.
|
|
1329
|
+
const suppliedPath = request.runnerRepoLocalPath.trim();
|
|
1330
|
+
if (!path.isAbsolute(suppliedPath)
|
|
1331
|
+
|| (process.platform === "win32" && path.parse(suppliedPath).root.length === 1)) {
|
|
1332
|
+
throw new Error("bridge_repo_mapping_mismatch: readiness requires a fully qualified checkout path.");
|
|
1333
|
+
}
|
|
1334
|
+
const localPath = path.resolve(configuredPath);
|
|
1335
|
+
const requestedPath = path.resolve(suppliedPath);
|
|
1336
|
+
const pathsMatch = process.platform === "win32"
|
|
1337
|
+
? localPath.toLowerCase() === requestedPath.toLowerCase()
|
|
1338
|
+
: localPath === requestedPath;
|
|
1339
|
+
if (!pathsMatch) {
|
|
1340
|
+
throw new Error("bridge_repo_mapping_mismatch: readiness checkout differs from the locally configured repository.");
|
|
1341
|
+
}
|
|
1342
|
+
return {
|
|
1343
|
+
...request,
|
|
1344
|
+
runnerId: this.config.runnerId,
|
|
1345
|
+
runnerRepoLocalPath: localPath,
|
|
1346
|
+
cloneOnDemand: false,
|
|
1347
|
+
repoRoot: path.dirname(localPath),
|
|
1348
|
+
cleanupWorktrees: this.config.cleanupWorktrees,
|
|
1349
|
+
};
|
|
1350
|
+
}
|
|
542
1351
|
normalizeDispatch(dispatch) {
|
|
543
|
-
const mapping = findRepoMapping(dispatch, repoMappingsFor(this.config));
|
|
1352
|
+
const mapping = findRepoMapping(dispatch.repo, repoMappingsFor(this.config));
|
|
544
1353
|
const localPath = mapping?.localPath?.trim();
|
|
545
1354
|
const repoRoot = localPath ? path.dirname(path.resolve(localPath)) : this.config.defaultRepoRoot;
|
|
546
1355
|
const qaLaunchTarget = normalizeQaAppLaunchTarget(mapping?.qaAppLaunchTarget) ?? dispatch.qaBrowser?.launchTarget ?? null;
|
|
@@ -569,24 +1378,105 @@ export class ForgeBridgeService {
|
|
|
569
1378
|
fencingToken: dispatch.lease.fencingToken,
|
|
570
1379
|
dispatchAttemptId: dispatch.sandbox?.dispatchAttemptId ?? null,
|
|
571
1380
|
runnerInstanceId: dispatch.sandbox?.runnerInstanceId ?? null,
|
|
572
|
-
|
|
1381
|
+
// Every terminal writer, including early admission failures and exception
|
|
1382
|
+
// paths, crosses the same redaction boundary before outbox persistence.
|
|
1383
|
+
result: redactRunOnceResultPacket(result, exactSecrets(dispatch, this.config))
|
|
573
1384
|
};
|
|
574
|
-
await this.sendResultPayload(payload);
|
|
1385
|
+
await this.sendResultPayload(payload, () => this.releaseActiveSessionAndRefreshCapacity(dispatch.lease.id, dispatch));
|
|
575
1386
|
}
|
|
576
1387
|
async sendLegacyResult(leaseId, fencingToken, result) {
|
|
577
1388
|
await this.sendResultPayload({ leaseId, fencingToken, result });
|
|
578
1389
|
}
|
|
579
|
-
async sendResultPayload(payload) {
|
|
1390
|
+
async sendResultPayload(payload, beforeDelivery) {
|
|
1391
|
+
// Register before persistence: replay can observe a record as soon as the
|
|
1392
|
+
// durable write becomes visible, even before persist() has returned.
|
|
1393
|
+
let releaseHandoff;
|
|
1394
|
+
const priorHandoff = this.dispatchResultHandoffs.get(payload.leaseId);
|
|
1395
|
+
const handoff = new Promise(resolve => { releaseHandoff = resolve; });
|
|
1396
|
+
this.dispatchResultHandoffs.set(payload.leaseId, handoff);
|
|
1397
|
+
let persisted = false;
|
|
1398
|
+
try {
|
|
1399
|
+
try {
|
|
1400
|
+
await this.dispatchResultOutbox.persist(payload);
|
|
1401
|
+
persisted = true;
|
|
1402
|
+
this.dispatchResultGeneration += 1;
|
|
1403
|
+
}
|
|
1404
|
+
catch (error) {
|
|
1405
|
+
logBridgeStatus(`Dispatch result ${payload.leaseId} could not be persisted; attempting direct delivery: ${errorMessage(error)}`);
|
|
1406
|
+
}
|
|
1407
|
+
await priorHandoff;
|
|
1408
|
+
await beforeDelivery?.();
|
|
1409
|
+
}
|
|
1410
|
+
finally {
|
|
1411
|
+
releaseHandoff();
|
|
1412
|
+
if (this.dispatchResultHandoffs.get(payload.leaseId) === handoff)
|
|
1413
|
+
this.dispatchResultHandoffs.delete(payload.leaseId);
|
|
1414
|
+
}
|
|
1415
|
+
if (!persisted) {
|
|
1416
|
+
await this.sendResultPayloadDirect(payload);
|
|
1417
|
+
return;
|
|
1418
|
+
}
|
|
1419
|
+
await this.flushDispatchResults().catch(error => {
|
|
1420
|
+
logBridgeStatus(`Dispatch result ${payload.leaseId} remains queued for replay: ${errorMessage(error)}`);
|
|
1421
|
+
});
|
|
1422
|
+
}
|
|
1423
|
+
async sendResultPayloadDirect(payload) {
|
|
580
1424
|
try {
|
|
581
1425
|
await this.invokeAck("DispatchResult", payload);
|
|
582
1426
|
}
|
|
583
1427
|
catch (error) {
|
|
584
|
-
if (isDispatchNoLongerPending(error))
|
|
1428
|
+
if (isDispatchNoLongerPending(error))
|
|
585
1429
|
return;
|
|
586
|
-
}
|
|
587
1430
|
throw error;
|
|
588
1431
|
}
|
|
589
1432
|
}
|
|
1433
|
+
async flushDispatchResults(force = false) {
|
|
1434
|
+
if (!force && this.dispatchResultGeneration === this.dispatchResultFlushedGeneration)
|
|
1435
|
+
return;
|
|
1436
|
+
if (this.dispatchResultFlush) {
|
|
1437
|
+
await this.dispatchResultFlush;
|
|
1438
|
+
if (!force && this.dispatchResultGeneration === this.dispatchResultFlushedGeneration)
|
|
1439
|
+
return;
|
|
1440
|
+
}
|
|
1441
|
+
const flush = this.flushDispatchResultsCore();
|
|
1442
|
+
this.dispatchResultFlush = flush;
|
|
1443
|
+
try {
|
|
1444
|
+
await flush;
|
|
1445
|
+
}
|
|
1446
|
+
finally {
|
|
1447
|
+
if (this.dispatchResultFlush === flush)
|
|
1448
|
+
this.dispatchResultFlush = null;
|
|
1449
|
+
}
|
|
1450
|
+
}
|
|
1451
|
+
async flushDispatchResultsCore() {
|
|
1452
|
+
const startedGeneration = this.dispatchResultGeneration;
|
|
1453
|
+
while (true) {
|
|
1454
|
+
const records = await this.dispatchResultOutbox.pending();
|
|
1455
|
+
for (const pending of records) {
|
|
1456
|
+
// Periodic heartbeats and reconnect replay share the same release
|
|
1457
|
+
// barrier as the terminal writer. Restart replay has no live handoff.
|
|
1458
|
+
while (this.dispatchResultHandoffs.has(pending.payload.leaseId)) {
|
|
1459
|
+
await this.dispatchResultHandoffs.get(pending.payload.leaseId);
|
|
1460
|
+
}
|
|
1461
|
+
try {
|
|
1462
|
+
await this.sendResultPayloadDirect(pending.payload);
|
|
1463
|
+
await this.dispatchResultOutbox.acknowledge(pending.payload.leaseId);
|
|
1464
|
+
}
|
|
1465
|
+
catch (error) {
|
|
1466
|
+
if (isDispatchNoLongerPending(error)) {
|
|
1467
|
+
await this.dispatchResultOutbox.acknowledge(pending.payload.leaseId);
|
|
1468
|
+
continue;
|
|
1469
|
+
}
|
|
1470
|
+
throw error;
|
|
1471
|
+
}
|
|
1472
|
+
}
|
|
1473
|
+
if (records.length < 3)
|
|
1474
|
+
break;
|
|
1475
|
+
}
|
|
1476
|
+
if (this.dispatchResultGeneration === startedGeneration) {
|
|
1477
|
+
this.dispatchResultFlushedGeneration = startedGeneration;
|
|
1478
|
+
}
|
|
1479
|
+
}
|
|
590
1480
|
startHeartbeat() {
|
|
591
1481
|
this.stopHeartbeat();
|
|
592
1482
|
this.consecutiveHeartbeatFailures = 0;
|
|
@@ -605,6 +1495,8 @@ export class ForgeBridgeService {
|
|
|
605
1495
|
return;
|
|
606
1496
|
this.heartbeatInFlight = true;
|
|
607
1497
|
try {
|
|
1498
|
+
await this.flushDispatchResults();
|
|
1499
|
+
await this.flushReadinessResults();
|
|
608
1500
|
await this.sendHeartbeat();
|
|
609
1501
|
this.consecutiveHeartbeatFailures = 0;
|
|
610
1502
|
}
|
|
@@ -637,17 +1529,20 @@ export class ForgeBridgeService {
|
|
|
637
1529
|
}
|
|
638
1530
|
expireLeases() {
|
|
639
1531
|
const now = Date.now();
|
|
640
|
-
|
|
641
|
-
if (Date.parse(reservation.expiresAt) <= now || Date.parse(reservation.reservedUntil) <= now) {
|
|
642
|
-
this.reservations.delete(attemptId);
|
|
643
|
-
}
|
|
644
|
-
}
|
|
1532
|
+
this.expireReservations(now);
|
|
645
1533
|
for (const active of this.activeSessions.values()) {
|
|
646
1534
|
if (Date.parse(active.leaseExpiresAt) <= now && !active.controller.signal.aborted) {
|
|
647
1535
|
active.controller.abort("dispatch lease expired");
|
|
648
1536
|
}
|
|
649
1537
|
}
|
|
650
1538
|
}
|
|
1539
|
+
expireReservations(now) {
|
|
1540
|
+
for (const [attemptId, reservation] of this.reservations) {
|
|
1541
|
+
if (Date.parse(reservation.expiresAt) <= now || Date.parse(reservation.reservedUntil) <= now) {
|
|
1542
|
+
this.reservations.delete(attemptId);
|
|
1543
|
+
}
|
|
1544
|
+
}
|
|
1545
|
+
}
|
|
651
1546
|
stopHeartbeat() {
|
|
652
1547
|
if (this.heartbeatTimer) {
|
|
653
1548
|
clearInterval(this.heartbeatTimer);
|
|
@@ -676,6 +1571,7 @@ export class ForgeBridgeService {
|
|
|
676
1571
|
runnerId: this.config.runnerId,
|
|
677
1572
|
runnerInstanceId: this.config.runnerInstanceId,
|
|
678
1573
|
});
|
|
1574
|
+
this.rememberRecoveredHostAttempts(hostReap);
|
|
679
1575
|
if (hostReap && (hostReap.reaped > 0 || hostReap.degraded > 0)) {
|
|
680
1576
|
logBridgeStatus(`Managed-host maintenance reaped ${hostReap.reaped} stale attempt(s) and left ${hostReap.degraded} degraded attempt(s) for reconciliation.`);
|
|
681
1577
|
}
|
|
@@ -716,22 +1612,50 @@ export class ForgeBridgeService {
|
|
|
716
1612
|
}
|
|
717
1613
|
}
|
|
718
1614
|
async sendHeartbeat() {
|
|
1615
|
+
// Heartbeats are the recovery path for capacity-blocked work. Refresh the
|
|
1616
|
+
// dynamic sample before advertising it, while reusing the static Docker
|
|
1617
|
+
// capability snapshot established at startup or by an offer.
|
|
1618
|
+
if (this.config.dockerEnabled)
|
|
1619
|
+
await this.preflight.refreshCapacity();
|
|
1620
|
+
// A due probe can start a Docker Desktop recovery that runs for minutes.
|
|
1621
|
+
// Wait only as long as a normal probe takes, then advertise the last
|
|
1622
|
+
// snapshot so the server doesn't mark the runner offline meanwhile.
|
|
1623
|
+
const probeWait = new AbortController();
|
|
1624
|
+
await Promise.race([
|
|
1625
|
+
this.dockerEngineProbe.refresh().catch(() => undefined),
|
|
1626
|
+
delay(positiveFiniteNumber(this.config.heartbeatDockerProbeWaitMs ?? 15_000, 15_000), probeWait.signal),
|
|
1627
|
+
]);
|
|
1628
|
+
probeWait.abort();
|
|
1629
|
+
await this.sendHeartbeatSnapshot();
|
|
1630
|
+
}
|
|
1631
|
+
async sendHeartbeatSnapshot(signal) {
|
|
719
1632
|
await this.invokeAck("Heartbeat", {
|
|
720
1633
|
status: "online",
|
|
721
1634
|
capabilities: this.capabilities(),
|
|
722
1635
|
version: this.config.version,
|
|
723
1636
|
protocolVersion: this.config.protocolVersion
|
|
724
|
-
});
|
|
1637
|
+
}, { signal });
|
|
725
1638
|
}
|
|
726
1639
|
capabilities() {
|
|
727
1640
|
const preflight = this.preflight.current();
|
|
1641
|
+
const dockerEngine = this.dockerEngineProbe.current();
|
|
728
1642
|
const runtimeState = this.sandboxRuntime.snapshot();
|
|
729
1643
|
const reserved = [...this.reservations.values()].filter(reservation => reservation.sandboxed).length;
|
|
730
1644
|
const used = this.usedSandboxResources(runtimeState.degraded);
|
|
731
1645
|
const activeResources = this.activeSandboxResources(runtimeState.degraded);
|
|
732
1646
|
const capacity = this.sandboxResourceCapacity(preflight, activeResources);
|
|
1647
|
+
const memoryAdmission = preflight ? buildMemoryAdmissionDiagnostics({
|
|
1648
|
+
sampledAt: preflight.sampledAt,
|
|
1649
|
+
totalMemoryBytes: preflight.totalMemoryBytes,
|
|
1650
|
+
rawAvailableMemoryBytes: preflight.rawAvailableMemoryBytes,
|
|
1651
|
+
stabilizedAvailableMemoryBytes: preflight.availableMemoryBytes,
|
|
1652
|
+
controlPlaneMemoryBytes: process.memoryUsage().rss,
|
|
1653
|
+
reserveMemoryBytes: preflight.hostReserveBytes,
|
|
1654
|
+
committedMemoryBytes: used.memoryBytes,
|
|
1655
|
+
pendingCommitmentMemoryBytes: Math.max(0, used.memoryBytes - activeResources.memoryBytes),
|
|
1656
|
+
}) : null;
|
|
733
1657
|
return {
|
|
734
|
-
schemaVersion:
|
|
1658
|
+
schemaVersion: 6,
|
|
735
1659
|
activeSessionCount: this.activeSessions.size,
|
|
736
1660
|
cliProviders: this.config.cliProviders,
|
|
737
1661
|
providerStatuses: this.config.providerStatuses,
|
|
@@ -741,6 +1665,19 @@ export class ForgeBridgeService {
|
|
|
741
1665
|
sandboxProfiles: this.config.sandboxProfiles,
|
|
742
1666
|
runnerInstanceId: this.config.runnerInstanceId,
|
|
743
1667
|
trustDomainId: this.config.trustDomainId,
|
|
1668
|
+
executionFacilities: dockerEngine?.available ? ["docker_engine"] : [],
|
|
1669
|
+
dockerEngine,
|
|
1670
|
+
toolchains: {
|
|
1671
|
+
...this.hostCapabilities.toolchains,
|
|
1672
|
+
docker: {
|
|
1673
|
+
available: dockerEngine?.available === true,
|
|
1674
|
+
executable: this.config.dockerExecutable,
|
|
1675
|
+
version: dockerEngine?.runtimeVersion ?? null,
|
|
1676
|
+
},
|
|
1677
|
+
},
|
|
1678
|
+
playwrightProjects: this.hostCapabilities.playwrightProjects,
|
|
1679
|
+
browserLaunch: this.hostCapabilities.browserLaunch,
|
|
1680
|
+
providerConnectivity: this.hostCapabilities.providerConnectivity,
|
|
744
1681
|
maxConcurrentSandboxes: capacity.sandboxSlots,
|
|
745
1682
|
reservedSandboxCount: reserved,
|
|
746
1683
|
activeSandboxCount: runtimeState.active,
|
|
@@ -752,7 +1689,7 @@ export class ForgeBridgeService {
|
|
|
752
1689
|
allowsOpenPublicSandbox: this.config.allowOpenPublicSandbox,
|
|
753
1690
|
sandboxPublicationModes: [this.config.sandboxPublicationMode ?? "shared_filesystem"],
|
|
754
1691
|
sandboxAuthModes: sandboxAuthModesFor(this.config),
|
|
755
|
-
availableMemoryBytes:
|
|
1692
|
+
availableMemoryBytes: memoryAdmission?.availableForAdmissionBytes ?? null,
|
|
756
1693
|
availableCpuMillis: preflight ? Math.max(0, capacity.cpuMillis - used.cpuMillis) : null,
|
|
757
1694
|
availableWorkspaceBytes: preflight ? Math.max(0, capacity.workspaceBytes - used.workspaceBytes) : null,
|
|
758
1695
|
diskLow: preflight?.diskLow ?? null,
|
|
@@ -766,8 +1703,11 @@ export class ForgeBridgeService {
|
|
|
766
1703
|
hostReserveBytes: preflight.hostReserveBytes,
|
|
767
1704
|
activeReservationBytes: used.memoryBytes,
|
|
768
1705
|
daemonTotalBytes: preflight.totalMemoryBytes,
|
|
769
|
-
daemonAvailableBytes: preflight.
|
|
770
|
-
|
|
1706
|
+
daemonAvailableBytes: preflight.rawDaemonAvailableMemoryBytes ?? 0,
|
|
1707
|
+
hostAvailableBytes: preflight.rawHostAvailableMemoryBytes,
|
|
1708
|
+
conservativeAvailableBytes: preflight.rawAvailableMemoryBytes,
|
|
1709
|
+
availableForAdmissionBytes: memoryAdmission.availableForAdmissionBytes,
|
|
1710
|
+
memoryAdmission: memoryAdmission,
|
|
771
1711
|
} : null,
|
|
772
1712
|
hostRuntime: this.sessions.runtimeCapacity?.() ?? null,
|
|
773
1713
|
supportsApiSecretGrant: this.config.supportsApiSecretGrant,
|
|
@@ -776,9 +1716,20 @@ export class ForgeBridgeService {
|
|
|
776
1716
|
supportsRunnerAuthoredPhaseArtifacts: this.config.supportsRunnerAuthoredPhaseArtifacts,
|
|
777
1717
|
supportsDraftRunOnce: this.config.supportsDraftRunOnce,
|
|
778
1718
|
supportsDraftAttachments: this.config.supportsDraftAttachments,
|
|
779
|
-
supportsDraftImageAttachments: this.config.supportsDraftImageAttachments
|
|
1719
|
+
supportsDraftImageAttachments: this.config.supportsDraftImageAttachments,
|
|
1720
|
+
bridgeChat: bridgeChatCapabilityFor(this.config.cliProviders, this.config.providerStatuses, this.config.bridgeChat)
|
|
780
1721
|
};
|
|
781
1722
|
}
|
|
1723
|
+
async requiredCapabilityFailure(dispatch) {
|
|
1724
|
+
const requiredCapabilities = dispatch.qaScope?.required_capabilities ?? [];
|
|
1725
|
+
if (!requiredCapabilities.includes("docker_engine"))
|
|
1726
|
+
return null;
|
|
1727
|
+
const executionMode = dispatch.sandbox?.executionMode;
|
|
1728
|
+
if (executionMode === "docker") {
|
|
1729
|
+
return dockerEngineRequirementFailure(requiredCapabilities, executionMode, null);
|
|
1730
|
+
}
|
|
1731
|
+
return dockerEngineRequirementFailure(requiredCapabilities, executionMode, await this.dockerEngineProbe.refresh(true));
|
|
1732
|
+
}
|
|
782
1733
|
usedSandboxResources(degradedReservations) {
|
|
783
1734
|
const resources = [
|
|
784
1735
|
...[...this.reservations.values()]
|
|
@@ -811,9 +1762,7 @@ export class ForgeBridgeService {
|
|
|
811
1762
|
workerMemoryLimitBytes: 0,
|
|
812
1763
|
serviceMemoryLimitBytes: 0,
|
|
813
1764
|
runtimeOverheadBytes: 0,
|
|
814
|
-
memoryBytes:
|
|
815
|
-
- (this.config.sandboxHostMemoryReserveBytes ?? 2 * 1024 ** 3)
|
|
816
|
-
+ activeResources.memoryBytes),
|
|
1765
|
+
memoryBytes: sandboxMemoryCommitmentCapacity(preflight, activeResources.memoryBytes),
|
|
817
1766
|
workspaceBytes: Math.max(0, preflight.availableWorkspaceBytes
|
|
818
1767
|
- this.config.minimumWorkspaceBytes
|
|
819
1768
|
+ activeResources.workspaceBytes),
|
|
@@ -821,22 +1770,96 @@ export class ForgeBridgeService {
|
|
|
821
1770
|
buildSlots: this.config.maxConcurrentSandboxBuilds
|
|
822
1771
|
};
|
|
823
1772
|
}
|
|
824
|
-
async invokeAck(methodName, payload) {
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
1773
|
+
async invokeAck(methodName, payload, options = {}) {
|
|
1774
|
+
let storageRecoveryAttempt = 0;
|
|
1775
|
+
let callbackTimeoutAttempt = 0;
|
|
1776
|
+
while (true) {
|
|
1777
|
+
if (options.signal?.aborted) {
|
|
1778
|
+
throw new Error(String(options.signal.reason ?? `${methodName} callback was cancelled.`));
|
|
1779
|
+
}
|
|
1780
|
+
const connection = this.connection;
|
|
1781
|
+
if (!connection) {
|
|
1782
|
+
throw new Error("Bridge is not connected.");
|
|
1783
|
+
}
|
|
1784
|
+
await this.waitForTransportRecovery(connection, this.transportRecovery, options.signal);
|
|
1785
|
+
const recoveryGeneration = this.transportRecoveryGeneration;
|
|
1786
|
+
try {
|
|
1787
|
+
const ack = await waitForCallbackAck(connection.invoke(methodName, payload), methodName, positiveFiniteNumber(this.config.callbackAckTimeoutMs ?? 30_000, 30_000), options.signal);
|
|
1788
|
+
if (!ack.accepted) {
|
|
1789
|
+
throw new BridgeCallbackRejectedError(ack.code ?? null, ack.message ?? ack.code ?? `${methodName} callback was rejected.`);
|
|
1790
|
+
}
|
|
1791
|
+
return ack;
|
|
1792
|
+
}
|
|
1793
|
+
catch (error) {
|
|
1794
|
+
if (options.signal?.aborted)
|
|
1795
|
+
throw error;
|
|
1796
|
+
if (connection !== this.connection || this.stopped)
|
|
1797
|
+
throw error;
|
|
1798
|
+
const recovery = this.transportRecovery;
|
|
1799
|
+
if (recovery && recovery.generation > recoveryGeneration) {
|
|
1800
|
+
await this.waitForTransportRecovery(connection, recovery, options.signal);
|
|
1801
|
+
continue;
|
|
1802
|
+
}
|
|
1803
|
+
if (isProviderStorageUnavailable(error)) {
|
|
1804
|
+
const backoff = this.config.reconnectBackoffMs[Math.min(storageRecoveryAttempt, this.config.reconnectBackoffMs.length - 1)] ?? 5000;
|
|
1805
|
+
storageRecoveryAttempt += 1;
|
|
1806
|
+
logBridgeStatus(`${methodName} reached Forge while storage was recovering. Retrying callback ${storageRecoveryAttempt} in ${formatDelay(backoff)}...`);
|
|
1807
|
+
await delay(backoff, options.signal);
|
|
1808
|
+
continue;
|
|
1809
|
+
}
|
|
1810
|
+
if (options.replayCallbackTimeout && isBridgeCallbackTimeout(error)) {
|
|
1811
|
+
const backoff = this.config.reconnectBackoffMs[Math.min(callbackTimeoutAttempt, this.config.reconnectBackoffMs.length - 1)] ?? 5000;
|
|
1812
|
+
callbackTimeoutAttempt += 1;
|
|
1813
|
+
logBridgeStatus(`${methodName} acknowledgement timed out. Replaying the idempotent callback ${callbackTimeoutAttempt} in ${formatDelay(backoff)}...`);
|
|
1814
|
+
await delay(backoff, options.signal);
|
|
1815
|
+
continue;
|
|
1816
|
+
}
|
|
1817
|
+
throw error;
|
|
1818
|
+
}
|
|
828
1819
|
}
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
1820
|
+
}
|
|
1821
|
+
beginTransportRecovery() {
|
|
1822
|
+
if (this.transportRecovery && !this.transportRecovery.completed)
|
|
1823
|
+
return;
|
|
1824
|
+
let resolve = () => undefined;
|
|
1825
|
+
const promise = new Promise((complete) => { resolve = complete; });
|
|
1826
|
+
this.transportRecoveryGeneration += 1;
|
|
1827
|
+
this.transportRecovery = {
|
|
1828
|
+
generation: this.transportRecoveryGeneration,
|
|
1829
|
+
promise,
|
|
1830
|
+
resolve,
|
|
1831
|
+
completed: false,
|
|
1832
|
+
error: null,
|
|
1833
|
+
};
|
|
1834
|
+
}
|
|
1835
|
+
completeTransportRecovery(error = null) {
|
|
1836
|
+
const recovery = this.transportRecovery;
|
|
1837
|
+
if (!recovery || recovery.completed)
|
|
1838
|
+
return;
|
|
1839
|
+
recovery.completed = true;
|
|
1840
|
+
recovery.error = error;
|
|
1841
|
+
recovery.resolve();
|
|
1842
|
+
}
|
|
1843
|
+
async waitForTransportRecovery(connection, recovery, signal) {
|
|
1844
|
+
if (!recovery)
|
|
1845
|
+
return;
|
|
1846
|
+
if (recovery.completed) {
|
|
1847
|
+
if (recovery.error)
|
|
1848
|
+
throw recovery.error;
|
|
1849
|
+
return;
|
|
832
1850
|
}
|
|
833
|
-
|
|
1851
|
+
await waitForPromiseOrAbort(recovery.promise, signal);
|
|
1852
|
+
signal?.throwIfAborted();
|
|
1853
|
+
if (connection !== this.connection)
|
|
1854
|
+
throw new Error("Bridge connection changed while recovering.");
|
|
1855
|
+
if (recovery.error)
|
|
1856
|
+
throw recovery.error;
|
|
834
1857
|
}
|
|
835
1858
|
async waitForClose(signal) {
|
|
836
1859
|
if (!this.closed) {
|
|
837
1860
|
return;
|
|
838
1861
|
}
|
|
839
|
-
await
|
|
1862
|
+
await waitForPromiseOrAbort(this.closed, signal);
|
|
840
1863
|
}
|
|
841
1864
|
async stopConnection() {
|
|
842
1865
|
this.stopHeartbeat();
|
|
@@ -862,11 +1885,23 @@ function isDispatchNoLongerPending(error) {
|
|
|
862
1885
|
}
|
|
863
1886
|
return error instanceof Error && error.message === "Dispatch is no longer pending.";
|
|
864
1887
|
}
|
|
865
|
-
function
|
|
1888
|
+
function isReadinessNoLongerPending(error) {
|
|
1889
|
+
return error instanceof BridgeCallbackRejectedError
|
|
1890
|
+
&& (error.code === "bridge.stale_readiness_probe" || error.code === "bridge.stale_runner_instance");
|
|
1891
|
+
}
|
|
1892
|
+
function isProviderStorageUnavailable(error) {
|
|
1893
|
+
const message = errorMessage(error).toLowerCase();
|
|
1894
|
+
return message.includes("provider_storage_unavailable")
|
|
1895
|
+
|| (message.includes("storage is recovering") && message.includes("503"));
|
|
1896
|
+
}
|
|
1897
|
+
function isBridgeCallbackTimeout(error) {
|
|
1898
|
+
return errorMessage(error).startsWith("bridge_callback_timeout:");
|
|
1899
|
+
}
|
|
1900
|
+
function createDefaultSessionService(sandboxStateRoot, sandboxExecutionRoot, dockerEngineProbe) {
|
|
866
1901
|
const rawProcessRunner = new StreamingProviderProcessRunner();
|
|
867
1902
|
const processSupervisors = new AttemptProcessSupervisorRegistry();
|
|
868
1903
|
const coordinatedProcessRunner = new AttemptCoordinatedProviderProcessRunner(processSupervisors, rawProcessRunner);
|
|
869
|
-
return new BridgeSessionService(new GitWorktreeService(undefined, new RepositoryCoordinatorLock(sandboxStateRoot)), new RoutingProviderCliService(new ProcessBackedProviderCliService(coordinatedProcessRunner)), new HostRuntimeLeaseManager(sandboxStateRoot, undefined, undefined, processSupervisors, rawProcessRunner));
|
|
1904
|
+
return new BridgeSessionService(new GitWorktreeService(undefined, new RepositoryCoordinatorLock(sandboxStateRoot)), new RoutingProviderCliService(new ProcessBackedProviderCliService(coordinatedProcessRunner, undefined, undefined, undefined, new PrePublicationValidator(undefined, dockerEngineProbe, readNativeTestPostgresFallback))), new HostRuntimeLeaseManager(sandboxStateRoot, undefined, undefined, processSupervisors, rawProcessRunner, undefined, undefined, undefined, undefined, undefined, sandboxExecutionRoot));
|
|
870
1905
|
}
|
|
871
1906
|
export class SignalRBridgeConnectionFactory {
|
|
872
1907
|
create(config) {
|
|
@@ -890,10 +1925,11 @@ export class SignalRBridgeConnection {
|
|
|
890
1925
|
Authorization: `Bearer ${config.deviceToken}`
|
|
891
1926
|
}
|
|
892
1927
|
})
|
|
1928
|
+
.withAutomaticReconnect([...config.reconnectBackoffMs])
|
|
893
1929
|
.configureLogging(LogLevel.Warning)
|
|
894
1930
|
.build();
|
|
895
1931
|
this.connection.on("Superseded", () => {
|
|
896
|
-
void this.connection.stop();
|
|
1932
|
+
void this.connection.stop().catch(() => undefined);
|
|
897
1933
|
});
|
|
898
1934
|
}
|
|
899
1935
|
start() {
|
|
@@ -905,11 +1941,24 @@ export class SignalRBridgeConnection {
|
|
|
905
1941
|
onClose(handler) {
|
|
906
1942
|
this.connection.onclose(handler);
|
|
907
1943
|
}
|
|
1944
|
+
onReconnecting(handler) {
|
|
1945
|
+
this.connection.onreconnecting(handler);
|
|
1946
|
+
}
|
|
1947
|
+
onReconnected(handler) {
|
|
1948
|
+
this.connection.onreconnected((connectionId) => {
|
|
1949
|
+
void Promise.resolve(handler(connectionId)).catch(() => undefined);
|
|
1950
|
+
});
|
|
1951
|
+
}
|
|
908
1952
|
onDispatch(handler) {
|
|
909
1953
|
this.connection.on("Dispatch", (raw) => {
|
|
910
1954
|
void handler(raw).catch(() => undefined);
|
|
911
1955
|
});
|
|
912
1956
|
}
|
|
1957
|
+
onChatTurn(handler) {
|
|
1958
|
+
this.connection.on("ChatTurn", (raw) => {
|
|
1959
|
+
void handler(raw).catch(() => undefined);
|
|
1960
|
+
});
|
|
1961
|
+
}
|
|
913
1962
|
onOffer(handler) {
|
|
914
1963
|
this.connection.on("Offer", (raw) => {
|
|
915
1964
|
void handler(raw).catch(() => undefined);
|
|
@@ -947,9 +1996,7 @@ export class SignalRBridgeConnection {
|
|
|
947
1996
|
export function defaultBridgeServiceConfig(overrides = {}) {
|
|
948
1997
|
const defaults = defaultForgeBridgeConfig();
|
|
949
1998
|
const cliProviders = splitCsv(process.env.FORGE_CLI_PROVIDERS ?? "codex");
|
|
950
|
-
const
|
|
951
|
-
? 2 * 1024 ** 3
|
|
952
|
-
: 1024 ** 3;
|
|
1999
|
+
const configuredSandboxHostMemoryReserveBytes = optionalPositiveFiniteNumber(process.env.FORGE_SANDBOX_HOST_MEMORY_RESERVE_BYTES);
|
|
953
2000
|
const sandboxPublicationMode = process.env.FORGE_SANDBOX_PUBLICATION_MODE === "bridge_push" ? "bridge_push" : "shared_filesystem";
|
|
954
2001
|
const config = {
|
|
955
2002
|
serverUrl: process.env.FORGE_URL ?? "http://localhost:5208",
|
|
@@ -964,12 +2011,19 @@ export function defaultBridgeServiceConfig(overrides = {}) {
|
|
|
964
2011
|
sandboxPublicationMode,
|
|
965
2012
|
dockerExecutable: process.env.FORGE_DOCKER_EXECUTABLE ?? "docker",
|
|
966
2013
|
dockerContext: process.env.FORGE_DOCKER_CONTEXT ?? "",
|
|
2014
|
+
dockerDesktopRecoveryEnabled: process.env.FORGE_DOCKER_DESKTOP_AUTO_RECOVERY === "true",
|
|
2015
|
+
dockerRecoveryFailureThreshold: positiveFiniteNumber(Number(process.env.FORGE_DOCKER_RECOVERY_FAILURE_THRESHOLD ?? 3), 3),
|
|
2016
|
+
dockerRecoveryCooldownMs: positiveFiniteNumber(Number(process.env.FORGE_DOCKER_RECOVERY_COOLDOWN_MS ?? 10 * 60_000), 10 * 60_000),
|
|
2017
|
+
dockerRecoveryStartupTimeoutMs: positiveFiniteNumber(Number(process.env.FORGE_DOCKER_RECOVERY_STARTUP_TIMEOUT_MS ?? 2 * 60_000), 2 * 60_000),
|
|
967
2018
|
sandboxStateRoot: process.env.FORGE_SANDBOX_STATE_ROOT ?? path.join(os.homedir(), ".forge", "bridge", "sandboxes"),
|
|
2019
|
+
sandboxExecutionRoot: defaultHostExecutionRoot(),
|
|
968
2020
|
requireRootlessDocker: process.env.FORGE_REQUIRE_ROOTLESS_DOCKER === "true" || process.platform === "linux",
|
|
969
2021
|
diskHighWatermarkPercent: positiveFiniteNumber(Number(process.env.FORGE_DISK_HIGH_WATERMARK_PERCENT ?? 85), 85),
|
|
970
2022
|
diskLowWatermarkPercent: positiveFiniteNumber(Number(process.env.FORGE_DISK_LOW_WATERMARK_PERCENT ?? 75), 75),
|
|
971
2023
|
minimumWorkspaceBytes: positiveFiniteNumber(Number(process.env.FORGE_MINIMUM_WORKSPACE_BYTES ?? 10 * 1024 ** 3), 10 * 1024 ** 3),
|
|
972
|
-
|
|
2024
|
+
// Undefined means preflight derives the bounded reserve from the actual
|
|
2025
|
+
// Docker/host capacity domain after sampling it.
|
|
2026
|
+
sandboxHostMemoryReserveBytes: configuredSandboxHostMemoryReserveBytes,
|
|
973
2027
|
defaultSandboxWorkerMemoryLimitBytes: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_DEFAULT_WORKER_MEMORY_LIMIT_BYTES ?? 2 * 1024 ** 3), 2 * 1024 ** 3),
|
|
974
2028
|
defaultSandboxServiceMemoryLimitBytes: nonNegativeFiniteNumber(Number(process.env.FORGE_SANDBOX_DEFAULT_SERVICE_MEMORY_LIMIT_BYTES ?? 0), 0),
|
|
975
2029
|
sandboxRuntimeOverheadBytes: Math.max(128 * 1024 ** 2, nonNegativeFiniteNumber(Number(process.env.FORGE_SANDBOX_RUNTIME_OVERHEAD_BYTES ?? 128 * 1024 ** 2), 128 * 1024 ** 2)),
|
|
@@ -995,10 +2049,12 @@ export function defaultBridgeServiceConfig(overrides = {}) {
|
|
|
995
2049
|
supportsDraftRunOnce: true,
|
|
996
2050
|
supportsDraftAttachments: true,
|
|
997
2051
|
supportsDraftImageAttachments: supportsDraftImageAttachmentsFor(cliProviders),
|
|
2052
|
+
bridgeChat: bridgeChatCapabilityFor(cliProviders, providerStatusesFor(cliProviders)),
|
|
998
2053
|
version: process.env.FORGE_BRIDGE_VERSION ?? forgeBridgeVersion,
|
|
999
|
-
protocolVersion:
|
|
2054
|
+
protocolVersion: 4,
|
|
1000
2055
|
minServerVersion: null,
|
|
1001
2056
|
heartbeatIntervalMs: positiveFiniteNumber(Number(process.env.FORGE_HEARTBEAT_INTERVAL_MS ?? 30_000), 30_000),
|
|
2057
|
+
callbackAckTimeoutMs: positiveFiniteNumber(Number(process.env.FORGE_CALLBACK_ACK_TIMEOUT_MS ?? 30_000), 30_000),
|
|
1002
2058
|
reconnectBackoffMs: [1000, 2000, 5000, 10000, 30000],
|
|
1003
2059
|
...overrides
|
|
1004
2060
|
};
|
|
@@ -1007,7 +2063,9 @@ export function defaultBridgeServiceConfig(overrides = {}) {
|
|
|
1007
2063
|
providerStatuses: overrides.providerStatuses ?? providerStatusesFor(config.cliProviders),
|
|
1008
2064
|
sandboxProfiles: config.sandboxProfiles ?? defaultSandboxProfiles(config.cliProviders),
|
|
1009
2065
|
supportsDraftImageAttachments: (overrides.supportsDraftImageAttachments ?? true) && supportsDraftImageAttachmentsFor(config.cliProviders),
|
|
1010
|
-
|
|
2066
|
+
bridgeChat: overrides.bridgeChat ?? bridgeChatCapabilityFor(config.cliProviders, overrides.providerStatuses ?? providerStatusesFor(config.cliProviders), config.bridgeChat),
|
|
2067
|
+
heartbeatIntervalMs: positiveFiniteNumber(config.heartbeatIntervalMs, 30_000),
|
|
2068
|
+
callbackAckTimeoutMs: positiveFiniteNumber(config.callbackAckTimeoutMs ?? 30_000, 30_000),
|
|
1011
2069
|
};
|
|
1012
2070
|
}
|
|
1013
2071
|
export function resolveBridgeRunnerHubUrl(serverUrl) {
|
|
@@ -1022,6 +2080,12 @@ function normalizeServerUrl(serverUrl) {
|
|
|
1022
2080
|
function positiveFiniteNumber(value, fallback) {
|
|
1023
2081
|
return Number.isFinite(value) && value > 0 ? value : fallback;
|
|
1024
2082
|
}
|
|
2083
|
+
function optionalPositiveFiniteNumber(value) {
|
|
2084
|
+
if (value === undefined || value.trim().length === 0)
|
|
2085
|
+
return undefined;
|
|
2086
|
+
const parsed = Number(value);
|
|
2087
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
|
|
2088
|
+
}
|
|
1025
2089
|
function nonNegativeFiniteNumber(value, fallback) {
|
|
1026
2090
|
return Number.isFinite(value) && value >= 0 ? value : fallback;
|
|
1027
2091
|
}
|
|
@@ -1126,15 +2190,32 @@ function mappingKeys(mapping) {
|
|
|
1126
2190
|
mapping.localPath ? `path:${path.resolve(mapping.localPath).toLowerCase()}` : null
|
|
1127
2191
|
].filter((key) => Boolean(key));
|
|
1128
2192
|
}
|
|
1129
|
-
function findRepoMapping(
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
2193
|
+
function findRepoMapping(repo, mappings) {
|
|
2194
|
+
const usable = mappings.filter(mapping => Boolean(mapping.localPath?.trim()));
|
|
2195
|
+
const selectUnique = (candidates, identity) => {
|
|
2196
|
+
if (candidates.length > 1) {
|
|
2197
|
+
throw new Error(`bridge_repo_mapping_ambiguous: multiple local checkouts match ${identity}.`);
|
|
1133
2198
|
}
|
|
1134
|
-
return
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
2199
|
+
return candidates[0];
|
|
2200
|
+
};
|
|
2201
|
+
// Repository identity is authoritative. A same-name mapping for a different
|
|
2202
|
+
// repository must never win merely because it appears first in configuration.
|
|
2203
|
+
const byId = usable.filter(mapping => equalsIgnoreCase(mapping.repoId, repo.id));
|
|
2204
|
+
if (byId.length > 0)
|
|
2205
|
+
return selectUnique(byId, `repo id '${repo.id}'`);
|
|
2206
|
+
const byRemote = repo.remoteUrl?.trim()
|
|
2207
|
+
? usable.filter(mapping => !mapping.repoId?.trim()
|
|
2208
|
+
&& Boolean(mapping.remoteUrl?.trim())
|
|
2209
|
+
&& equalsIgnoreCase(mapping.remoteUrl, repo.remoteUrl))
|
|
2210
|
+
: [];
|
|
2211
|
+
if (byRemote.length > 0)
|
|
2212
|
+
return selectUnique(byRemote, `remote '${repo.remoteUrl}'`);
|
|
2213
|
+
const byName = usable.filter(mapping => !mapping.repoId?.trim()
|
|
2214
|
+
&& !mapping.remoteUrl?.trim()
|
|
2215
|
+
&& equalsIgnoreCase(mapping.name, repo.name));
|
|
2216
|
+
return byName.length > 0
|
|
2217
|
+
? selectUnique(byName, `repo name '${repo.name}'`)
|
|
2218
|
+
: undefined;
|
|
1138
2219
|
}
|
|
1139
2220
|
function normalizeQaAppLaunchTarget(target) {
|
|
1140
2221
|
if (!target) {
|
|
@@ -1183,6 +2264,10 @@ function invalidDispatchSummary(error) {
|
|
|
1183
2264
|
const detail = error instanceof Error ? error.message : String(error);
|
|
1184
2265
|
return `Invalid bridge dispatch packet: ${detail}`;
|
|
1185
2266
|
}
|
|
2267
|
+
function invalidChatTurnSummary(error) {
|
|
2268
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
2269
|
+
return `Invalid Bridge Chat turn packet: ${detail}`;
|
|
2270
|
+
}
|
|
1186
2271
|
function errorMessage(error) {
|
|
1187
2272
|
return error instanceof Error ? error.message : String(error);
|
|
1188
2273
|
}
|
|
@@ -1192,29 +2277,328 @@ function formatDelay(ms) {
|
|
|
1192
2277
|
}
|
|
1193
2278
|
return `${ms}ms`;
|
|
1194
2279
|
}
|
|
2280
|
+
async function waitForCallbackAck(callback, methodName, timeoutMs, signal) {
|
|
2281
|
+
let timeout = null;
|
|
2282
|
+
let rejectAbort = null;
|
|
2283
|
+
const expired = new Promise((_resolve, reject) => {
|
|
2284
|
+
timeout = setTimeout(() => {
|
|
2285
|
+
reject(new Error(`bridge_callback_timeout: ${methodName} callback was not acknowledged within ${formatDelay(timeoutMs)}.`));
|
|
2286
|
+
}, timeoutMs);
|
|
2287
|
+
});
|
|
2288
|
+
const aborted = new Promise((_resolve, reject) => {
|
|
2289
|
+
rejectAbort = reject;
|
|
2290
|
+
});
|
|
2291
|
+
const onAbort = () => rejectAbort?.(new Error(String(signal?.reason ?? `${methodName} callback was cancelled.`)));
|
|
2292
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
2293
|
+
if (signal?.aborted)
|
|
2294
|
+
onAbort();
|
|
2295
|
+
try {
|
|
2296
|
+
return await Promise.race(signal ? [callback, expired, aborted] : [callback, expired]);
|
|
2297
|
+
}
|
|
2298
|
+
finally {
|
|
2299
|
+
if (timeout)
|
|
2300
|
+
clearTimeout(timeout);
|
|
2301
|
+
signal?.removeEventListener("abort", onAbort);
|
|
2302
|
+
}
|
|
2303
|
+
}
|
|
1195
2304
|
function logBridgeStatus(message) {
|
|
1196
2305
|
process.stderr.write(`[${new Date().toISOString()}] ${message}\n`);
|
|
1197
2306
|
}
|
|
1198
|
-
function delay(ms, signal) {
|
|
2307
|
+
export function delay(ms, signal) {
|
|
1199
2308
|
if (signal?.aborted) {
|
|
1200
2309
|
return Promise.resolve();
|
|
1201
2310
|
}
|
|
1202
2311
|
return new Promise((resolve) => {
|
|
1203
|
-
const
|
|
1204
|
-
signal?.addEventListener("abort", () => {
|
|
2312
|
+
const complete = () => {
|
|
1205
2313
|
clearTimeout(timeout);
|
|
2314
|
+
signal?.removeEventListener("abort", complete);
|
|
1206
2315
|
resolve();
|
|
1207
|
-
}
|
|
2316
|
+
};
|
|
2317
|
+
const timeout = setTimeout(complete, ms);
|
|
2318
|
+
signal?.addEventListener("abort", complete, { once: true });
|
|
2319
|
+
if (signal?.aborted)
|
|
2320
|
+
complete();
|
|
1208
2321
|
});
|
|
1209
2322
|
}
|
|
1210
|
-
function
|
|
2323
|
+
export async function waitForPromiseOrAbort(promise, signal) {
|
|
1211
2324
|
if (!signal) {
|
|
1212
|
-
|
|
2325
|
+
await promise;
|
|
2326
|
+
return;
|
|
1213
2327
|
}
|
|
1214
2328
|
if (signal.aborted) {
|
|
1215
|
-
return
|
|
2329
|
+
return;
|
|
2330
|
+
}
|
|
2331
|
+
let resolveAbort;
|
|
2332
|
+
const abort = new Promise(resolve => { resolveAbort = resolve; });
|
|
2333
|
+
signal.addEventListener("abort", resolveAbort, { once: true });
|
|
2334
|
+
try {
|
|
2335
|
+
await Promise.race([promise, abort]);
|
|
2336
|
+
}
|
|
2337
|
+
finally {
|
|
2338
|
+
signal.removeEventListener("abort", resolveAbort);
|
|
2339
|
+
}
|
|
2340
|
+
}
|
|
2341
|
+
// Matches the API Hello bound on settled recovered host attempts.
|
|
2342
|
+
const maxRecoveredHostAttempts = 128;
|
|
2343
|
+
const defaultChatTurnLeaseMs = 60 * 60 * 1000;
|
|
2344
|
+
const maxChatEventContentChars = 4000;
|
|
2345
|
+
const maxChatTerminalTextChars = 120_000;
|
|
2346
|
+
const maxChatExactSecretChars = 8 * 1024;
|
|
2347
|
+
const chatTurnMetadataFileName = "forge-chat-metadata.json";
|
|
2348
|
+
async function materializeChatTurnMetadata(packet, workspacePath) {
|
|
2349
|
+
if (packet.attachmentHandoffs.length === 0 && packet.actionBrokerGrants.length === 0) {
|
|
2350
|
+
return;
|
|
2351
|
+
}
|
|
2352
|
+
await fs.promises.writeFile(path.join(workspacePath, chatTurnMetadataFileName), `${JSON.stringify({
|
|
2353
|
+
schemaVersion: 1,
|
|
2354
|
+
attachmentHandoffs: packet.attachmentHandoffs.map((handoff) => ({
|
|
2355
|
+
turnId: handoff.turnId,
|
|
2356
|
+
attachmentId: handoff.attachmentId,
|
|
2357
|
+
tempFileToken: handoff.tempFileToken,
|
|
2358
|
+
mimeType: handoff.mimeType,
|
|
2359
|
+
sizeBytes: handoff.sizeBytes,
|
|
2360
|
+
sha256: handoff.sha256,
|
|
2361
|
+
fileNameDisplay: handoff.fileNameDisplay,
|
|
2362
|
+
expiresAt: handoff.expiresAt,
|
|
2363
|
+
status: handoff.status,
|
|
2364
|
+
})),
|
|
2365
|
+
actionBrokerGrants: packet.actionBrokerGrants.map((grant) => ({
|
|
2366
|
+
grantId: grant.grantId,
|
|
2367
|
+
turnId: grant.turnId,
|
|
2368
|
+
leaseId: grant.leaseId,
|
|
2369
|
+
fencingToken: grant.fencingToken,
|
|
2370
|
+
allowedScopes: grant.allowedScopes,
|
|
2371
|
+
expiresAt: grant.expiresAt,
|
|
2372
|
+
redactionStatus: grant.redactionStatus,
|
|
2373
|
+
nonceHeldByBridge: true,
|
|
2374
|
+
})),
|
|
2375
|
+
}, null, 2)}\n`, "utf8");
|
|
2376
|
+
}
|
|
2377
|
+
function chatTurnLeaseExpiresAt(packet, nowMs = Date.now()) {
|
|
2378
|
+
const defaultExpiresAtMs = nowMs + defaultChatTurnLeaseMs;
|
|
2379
|
+
const metadataExpiresAtMs = [
|
|
2380
|
+
...packet.attachmentHandoffs.map(handoff => Date.parse(handoff.expiresAt)),
|
|
2381
|
+
...packet.actionBrokerGrants.map(grant => Date.parse(grant.expiresAt)),
|
|
2382
|
+
].filter(Number.isFinite);
|
|
2383
|
+
return new Date(Math.min(defaultExpiresAtMs, ...metadataExpiresAtMs)).toISOString();
|
|
2384
|
+
}
|
|
2385
|
+
async function cleanupChatTurnWorkspace(workspacePath, redactionSecrets) {
|
|
2386
|
+
try {
|
|
2387
|
+
await fs.promises.rm(workspacePath, { recursive: true, force: true, maxRetries: 3 });
|
|
2388
|
+
return {
|
|
2389
|
+
status: "completed",
|
|
2390
|
+
eventType: "chat_turn_cleanup_completed",
|
|
2391
|
+
role: "runner",
|
|
2392
|
+
kind: "checkpoint",
|
|
2393
|
+
content: "Bridge Chat workspace cleanup completed.",
|
|
2394
|
+
};
|
|
2395
|
+
}
|
|
2396
|
+
catch (error) {
|
|
2397
|
+
return {
|
|
2398
|
+
status: "failed",
|
|
2399
|
+
eventType: "chat_turn_cleanup_failed",
|
|
2400
|
+
role: "runner",
|
|
2401
|
+
kind: "error",
|
|
2402
|
+
content: "Bridge Chat workspace cleanup failed.",
|
|
2403
|
+
message: clip(redactText(errorMessage(error), redactionSecrets), 1000),
|
|
2404
|
+
};
|
|
2405
|
+
}
|
|
2406
|
+
}
|
|
2407
|
+
function buildChatProviderCommand(packet, cwd) {
|
|
2408
|
+
const provider = packet.providerKind.trim().toLowerCase();
|
|
2409
|
+
const model = packet.model.trim();
|
|
2410
|
+
const prompt = buildChatPrompt(packet);
|
|
2411
|
+
const resumeThreadId = packet.providerThreadId?.trim() || null;
|
|
2412
|
+
if (provider === "codex") {
|
|
2413
|
+
const args = ["-s", codexSandboxModeForPlatform(process.platform, "host"), "-a", "never", "exec"];
|
|
2414
|
+
if (resumeThreadId) {
|
|
2415
|
+
args.push("--cd", cwd, "resume");
|
|
2416
|
+
}
|
|
2417
|
+
args.push("--model", model, "--json");
|
|
2418
|
+
if (!resumeThreadId) {
|
|
2419
|
+
args.push("--cd", cwd);
|
|
2420
|
+
}
|
|
2421
|
+
args.push("--skip-git-repo-check");
|
|
2422
|
+
if (resumeThreadId) {
|
|
2423
|
+
args.push(resumeThreadId);
|
|
2424
|
+
}
|
|
2425
|
+
args.push("-");
|
|
2426
|
+
return { command: { command: "codex", args }, stdin: prompt };
|
|
2427
|
+
}
|
|
2428
|
+
if (provider === "cursor") {
|
|
2429
|
+
const args = ["--print", "--output-format", "stream-json", "--model", model, "--workspace", cwd];
|
|
2430
|
+
if (resumeThreadId) {
|
|
2431
|
+
args.push("--resume", resumeThreadId);
|
|
2432
|
+
}
|
|
2433
|
+
args.push("--yolo", "--approve-mcps", "--trust", prompt);
|
|
2434
|
+
return { command: { command: "cursor-agent", args } };
|
|
2435
|
+
}
|
|
2436
|
+
if (provider === "claude") {
|
|
2437
|
+
const args = ["-p", "--output-format", "stream-json", "--verbose", "--model", model, "--permission-mode", "bypassPermissions"];
|
|
2438
|
+
if (resumeThreadId) {
|
|
2439
|
+
args.push("--resume", resumeThreadId);
|
|
2440
|
+
}
|
|
2441
|
+
return { command: { command: "claude", args }, stdin: prompt };
|
|
2442
|
+
}
|
|
2443
|
+
return {
|
|
2444
|
+
command: {
|
|
2445
|
+
command: provider,
|
|
2446
|
+
args: ["--model", model, prompt],
|
|
2447
|
+
},
|
|
2448
|
+
};
|
|
2449
|
+
}
|
|
2450
|
+
function buildChatPrompt(packet) {
|
|
2451
|
+
const messages = packet.messages
|
|
2452
|
+
.map((message, index) => {
|
|
2453
|
+
const role = message.role.trim() || `message-${index + 1}`;
|
|
2454
|
+
return `[${role}]\n${message.content}`;
|
|
2455
|
+
})
|
|
2456
|
+
.join("\n\n");
|
|
2457
|
+
const contextRefs = packet.contextRefs.length === 0
|
|
2458
|
+
? ""
|
|
2459
|
+
: `\n\n# Context References\n\n${packet.contextRefs.map((ref, index) => {
|
|
2460
|
+
const title = ref.title?.trim() ? `\nTitle: ${ref.title.trim()}` : "";
|
|
2461
|
+
const summary = ref.summary?.trim() ? `\nSummary: ${ref.summary.trim()}` : "";
|
|
2462
|
+
return `${index + 1}. ${ref.kind}:${ref.refId}${title}${summary}`;
|
|
2463
|
+
}).join("\n\n")}`;
|
|
2464
|
+
const continuation = packet.providerThreadId
|
|
2465
|
+
? "\n\nContinue the existing provider thread identified by the bridge; do not restart the conversation context."
|
|
2466
|
+
: "";
|
|
2467
|
+
const boundedMetadata = buildChatBoundedMetadataPrompt(packet);
|
|
2468
|
+
return `You are handling a Forge Bridge Chat turn.
|
|
2469
|
+
|
|
2470
|
+
Conversation ID: ${packet.conversationId}
|
|
2471
|
+
Turn ID: ${packet.turnId}
|
|
2472
|
+
Provider: ${packet.providerKind}
|
|
2473
|
+
Model: ${packet.model}${continuation}${boundedMetadata}
|
|
2474
|
+
|
|
2475
|
+
# Conversation
|
|
2476
|
+
|
|
2477
|
+
${messages}${contextRefs}
|
|
2478
|
+
`;
|
|
2479
|
+
}
|
|
2480
|
+
function buildChatBoundedMetadataPrompt(packet) {
|
|
2481
|
+
if (packet.attachmentHandoffs.length === 0 && packet.actionBrokerGrants.length === 0) {
|
|
2482
|
+
return "";
|
|
2483
|
+
}
|
|
2484
|
+
const sections = [
|
|
2485
|
+
`\n\n# Bounded Handoff Metadata\n\nMetadata file: ${chatTurnMetadataFileName}\n\nThe metadata file is turn-scoped and cleaned up by the bridge when the turn ends. It contains opaque handles only; attachment content, raw paths, and grant nonces are not included.`,
|
|
2486
|
+
];
|
|
2487
|
+
if (packet.attachmentHandoffs.length > 0) {
|
|
2488
|
+
sections.push(`Attachment handoffs:\n${packet.attachmentHandoffs.map((handoff, index) => [
|
|
2489
|
+
`${index + 1}. turnId: ${handoff.turnId}`,
|
|
2490
|
+
` attachmentId: ${handoff.attachmentId}`,
|
|
2491
|
+
` tempFileToken: ${handoff.tempFileToken}`,
|
|
2492
|
+
` mimeType: ${handoff.mimeType}`,
|
|
2493
|
+
` sizeBytes: ${handoff.sizeBytes}`,
|
|
2494
|
+
` sha256: ${handoff.sha256}`,
|
|
2495
|
+
` fileNameDisplay: ${handoff.fileNameDisplay}`,
|
|
2496
|
+
` expiresAt: ${handoff.expiresAt}`,
|
|
2497
|
+
` status: ${handoff.status}`,
|
|
2498
|
+
].join("\n")).join("\n\n")}`);
|
|
2499
|
+
}
|
|
2500
|
+
if (packet.actionBrokerGrants.length > 0) {
|
|
2501
|
+
sections.push(`Action-broker grants:\n${packet.actionBrokerGrants.map((grant, index) => [
|
|
2502
|
+
`${index + 1}. grantId: ${grant.grantId}`,
|
|
2503
|
+
` turnId: ${grant.turnId}`,
|
|
2504
|
+
` leaseId: ${grant.leaseId}`,
|
|
2505
|
+
` fencingToken: ${grant.fencingToken}`,
|
|
2506
|
+
` allowedScopes: ${grant.allowedScopes.join(", ")}`,
|
|
2507
|
+
` expiresAt: ${grant.expiresAt}`,
|
|
2508
|
+
` redactionStatus: ${grant.redactionStatus}`,
|
|
2509
|
+
" nonce: held by bridge",
|
|
2510
|
+
].join("\n")).join("\n\n")}`);
|
|
2511
|
+
}
|
|
2512
|
+
return sections.join("\n\n");
|
|
2513
|
+
}
|
|
2514
|
+
function chatProviderExitFailure(command, result, redactionSecrets) {
|
|
2515
|
+
const commandText = redactText(formatProviderCliCommandForCheckpoint(command), redactionSecrets);
|
|
2516
|
+
const stderr = clip(redactText(result.stderr.trim(), redactionSecrets), 1000);
|
|
2517
|
+
const stdout = clip(redactText(result.stdout.trim(), redactionSecrets), 1000);
|
|
2518
|
+
const diagnostics = [
|
|
2519
|
+
stderr ? `stderr: ${stderr}` : null,
|
|
2520
|
+
stdout ? `stdout: ${stdout}` : null,
|
|
2521
|
+
].filter(Boolean).join(" ");
|
|
2522
|
+
return new Error(`provider_failed: ${commandText} exited with code ${result.exitCode}.${diagnostics ? ` ${diagnostics}` : ""}`);
|
|
2523
|
+
}
|
|
2524
|
+
function chatAbortKind(reason) {
|
|
2525
|
+
const message = typeof reason === "string" ? reason.toLowerCase() : "";
|
|
2526
|
+
return message.includes("expired") ? "expired" : "cancelled";
|
|
2527
|
+
}
|
|
2528
|
+
function chatAbortPayload(reason, cleanupOutcome, packet) {
|
|
2529
|
+
const kind = chatAbortKind(reason);
|
|
2530
|
+
const message = typeof reason === "string" && reason.trim()
|
|
2531
|
+
? reason.trim()
|
|
2532
|
+
: (kind === "expired" ? "Bridge Chat turn expired." : "Bridge Chat turn cancelled.");
|
|
2533
|
+
return {
|
|
2534
|
+
code: kind === "expired" ? "chat_turn_expired" : "chat_turn_cancelled",
|
|
2535
|
+
message: clip(redactText(message, packet ? chatRedactionSecrets(packet) : []), 1000),
|
|
2536
|
+
cleanup: cleanupOutcome,
|
|
2537
|
+
};
|
|
2538
|
+
}
|
|
2539
|
+
function providerThreadIdFromEvent(event) {
|
|
2540
|
+
if (!event.data) {
|
|
2541
|
+
return null;
|
|
2542
|
+
}
|
|
2543
|
+
try {
|
|
2544
|
+
const data = JSON.parse(event.data);
|
|
2545
|
+
for (const key of ["threadId", "thread_id", "sessionId", "session_id", "providerThreadId"]) {
|
|
2546
|
+
const value = data[key];
|
|
2547
|
+
if (typeof value === "string") {
|
|
2548
|
+
const clean = value.trim();
|
|
2549
|
+
if (clean.length > 0 && clean.length <= bridgeChatMaxProviderThreadIdChars) {
|
|
2550
|
+
return clean;
|
|
2551
|
+
}
|
|
2552
|
+
}
|
|
2553
|
+
}
|
|
2554
|
+
}
|
|
2555
|
+
catch {
|
|
2556
|
+
return null;
|
|
1216
2557
|
}
|
|
1217
|
-
return
|
|
2558
|
+
return null;
|
|
2559
|
+
}
|
|
2560
|
+
function chatRedactionSecrets(packet) {
|
|
2561
|
+
return [
|
|
2562
|
+
packet.chatRunId,
|
|
2563
|
+
packet.conversationId,
|
|
2564
|
+
packet.turnId,
|
|
2565
|
+
packet.leaseId,
|
|
2566
|
+
packet.fencingToken,
|
|
2567
|
+
packet.runnerInstanceId,
|
|
2568
|
+
packet.providerThreadId,
|
|
2569
|
+
...packet.attachmentHandoffs.flatMap((handoff) => [
|
|
2570
|
+
handoff.attachmentId,
|
|
2571
|
+
handoff.tempFileToken,
|
|
2572
|
+
]),
|
|
2573
|
+
...packet.actionBrokerGrants.flatMap((grant) => [
|
|
2574
|
+
grant.grantId,
|
|
2575
|
+
grant.nonce,
|
|
2576
|
+
]),
|
|
2577
|
+
...packet.messages.flatMap((message) => boundedChatSecretFragments(message.content)),
|
|
2578
|
+
...packet.contextRefs.flatMap((ref) => [
|
|
2579
|
+
...boundedChatSecretFragments(ref.title ?? ""),
|
|
2580
|
+
...boundedChatSecretFragments(ref.summary ?? ""),
|
|
2581
|
+
]),
|
|
2582
|
+
].filter((value) => typeof value === "string" && value.length > 0);
|
|
2583
|
+
}
|
|
2584
|
+
function boundedChatSecretFragments(value) {
|
|
2585
|
+
const clean = value.trim();
|
|
2586
|
+
if (!clean) {
|
|
2587
|
+
return [];
|
|
2588
|
+
}
|
|
2589
|
+
if (clean.length <= maxChatExactSecretChars) {
|
|
2590
|
+
return [clean];
|
|
2591
|
+
}
|
|
2592
|
+
const half = Math.floor(maxChatExactSecretChars / 2);
|
|
2593
|
+
return [clean.slice(0, half), clean.slice(clean.length - half)];
|
|
2594
|
+
}
|
|
2595
|
+
function safePathComponent(value) {
|
|
2596
|
+
const clean = value.trim().replace(/[^a-zA-Z0-9_.-]+/gu, "-").replace(/^-+|-+$/gu, "");
|
|
2597
|
+
return clean || "chat-turn";
|
|
2598
|
+
}
|
|
2599
|
+
function containsIgnoreCase(values, value) {
|
|
2600
|
+
const normalized = value.trim().toLowerCase();
|
|
2601
|
+
return values.some((candidate) => candidate.trim().toLowerCase() === normalized);
|
|
1218
2602
|
}
|
|
1219
2603
|
export function canReapManagedSandboxes(preflight) {
|
|
1220
2604
|
return canRunDockerMaintenance(preflight);
|
|
@@ -1234,24 +2618,15 @@ function sandboxNetworkPolicyFailure(dispatch, config) {
|
|
|
1234
2618
|
}
|
|
1235
2619
|
return null;
|
|
1236
2620
|
}
|
|
1237
|
-
function
|
|
1238
|
-
if (
|
|
1239
|
-
return
|
|
1240
|
-
if (
|
|
1241
|
-
return "
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
if (summary.startsWith("host_runtime_listener_timeout:"))
|
|
1247
|
-
return "host_runtime_listener_timeout";
|
|
1248
|
-
if (summary.startsWith("host_runtime_http_readiness_timeout:"))
|
|
1249
|
-
return "host_runtime_http_readiness_timeout";
|
|
1250
|
-
if (summary.startsWith("attempt_process_supervisor_start_failed:"))
|
|
1251
|
-
return "attempt_process_supervisor_start_failed";
|
|
1252
|
-
if (summary.startsWith("attempt_process_supervisor_start_timeout:"))
|
|
1253
|
-
return "attempt_process_supervisor_start_timeout";
|
|
1254
|
-
return null;
|
|
2621
|
+
export function dockerEngineRequirementFailure(requiredCapabilities, executionMode, capability) {
|
|
2622
|
+
if (!requiredCapabilities.includes("docker_engine"))
|
|
2623
|
+
return null;
|
|
2624
|
+
if (executionMode === "docker") {
|
|
2625
|
+
return "docker_engine_unavailable: docker_engine is not exposed inside Forge agent sandboxes; use a managed host runner with Docker-daemon access.";
|
|
2626
|
+
}
|
|
2627
|
+
return capability?.available
|
|
2628
|
+
? null
|
|
2629
|
+
: `docker_engine_unavailable: ${capability?.unavailableReason ?? "Docker daemon is unreachable from the bridge process."}`;
|
|
1255
2630
|
}
|
|
1256
2631
|
function parseReadinessProbe(value) {
|
|
1257
2632
|
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
@@ -1279,6 +2654,24 @@ function parseReadinessProbe(value) {
|
|
|
1279
2654
|
}
|
|
1280
2655
|
return request;
|
|
1281
2656
|
}
|
|
2657
|
+
function deferredReadinessResult(request, runnerInstanceId) {
|
|
2658
|
+
return {
|
|
2659
|
+
schemaVersion: 1,
|
|
2660
|
+
requestId: request.requestId,
|
|
2661
|
+
fencingToken: request.fencingToken,
|
|
2662
|
+
runnerInstanceId,
|
|
2663
|
+
status: "failed",
|
|
2664
|
+
reasonCode: "host_runtime_probe_deferred",
|
|
2665
|
+
reasonMessage: "Foreground agent work is active; Forge deferred background runtime readiness so it cannot consume the agent's host capacity.",
|
|
2666
|
+
checkedAt: new Date().toISOString(),
|
|
2667
|
+
};
|
|
2668
|
+
}
|
|
2669
|
+
function readinessDeferralReason(signal) {
|
|
2670
|
+
if (!signal.aborted)
|
|
2671
|
+
return null;
|
|
2672
|
+
const reason = errorMessage(signal.reason);
|
|
2673
|
+
return reason.startsWith("host_runtime_probe_deferred:") ? reason : null;
|
|
2674
|
+
}
|
|
1282
2675
|
function sandboxProviderNetworkFailure(dispatch, config) {
|
|
1283
2676
|
const sandbox = dispatch.sandbox;
|
|
1284
2677
|
if (!sandbox || sandbox.networkPolicy.mode !== "offline")
|
|
@@ -1308,6 +2701,15 @@ function sandboxAuthFailure(dispatch, config) {
|
|
|
1308
2701
|
const doctor = adapter.doctor();
|
|
1309
2702
|
return doctor.available ? null : doctor.message;
|
|
1310
2703
|
}
|
|
2704
|
+
function rejectedOfferReservation(code, message) {
|
|
2705
|
+
return {
|
|
2706
|
+
accepted: false,
|
|
2707
|
+
code,
|
|
2708
|
+
message,
|
|
2709
|
+
reservedUntil: null,
|
|
2710
|
+
resources: null,
|
|
2711
|
+
};
|
|
2712
|
+
}
|
|
1311
2713
|
export function canReserveSandboxResources(used, requested, capacity) {
|
|
1312
2714
|
return used.sandboxSlots + requested.sandboxSlots <= capacity.sandboxSlots
|
|
1313
2715
|
&& used.cpuMillis + requested.cpuMillis <= capacity.cpuMillis
|
|
@@ -1316,6 +2718,14 @@ export function canReserveSandboxResources(used, requested, capacity) {
|
|
|
1316
2718
|
&& used.pids + requested.pids <= capacity.pids
|
|
1317
2719
|
&& used.buildSlots + requested.buildSlots <= capacity.buildSlots;
|
|
1318
2720
|
}
|
|
2721
|
+
export function sandboxMemoryCommitmentCapacity(preflight, activeMemoryBytes) {
|
|
2722
|
+
const ambientCapacity = preflight.availableMemoryBytes
|
|
2723
|
+
- preflight.hostReserveBytes
|
|
2724
|
+
+ activeMemoryBytes;
|
|
2725
|
+
const commitmentCapacity = preflight.totalMemoryBytes
|
|
2726
|
+
- preflight.hostReserveBytes;
|
|
2727
|
+
return Math.max(0, Math.min(ambientCapacity, commitmentCapacity));
|
|
2728
|
+
}
|
|
1319
2729
|
export function sandboxCapacityFailureMessage(used, requested, capacity, preflight, config) {
|
|
1320
2730
|
const availableSlots = Math.max(0, capacity.sandboxSlots - used.sandboxSlots);
|
|
1321
2731
|
if (requested.sandboxSlots > availableSlots) {
|
|
@@ -1346,14 +2756,6 @@ export function sandboxCapacityFailureMessage(used, requested, capacity, preflig
|
|
|
1346
2756
|
const availableBuildSlots = Math.max(0, capacity.buildSlots - used.buildSlots);
|
|
1347
2757
|
return `Sandbox build capacity is insufficient: requested ${requested.buildSlots}, available ${availableBuildSlots}, ${used.buildSlots} already reserved.`;
|
|
1348
2758
|
}
|
|
1349
|
-
function formatBytes(bytes) {
|
|
1350
|
-
if (!Number.isFinite(bytes) || bytes <= 0)
|
|
1351
|
-
return "0 B";
|
|
1352
|
-
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
1353
|
-
const power = Math.min(units.length - 1, Math.floor(Math.log(bytes) / Math.log(1024)));
|
|
1354
|
-
const value = bytes / 1024 ** power;
|
|
1355
|
-
return `${value >= 10 || power === 0 ? value.toFixed(0) : value.toFixed(2)} ${units[power]}`;
|
|
1356
|
-
}
|
|
1357
2759
|
function sumSandboxResources(resources) {
|
|
1358
2760
|
return resources.reduce((total, resource) => ({
|
|
1359
2761
|
sandboxSlots: total.sandboxSlots + resource.sandboxSlots,
|
|
@@ -1425,8 +2827,16 @@ export async function resolveSandboxResourcesFromRepo(dispatch, baseRuntimeOverh
|
|
|
1425
2827
|
throw new Error("The approved immutable sandbox contract is unavailable in the mapped repository.");
|
|
1426
2828
|
}
|
|
1427
2829
|
const contract = validateSandboxContract(contractResult.stdout);
|
|
2830
|
+
const workspaceBytes = await estimateWorkspaceBytesFromRepo(repoPath, {
|
|
2831
|
+
role: dispatch.execution.role,
|
|
2832
|
+
serviceCount: contract.contract.services.length,
|
|
2833
|
+
browser: contract.contract.browser,
|
|
2834
|
+
});
|
|
1428
2835
|
return {
|
|
1429
|
-
resources: resolveDynamicSandboxResources(
|
|
2836
|
+
resources: resolveDynamicSandboxResources({
|
|
2837
|
+
...sandbox.requestedResources,
|
|
2838
|
+
workspaceBytes: Math.max(sandbox.requestedResources.workspaceBytes, workspaceBytes),
|
|
2839
|
+
}, contract.contract.services.length, sandbox.networkPolicy.mode, baseRuntimeOverheadBytes),
|
|
1430
2840
|
sandboxConfigCommitSha: configCommitSha,
|
|
1431
2841
|
};
|
|
1432
2842
|
}
|