@loucompanion/forge-bridge 0.1.3-dev.fbe15023cdfe → 0.1.4-dev.1ff37038366e
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +88 -2
- package/dist/bridge/bridge-chat.types.d.ts +111 -0
- package/dist/bridge/bridge-chat.types.js +401 -0
- package/dist/bridge/bridge.base.d.ts +4 -1
- package/dist/bridge/bridge.service.d.ts +72 -3
- package/dist/bridge/bridge.service.js +1563 -152
- package/dist/bridge/bridge.types.d.ts +46 -1
- package/dist/bridge/dispatch-result-outbox.d.ts +25 -0
- package/dist/bridge/dispatch-result-outbox.js +62 -0
- package/dist/bridge/dispatch-result-worker.d.ts +2 -0
- package/dist/bridge/dispatch-result-worker.js +122 -0
- package/dist/bridge/internals.d.ts +2 -0
- package/dist/bridge/internals.js +1 -0
- package/dist/bridge/readiness-result-outbox.d.ts +21 -0
- package/dist/bridge/readiness-result-outbox.js +54 -0
- package/dist/bridge/readiness-result-worker.d.ts +2 -0
- package/dist/bridge/readiness-result-worker.js +121 -0
- package/dist/bridge/runner-host-capabilities.d.ts +15 -0
- package/dist/bridge/runner-host-capabilities.js +93 -0
- package/dist/cleanup/cleanup.service.d.ts +1 -0
- package/dist/cleanup/cleanup.service.js +70 -9
- package/dist/cli/cli.help.js +2 -2
- package/dist/cli/cli.service.d.ts +14 -6
- package/dist/cli/cli.service.js +75 -16
- package/dist/cli/local-config.d.ts +1 -0
- package/dist/cli/local-config.js +4 -0
- package/dist/runtime/aspnet-runtime-prebuild.d.ts +36 -0
- package/dist/runtime/aspnet-runtime-prebuild.js +341 -0
- package/dist/runtime/attempt-database-broker.d.ts +89 -0
- package/dist/runtime/attempt-database-broker.js +390 -0
- package/dist/runtime/attempt-node-runtime-launcher.d.ts +2 -1
- package/dist/runtime/attempt-node-runtime-launcher.js +57 -18
- package/dist/runtime/attempt-path-operation.d.ts +21 -0
- package/dist/runtime/attempt-path-operation.js +94 -0
- package/dist/runtime/attempt-private-directory.d.ts +6 -0
- package/dist/runtime/attempt-private-directory.js +54 -0
- package/dist/runtime/attempt-process-supervisor.d.ts +11 -0
- package/dist/runtime/attempt-process-supervisor.js +67 -20
- package/dist/runtime/canonical-source-file.d.ts +3 -0
- package/dist/runtime/canonical-source-file.js +54 -0
- package/dist/runtime/docker-engine-probe.d.ts +90 -0
- package/dist/runtime/docker-engine-probe.js +333 -0
- package/dist/runtime/generated-output-reset.d.ts +10 -0
- package/dist/runtime/generated-output-reset.js +62 -8
- package/dist/runtime/host-database-capability-boundary.d.ts +16 -0
- package/dist/runtime/host-database-capability-boundary.js +62 -0
- package/dist/runtime/host-diagnostics.d.ts +11 -0
- package/dist/runtime/host-diagnostics.js +191 -0
- package/dist/runtime/host-execution-root.d.ts +4 -0
- package/dist/runtime/host-execution-root.js +39 -0
- package/dist/runtime/host-listener-inspector.d.ts +8 -1
- package/dist/runtime/host-listener-inspector.js +121 -60
- package/dist/runtime/host-runtime-contract.d.ts +12 -1
- package/dist/runtime/host-runtime-contract.js +168 -55
- package/dist/runtime/host-runtime-lease.d.ts +92 -8
- package/dist/runtime/host-runtime-lease.js +1141 -114
- package/dist/runtime/host-runtime-promotion.d.ts +12 -3
- package/dist/runtime/host-runtime-promotion.js +59 -12
- package/dist/runtime/memory-capacity.d.ts +70 -0
- package/dist/runtime/memory-capacity.js +123 -0
- package/dist/runtime/native-test-postgres-fallback.d.ts +25 -0
- package/dist/runtime/native-test-postgres-fallback.js +91 -0
- package/dist/runtime/node-host-runtime-adapter.d.ts +3 -0
- package/dist/runtime/node-host-runtime-adapter.js +28 -11
- package/dist/runtime/pinned-directory-process.d.ts +9 -0
- package/dist/runtime/pinned-directory-process.js +61 -0
- package/dist/runtime/pinned-source-directory.d.ts +17 -0
- package/dist/runtime/pinned-source-directory.js +94 -0
- package/dist/runtime/playwright-host-runtime-adapter.d.ts +1 -0
- package/dist/runtime/playwright-host-runtime-adapter.js +53 -5
- package/dist/runtime/prepared-runtime-identity.d.ts +6 -0
- package/dist/runtime/prepared-runtime-identity.js +218 -0
- package/dist/runtime/prepared-runtime-readiness.d.ts +30 -0
- package/dist/runtime/prepared-runtime-readiness.js +285 -0
- package/dist/runtime/runtime-isolation-lease.js +4 -1
- package/dist/runtime/workspace-resource-estimator.d.ts +17 -0
- package/dist/runtime/workspace-resource-estimator.js +174 -0
- package/dist/sandbox/preflight.d.ts +31 -1
- package/dist/sandbox/preflight.js +237 -51
- package/dist/sandbox/repository-lock.d.ts +2 -1
- package/dist/sandbox/repository-lock.js +17 -8
- package/dist/sandbox/runtime.d.ts +6 -1
- package/dist/sandbox/runtime.js +132 -103
- package/dist/sandbox/sandbox.types.d.ts +7 -0
- package/dist/sandbox/workspace.service.js +2 -0
- package/dist/session/managed-attempt-tools.d.ts +17 -1
- package/dist/session/managed-attempt-tools.js +485 -31
- package/dist/session/managed-build-coordinator.d.ts +5 -0
- package/dist/session/managed-build-coordinator.js +105 -0
- package/dist/session/managed-build-receipt.d.ts +21 -0
- package/dist/session/managed-build-receipt.js +337 -0
- package/dist/session/managed-verification-timeouts.d.ts +8 -0
- package/dist/session/managed-verification-timeouts.js +17 -0
- package/dist/session/phase-completion.artifacts.d.ts +17 -0
- package/dist/session/phase-completion.artifacts.js +224 -22
- package/dist/session/phase-completion.d.ts +11 -1
- package/dist/session/phase-completion.helpers.d.ts +5 -0
- package/dist/session/phase-completion.helpers.js +30 -7
- package/dist/session/phase-completion.js +14 -3
- package/dist/session/phase-completion.prompts.js +17 -18
- package/dist/session/phase-completion.runtime.d.ts +3 -1
- package/dist/session/phase-completion.runtime.js +126 -17
- package/dist/session/phase-completion.schemas.js +49 -2
- package/dist/session/phase-completion.validation.d.ts +1 -0
- package/dist/session/phase-completion.validation.js +111 -21
- package/dist/session/pre-publication-validation.d.ts +75 -0
- package/dist/session/pre-publication-validation.js +661 -0
- package/dist/session/provider-cli/bridge-cli.conformance-fixtures.d.ts +72 -0
- package/dist/session/provider-cli/bridge-cli.conformance-fixtures.js +462 -0
- package/dist/session/provider-cli/ci-repair-readiness.d.ts +8 -0
- package/dist/session/provider-cli/ci-repair-readiness.js +23 -3
- package/dist/session/provider-cli/provider-cli.artifacts.d.ts +4 -2
- package/dist/session/provider-cli/provider-cli.artifacts.js +47 -13
- package/dist/session/provider-cli/provider-cli.deterministic.js +34 -2
- package/dist/session/provider-cli/provider-cli.process.d.ts +51 -0
- package/dist/session/provider-cli/provider-cli.process.js +422 -22
- package/dist/session/provider-cli/provider-cli.prompts.d.ts +5 -0
- package/dist/session/provider-cli/provider-cli.prompts.js +136 -16
- package/dist/session/provider-cli/provider-cli.repair-epoch.d.ts +6 -0
- package/dist/session/provider-cli/provider-cli.repair-epoch.js +11 -0
- package/dist/session/provider-cli/provider-cli.runtime.d.ts +8 -1
- package/dist/session/provider-cli/provider-cli.runtime.js +43 -3
- package/dist/session/provider-cli/provider-cli.service.d.ts +30 -5
- package/dist/session/provider-cli/provider-cli.service.js +771 -68
- package/dist/session/provider-cli/provider-cli.types.d.ts +2 -0
- package/dist/session/provider-cli/provider-output-state.d.ts +10 -0
- package/dist/session/provider-cli/provider-output-state.js +361 -17
- package/dist/session/provider-cli/provider-usage.d.ts +7 -0
- package/dist/session/provider-cli/provider-usage.js +28 -5
- package/dist/session/session-redaction.js +434 -9
- package/dist/session/session.service.js +145 -23
- package/dist/session/session.types.d.ts +147 -2
- package/dist/session/session.types.js +587 -7
- package/dist/session/unreferenced-exports.d.ts +12 -0
- package/dist/session/unreferenced-exports.js +59 -0
- package/dist/shared/format/format-bytes.d.ts +1 -0
- package/dist/shared/format/format-bytes.js +9 -0
- package/dist/shared/git/git.js +2 -1
- package/dist/shared/path/path.d.ts +1 -0
- package/dist/shared/path/path.js +18 -0
- package/dist/shared/process/dotnet-toolchain.d.ts +17 -0
- package/dist/shared/process/dotnet-toolchain.js +307 -0
- package/dist/shared/process/host-process-supervisor.d.ts +2 -0
- package/dist/shared/process/host-process-supervisor.js +57 -9
- package/dist/shared/process/process.d.ts +2 -0
- package/dist/shared/process/process.js +12 -7
- package/dist/shared/provider-capabilities.d.ts +3 -0
- package/dist/shared/provider-capabilities.js +85 -0
- package/dist/shared/redaction/redaction.d.ts +1 -0
- package/dist/shared/redaction/redaction.js +39 -3
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/worktree/dependency-warmer.d.ts +41 -9
- package/dist/worktree/dependency-warmer.js +497 -22
- package/dist/worktree/index.d.ts +1 -1
- package/dist/worktree/story-qa-reference-hydrator.d.ts +4 -0
- package/dist/worktree/story-qa-reference-hydrator.js +253 -0
- package/dist/worktree/worktree.service.d.ts +7 -0
- package/dist/worktree/worktree.service.js +253 -21
- package/dist/worktree/worktree.types.d.ts +7 -1
- package/package.json +7 -5
- package/runtime/windows-job-supervisor.ps1 +380 -14
|
@@ -4,25 +4,55 @@ import path from "node:path";
|
|
|
4
4
|
import { randomUUID } from "node:crypto";
|
|
5
5
|
import os from "node:os";
|
|
6
6
|
import { defaultForgeBridgeConfig } from "../config.js";
|
|
7
|
-
import { supportsDraftImageAttachmentsFor } from "../shared/provider-capabilities.js";
|
|
7
|
+
import { bridgeChatCapabilityFor, supportsDraftImageAttachmentsFor } from "../shared/provider-capabilities.js";
|
|
8
8
|
import { redactText } from "../shared/redaction/redaction.js";
|
|
9
9
|
import { BridgeSessionService } from "../session/session.service.js";
|
|
10
10
|
import { ProcessBackedProviderCliService, RoutingProviderCliService, StreamingProviderProcessRunner, } from "../session/provider-cli/provider-cli.service.js";
|
|
11
|
+
import { codexSandboxModeForPlatform, formatProviderCliCommandForCheckpoint } from "../session/provider-cli/provider-cli.prompts.js";
|
|
12
|
+
import { clip, createProviderOutputState } from "../session/provider-cli/provider-output-state.js";
|
|
11
13
|
import { redactRunnerDispatchEvent, redactRunOnceResultPacket } from "../session/session-redaction.js";
|
|
12
|
-
import {
|
|
14
|
+
import { PrePublicationValidator } from "../session/pre-publication-validation.js";
|
|
15
|
+
import { HostRuntimeLeaseManager, } from "../runtime/host-runtime-lease.js";
|
|
16
|
+
import { assertHostDatabaseCapabilityBoundary, establishHostDatabaseCapabilityBoundary, } from "../runtime/host-database-capability-boundary.js";
|
|
17
|
+
import { defaultHostExecutionRoot } from "../runtime/host-execution-root.js";
|
|
18
|
+
import { buildMemoryAdmissionDiagnostics } from "../runtime/memory-capacity.js";
|
|
19
|
+
import { DockerEngineProbe } from "../runtime/docker-engine-probe.js";
|
|
20
|
+
import { createNativeTestClusterDatabaseBroker } from "../runtime/attempt-database-broker.js";
|
|
21
|
+
import { readNativeTestPostgresFallback } from "../runtime/native-test-postgres-fallback.js";
|
|
22
|
+
import { estimateWorkspaceBytesFromRepo } from "../runtime/workspace-resource-estimator.js";
|
|
13
23
|
import { GitWorktreeService } from "../worktree/worktree.service.js";
|
|
14
24
|
import { RepositoryCoordinatorLock } from "../sandbox/repository-lock.js";
|
|
15
25
|
import { AttemptCoordinatedProviderProcessRunner, AttemptProcessSupervisorRegistry, } from "../runtime/attempt-process-supervisor-registry.js";
|
|
16
|
-
import { failureResult, parseRunOnceDispatchPacket } from "../session/session.types.js";
|
|
26
|
+
import { failureFactsForCode, failureFactsFromError, failureResult, parseRunOnceDispatchPacket } from "../session/session.types.js";
|
|
17
27
|
import { forgeBridgeVersion } from "../version.js";
|
|
18
28
|
import { canRunDockerMaintenance, DockerPreflight } from "../sandbox/preflight.js";
|
|
19
29
|
import { validateSandboxContract } from "../sandbox/contract.js";
|
|
20
30
|
import { runSandboxProcess } from "../sandbox/process.js";
|
|
21
31
|
import { DockerSandboxRuntime, selectSandboxConfigRefs } from "../sandbox/runtime.js";
|
|
22
32
|
import { codexSandboxProviderAuthAdapter, sandboxProviderAuthAdapterFor, sandboxProviderAuthUnavailableMessage } from "../sandbox/provider-auth/codex.js";
|
|
33
|
+
import { assertBridgeChatCallbackPayload, bridgeChatMaxProviderThreadIdChars, bridgeChatProtocolVersion, isTerminalChatTurnCallbackKind, parseChatTurnDispatchPacket, } from "./bridge-chat.types.js";
|
|
34
|
+
import { formatBytes } from "../shared/format/format-bytes.js";
|
|
35
|
+
import { emptyRunnerHostCapabilitySnapshot, ProcessRunnerHostCapabilityProbe, } from "./runner-host-capabilities.js";
|
|
36
|
+
import { FileReadinessResultOutbox } from "./readiness-result-outbox.js";
|
|
37
|
+
import { FileDispatchResultOutbox } from "./dispatch-result-outbox.js";
|
|
38
|
+
/**
|
|
39
|
+
* Protocol-v2 events are acknowledged one at a time before runOnce returns.
|
|
40
|
+
* Sending the complete transcript again in DispatchResult made long QA and CI
|
|
41
|
+
* sessions exceed SignalR's receive limit, disconnecting the bridge after the
|
|
42
|
+
* provider had already completed. The API combines its acknowledged streamed
|
|
43
|
+
* events with the terminal result, so the callback only needs the outcome and
|
|
44
|
+
* artifacts. Protocol v1 retains the transcript for legacy servers.
|
|
45
|
+
*/
|
|
46
|
+
export function compactTerminalResult(dispatch, result) {
|
|
47
|
+
return dispatch.schemaVersion >= 2 && result.events.length > 0
|
|
48
|
+
? { ...result, events: [] }
|
|
49
|
+
: result;
|
|
50
|
+
}
|
|
23
51
|
export class ForgeBridgeService {
|
|
24
52
|
config;
|
|
25
53
|
connections;
|
|
54
|
+
hostCapabilityProbe;
|
|
55
|
+
chatProcessRunner;
|
|
26
56
|
connection = null;
|
|
27
57
|
heartbeatTimer = null;
|
|
28
58
|
heartbeatInFlight = false;
|
|
@@ -32,18 +62,40 @@ export class ForgeBridgeService {
|
|
|
32
62
|
sandboxMaintenanceTimer = null;
|
|
33
63
|
sandboxMaintenanceRunning = false;
|
|
34
64
|
closed = null;
|
|
65
|
+
transportRecovery = null;
|
|
66
|
+
transportRecoveryGeneration = 0;
|
|
35
67
|
stopped = false;
|
|
36
68
|
activeSessions = new Map();
|
|
37
69
|
reservations = new Map();
|
|
38
70
|
activeReadinessProbes = new Map();
|
|
71
|
+
readinessResultOutbox;
|
|
72
|
+
readinessResultFlush = null;
|
|
73
|
+
dispatchResultOutbox;
|
|
74
|
+
dispatchResultFlush = null;
|
|
75
|
+
dispatchResultGeneration = 0;
|
|
76
|
+
dispatchResultFlushedGeneration = -1;
|
|
77
|
+
dispatchResultHandoffs = new Map();
|
|
39
78
|
preflight;
|
|
79
|
+
dockerEngineProbe;
|
|
40
80
|
sandboxRuntime;
|
|
41
81
|
sandboxResourceResolver;
|
|
42
82
|
sessions;
|
|
43
|
-
|
|
83
|
+
enforceHostDatabaseCapabilityBoundary;
|
|
84
|
+
hostCapabilities = emptyRunnerHostCapabilitySnapshot();
|
|
85
|
+
// Reaping deletes the stale ledger, so its stopped-process proof is held here
|
|
86
|
+
// until a Hello carrying it is accepted. A bridge restart forgets it, which
|
|
87
|
+
// leaves those attempts on Forge's conservative unknown-outcome path.
|
|
88
|
+
recoveredHostAttempts = new Map();
|
|
89
|
+
hostCapabilitiesDetected = false;
|
|
90
|
+
constructor(config, sessions = undefined, connections = new SignalRBridgeConnectionFactory(), preflight, sandboxRuntime, sandboxResourceResolver, dockerEngineProbe, hostCapabilityProbe = new ProcessRunnerHostCapabilityProbe(), readinessResultOutbox, chatProcessRunner = new StreamingProviderProcessRunner(), dispatchResultOutbox) {
|
|
44
91
|
this.config = config;
|
|
45
92
|
this.connections = connections;
|
|
46
|
-
this.
|
|
93
|
+
this.hostCapabilityProbe = hostCapabilityProbe;
|
|
94
|
+
this.chatProcessRunner = chatProcessRunner;
|
|
95
|
+
this.enforceHostDatabaseCapabilityBoundary = sessions === undefined;
|
|
96
|
+
if (this.enforceHostDatabaseCapabilityBoundary) {
|
|
97
|
+
establishHostDatabaseCapabilityBoundary();
|
|
98
|
+
}
|
|
47
99
|
this.preflight = preflight ?? new DockerPreflight({
|
|
48
100
|
dockerExecutable: config.dockerExecutable,
|
|
49
101
|
dockerContext: config.dockerContext,
|
|
@@ -59,17 +111,36 @@ export class ForgeBridgeService {
|
|
|
59
111
|
runtimeOverheadBytes: config.sandboxRuntimeOverheadBytes,
|
|
60
112
|
requireRootless: config.requireRootlessDocker
|
|
61
113
|
});
|
|
114
|
+
this.dockerEngineProbe = dockerEngineProbe ?? new DockerEngineProbe({
|
|
115
|
+
dockerExecutable: config.dockerExecutable,
|
|
116
|
+
dockerContext: config.dockerContext,
|
|
117
|
+
recovery: {
|
|
118
|
+
enabled: config.dockerDesktopRecoveryEnabled,
|
|
119
|
+
failureThreshold: config.dockerRecoveryFailureThreshold,
|
|
120
|
+
cooldownMs: config.dockerRecoveryCooldownMs,
|
|
121
|
+
startupTimeoutMs: config.dockerRecoveryStartupTimeoutMs,
|
|
122
|
+
canRecover: () => this.activeSessions.size === 0 && this.reservations.size === 0,
|
|
123
|
+
},
|
|
124
|
+
onStatus: logBridgeStatus,
|
|
125
|
+
});
|
|
126
|
+
this.sessions = sessions ?? createDefaultSessionService(config.sandboxStateRoot, config.sandboxExecutionRoot, this.dockerEngineProbe);
|
|
62
127
|
this.sandboxRuntime = sandboxRuntime ?? new DockerSandboxRuntime(config, (request, signal) => this.redeemPublishGrant(request, signal));
|
|
63
128
|
this.sandboxResourceResolver = sandboxResourceResolver
|
|
64
129
|
?? (dispatch => resolveSandboxResourcesFromRepo(dispatch, this.config.sandboxRuntimeOverheadBytes));
|
|
130
|
+
this.readinessResultOutbox = readinessResultOutbox ?? new FileReadinessResultOutbox(config.sandboxStateRoot);
|
|
131
|
+
this.dispatchResultOutbox = dispatchResultOutbox ?? new FileDispatchResultOutbox(config.sandboxStateRoot);
|
|
65
132
|
}
|
|
66
133
|
async connectOnce() {
|
|
67
134
|
await this.stopConnection();
|
|
68
135
|
this.stopped = false;
|
|
136
|
+
if (this.enforceHostDatabaseCapabilityBoundary) {
|
|
137
|
+
assertHostDatabaseCapabilityBoundary();
|
|
138
|
+
}
|
|
69
139
|
const hostReap = await this.sessions.reapHostRuntime?.({
|
|
70
140
|
runnerId: this.config.runnerId,
|
|
71
141
|
runnerInstanceId: this.config.runnerInstanceId,
|
|
72
142
|
});
|
|
143
|
+
this.rememberRecoveredHostAttempts(hostReap);
|
|
73
144
|
if (hostReap && (hostReap.reaped > 0 || hostReap.degraded > 0)) {
|
|
74
145
|
logBridgeStatus(`Managed-host recovery reaped ${hostReap.reaped} stale attempt(s), protected ${hostReap.reusedPidsProtected} reused PID(s), and left ${hostReap.degraded} degraded attempt(s) for reconciliation.`);
|
|
75
146
|
}
|
|
@@ -80,17 +151,54 @@ export class ForgeBridgeService {
|
|
|
80
151
|
await this.preflight.refresh();
|
|
81
152
|
}
|
|
82
153
|
}
|
|
154
|
+
await this.dockerEngineProbe.refresh(true);
|
|
155
|
+
if (!this.hostCapabilitiesDetected) {
|
|
156
|
+
this.hostCapabilities = await this.hostCapabilityProbe.detect(this.config);
|
|
157
|
+
this.hostCapabilitiesDetected = true;
|
|
158
|
+
}
|
|
83
159
|
const connection = this.connections.create(this.config);
|
|
84
160
|
this.connection = connection;
|
|
161
|
+
this.transportRecovery = null;
|
|
85
162
|
this.closed = new Promise((resolve) => {
|
|
86
|
-
connection.onClose(() => {
|
|
163
|
+
connection.onClose((error) => {
|
|
87
164
|
this.stopHeartbeat();
|
|
165
|
+
this.completeTransportRecovery(error ?? new Error("bridge connection closed"));
|
|
88
166
|
this.abortActiveSessions("bridge connection closed");
|
|
89
167
|
this.abortActiveReadinessProbes("bridge connection closed");
|
|
90
168
|
resolve();
|
|
91
169
|
});
|
|
92
170
|
});
|
|
171
|
+
connection.onReconnecting?.((error) => {
|
|
172
|
+
if (connection !== this.connection || this.stopped)
|
|
173
|
+
return;
|
|
174
|
+
this.stopHeartbeat();
|
|
175
|
+
this.beginTransportRecovery();
|
|
176
|
+
logBridgeStatus(`Forge connection interrupted${error ? `: ${errorMessage(error)}` : ""}. Reconnecting in place; active dispatches remain supervised...`);
|
|
177
|
+
});
|
|
178
|
+
connection.onReconnected?.(async () => {
|
|
179
|
+
if (connection !== this.connection || this.stopped)
|
|
180
|
+
return;
|
|
181
|
+
try {
|
|
182
|
+
await this.registerConnection(connection);
|
|
183
|
+
this.completeTransportRecovery();
|
|
184
|
+
this.startHeartbeat();
|
|
185
|
+
await this.flushDispatchResults(true).catch(error => {
|
|
186
|
+
logBridgeStatus(`Dispatch result replay deferred after reconnect: ${errorMessage(error)}`);
|
|
187
|
+
});
|
|
188
|
+
await this.flushReadinessResults().catch(error => {
|
|
189
|
+
logBridgeStatus(`Readiness result replay deferred after reconnect: ${errorMessage(error)}`);
|
|
190
|
+
});
|
|
191
|
+
await this.flushCleanupReports().catch(() => undefined);
|
|
192
|
+
logBridgeStatus("Forge connection restored; active dispatches remain attached to this runner process.");
|
|
193
|
+
}
|
|
194
|
+
catch (error) {
|
|
195
|
+
this.completeTransportRecovery(error);
|
|
196
|
+
logBridgeStatus(`Forge reconnect registration failed: ${errorMessage(error)}. Closing the connection for a clean retry...`);
|
|
197
|
+
await connection.stop().catch(() => undefined);
|
|
198
|
+
}
|
|
199
|
+
});
|
|
93
200
|
connection.onDispatch((dispatch) => this.handleDispatch(dispatch));
|
|
201
|
+
connection.onChatTurn?.((dispatch) => this.handleChatTurn(dispatch));
|
|
94
202
|
connection.onOffer?.((dispatch) => this.handleOffer(dispatch));
|
|
95
203
|
connection.onStart?.((dispatch) => this.handleStart(dispatch));
|
|
96
204
|
connection.onLeaseUpdate?.((request) => this.handleLeaseUpdate(request));
|
|
@@ -98,18 +206,67 @@ export class ForgeBridgeService {
|
|
|
98
206
|
connection.onReap?.((request) => this.handleReap(request));
|
|
99
207
|
connection.onReadinessProbe?.((request) => this.handleReadinessProbe(request));
|
|
100
208
|
await connection.start();
|
|
101
|
-
await this.
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
209
|
+
await this.registerConnection(connection);
|
|
210
|
+
await this.flushDispatchResults(true).catch(error => {
|
|
211
|
+
logBridgeStatus(`Dispatch result replay deferred: ${errorMessage(error)}`);
|
|
212
|
+
});
|
|
213
|
+
await this.flushReadinessResults().catch(error => {
|
|
214
|
+
logBridgeStatus(`Readiness result replay deferred: ${errorMessage(error)}`);
|
|
107
215
|
});
|
|
108
216
|
await this.flushCleanupReports();
|
|
109
217
|
this.startHeartbeat();
|
|
110
218
|
this.startLeaseWatchdog();
|
|
111
219
|
this.startSandboxMaintenance();
|
|
112
220
|
}
|
|
221
|
+
rememberRecoveredHostAttempts(report) {
|
|
222
|
+
for (const attempt of report?.recoveredAttempts ?? []) {
|
|
223
|
+
if (this.recoveredHostAttempts.size >= maxRecoveredHostAttempts)
|
|
224
|
+
break;
|
|
225
|
+
this.recoveredHostAttempts.set(attempt.dispatchAttemptId, attempt);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
async recoveredHostAttemptsForHello() {
|
|
229
|
+
if (this.recoveredHostAttempts.size === 0)
|
|
230
|
+
return [];
|
|
231
|
+
// A durable terminal result outranks the stopped-process proof: it replays
|
|
232
|
+
// after Hello and Forge accepts the provider's real outcome instead.
|
|
233
|
+
const pendingResults = new Set((await this.dispatchResultOutbox.pending())
|
|
234
|
+
.map(entry => entry.payload.dispatchAttemptId)
|
|
235
|
+
.filter((id) => typeof id === "string"));
|
|
236
|
+
return [...this.recoveredHostAttempts.values()]
|
|
237
|
+
.filter(attempt => !pendingResults.has(attempt.dispatchAttemptId));
|
|
238
|
+
}
|
|
239
|
+
async registerConnection(connection) {
|
|
240
|
+
let storageRecoveryAttempt = 0;
|
|
241
|
+
while (true) {
|
|
242
|
+
try {
|
|
243
|
+
const recoveredHostAttempts = await this.recoveredHostAttemptsForHello();
|
|
244
|
+
const ack = await connection.invoke("Hello", {
|
|
245
|
+
version: this.config.version,
|
|
246
|
+
protocolVersion: this.config.protocolVersion,
|
|
247
|
+
minServerVersion: this.config.minServerVersion ?? null,
|
|
248
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
249
|
+
capabilities: this.capabilities(),
|
|
250
|
+
recoveredHostAttempts,
|
|
251
|
+
});
|
|
252
|
+
if (!ack.accepted) {
|
|
253
|
+
throw new BridgeCallbackRejectedError(ack.code ?? null, ack.message ?? ack.code ?? "Hello callback was rejected.");
|
|
254
|
+
}
|
|
255
|
+
this.recoveredHostAttempts.clear();
|
|
256
|
+
logBridgeStatus(`Registered runner ${this.config.runnerId}, instance ${this.config.runnerInstanceId}, build ${this.config.version}, protocol ${this.config.protocolVersion}.`);
|
|
257
|
+
return;
|
|
258
|
+
}
|
|
259
|
+
catch (error) {
|
|
260
|
+
if (!isProviderStorageUnavailable(error) || storageRecoveryAttempt >= this.config.reconnectBackoffMs.length) {
|
|
261
|
+
throw error;
|
|
262
|
+
}
|
|
263
|
+
const backoff = this.config.reconnectBackoffMs[storageRecoveryAttempt] ?? 5000;
|
|
264
|
+
storageRecoveryAttempt += 1;
|
|
265
|
+
logBridgeStatus(`Hello reached Forge while storage was recovering. Retrying registration ${storageRecoveryAttempt} in ${formatDelay(backoff)}...`);
|
|
266
|
+
await delay(backoff);
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
}
|
|
113
270
|
async run(signal) {
|
|
114
271
|
let attempt = 0;
|
|
115
272
|
try {
|
|
@@ -151,24 +308,351 @@ export class ForgeBridgeService {
|
|
|
151
308
|
await this.stopConnection();
|
|
152
309
|
await this.preflight.release();
|
|
153
310
|
}
|
|
154
|
-
async
|
|
155
|
-
const
|
|
311
|
+
async handleChatTurn(rawDispatch) {
|
|
312
|
+
const packet = this.parseChatTurnOrReport(rawDispatch);
|
|
313
|
+
if (!packet) {
|
|
314
|
+
return;
|
|
315
|
+
}
|
|
316
|
+
if (packet.runnerInstanceId !== this.config.runnerInstanceId) {
|
|
317
|
+
logBridgeStatus(`Rejected Bridge Chat turn ${packet.turnId}: stale runner instance.`);
|
|
318
|
+
return;
|
|
319
|
+
}
|
|
320
|
+
const capabilityFailure = this.requiredChatCapabilityFailure(packet);
|
|
321
|
+
if (capabilityFailure) {
|
|
322
|
+
await this.sendStandaloneChatTerminal(packet, "failed", {
|
|
323
|
+
code: "bridge_chat.runner_unavailable",
|
|
324
|
+
message: capabilityFailure,
|
|
325
|
+
}).catch(() => undefined);
|
|
326
|
+
return;
|
|
327
|
+
}
|
|
328
|
+
const existing = this.activeSessions.get(packet.leaseId);
|
|
329
|
+
if (existing) {
|
|
330
|
+
if (existing.kind === "chat"
|
|
331
|
+
&& existing.fencingToken === packet.fencingToken
|
|
332
|
+
&& existing.chatRunId === packet.chatRunId
|
|
333
|
+
&& existing.conversationId === packet.conversationId
|
|
334
|
+
&& existing.turnId === packet.turnId) {
|
|
335
|
+
return;
|
|
336
|
+
}
|
|
337
|
+
await this.sendStandaloneChatTerminal(packet, "failed", {
|
|
338
|
+
code: "bridge_chat.stale_fencing_token",
|
|
339
|
+
message: "Bridge Chat turn fence conflicts with an active bridge session.",
|
|
340
|
+
}).catch(() => undefined);
|
|
341
|
+
return;
|
|
342
|
+
}
|
|
343
|
+
const controller = new AbortController();
|
|
344
|
+
const active = {
|
|
345
|
+
leaseId: packet.leaseId,
|
|
346
|
+
kind: "chat",
|
|
347
|
+
fencingToken: packet.fencingToken,
|
|
348
|
+
dispatchAttemptId: null,
|
|
349
|
+
runnerInstanceId: packet.runnerInstanceId,
|
|
350
|
+
leaseExpiresAt: chatTurnLeaseExpiresAt(packet),
|
|
351
|
+
sandboxed: false,
|
|
352
|
+
resources: zeroSandboxResources(),
|
|
353
|
+
controller,
|
|
354
|
+
chatRunId: packet.chatRunId,
|
|
355
|
+
conversationId: packet.conversationId,
|
|
356
|
+
turnId: packet.turnId,
|
|
357
|
+
providerThreadId: packet.providerThreadId ?? null,
|
|
358
|
+
chatSequence: 0,
|
|
359
|
+
terminalCallbackAttempted: false,
|
|
360
|
+
};
|
|
361
|
+
this.activeSessions.set(packet.leaseId, active);
|
|
362
|
+
let workspacePath = null;
|
|
363
|
+
let cleanupAttempted = false;
|
|
364
|
+
let cleanupOutcome = null;
|
|
365
|
+
let cleanupReported = false;
|
|
366
|
+
const cleanup = async () => {
|
|
367
|
+
if (!workspacePath || cleanupAttempted)
|
|
368
|
+
return null;
|
|
369
|
+
cleanupAttempted = true;
|
|
370
|
+
cleanupOutcome = await cleanupChatTurnWorkspace(workspacePath, chatRedactionSecrets(packet));
|
|
371
|
+
return cleanupOutcome;
|
|
372
|
+
};
|
|
373
|
+
const reportCleanup = async (outcome) => {
|
|
374
|
+
if (!outcome || cleanupReported)
|
|
375
|
+
return;
|
|
376
|
+
cleanupReported = true;
|
|
377
|
+
await this.sendChatCallback(active, "output_delta", outcome).catch(() => undefined);
|
|
378
|
+
};
|
|
379
|
+
try {
|
|
380
|
+
workspacePath = await this.createChatTurnWorkspace(packet);
|
|
381
|
+
await materializeChatTurnMetadata(packet, workspacePath);
|
|
382
|
+
const run = await this.runChatProviderProcess(packet, active, workspacePath);
|
|
383
|
+
cleanupOutcome = await cleanup();
|
|
384
|
+
await reportCleanup(cleanupOutcome);
|
|
385
|
+
if (controller.signal.aborted) {
|
|
386
|
+
await this.sendChatAbortTerminal(packet, active, cleanupOutcome);
|
|
387
|
+
return;
|
|
388
|
+
}
|
|
389
|
+
if (cleanupOutcome?.status === "failed") {
|
|
390
|
+
throw new Error(`chat_turn_cleanup_failed: ${cleanupOutcome.message}`);
|
|
391
|
+
}
|
|
392
|
+
if (run.result.exitCode !== 0) {
|
|
393
|
+
throw chatProviderExitFailure(run.command, run.result, chatRedactionSecrets(packet));
|
|
394
|
+
}
|
|
395
|
+
await this.sendChatCallback(active, "output_completed", {
|
|
396
|
+
code: "output_completed",
|
|
397
|
+
text: clip(redactText(run.output.finalMessage ?? "", chatRedactionSecrets(packet)), maxChatTerminalTextChars),
|
|
398
|
+
provider: packet.providerKind,
|
|
399
|
+
model: packet.model,
|
|
400
|
+
cleanup: cleanupOutcome,
|
|
401
|
+
});
|
|
402
|
+
}
|
|
403
|
+
catch (error) {
|
|
404
|
+
cleanupOutcome = cleanupOutcome ?? await cleanup();
|
|
405
|
+
if (active.terminalCallbackAttempted) {
|
|
406
|
+
return;
|
|
407
|
+
}
|
|
408
|
+
await reportCleanup(cleanupOutcome);
|
|
409
|
+
const payload = controller.signal.aborted
|
|
410
|
+
? chatAbortPayload(controller.signal.reason, cleanupOutcome, packet)
|
|
411
|
+
: {
|
|
412
|
+
code: "provider_failed",
|
|
413
|
+
message: clip(redactText(errorMessage(error), chatRedactionSecrets(packet)), 2000),
|
|
414
|
+
cleanup: cleanupOutcome,
|
|
415
|
+
};
|
|
416
|
+
const kind = controller.signal.aborted ? chatAbortKind(controller.signal.reason) : "failed";
|
|
417
|
+
await this.sendChatCallback(active, kind, payload).catch(callbackError => {
|
|
418
|
+
logBridgeStatus(`Bridge Chat terminal callback for turn ${packet.turnId} was not accepted: ${errorMessage(callbackError)}`);
|
|
419
|
+
});
|
|
420
|
+
}
|
|
421
|
+
finally {
|
|
422
|
+
await this.releaseActiveSessionAndRefreshCapacity(packet.leaseId);
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
async createChatTurnWorkspace(packet) {
|
|
426
|
+
const root = path.join(this.config.sandboxStateRoot, "chat-turns");
|
|
427
|
+
await fs.promises.mkdir(root, { recursive: true });
|
|
428
|
+
return await fs.promises.mkdtemp(path.join(root, `${safePathComponent(packet.turnId)}-`));
|
|
429
|
+
}
|
|
430
|
+
async runChatProviderProcess(packet, active, cwd) {
|
|
431
|
+
const output = createProviderOutputState(packet.providerKind);
|
|
432
|
+
const plan = buildChatProviderCommand(packet, cwd);
|
|
433
|
+
const redactionSecrets = chatRedactionSecrets(packet);
|
|
434
|
+
let started = false;
|
|
435
|
+
const result = await this.chatProcessRunner.run({
|
|
436
|
+
command: plan.command.command,
|
|
437
|
+
args: plan.command.args,
|
|
438
|
+
cwd,
|
|
439
|
+
env: {
|
|
440
|
+
FORGE_BRIDGE_CHAT_RUN_ID: packet.chatRunId,
|
|
441
|
+
FORGE_BRIDGE_CHAT_TURN_ID: packet.turnId,
|
|
442
|
+
},
|
|
443
|
+
stdin: plan.stdin,
|
|
444
|
+
}, {
|
|
445
|
+
onStarted: async () => {
|
|
446
|
+
started = true;
|
|
447
|
+
await this.sendChatCallback(active, "turn_started", {
|
|
448
|
+
status: "running",
|
|
449
|
+
provider: packet.providerKind,
|
|
450
|
+
model: packet.model,
|
|
451
|
+
command: redactText(formatProviderCliCommandForCheckpoint(plan.command), redactionSecrets),
|
|
452
|
+
});
|
|
453
|
+
},
|
|
454
|
+
onStdoutLine: async (line) => {
|
|
455
|
+
for (const event of output.consume(line)) {
|
|
456
|
+
await this.sendChatProviderEvent(packet, active, event, output.providerThreadId);
|
|
457
|
+
}
|
|
458
|
+
},
|
|
459
|
+
}, active.controller.signal);
|
|
460
|
+
if (started && !active.controller.signal.aborted) {
|
|
461
|
+
await this.sendChatCallback(active, "output_delta", {
|
|
462
|
+
eventType: "provider_cli_completed",
|
|
463
|
+
role: "runner",
|
|
464
|
+
kind: "checkpoint",
|
|
465
|
+
content: "Bridge Chat provider process exited.",
|
|
466
|
+
}).catch(() => undefined);
|
|
467
|
+
}
|
|
468
|
+
return { command: plan.command, output, result };
|
|
469
|
+
}
|
|
470
|
+
async sendChatProviderEvent(packet, active, event, providerThreadId) {
|
|
471
|
+
if (providerThreadId) {
|
|
472
|
+
active.providerThreadId = providerThreadId;
|
|
473
|
+
}
|
|
474
|
+
else if (event.eventType === "provider_thread_started") {
|
|
475
|
+
active.providerThreadId = providerThreadIdFromEvent(event) ?? active.providerThreadId;
|
|
476
|
+
}
|
|
477
|
+
if (event.eventType === "provider_tool_heartbeat") {
|
|
478
|
+
return;
|
|
479
|
+
}
|
|
480
|
+
const redactionSecrets = chatRedactionSecrets(packet);
|
|
481
|
+
const payload = {
|
|
482
|
+
role: event.role,
|
|
483
|
+
kind: event.kind,
|
|
484
|
+
eventType: event.eventType ?? "provider_event",
|
|
485
|
+
content: clip(redactText(event.content, redactionSecrets), maxChatEventContentChars),
|
|
486
|
+
data: event.data ? clip(redactText(event.data, redactionSecrets), maxChatEventContentChars) : undefined,
|
|
487
|
+
};
|
|
488
|
+
await this.sendChatCallback(active, "output_delta", payload);
|
|
489
|
+
}
|
|
490
|
+
async sendChatAbortTerminal(packet, active, cleanupOutcome) {
|
|
491
|
+
await this.sendChatCallback(active, chatAbortKind(active.controller.signal.reason), chatAbortPayload(active.controller.signal.reason, cleanupOutcome, packet));
|
|
492
|
+
}
|
|
493
|
+
async sendStandaloneChatTerminal(packet, kind, payload) {
|
|
494
|
+
const active = {
|
|
495
|
+
leaseId: packet.leaseId,
|
|
496
|
+
kind: "chat",
|
|
497
|
+
fencingToken: packet.fencingToken,
|
|
498
|
+
dispatchAttemptId: null,
|
|
499
|
+
runnerInstanceId: packet.runnerInstanceId,
|
|
500
|
+
leaseExpiresAt: new Date(Date.now() + defaultChatTurnLeaseMs).toISOString(),
|
|
501
|
+
sandboxed: false,
|
|
502
|
+
resources: zeroSandboxResources(),
|
|
503
|
+
controller: new AbortController(),
|
|
504
|
+
chatRunId: packet.chatRunId,
|
|
505
|
+
conversationId: packet.conversationId,
|
|
506
|
+
turnId: packet.turnId,
|
|
507
|
+
providerThreadId: packet.providerThreadId ?? null,
|
|
508
|
+
chatSequence: 0,
|
|
509
|
+
terminalCallbackAttempted: false,
|
|
510
|
+
};
|
|
511
|
+
await this.sendChatCallback(active, kind, payload);
|
|
512
|
+
}
|
|
513
|
+
async sendChatCallback(active, kind, payload = null) {
|
|
514
|
+
if (active.kind !== "chat" || !active.chatRunId || !active.conversationId || !active.turnId) {
|
|
515
|
+
throw new Error("Bridge Chat callback requires an active chat turn.");
|
|
516
|
+
}
|
|
517
|
+
const terminal = isTerminalChatTurnCallbackKind(kind);
|
|
518
|
+
if (terminal && active.terminalCallbackAttempted) {
|
|
519
|
+
return;
|
|
520
|
+
}
|
|
521
|
+
assertBridgeChatCallbackPayload(payload);
|
|
522
|
+
const sequence = (active.chatSequence ?? 0) + 1;
|
|
523
|
+
const envelope = {
|
|
524
|
+
callbackId: randomUUID(),
|
|
525
|
+
chatRunId: active.chatRunId,
|
|
526
|
+
conversationId: active.conversationId,
|
|
527
|
+
turnId: active.turnId,
|
|
528
|
+
leaseId: active.leaseId,
|
|
529
|
+
fencingToken: active.fencingToken,
|
|
530
|
+
runnerId: this.config.runnerId,
|
|
531
|
+
runnerInstanceId: active.runnerInstanceId ?? this.config.runnerInstanceId,
|
|
532
|
+
sequence,
|
|
533
|
+
eventType: kind,
|
|
534
|
+
redactionStatus: "redacted",
|
|
535
|
+
providerThreadId: active.providerThreadId ?? null,
|
|
536
|
+
payload,
|
|
537
|
+
occurredAt: new Date().toISOString(),
|
|
538
|
+
protocolVersion: bridgeChatProtocolVersion,
|
|
539
|
+
};
|
|
540
|
+
if (terminal) {
|
|
541
|
+
active.terminalCallbackAttempted = true;
|
|
542
|
+
}
|
|
543
|
+
await this.invokeChatCallbackWithRetry(envelope);
|
|
544
|
+
active.chatSequence = sequence;
|
|
545
|
+
}
|
|
546
|
+
async invokeChatCallbackWithRetry(envelope) {
|
|
547
|
+
const maxAttempts = 3;
|
|
548
|
+
for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
|
|
549
|
+
try {
|
|
550
|
+
await this.invokeAck("ChatTurnCallback", envelope);
|
|
551
|
+
return;
|
|
552
|
+
}
|
|
553
|
+
catch (error) {
|
|
554
|
+
if (error instanceof BridgeCallbackRejectedError || attempt === maxAttempts - 1) {
|
|
555
|
+
throw error;
|
|
556
|
+
}
|
|
557
|
+
await delay(200 * (attempt + 1));
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
requiredChatCapabilityFailure(packet) {
|
|
562
|
+
const capability = this.bridgeChatCapability();
|
|
563
|
+
if (capability.availability === "unavailable" || !capability.supportsTextStreaming) {
|
|
564
|
+
return capability.unavailableReason ?? "Bridge Chat text capability is unavailable.";
|
|
565
|
+
}
|
|
566
|
+
if (capability.supportedProviders.length > 0
|
|
567
|
+
&& !containsIgnoreCase(capability.supportedProviders, packet.providerKind)) {
|
|
568
|
+
return `Bridge Chat provider '${packet.providerKind}' is not advertised by this bridge.`;
|
|
569
|
+
}
|
|
570
|
+
if (capability.supportedModels.length > 0
|
|
571
|
+
&& !containsIgnoreCase(capability.supportedModels, packet.model)) {
|
|
572
|
+
return `Bridge Chat model '${packet.model}' is not advertised by this bridge.`;
|
|
573
|
+
}
|
|
574
|
+
if (packet.requestedCapabilities.includes("cancel") && !capability.supportsCancel) {
|
|
575
|
+
return "Bridge Chat cancellation is not advertised by this bridge.";
|
|
576
|
+
}
|
|
577
|
+
if (packet.requestedCapabilities.includes("resume") && !capability.supportsResume) {
|
|
578
|
+
return "Bridge Chat resume is not advertised by this bridge.";
|
|
579
|
+
}
|
|
580
|
+
if (packet.requestedCapabilities.includes("thread_continuity")
|
|
581
|
+
&& !capability.supportsProviderThreadContinuity) {
|
|
582
|
+
return "Bridge Chat provider thread continuity is not advertised by this bridge.";
|
|
583
|
+
}
|
|
584
|
+
if (packet.requestedCapabilities.includes("attachment_handoffs") && !capability.supportsAttachments) {
|
|
585
|
+
return "Bridge Chat attachment handoffs are not advertised by this bridge.";
|
|
586
|
+
}
|
|
587
|
+
if (packet.requestedCapabilities.includes("action_broker_grants") && !capability.supportsActionBrokerGrants) {
|
|
588
|
+
return "Bridge Chat action-broker grants are not advertised by this bridge.";
|
|
589
|
+
}
|
|
590
|
+
return null;
|
|
591
|
+
}
|
|
592
|
+
bridgeChatCapability() {
|
|
593
|
+
return bridgeChatCapabilityFor(this.config.cliProviders, this.config.providerStatuses, this.config.bridgeChat);
|
|
594
|
+
}
|
|
595
|
+
async handleDispatch(rawDispatch, reservationHandoff) {
|
|
596
|
+
const dispatch = await this.parseDispatchOrReport(rawDispatch);
|
|
156
597
|
if (!dispatch) {
|
|
598
|
+
this.releaseReservedDispatchHandoff(reservationHandoff);
|
|
599
|
+
return;
|
|
600
|
+
}
|
|
601
|
+
if (this.enforceHostDatabaseCapabilityBoundary) {
|
|
602
|
+
try {
|
|
603
|
+
assertHostDatabaseCapabilityBoundary();
|
|
604
|
+
}
|
|
605
|
+
catch (error) {
|
|
606
|
+
const message = errorMessage(error);
|
|
607
|
+
await this.sendResult(dispatch, failureResult(message, [{
|
|
608
|
+
role: "runner",
|
|
609
|
+
kind: "error",
|
|
610
|
+
content: message,
|
|
611
|
+
eventType: "host_database_capability_forbidden",
|
|
612
|
+
}], failureFactsFromError(error, { source: "bridge" }))).catch(() => undefined);
|
|
613
|
+
this.releaseReservedDispatchHandoff(reservationHandoff);
|
|
614
|
+
return;
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
let capabilityFailure;
|
|
618
|
+
try {
|
|
619
|
+
capabilityFailure = await this.requiredCapabilityFailure(dispatch);
|
|
620
|
+
}
|
|
621
|
+
catch (error) {
|
|
622
|
+
this.releaseReservedDispatchHandoff(reservationHandoff);
|
|
623
|
+
throw error;
|
|
624
|
+
}
|
|
625
|
+
if (capabilityFailure) {
|
|
626
|
+
await this.sendResult(dispatch, failureResult(capabilityFailure, [{
|
|
627
|
+
role: "runner",
|
|
628
|
+
kind: "error",
|
|
629
|
+
content: capabilityFailure,
|
|
630
|
+
eventType: "docker_engine_unavailable",
|
|
631
|
+
}], failureFactsForCode("docker_engine_unavailable"))).catch(() => undefined);
|
|
632
|
+
this.releaseReservedDispatchHandoff(reservationHandoff);
|
|
157
633
|
return;
|
|
158
634
|
}
|
|
159
635
|
const leaseId = dispatch.lease.id;
|
|
160
636
|
const fencingToken = dispatch.lease.fencingToken;
|
|
161
637
|
const redactionSecrets = exactSecrets(dispatch, this.config);
|
|
638
|
+
if (reservationHandoff && !this.reservedDispatchHandoffIsCurrent(reservationHandoff, dispatch)) {
|
|
639
|
+
await this.sendResult(dispatch, failureResult("Start rejected because its reservation is missing or stale.", [], failureFactsForCode("bridge.reservation_stale"))).catch(() => undefined);
|
|
640
|
+
return;
|
|
641
|
+
}
|
|
162
642
|
const existing = this.activeSessions.get(leaseId);
|
|
163
643
|
if (existing) {
|
|
164
644
|
if (existing.fencingToken === fencingToken) {
|
|
645
|
+
this.releaseReservedDispatchHandoff(reservationHandoff);
|
|
165
646
|
return;
|
|
166
647
|
}
|
|
167
|
-
await this.sendResult(dispatch, failureResult("Dispatch fence conflicts with an active bridge session.")).catch(() => undefined);
|
|
648
|
+
await this.sendResult(dispatch, failureResult("Dispatch fence conflicts with an active bridge session.", [], failureFactsForCode("bridge.dispatch_fence_conflict"))).catch(() => undefined);
|
|
649
|
+
this.releaseReservedDispatchHandoff(reservationHandoff);
|
|
168
650
|
return;
|
|
169
651
|
}
|
|
170
|
-
const controller = new AbortController();
|
|
652
|
+
const controller = reservationHandoff?.controller ?? new AbortController();
|
|
171
653
|
this.activeSessions.set(leaseId, {
|
|
654
|
+
leaseId,
|
|
655
|
+
kind: "dispatch",
|
|
172
656
|
fencingToken,
|
|
173
657
|
dispatchAttemptId: dispatch.sandbox?.dispatchAttemptId ?? null,
|
|
174
658
|
runnerInstanceId: dispatch.sandbox?.runnerInstanceId ?? null,
|
|
@@ -179,6 +663,18 @@ export class ForgeBridgeService {
|
|
|
179
663
|
: zeroSandboxResources(),
|
|
180
664
|
controller
|
|
181
665
|
});
|
|
666
|
+
if (reservationHandoff) {
|
|
667
|
+
// Promotion is synchronous with active-session registration. A qualified
|
|
668
|
+
// pre-start Cancel therefore sees either the cancellable reservation or
|
|
669
|
+
// the fully active owner; there is no cancellation-blind handoff gap.
|
|
670
|
+
this.releaseReservedDispatchHandoff(reservationHandoff);
|
|
671
|
+
}
|
|
672
|
+
// Server-requested runtime readiness is background maintenance. A probe may
|
|
673
|
+
// have passed admission during the narrow handoff between prepared-runtime
|
|
674
|
+
// verification and the foreground attempt's final admission. Preempt it as
|
|
675
|
+
// soon as real agent work arrives so the probe cannot consume the memory
|
|
676
|
+
// headroom that the foreground attempt is waiting to acquire.
|
|
677
|
+
this.deferActiveReadinessProbes();
|
|
182
678
|
try {
|
|
183
679
|
const normalized = this.normalizeDispatch(dispatch);
|
|
184
680
|
if (normalized.sandbox) {
|
|
@@ -205,7 +701,10 @@ export class ForgeBridgeService {
|
|
|
205
701
|
dispatchAttemptId: dispatch.sandbox.dispatchAttemptId,
|
|
206
702
|
runnerInstanceId: dispatch.sandbox.runnerInstanceId
|
|
207
703
|
} : {}),
|
|
208
|
-
event: redactRunnerDispatchEvent(event, redactionSecrets)
|
|
704
|
+
event: redactRunnerDispatchEvent({ ...event, eventId: event.eventId ?? randomUUID() }, redactionSecrets)
|
|
705
|
+
}, {
|
|
706
|
+
replayCallbackTimeout: true,
|
|
707
|
+
signal: controller.signal,
|
|
209
708
|
});
|
|
210
709
|
}
|
|
211
710
|
catch (error) {
|
|
@@ -223,23 +722,86 @@ export class ForgeBridgeService {
|
|
|
223
722
|
if (normalized.sandbox?.executionMode === "docker") {
|
|
224
723
|
await this.flushCleanupReports().catch(() => undefined);
|
|
225
724
|
}
|
|
226
|
-
|
|
725
|
+
const redactedResult = redactRunOnceResultPacket(result, redactionSecrets);
|
|
726
|
+
await this.sendResult(dispatch, compactTerminalResult(dispatch, redactedResult));
|
|
227
727
|
}
|
|
228
728
|
catch (error) {
|
|
229
729
|
const summary = controller.signal.aborted
|
|
230
730
|
? `Dispatch cancelled: ${String(controller.signal.reason ?? "cancelled")}`
|
|
231
731
|
: redactText(error instanceof Error ? error.message : String(error), redactionSecrets);
|
|
232
|
-
const
|
|
732
|
+
const failureFacts = failureFactsFromError(error, {
|
|
733
|
+
cancelled: controller.signal.aborted,
|
|
734
|
+
source: "bridge",
|
|
735
|
+
});
|
|
736
|
+
const failureEventType = failureFacts.category === "unknown" ? null : failureFacts.code;
|
|
233
737
|
const failureEvents = failureEventType ? [{
|
|
234
738
|
role: "runner",
|
|
235
739
|
kind: "error",
|
|
236
740
|
content: summary,
|
|
237
741
|
eventType: failureEventType,
|
|
238
742
|
}] : [];
|
|
239
|
-
await this.sendResult(dispatch, failureResult(summary, failureEvents)).catch(() => undefined);
|
|
743
|
+
await this.sendResult(dispatch, failureResult(summary, failureEvents, failureFacts)).catch(() => undefined);
|
|
240
744
|
}
|
|
241
745
|
finally {
|
|
242
|
-
this.
|
|
746
|
+
await this.releaseActiveSessionAndRefreshCapacity(leaseId);
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
reservedDispatchHandoffIsCurrent(handoff, dispatch) {
|
|
750
|
+
const sandbox = dispatch.sandbox;
|
|
751
|
+
const reservation = handoff.reservation;
|
|
752
|
+
return sandbox !== null
|
|
753
|
+
&& sandbox !== undefined
|
|
754
|
+
&& sandbox.dispatchAttemptId === handoff.dispatchAttemptId
|
|
755
|
+
&& this.reservations.get(handoff.dispatchAttemptId) === reservation
|
|
756
|
+
&& reservation.startClaim === handoff
|
|
757
|
+
&& !handoff.controller.signal.aborted
|
|
758
|
+
&& reservation.leaseId === dispatch.lease.id
|
|
759
|
+
&& reservation.fencingToken === dispatch.lease.fencingToken
|
|
760
|
+
&& reservation.runnerInstanceId === sandbox.runnerInstanceId
|
|
761
|
+
&& Date.parse(reservation.reservedUntil) > Date.now();
|
|
762
|
+
}
|
|
763
|
+
releaseReservedDispatchHandoff(handoff) {
|
|
764
|
+
if (!handoff
|
|
765
|
+
|| this.reservations.get(handoff.dispatchAttemptId) !== handoff.reservation
|
|
766
|
+
|| handoff.reservation.startClaim !== handoff)
|
|
767
|
+
return;
|
|
768
|
+
this.reservations.delete(handoff.dispatchAttemptId);
|
|
769
|
+
}
|
|
770
|
+
async releaseActiveSessionAndRefreshCapacity(leaseId, expectedDispatch) {
|
|
771
|
+
if (expectedDispatch) {
|
|
772
|
+
const active = this.activeSessions.get(leaseId);
|
|
773
|
+
// Admission failures may arrive after a newer attempt starts on the same
|
|
774
|
+
// lease. A terminal callback can release only the dispatch it identifies.
|
|
775
|
+
if (!active || active.kind !== "dispatch"
|
|
776
|
+
|| active.fencingToken !== expectedDispatch.lease.fencingToken
|
|
777
|
+
|| active.dispatchAttemptId !== (expectedDispatch.sandbox?.dispatchAttemptId ?? null)
|
|
778
|
+
|| active.runnerInstanceId !== (expectedDispatch.sandbox?.runnerInstanceId ?? null))
|
|
779
|
+
return;
|
|
780
|
+
}
|
|
781
|
+
if (!this.activeSessions.delete(leaseId))
|
|
782
|
+
return;
|
|
783
|
+
// Publish released host/runtime capacity before the terminal result can
|
|
784
|
+
// enqueue the next pipeline phase. Otherwise the server may reject that
|
|
785
|
+
// handoff from the previous periodic heartbeat's active reservation.
|
|
786
|
+
// Publish the updated reservation counts from cached samples only. Probes
|
|
787
|
+
// belong to periodic/admission heartbeats and must not outlive this bounded
|
|
788
|
+
// terminal handoff. Storage recovery must not block durable result delivery.
|
|
789
|
+
const controller = new AbortController();
|
|
790
|
+
let timer;
|
|
791
|
+
try {
|
|
792
|
+
await Promise.race([
|
|
793
|
+
this.sendHeartbeatSnapshot(controller.signal).catch(() => undefined),
|
|
794
|
+
new Promise(resolve => {
|
|
795
|
+
timer = setTimeout(() => {
|
|
796
|
+
controller.abort("terminal capacity refresh deadline");
|
|
797
|
+
resolve();
|
|
798
|
+
}, Math.min(5000, positiveFiniteNumber(this.config.callbackAckTimeoutMs ?? 30_000, 30_000)));
|
|
799
|
+
}),
|
|
800
|
+
]);
|
|
801
|
+
}
|
|
802
|
+
finally {
|
|
803
|
+
if (timer)
|
|
804
|
+
clearTimeout(timer);
|
|
243
805
|
}
|
|
244
806
|
}
|
|
245
807
|
async handleOffer(rawDispatch) {
|
|
@@ -247,20 +809,19 @@ export class ForgeBridgeService {
|
|
|
247
809
|
if (!envelope) {
|
|
248
810
|
return;
|
|
249
811
|
}
|
|
250
|
-
const dispatch = this.parseDispatchOrReport(envelope.dispatch);
|
|
812
|
+
const dispatch = await this.parseDispatchOrReport(envelope.dispatch);
|
|
251
813
|
if (!dispatch?.sandbox
|
|
252
814
|
|| dispatch.sandbox.dispatchAttemptId !== envelope.dispatchAttemptId
|
|
253
815
|
|| dispatch.lease.id !== envelope.leaseId
|
|
254
816
|
|| dispatch.lease.fencingToken !== envelope.fencingToken)
|
|
255
817
|
return;
|
|
256
818
|
const sandbox = dispatch.sandbox;
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
this.reservations.delete(sandbox.dispatchAttemptId);
|
|
260
|
-
existing = undefined;
|
|
261
|
-
}
|
|
819
|
+
this.expireReservations(Date.now());
|
|
820
|
+
const existing = this.reservations.get(sandbox.dispatchAttemptId);
|
|
262
821
|
if (existing) {
|
|
263
|
-
const sameFence = existing.
|
|
822
|
+
const sameFence = existing.leaseId === dispatch.lease.id
|
|
823
|
+
&& existing.fencingToken === dispatch.lease.fencingToken
|
|
824
|
+
&& existing.runnerInstanceId === sandbox.runnerInstanceId;
|
|
264
825
|
await this.sendOfferDecision(dispatch, sameFence, sameFence ? null : "stale_fence", sameFence ? null : "Offer fence conflicts with the existing reservation.", sameFence ? existing.reservedUntil : null, sameFence ? existing.resources : null);
|
|
265
826
|
return;
|
|
266
827
|
}
|
|
@@ -287,7 +848,7 @@ export class ForgeBridgeService {
|
|
|
287
848
|
}
|
|
288
849
|
const providerNetworkFailure = sandboxProviderNetworkFailure(dispatch, this.config);
|
|
289
850
|
if (providerNetworkFailure) {
|
|
290
|
-
await this.sendOfferDecision(dispatch, false, "
|
|
851
|
+
await this.sendOfferDecision(dispatch, false, "provider_network_policy_denied", providerNetworkFailure);
|
|
291
852
|
return;
|
|
292
853
|
}
|
|
293
854
|
const authFailure = sandboxAuthFailure(dispatch, this.config);
|
|
@@ -296,15 +857,12 @@ export class ForgeBridgeService {
|
|
|
296
857
|
return;
|
|
297
858
|
}
|
|
298
859
|
}
|
|
299
|
-
const runtimeState = this.sandboxRuntime.snapshot();
|
|
300
|
-
const usedResources = this.usedSandboxResources(runtimeState.degraded);
|
|
301
|
-
const activeResources = this.activeSandboxResources(runtimeState.degraded);
|
|
302
|
-
const capacity = this.sandboxResourceCapacity(preflight, activeResources);
|
|
303
860
|
let requestedResources = sandbox.requestedResources;
|
|
304
861
|
let sandboxConfigCommitSha = sandbox.sandboxConfigCommitSha ?? null;
|
|
862
|
+
const normalizedDispatch = this.normalizeDispatch(dispatch);
|
|
305
863
|
if (sandbox.executionMode === "docker") {
|
|
306
864
|
try {
|
|
307
|
-
const profile = await this.sandboxResourceResolver(
|
|
865
|
+
const profile = await this.sandboxResourceResolver(normalizedDispatch);
|
|
308
866
|
requestedResources = profile.resources;
|
|
309
867
|
sandboxConfigCommitSha = profile.sandboxConfigCommitSha;
|
|
310
868
|
}
|
|
@@ -313,73 +871,166 @@ export class ForgeBridgeService {
|
|
|
313
871
|
return;
|
|
314
872
|
}
|
|
315
873
|
}
|
|
316
|
-
if (
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
874
|
+
else if (normalizedDispatch.repo.serverLocalPath) {
|
|
875
|
+
// Host execution uses the same project-derived disk reservation as Docker.
|
|
876
|
+
// The API supplies a placement-time estimate; the runner can safely raise it
|
|
877
|
+
// from its authoritative checkout before admitting the attempt.
|
|
878
|
+
const measuredWorkspaceBytes = await estimateWorkspaceBytesFromRepo(normalizedDispatch.repo.serverLocalPath, { role: normalizedDispatch.execution.role }).catch(() => null);
|
|
879
|
+
if (measuredWorkspaceBytes !== null) {
|
|
880
|
+
requestedResources = {
|
|
881
|
+
...requestedResources,
|
|
882
|
+
workspaceBytes: Math.max(requestedResources.workspaceBytes, measuredWorkspaceBytes),
|
|
883
|
+
};
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
// Resource profiling can await filesystem and Docker work. Admission needs
|
|
887
|
+
// a dynamic host/daemon sample that began after profiling completed, so an
|
|
888
|
+
// older in-flight heartbeat inspection cannot supply the commit decision.
|
|
889
|
+
const admissionPreflight = sandbox.executionMode === "docker"
|
|
890
|
+
? await this.preflight.refreshCapacityForAdmission()
|
|
891
|
+
: preflight;
|
|
892
|
+
// Resource profiling and preflight may await slow filesystem or Docker I/O.
|
|
893
|
+
// Re-read every mutable capacity input only after that work, then decide and
|
|
894
|
+
// commit synchronously. JavaScript cannot interleave another Offer handler
|
|
895
|
+
// inside this non-async method, so the ledger check and insertion are one
|
|
896
|
+
// atomic event-loop turn without holding a lock across external I/O.
|
|
897
|
+
const decision = this.commitOfferReservation(dispatch, requestedResources, admissionPreflight);
|
|
898
|
+
await this.sendOfferDecision(dispatch, decision.accepted, decision.code, decision.message, decision.reservedUntil, decision.resources, sandboxConfigCommitSha);
|
|
899
|
+
}
|
|
900
|
+
commitOfferReservation(dispatch, requestedResources, preparedPreflight) {
|
|
901
|
+
const sandbox = dispatch.sandbox;
|
|
902
|
+
const now = Date.now();
|
|
903
|
+
this.expireReservations(now);
|
|
904
|
+
const existing = this.reservations.get(sandbox.dispatchAttemptId);
|
|
905
|
+
if (existing) {
|
|
906
|
+
const sameFence = existing.leaseId === dispatch.lease.id
|
|
907
|
+
&& existing.fencingToken === dispatch.lease.fencingToken
|
|
908
|
+
&& existing.runnerInstanceId === sandbox.runnerInstanceId;
|
|
909
|
+
return {
|
|
910
|
+
accepted: sameFence,
|
|
911
|
+
code: sameFence ? null : "stale_fence",
|
|
912
|
+
message: sameFence ? null : "Offer fence conflicts with the existing reservation.",
|
|
913
|
+
reservedUntil: sameFence ? existing.reservedUntil : null,
|
|
914
|
+
resources: sameFence ? existing.resources : null,
|
|
915
|
+
};
|
|
916
|
+
}
|
|
917
|
+
if (Date.parse(dispatch.lease.expiresAt) <= now) {
|
|
918
|
+
return rejectedOfferReservation("lease_expired", "Offer lease is already expired.");
|
|
320
919
|
}
|
|
321
|
-
|
|
322
|
-
|
|
920
|
+
if (sandbox.executionMode === "docker") {
|
|
921
|
+
// A concurrent heartbeat/preflight refresh may have produced a newer
|
|
922
|
+
// conservative dual-domain sample while this offer was being profiled.
|
|
923
|
+
const admissionPreflight = this.preflight.current() ?? preparedPreflight;
|
|
924
|
+
if (!this.config.dockerEnabled || !admissionPreflight?.healthy) {
|
|
925
|
+
return rejectedOfferReservation(admissionPreflight?.unavailableCode ?? "docker_unavailable", admissionPreflight?.unavailableReason ?? "Docker sandbox preflight is unavailable.");
|
|
926
|
+
}
|
|
927
|
+
const runtimeState = this.sandboxRuntime.snapshot();
|
|
928
|
+
const usedResources = this.usedSandboxResources(runtimeState.degraded);
|
|
929
|
+
const activeResources = this.activeSandboxResources(runtimeState.degraded);
|
|
930
|
+
const capacity = this.sandboxResourceCapacity(admissionPreflight, activeResources);
|
|
931
|
+
if (!canReserveSandboxResources(usedResources, requestedResources, capacity)) {
|
|
932
|
+
return rejectedOfferReservation("capacity_unavailable", sandboxCapacityFailureMessage(usedResources, requestedResources, capacity, admissionPreflight, this.config));
|
|
933
|
+
}
|
|
934
|
+
}
|
|
935
|
+
const reservedUntil = new Date(Math.min(Date.parse(dispatch.lease.expiresAt), now + 30_000)).toISOString();
|
|
936
|
+
const reservation = {
|
|
323
937
|
leaseId: dispatch.lease.id,
|
|
324
938
|
fencingToken: dispatch.lease.fencingToken,
|
|
325
939
|
runnerInstanceId: sandbox.runnerInstanceId,
|
|
326
940
|
expiresAt: dispatch.lease.expiresAt,
|
|
327
941
|
reservedUntil,
|
|
328
942
|
sandboxed: sandbox.executionMode === "docker",
|
|
329
|
-
resources: requestedResources
|
|
330
|
-
}
|
|
331
|
-
|
|
943
|
+
resources: requestedResources,
|
|
944
|
+
};
|
|
945
|
+
this.reservations.set(sandbox.dispatchAttemptId, reservation);
|
|
946
|
+
return {
|
|
947
|
+
accepted: true,
|
|
948
|
+
code: null,
|
|
949
|
+
message: null,
|
|
950
|
+
reservedUntil,
|
|
951
|
+
resources: requestedResources,
|
|
952
|
+
};
|
|
332
953
|
}
|
|
333
954
|
async handleStart(rawDispatch) {
|
|
334
955
|
const envelope = parseStartEnvelope(rawDispatch);
|
|
335
956
|
if (!envelope) {
|
|
336
957
|
return;
|
|
337
958
|
}
|
|
338
|
-
let dispatch = this.parseDispatchOrReport(envelope.dispatch);
|
|
959
|
+
let dispatch = await this.parseDispatchOrReport(envelope.dispatch);
|
|
339
960
|
if (!dispatch?.sandbox
|
|
340
961
|
|| dispatch.sandbox.dispatchAttemptId !== envelope.dispatchAttemptId
|
|
341
962
|
|| dispatch.lease.id !== envelope.leaseId
|
|
342
963
|
|| dispatch.lease.fencingToken !== envelope.fencingToken)
|
|
343
964
|
return;
|
|
965
|
+
const active = this.activeSessions.get(dispatch.lease.id);
|
|
966
|
+
if (active?.kind === "dispatch"
|
|
967
|
+
&& active.fencingToken === dispatch.lease.fencingToken
|
|
968
|
+
&& active.dispatchAttemptId === dispatch.sandbox.dispatchAttemptId
|
|
969
|
+
&& active.runnerInstanceId === dispatch.sandbox.runnerInstanceId) {
|
|
970
|
+
return;
|
|
971
|
+
}
|
|
344
972
|
const reservation = this.reservations.get(dispatch.sandbox.dispatchAttemptId);
|
|
345
973
|
if (!reservation
|
|
346
974
|
|| reservation.leaseId !== dispatch.lease.id
|
|
347
975
|
|| reservation.fencingToken !== dispatch.lease.fencingToken
|
|
348
976
|
|| reservation.runnerInstanceId !== dispatch.sandbox.runnerInstanceId
|
|
349
977
|
|| Date.parse(reservation.reservedUntil) <= Date.now()) {
|
|
350
|
-
await this.sendResult(dispatch, failureResult("Start rejected because its reservation is missing or stale.")).catch(() => undefined);
|
|
978
|
+
await this.sendResult(dispatch, failureResult("Start rejected because its reservation is missing or stale.", [], failureFactsForCode("bridge.reservation_stale"))).catch(() => undefined);
|
|
351
979
|
return;
|
|
352
980
|
}
|
|
981
|
+
// Claim the exact reservation before redemption or any other post-validation
|
|
982
|
+
// await. A duplicate Start for the same reservation is an idempotent replay:
|
|
983
|
+
// it must neither redeem a one-time grant nor disturb the original owner.
|
|
984
|
+
if (reservation.startClaim)
|
|
985
|
+
return;
|
|
986
|
+
const startClaim = {
|
|
987
|
+
dispatchAttemptId: envelope.dispatchAttemptId,
|
|
988
|
+
reservation,
|
|
989
|
+
controller: new AbortController(),
|
|
990
|
+
};
|
|
991
|
+
reservation.startClaim = startClaim;
|
|
353
992
|
if (dispatch.apiKeySecretGrant) {
|
|
354
|
-
this.
|
|
355
|
-
await this.sendResult(dispatch, failureResult("Start rejected because protocol-v2 packets may not contain a plaintext provider grant.")).catch(() => undefined);
|
|
993
|
+
this.releaseReservedDispatchHandoff(startClaim);
|
|
994
|
+
await this.sendResult(dispatch, failureResult("Start rejected because protocol-v2 packets may not contain a plaintext provider grant.", [], failureFactsForCode("contract_plaintext_secret_rejected"))).catch(() => undefined);
|
|
356
995
|
return;
|
|
357
996
|
}
|
|
358
997
|
if (dispatch.apiKeySecretGrantHandle) {
|
|
359
|
-
|
|
998
|
+
let response;
|
|
999
|
+
try {
|
|
1000
|
+
response = await this.redeemSecretGrant(dispatch);
|
|
1001
|
+
}
|
|
1002
|
+
catch (error) {
|
|
1003
|
+
this.releaseReservedDispatchHandoff(startClaim);
|
|
1004
|
+
throw error;
|
|
1005
|
+
}
|
|
1006
|
+
// Redemption may yield while a qualified Cancel releases this reservation.
|
|
1007
|
+
// Never launch from it or remove a successor reservation after that wait.
|
|
1008
|
+
if (!this.reservedDispatchHandoffIsCurrent(startClaim, dispatch)) {
|
|
1009
|
+
await this.sendResult(dispatch, failureResult("Start rejected because its reservation is missing or stale.", [], failureFactsForCode("bridge.reservation_stale"))).catch(() => undefined);
|
|
1010
|
+
return;
|
|
1011
|
+
}
|
|
360
1012
|
const grant = response.grant;
|
|
361
1013
|
if (!response.accepted
|
|
362
1014
|
|| !grant
|
|
363
1015
|
|| grant.grantId !== dispatch.apiKeySecretGrantHandle.grantId
|
|
364
1016
|
|| grant.provider !== dispatch.apiKeySecretGrantHandle.provider
|
|
365
1017
|
|| Date.parse(grant.expiresAt) <= Date.now()) {
|
|
366
|
-
this.
|
|
367
|
-
await this.sendResult(dispatch, failureResult(response.message ?? "Secret grant redemption was rejected.")).catch(() => undefined);
|
|
1018
|
+
this.releaseReservedDispatchHandoff(startClaim);
|
|
1019
|
+
await this.sendResult(dispatch, failureResult(response.message ?? "Secret grant redemption was rejected.", [], failureFactsForCode("bridge.api_key_secret_grant_denied"))).catch(() => undefined);
|
|
368
1020
|
return;
|
|
369
1021
|
}
|
|
370
1022
|
dispatch = { ...dispatch, apiKeySecretGrant: grant, apiKeySecretGrantHandle: null };
|
|
371
1023
|
}
|
|
372
1024
|
else if (dispatch.execution.requiresApiKeySecretGrant) {
|
|
373
|
-
this.
|
|
374
|
-
await this.sendResult(dispatch, failureResult("Start rejected because the provider secret grant handle is missing.")).catch(() => undefined);
|
|
1025
|
+
this.releaseReservedDispatchHandoff(startClaim);
|
|
1026
|
+
await this.sendResult(dispatch, failureResult("Start rejected because the provider secret grant handle is missing.", [], failureFactsForCode("bridge.api_key_secret_grant_missing"))).catch(() => undefined);
|
|
375
1027
|
return;
|
|
376
1028
|
}
|
|
377
1029
|
dispatch = {
|
|
378
1030
|
...dispatch,
|
|
379
1031
|
sandbox: { ...dispatch.sandbox, requestedResources: reservation.resources },
|
|
380
1032
|
};
|
|
381
|
-
this.
|
|
382
|
-
await this.handleDispatch(dispatch);
|
|
1033
|
+
await this.handleDispatch(dispatch, startClaim);
|
|
383
1034
|
}
|
|
384
1035
|
async redeemSecretGrant(dispatch) {
|
|
385
1036
|
const sandbox = dispatch.sandbox;
|
|
@@ -443,7 +1094,7 @@ export class ForgeBridgeService {
|
|
|
443
1094
|
};
|
|
444
1095
|
await this.invokeAck("OfferDecision", payload);
|
|
445
1096
|
}
|
|
446
|
-
parseDispatchOrReport(rawDispatch) {
|
|
1097
|
+
async parseDispatchOrReport(rawDispatch) {
|
|
447
1098
|
try {
|
|
448
1099
|
return parseRunOnceDispatchPacket(rawDispatch);
|
|
449
1100
|
}
|
|
@@ -453,24 +1104,78 @@ export class ForgeBridgeService {
|
|
|
453
1104
|
return null;
|
|
454
1105
|
}
|
|
455
1106
|
const summary = invalidDispatchSummary(error);
|
|
456
|
-
|
|
1107
|
+
await this.sendLegacyResult(identifiers.leaseId, identifiers.fencingToken, failureResult(summary, [
|
|
457
1108
|
{
|
|
458
1109
|
role: "runner",
|
|
459
1110
|
kind: "error",
|
|
460
1111
|
content: summary,
|
|
461
1112
|
eventType: "invalid_dispatch"
|
|
462
1113
|
}
|
|
463
|
-
])).catch(() => undefined);
|
|
1114
|
+
], failureFactsForCode("invalid_dispatch"))).catch(() => undefined);
|
|
1115
|
+
return null;
|
|
1116
|
+
}
|
|
1117
|
+
}
|
|
1118
|
+
parseChatTurnOrReport(rawDispatch) {
|
|
1119
|
+
try {
|
|
1120
|
+
return parseChatTurnDispatchPacket(rawDispatch, this.bridgeChatCapability());
|
|
1121
|
+
}
|
|
1122
|
+
catch (error) {
|
|
1123
|
+
logBridgeStatus(invalidChatTurnSummary(error));
|
|
464
1124
|
return null;
|
|
465
1125
|
}
|
|
466
1126
|
}
|
|
467
1127
|
async handleCancel(request) {
|
|
1128
|
+
if (request.dispatchAttemptId !== undefined && request.dispatchAttemptId !== null) {
|
|
1129
|
+
// Qualified cancellation is reservation-only. In particular, a delayed
|
|
1130
|
+
// pre-start Cancel must not abort an active session if Start won the race.
|
|
1131
|
+
const reservation = this.reservations.get(request.dispatchAttemptId);
|
|
1132
|
+
if (reservation
|
|
1133
|
+
&& request.runnerId === this.config.runnerId
|
|
1134
|
+
&& request.runnerInstanceId === this.config.runnerInstanceId
|
|
1135
|
+
&& reservation.runnerInstanceId === request.runnerInstanceId
|
|
1136
|
+
&& reservation.leaseId === request.leaseId
|
|
1137
|
+
&& reservation.fencingToken === request.fencingToken) {
|
|
1138
|
+
reservation.startClaim?.controller.abort(request.reason ?? "cancelled by server");
|
|
1139
|
+
this.reservations.delete(request.dispatchAttemptId);
|
|
1140
|
+
}
|
|
1141
|
+
return;
|
|
1142
|
+
}
|
|
468
1143
|
const active = this.activeSessions.get(request.leaseId);
|
|
469
1144
|
if (!active || active.fencingToken !== request.fencingToken) {
|
|
470
1145
|
return;
|
|
471
1146
|
}
|
|
1147
|
+
if (request.runnerId && request.runnerId !== this.config.runnerId) {
|
|
1148
|
+
return;
|
|
1149
|
+
}
|
|
1150
|
+
if (request.runnerInstanceId && request.runnerInstanceId !== (active.runnerInstanceId ?? this.config.runnerInstanceId)) {
|
|
1151
|
+
return;
|
|
1152
|
+
}
|
|
1153
|
+
if (active.kind === "chat" && !this.chatCancelMatchesActiveTurn(active, request)) {
|
|
1154
|
+
return;
|
|
1155
|
+
}
|
|
472
1156
|
active.controller.abort(request.reason ?? "cancelled by server");
|
|
473
1157
|
}
|
|
1158
|
+
chatCancelMatchesActiveTurn(active, request) {
|
|
1159
|
+
if (request.protocolVersion && request.protocolVersion !== bridgeChatProtocolVersion) {
|
|
1160
|
+
return false;
|
|
1161
|
+
}
|
|
1162
|
+
try {
|
|
1163
|
+
assertBridgeChatCallbackPayload(request.payload, "cancel payload");
|
|
1164
|
+
}
|
|
1165
|
+
catch {
|
|
1166
|
+
return false;
|
|
1167
|
+
}
|
|
1168
|
+
if (request.chatRunId && request.chatRunId !== active.chatRunId)
|
|
1169
|
+
return false;
|
|
1170
|
+
if (request.conversationId && request.conversationId !== active.conversationId)
|
|
1171
|
+
return false;
|
|
1172
|
+
if (request.turnId && request.turnId !== active.turnId)
|
|
1173
|
+
return false;
|
|
1174
|
+
if (request.providerThreadId && active.providerThreadId && request.providerThreadId !== active.providerThreadId) {
|
|
1175
|
+
return false;
|
|
1176
|
+
}
|
|
1177
|
+
return true;
|
|
1178
|
+
}
|
|
474
1179
|
async handleReap(request) {
|
|
475
1180
|
if (request.runnerId !== this.config.runnerId
|
|
476
1181
|
|| request.runnerInstanceId !== this.config.runnerInstanceId)
|
|
@@ -492,32 +1197,43 @@ export class ForgeBridgeService {
|
|
|
492
1197
|
timer.unref?.();
|
|
493
1198
|
let result;
|
|
494
1199
|
try {
|
|
495
|
-
|
|
496
|
-
throw new Error("host_runtime_probe_unavailable: this bridge does not implement runtime readiness probes.");
|
|
497
|
-
}
|
|
498
|
-
const probe = await this.sessions.probeHostRuntime({
|
|
499
|
-
...request,
|
|
500
|
-
runnerId: this.config.runnerId,
|
|
501
|
-
repoRoot: this.config.defaultRepoRoot,
|
|
502
|
-
cleanupWorktrees: this.config.cleanupWorktrees,
|
|
503
|
-
}, controller.signal);
|
|
504
|
-
result = {
|
|
1200
|
+
const accepted = {
|
|
505
1201
|
schemaVersion: 1,
|
|
506
1202
|
requestId: request.requestId,
|
|
507
1203
|
fencingToken: request.fencingToken,
|
|
508
1204
|
runnerInstanceId: this.config.runnerInstanceId,
|
|
509
|
-
|
|
1205
|
+
acceptedAt: new Date().toISOString(),
|
|
510
1206
|
};
|
|
1207
|
+
await this.invokeAck("ReadinessAccepted", accepted);
|
|
1208
|
+
if (this.activeSessions.size > 0) {
|
|
1209
|
+
result = deferredReadinessResult(request, this.config.runnerInstanceId);
|
|
1210
|
+
}
|
|
1211
|
+
else if (!this.sessions.probeHostRuntime) {
|
|
1212
|
+
throw new Error("host_runtime_probe_unavailable: this bridge does not implement runtime readiness probes.");
|
|
1213
|
+
}
|
|
1214
|
+
else {
|
|
1215
|
+
const probe = await this.sessions.probeHostRuntime(this.normalizeReadinessProbe(request), controller.signal);
|
|
1216
|
+
result = readinessDeferralReason(controller.signal)
|
|
1217
|
+
? deferredReadinessResult(request, this.config.runnerInstanceId)
|
|
1218
|
+
: {
|
|
1219
|
+
schemaVersion: 1,
|
|
1220
|
+
requestId: request.requestId,
|
|
1221
|
+
fencingToken: request.fencingToken,
|
|
1222
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
1223
|
+
...probe,
|
|
1224
|
+
};
|
|
1225
|
+
}
|
|
511
1226
|
}
|
|
512
1227
|
catch (error) {
|
|
1228
|
+
const deferredReason = readinessDeferralReason(controller.signal);
|
|
513
1229
|
result = {
|
|
514
1230
|
schemaVersion: 1,
|
|
515
1231
|
requestId: request.requestId,
|
|
516
1232
|
fencingToken: request.fencingToken,
|
|
517
1233
|
runnerInstanceId: this.config.runnerInstanceId,
|
|
518
1234
|
status: "failed",
|
|
519
|
-
reasonCode: "host_runtime_probe_failed",
|
|
520
|
-
reasonMessage: errorMessage(error).slice(0, 1000),
|
|
1235
|
+
reasonCode: deferredReason ? "host_runtime_probe_deferred" : "host_runtime_probe_failed",
|
|
1236
|
+
reasonMessage: (deferredReason ?? errorMessage(error)).slice(0, 1000),
|
|
521
1237
|
checkedAt: new Date().toISOString(),
|
|
522
1238
|
};
|
|
523
1239
|
}
|
|
@@ -525,9 +1241,69 @@ export class ForgeBridgeService {
|
|
|
525
1241
|
clearTimeout(timer);
|
|
526
1242
|
this.activeReadinessProbes.delete(request.requestId);
|
|
527
1243
|
}
|
|
528
|
-
|
|
1244
|
+
try {
|
|
1245
|
+
await this.readinessResultOutbox.persist(result, request.expiresAt);
|
|
1246
|
+
}
|
|
1247
|
+
catch (error) {
|
|
1248
|
+
if (errorMessage(error).includes("runtime_readiness_result_conflict:")) {
|
|
1249
|
+
logBridgeStatus(`Readiness result ${request.requestId} rejected conflicting reuse of a durable request identity.`);
|
|
1250
|
+
await this.sendHeartbeat().catch(() => undefined);
|
|
1251
|
+
return;
|
|
1252
|
+
}
|
|
1253
|
+
logBridgeStatus(`Readiness result ${request.requestId} could not be persisted; attempting direct delivery: ${errorMessage(error)}`);
|
|
1254
|
+
await this.invokeAck("ReadinessResult", result).catch(deliveryError => {
|
|
1255
|
+
logBridgeStatus(`Readiness result ${request.requestId} could not be persisted or delivered: ${errorMessage(deliveryError)}`);
|
|
1256
|
+
});
|
|
1257
|
+
await this.sendHeartbeat().catch(() => undefined);
|
|
1258
|
+
return;
|
|
1259
|
+
}
|
|
1260
|
+
try {
|
|
1261
|
+
await this.flushReadinessResults();
|
|
1262
|
+
}
|
|
1263
|
+
catch (error) {
|
|
1264
|
+
logBridgeStatus(`Readiness result ${request.requestId} remains queued for replay: ${errorMessage(error)}`);
|
|
1265
|
+
}
|
|
529
1266
|
await this.sendHeartbeat().catch(() => undefined);
|
|
530
1267
|
}
|
|
1268
|
+
deferActiveReadinessProbes() {
|
|
1269
|
+
for (const controller of this.activeReadinessProbes.values()) {
|
|
1270
|
+
if (!controller.signal.aborted) {
|
|
1271
|
+
controller.abort("host_runtime_probe_deferred: foreground agent work took priority over background runtime readiness.");
|
|
1272
|
+
}
|
|
1273
|
+
}
|
|
1274
|
+
}
|
|
1275
|
+
async flushReadinessResults() {
|
|
1276
|
+
if (this.readinessResultFlush)
|
|
1277
|
+
return await this.readinessResultFlush;
|
|
1278
|
+
const flush = this.flushReadinessResultsCore();
|
|
1279
|
+
this.readinessResultFlush = flush;
|
|
1280
|
+
try {
|
|
1281
|
+
await flush;
|
|
1282
|
+
}
|
|
1283
|
+
finally {
|
|
1284
|
+
if (this.readinessResultFlush === flush)
|
|
1285
|
+
this.readinessResultFlush = null;
|
|
1286
|
+
}
|
|
1287
|
+
}
|
|
1288
|
+
async flushReadinessResultsCore() {
|
|
1289
|
+
for (const pending of await this.readinessResultOutbox.pending()) {
|
|
1290
|
+
if (Date.parse(pending.expiresAt) <= Date.now()) {
|
|
1291
|
+
await this.readinessResultOutbox.acknowledge(pending.result.requestId);
|
|
1292
|
+
continue;
|
|
1293
|
+
}
|
|
1294
|
+
try {
|
|
1295
|
+
await this.invokeAck("ReadinessResult", pending.result);
|
|
1296
|
+
await this.readinessResultOutbox.acknowledge(pending.result.requestId);
|
|
1297
|
+
}
|
|
1298
|
+
catch (error) {
|
|
1299
|
+
if (isReadinessNoLongerPending(error)) {
|
|
1300
|
+
await this.readinessResultOutbox.acknowledge(pending.result.requestId);
|
|
1301
|
+
continue;
|
|
1302
|
+
}
|
|
1303
|
+
throw error;
|
|
1304
|
+
}
|
|
1305
|
+
}
|
|
1306
|
+
}
|
|
531
1307
|
abortActiveSessions(reason) {
|
|
532
1308
|
for (const active of this.activeSessions.values()) {
|
|
533
1309
|
active.controller.abort(reason);
|
|
@@ -539,8 +1315,42 @@ export class ForgeBridgeService {
|
|
|
539
1315
|
controller.abort(reason);
|
|
540
1316
|
this.activeReadinessProbes.clear();
|
|
541
1317
|
}
|
|
1318
|
+
normalizeReadinessProbe(request) {
|
|
1319
|
+
const mapping = findRepoMapping({
|
|
1320
|
+
id: request.repoId,
|
|
1321
|
+
name: request.repoName,
|
|
1322
|
+
remoteUrl: request.remoteUrl,
|
|
1323
|
+
}, repoMappingsFor(this.config));
|
|
1324
|
+
const configuredPath = mapping?.localPath?.trim();
|
|
1325
|
+
if (!configuredPath) {
|
|
1326
|
+
throw new Error("bridge_repo_mapping_missing: readiness requires a matching local repository mapping.");
|
|
1327
|
+
}
|
|
1328
|
+
// Local configuration, not a path supplied by the server, grants checkout
|
|
1329
|
+
// authority. Equality also excludes a different child/sibling of this repo.
|
|
1330
|
+
const suppliedPath = request.runnerRepoLocalPath.trim();
|
|
1331
|
+
if (!path.isAbsolute(suppliedPath)
|
|
1332
|
+
|| (process.platform === "win32" && path.parse(suppliedPath).root.length === 1)) {
|
|
1333
|
+
throw new Error("bridge_repo_mapping_mismatch: readiness requires a fully qualified checkout path.");
|
|
1334
|
+
}
|
|
1335
|
+
const localPath = path.resolve(configuredPath);
|
|
1336
|
+
const requestedPath = path.resolve(suppliedPath);
|
|
1337
|
+
const pathsMatch = process.platform === "win32"
|
|
1338
|
+
? localPath.toLowerCase() === requestedPath.toLowerCase()
|
|
1339
|
+
: localPath === requestedPath;
|
|
1340
|
+
if (!pathsMatch) {
|
|
1341
|
+
throw new Error("bridge_repo_mapping_mismatch: readiness checkout differs from the locally configured repository.");
|
|
1342
|
+
}
|
|
1343
|
+
return {
|
|
1344
|
+
...request,
|
|
1345
|
+
runnerId: this.config.runnerId,
|
|
1346
|
+
runnerRepoLocalPath: localPath,
|
|
1347
|
+
cloneOnDemand: false,
|
|
1348
|
+
repoRoot: path.dirname(localPath),
|
|
1349
|
+
cleanupWorktrees: this.config.cleanupWorktrees,
|
|
1350
|
+
};
|
|
1351
|
+
}
|
|
542
1352
|
normalizeDispatch(dispatch) {
|
|
543
|
-
const mapping = findRepoMapping(dispatch, repoMappingsFor(this.config));
|
|
1353
|
+
const mapping = findRepoMapping(dispatch.repo, repoMappingsFor(this.config));
|
|
544
1354
|
const localPath = mapping?.localPath?.trim();
|
|
545
1355
|
const repoRoot = localPath ? path.dirname(path.resolve(localPath)) : this.config.defaultRepoRoot;
|
|
546
1356
|
const qaLaunchTarget = normalizeQaAppLaunchTarget(mapping?.qaAppLaunchTarget) ?? dispatch.qaBrowser?.launchTarget ?? null;
|
|
@@ -569,24 +1379,105 @@ export class ForgeBridgeService {
|
|
|
569
1379
|
fencingToken: dispatch.lease.fencingToken,
|
|
570
1380
|
dispatchAttemptId: dispatch.sandbox?.dispatchAttemptId ?? null,
|
|
571
1381
|
runnerInstanceId: dispatch.sandbox?.runnerInstanceId ?? null,
|
|
572
|
-
|
|
1382
|
+
// Every terminal writer, including early admission failures and exception
|
|
1383
|
+
// paths, crosses the same redaction boundary before outbox persistence.
|
|
1384
|
+
result: redactRunOnceResultPacket(result, exactSecrets(dispatch, this.config))
|
|
573
1385
|
};
|
|
574
|
-
await this.sendResultPayload(payload);
|
|
1386
|
+
await this.sendResultPayload(payload, () => this.releaseActiveSessionAndRefreshCapacity(dispatch.lease.id, dispatch));
|
|
575
1387
|
}
|
|
576
1388
|
async sendLegacyResult(leaseId, fencingToken, result) {
|
|
577
1389
|
await this.sendResultPayload({ leaseId, fencingToken, result });
|
|
578
1390
|
}
|
|
579
|
-
async sendResultPayload(payload) {
|
|
1391
|
+
async sendResultPayload(payload, beforeDelivery) {
|
|
1392
|
+
// Register before persistence: replay can observe a record as soon as the
|
|
1393
|
+
// durable write becomes visible, even before persist() has returned.
|
|
1394
|
+
let releaseHandoff;
|
|
1395
|
+
const priorHandoff = this.dispatchResultHandoffs.get(payload.leaseId);
|
|
1396
|
+
const handoff = new Promise(resolve => { releaseHandoff = resolve; });
|
|
1397
|
+
this.dispatchResultHandoffs.set(payload.leaseId, handoff);
|
|
1398
|
+
let persisted = false;
|
|
1399
|
+
try {
|
|
1400
|
+
try {
|
|
1401
|
+
await this.dispatchResultOutbox.persist(payload);
|
|
1402
|
+
persisted = true;
|
|
1403
|
+
this.dispatchResultGeneration += 1;
|
|
1404
|
+
}
|
|
1405
|
+
catch (error) {
|
|
1406
|
+
logBridgeStatus(`Dispatch result ${payload.leaseId} could not be persisted; attempting direct delivery: ${errorMessage(error)}`);
|
|
1407
|
+
}
|
|
1408
|
+
await priorHandoff;
|
|
1409
|
+
await beforeDelivery?.();
|
|
1410
|
+
}
|
|
1411
|
+
finally {
|
|
1412
|
+
releaseHandoff();
|
|
1413
|
+
if (this.dispatchResultHandoffs.get(payload.leaseId) === handoff)
|
|
1414
|
+
this.dispatchResultHandoffs.delete(payload.leaseId);
|
|
1415
|
+
}
|
|
1416
|
+
if (!persisted) {
|
|
1417
|
+
await this.sendResultPayloadDirect(payload);
|
|
1418
|
+
return;
|
|
1419
|
+
}
|
|
1420
|
+
await this.flushDispatchResults().catch(error => {
|
|
1421
|
+
logBridgeStatus(`Dispatch result ${payload.leaseId} remains queued for replay: ${errorMessage(error)}`);
|
|
1422
|
+
});
|
|
1423
|
+
}
|
|
1424
|
+
async sendResultPayloadDirect(payload) {
|
|
580
1425
|
try {
|
|
581
1426
|
await this.invokeAck("DispatchResult", payload);
|
|
582
1427
|
}
|
|
583
1428
|
catch (error) {
|
|
584
|
-
if (isDispatchNoLongerPending(error))
|
|
1429
|
+
if (isDispatchNoLongerPending(error))
|
|
585
1430
|
return;
|
|
586
|
-
}
|
|
587
1431
|
throw error;
|
|
588
1432
|
}
|
|
589
1433
|
}
|
|
1434
|
+
async flushDispatchResults(force = false) {
|
|
1435
|
+
if (!force && this.dispatchResultGeneration === this.dispatchResultFlushedGeneration)
|
|
1436
|
+
return;
|
|
1437
|
+
if (this.dispatchResultFlush) {
|
|
1438
|
+
await this.dispatchResultFlush;
|
|
1439
|
+
if (!force && this.dispatchResultGeneration === this.dispatchResultFlushedGeneration)
|
|
1440
|
+
return;
|
|
1441
|
+
}
|
|
1442
|
+
const flush = this.flushDispatchResultsCore();
|
|
1443
|
+
this.dispatchResultFlush = flush;
|
|
1444
|
+
try {
|
|
1445
|
+
await flush;
|
|
1446
|
+
}
|
|
1447
|
+
finally {
|
|
1448
|
+
if (this.dispatchResultFlush === flush)
|
|
1449
|
+
this.dispatchResultFlush = null;
|
|
1450
|
+
}
|
|
1451
|
+
}
|
|
1452
|
+
async flushDispatchResultsCore() {
|
|
1453
|
+
const startedGeneration = this.dispatchResultGeneration;
|
|
1454
|
+
while (true) {
|
|
1455
|
+
const records = await this.dispatchResultOutbox.pending();
|
|
1456
|
+
for (const pending of records) {
|
|
1457
|
+
// Periodic heartbeats and reconnect replay share the same release
|
|
1458
|
+
// barrier as the terminal writer. Restart replay has no live handoff.
|
|
1459
|
+
while (this.dispatchResultHandoffs.has(pending.payload.leaseId)) {
|
|
1460
|
+
await this.dispatchResultHandoffs.get(pending.payload.leaseId);
|
|
1461
|
+
}
|
|
1462
|
+
try {
|
|
1463
|
+
await this.sendResultPayloadDirect(pending.payload);
|
|
1464
|
+
await this.dispatchResultOutbox.acknowledge(pending.payload.leaseId);
|
|
1465
|
+
}
|
|
1466
|
+
catch (error) {
|
|
1467
|
+
if (isDispatchNoLongerPending(error)) {
|
|
1468
|
+
await this.dispatchResultOutbox.acknowledge(pending.payload.leaseId);
|
|
1469
|
+
continue;
|
|
1470
|
+
}
|
|
1471
|
+
throw error;
|
|
1472
|
+
}
|
|
1473
|
+
}
|
|
1474
|
+
if (records.length < 3)
|
|
1475
|
+
break;
|
|
1476
|
+
}
|
|
1477
|
+
if (this.dispatchResultGeneration === startedGeneration) {
|
|
1478
|
+
this.dispatchResultFlushedGeneration = startedGeneration;
|
|
1479
|
+
}
|
|
1480
|
+
}
|
|
590
1481
|
startHeartbeat() {
|
|
591
1482
|
this.stopHeartbeat();
|
|
592
1483
|
this.consecutiveHeartbeatFailures = 0;
|
|
@@ -605,6 +1496,8 @@ export class ForgeBridgeService {
|
|
|
605
1496
|
return;
|
|
606
1497
|
this.heartbeatInFlight = true;
|
|
607
1498
|
try {
|
|
1499
|
+
await this.flushDispatchResults();
|
|
1500
|
+
await this.flushReadinessResults();
|
|
608
1501
|
await this.sendHeartbeat();
|
|
609
1502
|
this.consecutiveHeartbeatFailures = 0;
|
|
610
1503
|
}
|
|
@@ -637,17 +1530,20 @@ export class ForgeBridgeService {
|
|
|
637
1530
|
}
|
|
638
1531
|
expireLeases() {
|
|
639
1532
|
const now = Date.now();
|
|
640
|
-
|
|
641
|
-
if (Date.parse(reservation.expiresAt) <= now || Date.parse(reservation.reservedUntil) <= now) {
|
|
642
|
-
this.reservations.delete(attemptId);
|
|
643
|
-
}
|
|
644
|
-
}
|
|
1533
|
+
this.expireReservations(now);
|
|
645
1534
|
for (const active of this.activeSessions.values()) {
|
|
646
1535
|
if (Date.parse(active.leaseExpiresAt) <= now && !active.controller.signal.aborted) {
|
|
647
1536
|
active.controller.abort("dispatch lease expired");
|
|
648
1537
|
}
|
|
649
1538
|
}
|
|
650
1539
|
}
|
|
1540
|
+
expireReservations(now) {
|
|
1541
|
+
for (const [attemptId, reservation] of this.reservations) {
|
|
1542
|
+
if (Date.parse(reservation.expiresAt) <= now || Date.parse(reservation.reservedUntil) <= now) {
|
|
1543
|
+
this.reservations.delete(attemptId);
|
|
1544
|
+
}
|
|
1545
|
+
}
|
|
1546
|
+
}
|
|
651
1547
|
stopHeartbeat() {
|
|
652
1548
|
if (this.heartbeatTimer) {
|
|
653
1549
|
clearInterval(this.heartbeatTimer);
|
|
@@ -676,6 +1572,7 @@ export class ForgeBridgeService {
|
|
|
676
1572
|
runnerId: this.config.runnerId,
|
|
677
1573
|
runnerInstanceId: this.config.runnerInstanceId,
|
|
678
1574
|
});
|
|
1575
|
+
this.rememberRecoveredHostAttempts(hostReap);
|
|
679
1576
|
if (hostReap && (hostReap.reaped > 0 || hostReap.degraded > 0)) {
|
|
680
1577
|
logBridgeStatus(`Managed-host maintenance reaped ${hostReap.reaped} stale attempt(s) and left ${hostReap.degraded} degraded attempt(s) for reconciliation.`);
|
|
681
1578
|
}
|
|
@@ -716,22 +1613,50 @@ export class ForgeBridgeService {
|
|
|
716
1613
|
}
|
|
717
1614
|
}
|
|
718
1615
|
async sendHeartbeat() {
|
|
1616
|
+
// Heartbeats are the recovery path for capacity-blocked work. Refresh the
|
|
1617
|
+
// dynamic sample before advertising it, while reusing the static Docker
|
|
1618
|
+
// capability snapshot established at startup or by an offer.
|
|
1619
|
+
if (this.config.dockerEnabled)
|
|
1620
|
+
await this.preflight.refreshCapacity();
|
|
1621
|
+
// A due probe can start a Docker Desktop recovery that runs for minutes.
|
|
1622
|
+
// Wait only as long as a normal probe takes, then advertise the last
|
|
1623
|
+
// snapshot so the server doesn't mark the runner offline meanwhile.
|
|
1624
|
+
const probeWait = new AbortController();
|
|
1625
|
+
await Promise.race([
|
|
1626
|
+
this.dockerEngineProbe.refresh().catch(() => undefined),
|
|
1627
|
+
delay(positiveFiniteNumber(this.config.heartbeatDockerProbeWaitMs ?? 15_000, 15_000), probeWait.signal),
|
|
1628
|
+
]);
|
|
1629
|
+
probeWait.abort();
|
|
1630
|
+
await this.sendHeartbeatSnapshot();
|
|
1631
|
+
}
|
|
1632
|
+
async sendHeartbeatSnapshot(signal) {
|
|
719
1633
|
await this.invokeAck("Heartbeat", {
|
|
720
1634
|
status: "online",
|
|
721
1635
|
capabilities: this.capabilities(),
|
|
722
1636
|
version: this.config.version,
|
|
723
1637
|
protocolVersion: this.config.protocolVersion
|
|
724
|
-
});
|
|
1638
|
+
}, { signal });
|
|
725
1639
|
}
|
|
726
1640
|
capabilities() {
|
|
727
1641
|
const preflight = this.preflight.current();
|
|
1642
|
+
const dockerEngine = this.dockerEngineProbe.current();
|
|
728
1643
|
const runtimeState = this.sandboxRuntime.snapshot();
|
|
729
1644
|
const reserved = [...this.reservations.values()].filter(reservation => reservation.sandboxed).length;
|
|
730
1645
|
const used = this.usedSandboxResources(runtimeState.degraded);
|
|
731
1646
|
const activeResources = this.activeSandboxResources(runtimeState.degraded);
|
|
732
1647
|
const capacity = this.sandboxResourceCapacity(preflight, activeResources);
|
|
1648
|
+
const memoryAdmission = preflight ? buildMemoryAdmissionDiagnostics({
|
|
1649
|
+
sampledAt: preflight.sampledAt,
|
|
1650
|
+
totalMemoryBytes: preflight.totalMemoryBytes,
|
|
1651
|
+
rawAvailableMemoryBytes: preflight.rawAvailableMemoryBytes,
|
|
1652
|
+
stabilizedAvailableMemoryBytes: preflight.availableMemoryBytes,
|
|
1653
|
+
controlPlaneMemoryBytes: process.memoryUsage().rss,
|
|
1654
|
+
reserveMemoryBytes: preflight.hostReserveBytes,
|
|
1655
|
+
committedMemoryBytes: used.memoryBytes,
|
|
1656
|
+
pendingCommitmentMemoryBytes: Math.max(0, used.memoryBytes - activeResources.memoryBytes),
|
|
1657
|
+
}) : null;
|
|
733
1658
|
return {
|
|
734
|
-
schemaVersion:
|
|
1659
|
+
schemaVersion: 6,
|
|
735
1660
|
activeSessionCount: this.activeSessions.size,
|
|
736
1661
|
cliProviders: this.config.cliProviders,
|
|
737
1662
|
providerStatuses: this.config.providerStatuses,
|
|
@@ -741,6 +1666,19 @@ export class ForgeBridgeService {
|
|
|
741
1666
|
sandboxProfiles: this.config.sandboxProfiles,
|
|
742
1667
|
runnerInstanceId: this.config.runnerInstanceId,
|
|
743
1668
|
trustDomainId: this.config.trustDomainId,
|
|
1669
|
+
executionFacilities: dockerEngine?.available ? ["docker_engine"] : [],
|
|
1670
|
+
dockerEngine,
|
|
1671
|
+
toolchains: {
|
|
1672
|
+
...this.hostCapabilities.toolchains,
|
|
1673
|
+
docker: {
|
|
1674
|
+
available: dockerEngine?.available === true,
|
|
1675
|
+
executable: this.config.dockerExecutable,
|
|
1676
|
+
version: dockerEngine?.runtimeVersion ?? null,
|
|
1677
|
+
},
|
|
1678
|
+
},
|
|
1679
|
+
playwrightProjects: this.hostCapabilities.playwrightProjects,
|
|
1680
|
+
browserLaunch: this.hostCapabilities.browserLaunch,
|
|
1681
|
+
providerConnectivity: this.hostCapabilities.providerConnectivity,
|
|
744
1682
|
maxConcurrentSandboxes: capacity.sandboxSlots,
|
|
745
1683
|
reservedSandboxCount: reserved,
|
|
746
1684
|
activeSandboxCount: runtimeState.active,
|
|
@@ -752,7 +1690,7 @@ export class ForgeBridgeService {
|
|
|
752
1690
|
allowsOpenPublicSandbox: this.config.allowOpenPublicSandbox,
|
|
753
1691
|
sandboxPublicationModes: [this.config.sandboxPublicationMode ?? "shared_filesystem"],
|
|
754
1692
|
sandboxAuthModes: sandboxAuthModesFor(this.config),
|
|
755
|
-
availableMemoryBytes:
|
|
1693
|
+
availableMemoryBytes: memoryAdmission?.availableForAdmissionBytes ?? null,
|
|
756
1694
|
availableCpuMillis: preflight ? Math.max(0, capacity.cpuMillis - used.cpuMillis) : null,
|
|
757
1695
|
availableWorkspaceBytes: preflight ? Math.max(0, capacity.workspaceBytes - used.workspaceBytes) : null,
|
|
758
1696
|
diskLow: preflight?.diskLow ?? null,
|
|
@@ -766,8 +1704,11 @@ export class ForgeBridgeService {
|
|
|
766
1704
|
hostReserveBytes: preflight.hostReserveBytes,
|
|
767
1705
|
activeReservationBytes: used.memoryBytes,
|
|
768
1706
|
daemonTotalBytes: preflight.totalMemoryBytes,
|
|
769
|
-
daemonAvailableBytes: preflight.
|
|
770
|
-
|
|
1707
|
+
daemonAvailableBytes: preflight.rawDaemonAvailableMemoryBytes ?? 0,
|
|
1708
|
+
hostAvailableBytes: preflight.rawHostAvailableMemoryBytes,
|
|
1709
|
+
conservativeAvailableBytes: preflight.rawAvailableMemoryBytes,
|
|
1710
|
+
availableForAdmissionBytes: memoryAdmission.availableForAdmissionBytes,
|
|
1711
|
+
memoryAdmission: memoryAdmission,
|
|
771
1712
|
} : null,
|
|
772
1713
|
hostRuntime: this.sessions.runtimeCapacity?.() ?? null,
|
|
773
1714
|
supportsApiSecretGrant: this.config.supportsApiSecretGrant,
|
|
@@ -776,9 +1717,20 @@ export class ForgeBridgeService {
|
|
|
776
1717
|
supportsRunnerAuthoredPhaseArtifacts: this.config.supportsRunnerAuthoredPhaseArtifacts,
|
|
777
1718
|
supportsDraftRunOnce: this.config.supportsDraftRunOnce,
|
|
778
1719
|
supportsDraftAttachments: this.config.supportsDraftAttachments,
|
|
779
|
-
supportsDraftImageAttachments: this.config.supportsDraftImageAttachments
|
|
1720
|
+
supportsDraftImageAttachments: this.config.supportsDraftImageAttachments,
|
|
1721
|
+
bridgeChat: bridgeChatCapabilityFor(this.config.cliProviders, this.config.providerStatuses, this.config.bridgeChat)
|
|
780
1722
|
};
|
|
781
1723
|
}
|
|
1724
|
+
async requiredCapabilityFailure(dispatch) {
|
|
1725
|
+
const requiredCapabilities = dispatch.qaScope?.required_capabilities ?? [];
|
|
1726
|
+
if (!requiredCapabilities.includes("docker_engine"))
|
|
1727
|
+
return null;
|
|
1728
|
+
const executionMode = dispatch.sandbox?.executionMode;
|
|
1729
|
+
if (executionMode === "docker") {
|
|
1730
|
+
return dockerEngineRequirementFailure(requiredCapabilities, executionMode, null);
|
|
1731
|
+
}
|
|
1732
|
+
return dockerEngineRequirementFailure(requiredCapabilities, executionMode, await this.dockerEngineProbe.refresh(true));
|
|
1733
|
+
}
|
|
782
1734
|
usedSandboxResources(degradedReservations) {
|
|
783
1735
|
const resources = [
|
|
784
1736
|
...[...this.reservations.values()]
|
|
@@ -811,9 +1763,7 @@ export class ForgeBridgeService {
|
|
|
811
1763
|
workerMemoryLimitBytes: 0,
|
|
812
1764
|
serviceMemoryLimitBytes: 0,
|
|
813
1765
|
runtimeOverheadBytes: 0,
|
|
814
|
-
memoryBytes:
|
|
815
|
-
- (this.config.sandboxHostMemoryReserveBytes ?? 2 * 1024 ** 3)
|
|
816
|
-
+ activeResources.memoryBytes),
|
|
1766
|
+
memoryBytes: sandboxMemoryCommitmentCapacity(preflight, activeResources.memoryBytes),
|
|
817
1767
|
workspaceBytes: Math.max(0, preflight.availableWorkspaceBytes
|
|
818
1768
|
- this.config.minimumWorkspaceBytes
|
|
819
1769
|
+ activeResources.workspaceBytes),
|
|
@@ -821,22 +1771,96 @@ export class ForgeBridgeService {
|
|
|
821
1771
|
buildSlots: this.config.maxConcurrentSandboxBuilds
|
|
822
1772
|
};
|
|
823
1773
|
}
|
|
824
|
-
async invokeAck(methodName, payload) {
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
1774
|
+
async invokeAck(methodName, payload, options = {}) {
|
|
1775
|
+
let storageRecoveryAttempt = 0;
|
|
1776
|
+
let callbackTimeoutAttempt = 0;
|
|
1777
|
+
while (true) {
|
|
1778
|
+
if (options.signal?.aborted) {
|
|
1779
|
+
throw new Error(String(options.signal.reason ?? `${methodName} callback was cancelled.`));
|
|
1780
|
+
}
|
|
1781
|
+
const connection = this.connection;
|
|
1782
|
+
if (!connection) {
|
|
1783
|
+
throw new Error("Bridge is not connected.");
|
|
1784
|
+
}
|
|
1785
|
+
await this.waitForTransportRecovery(connection, this.transportRecovery, options.signal);
|
|
1786
|
+
const recoveryGeneration = this.transportRecoveryGeneration;
|
|
1787
|
+
try {
|
|
1788
|
+
const ack = await waitForCallbackAck(connection.invoke(methodName, payload), methodName, positiveFiniteNumber(this.config.callbackAckTimeoutMs ?? 30_000, 30_000), options.signal);
|
|
1789
|
+
if (!ack.accepted) {
|
|
1790
|
+
throw new BridgeCallbackRejectedError(ack.code ?? null, ack.message ?? ack.code ?? `${methodName} callback was rejected.`);
|
|
1791
|
+
}
|
|
1792
|
+
return ack;
|
|
1793
|
+
}
|
|
1794
|
+
catch (error) {
|
|
1795
|
+
if (options.signal?.aborted)
|
|
1796
|
+
throw error;
|
|
1797
|
+
if (connection !== this.connection || this.stopped)
|
|
1798
|
+
throw error;
|
|
1799
|
+
const recovery = this.transportRecovery;
|
|
1800
|
+
if (recovery && recovery.generation > recoveryGeneration) {
|
|
1801
|
+
await this.waitForTransportRecovery(connection, recovery, options.signal);
|
|
1802
|
+
continue;
|
|
1803
|
+
}
|
|
1804
|
+
if (isProviderStorageUnavailable(error)) {
|
|
1805
|
+
const backoff = this.config.reconnectBackoffMs[Math.min(storageRecoveryAttempt, this.config.reconnectBackoffMs.length - 1)] ?? 5000;
|
|
1806
|
+
storageRecoveryAttempt += 1;
|
|
1807
|
+
logBridgeStatus(`${methodName} reached Forge while storage was recovering. Retrying callback ${storageRecoveryAttempt} in ${formatDelay(backoff)}...`);
|
|
1808
|
+
await delay(backoff, options.signal);
|
|
1809
|
+
continue;
|
|
1810
|
+
}
|
|
1811
|
+
if (options.replayCallbackTimeout && isBridgeCallbackTimeout(error)) {
|
|
1812
|
+
const backoff = this.config.reconnectBackoffMs[Math.min(callbackTimeoutAttempt, this.config.reconnectBackoffMs.length - 1)] ?? 5000;
|
|
1813
|
+
callbackTimeoutAttempt += 1;
|
|
1814
|
+
logBridgeStatus(`${methodName} acknowledgement timed out. Replaying the idempotent callback ${callbackTimeoutAttempt} in ${formatDelay(backoff)}...`);
|
|
1815
|
+
await delay(backoff, options.signal);
|
|
1816
|
+
continue;
|
|
1817
|
+
}
|
|
1818
|
+
throw error;
|
|
1819
|
+
}
|
|
828
1820
|
}
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
1821
|
+
}
|
|
1822
|
+
beginTransportRecovery() {
|
|
1823
|
+
if (this.transportRecovery && !this.transportRecovery.completed)
|
|
1824
|
+
return;
|
|
1825
|
+
let resolve = () => undefined;
|
|
1826
|
+
const promise = new Promise((complete) => { resolve = complete; });
|
|
1827
|
+
this.transportRecoveryGeneration += 1;
|
|
1828
|
+
this.transportRecovery = {
|
|
1829
|
+
generation: this.transportRecoveryGeneration,
|
|
1830
|
+
promise,
|
|
1831
|
+
resolve,
|
|
1832
|
+
completed: false,
|
|
1833
|
+
error: null,
|
|
1834
|
+
};
|
|
1835
|
+
}
|
|
1836
|
+
completeTransportRecovery(error = null) {
|
|
1837
|
+
const recovery = this.transportRecovery;
|
|
1838
|
+
if (!recovery || recovery.completed)
|
|
1839
|
+
return;
|
|
1840
|
+
recovery.completed = true;
|
|
1841
|
+
recovery.error = error;
|
|
1842
|
+
recovery.resolve();
|
|
1843
|
+
}
|
|
1844
|
+
async waitForTransportRecovery(connection, recovery, signal) {
|
|
1845
|
+
if (!recovery)
|
|
1846
|
+
return;
|
|
1847
|
+
if (recovery.completed) {
|
|
1848
|
+
if (recovery.error)
|
|
1849
|
+
throw recovery.error;
|
|
1850
|
+
return;
|
|
832
1851
|
}
|
|
833
|
-
|
|
1852
|
+
await waitForPromiseOrAbort(recovery.promise, signal);
|
|
1853
|
+
signal?.throwIfAborted();
|
|
1854
|
+
if (connection !== this.connection)
|
|
1855
|
+
throw new Error("Bridge connection changed while recovering.");
|
|
1856
|
+
if (recovery.error)
|
|
1857
|
+
throw recovery.error;
|
|
834
1858
|
}
|
|
835
1859
|
async waitForClose(signal) {
|
|
836
1860
|
if (!this.closed) {
|
|
837
1861
|
return;
|
|
838
1862
|
}
|
|
839
|
-
await
|
|
1863
|
+
await waitForPromiseOrAbort(this.closed, signal);
|
|
840
1864
|
}
|
|
841
1865
|
async stopConnection() {
|
|
842
1866
|
this.stopHeartbeat();
|
|
@@ -862,11 +1886,23 @@ function isDispatchNoLongerPending(error) {
|
|
|
862
1886
|
}
|
|
863
1887
|
return error instanceof Error && error.message === "Dispatch is no longer pending.";
|
|
864
1888
|
}
|
|
865
|
-
function
|
|
1889
|
+
function isReadinessNoLongerPending(error) {
|
|
1890
|
+
return error instanceof BridgeCallbackRejectedError
|
|
1891
|
+
&& (error.code === "bridge.stale_readiness_probe" || error.code === "bridge.stale_runner_instance");
|
|
1892
|
+
}
|
|
1893
|
+
function isProviderStorageUnavailable(error) {
|
|
1894
|
+
const message = errorMessage(error).toLowerCase();
|
|
1895
|
+
return message.includes("provider_storage_unavailable")
|
|
1896
|
+
|| (message.includes("storage is recovering") && message.includes("503"));
|
|
1897
|
+
}
|
|
1898
|
+
function isBridgeCallbackTimeout(error) {
|
|
1899
|
+
return errorMessage(error).startsWith("bridge_callback_timeout:");
|
|
1900
|
+
}
|
|
1901
|
+
function createDefaultSessionService(sandboxStateRoot, sandboxExecutionRoot, dockerEngineProbe) {
|
|
866
1902
|
const rawProcessRunner = new StreamingProviderProcessRunner();
|
|
867
1903
|
const processSupervisors = new AttemptProcessSupervisorRegistry();
|
|
868
1904
|
const coordinatedProcessRunner = new AttemptCoordinatedProviderProcessRunner(processSupervisors, rawProcessRunner);
|
|
869
|
-
return new BridgeSessionService(new GitWorktreeService(undefined, new RepositoryCoordinatorLock(sandboxStateRoot)), new RoutingProviderCliService(new ProcessBackedProviderCliService(coordinatedProcessRunner)), new HostRuntimeLeaseManager(sandboxStateRoot, undefined, undefined, processSupervisors, rawProcessRunner));
|
|
1905
|
+
return new BridgeSessionService(new GitWorktreeService(undefined, new RepositoryCoordinatorLock(sandboxStateRoot)), new RoutingProviderCliService(new ProcessBackedProviderCliService(coordinatedProcessRunner, undefined, undefined, undefined, new PrePublicationValidator(undefined, dockerEngineProbe, readNativeTestPostgresFallback))), new HostRuntimeLeaseManager(sandboxStateRoot, undefined, undefined, processSupervisors, rawProcessRunner, undefined, undefined, undefined, undefined, undefined, sandboxExecutionRoot, createNativeTestClusterDatabaseBroker(readNativeTestPostgresFallback, sandboxStateRoot)));
|
|
870
1906
|
}
|
|
871
1907
|
export class SignalRBridgeConnectionFactory {
|
|
872
1908
|
create(config) {
|
|
@@ -890,10 +1926,11 @@ export class SignalRBridgeConnection {
|
|
|
890
1926
|
Authorization: `Bearer ${config.deviceToken}`
|
|
891
1927
|
}
|
|
892
1928
|
})
|
|
1929
|
+
.withAutomaticReconnect([...config.reconnectBackoffMs])
|
|
893
1930
|
.configureLogging(LogLevel.Warning)
|
|
894
1931
|
.build();
|
|
895
1932
|
this.connection.on("Superseded", () => {
|
|
896
|
-
void this.connection.stop();
|
|
1933
|
+
void this.connection.stop().catch(() => undefined);
|
|
897
1934
|
});
|
|
898
1935
|
}
|
|
899
1936
|
start() {
|
|
@@ -905,11 +1942,24 @@ export class SignalRBridgeConnection {
|
|
|
905
1942
|
onClose(handler) {
|
|
906
1943
|
this.connection.onclose(handler);
|
|
907
1944
|
}
|
|
1945
|
+
onReconnecting(handler) {
|
|
1946
|
+
this.connection.onreconnecting(handler);
|
|
1947
|
+
}
|
|
1948
|
+
onReconnected(handler) {
|
|
1949
|
+
this.connection.onreconnected((connectionId) => {
|
|
1950
|
+
void Promise.resolve(handler(connectionId)).catch(() => undefined);
|
|
1951
|
+
});
|
|
1952
|
+
}
|
|
908
1953
|
onDispatch(handler) {
|
|
909
1954
|
this.connection.on("Dispatch", (raw) => {
|
|
910
1955
|
void handler(raw).catch(() => undefined);
|
|
911
1956
|
});
|
|
912
1957
|
}
|
|
1958
|
+
onChatTurn(handler) {
|
|
1959
|
+
this.connection.on("ChatTurn", (raw) => {
|
|
1960
|
+
void handler(raw).catch(() => undefined);
|
|
1961
|
+
});
|
|
1962
|
+
}
|
|
913
1963
|
onOffer(handler) {
|
|
914
1964
|
this.connection.on("Offer", (raw) => {
|
|
915
1965
|
void handler(raw).catch(() => undefined);
|
|
@@ -947,9 +1997,7 @@ export class SignalRBridgeConnection {
|
|
|
947
1997
|
export function defaultBridgeServiceConfig(overrides = {}) {
|
|
948
1998
|
const defaults = defaultForgeBridgeConfig();
|
|
949
1999
|
const cliProviders = splitCsv(process.env.FORGE_CLI_PROVIDERS ?? "codex");
|
|
950
|
-
const
|
|
951
|
-
? 2 * 1024 ** 3
|
|
952
|
-
: 1024 ** 3;
|
|
2000
|
+
const configuredSandboxHostMemoryReserveBytes = optionalPositiveFiniteNumber(process.env.FORGE_SANDBOX_HOST_MEMORY_RESERVE_BYTES);
|
|
953
2001
|
const sandboxPublicationMode = process.env.FORGE_SANDBOX_PUBLICATION_MODE === "bridge_push" ? "bridge_push" : "shared_filesystem";
|
|
954
2002
|
const config = {
|
|
955
2003
|
serverUrl: process.env.FORGE_URL ?? "http://localhost:5208",
|
|
@@ -964,12 +2012,19 @@ export function defaultBridgeServiceConfig(overrides = {}) {
|
|
|
964
2012
|
sandboxPublicationMode,
|
|
965
2013
|
dockerExecutable: process.env.FORGE_DOCKER_EXECUTABLE ?? "docker",
|
|
966
2014
|
dockerContext: process.env.FORGE_DOCKER_CONTEXT ?? "",
|
|
2015
|
+
dockerDesktopRecoveryEnabled: process.env.FORGE_DOCKER_DESKTOP_AUTO_RECOVERY === "true",
|
|
2016
|
+
dockerRecoveryFailureThreshold: positiveFiniteNumber(Number(process.env.FORGE_DOCKER_RECOVERY_FAILURE_THRESHOLD ?? 3), 3),
|
|
2017
|
+
dockerRecoveryCooldownMs: positiveFiniteNumber(Number(process.env.FORGE_DOCKER_RECOVERY_COOLDOWN_MS ?? 10 * 60_000), 10 * 60_000),
|
|
2018
|
+
dockerRecoveryStartupTimeoutMs: positiveFiniteNumber(Number(process.env.FORGE_DOCKER_RECOVERY_STARTUP_TIMEOUT_MS ?? 2 * 60_000), 2 * 60_000),
|
|
967
2019
|
sandboxStateRoot: process.env.FORGE_SANDBOX_STATE_ROOT ?? path.join(os.homedir(), ".forge", "bridge", "sandboxes"),
|
|
2020
|
+
sandboxExecutionRoot: defaultHostExecutionRoot(),
|
|
968
2021
|
requireRootlessDocker: process.env.FORGE_REQUIRE_ROOTLESS_DOCKER === "true" || process.platform === "linux",
|
|
969
2022
|
diskHighWatermarkPercent: positiveFiniteNumber(Number(process.env.FORGE_DISK_HIGH_WATERMARK_PERCENT ?? 85), 85),
|
|
970
2023
|
diskLowWatermarkPercent: positiveFiniteNumber(Number(process.env.FORGE_DISK_LOW_WATERMARK_PERCENT ?? 75), 75),
|
|
971
2024
|
minimumWorkspaceBytes: positiveFiniteNumber(Number(process.env.FORGE_MINIMUM_WORKSPACE_BYTES ?? 10 * 1024 ** 3), 10 * 1024 ** 3),
|
|
972
|
-
|
|
2025
|
+
// Undefined means preflight derives the bounded reserve from the actual
|
|
2026
|
+
// Docker/host capacity domain after sampling it.
|
|
2027
|
+
sandboxHostMemoryReserveBytes: configuredSandboxHostMemoryReserveBytes,
|
|
973
2028
|
defaultSandboxWorkerMemoryLimitBytes: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_DEFAULT_WORKER_MEMORY_LIMIT_BYTES ?? 2 * 1024 ** 3), 2 * 1024 ** 3),
|
|
974
2029
|
defaultSandboxServiceMemoryLimitBytes: nonNegativeFiniteNumber(Number(process.env.FORGE_SANDBOX_DEFAULT_SERVICE_MEMORY_LIMIT_BYTES ?? 0), 0),
|
|
975
2030
|
sandboxRuntimeOverheadBytes: Math.max(128 * 1024 ** 2, nonNegativeFiniteNumber(Number(process.env.FORGE_SANDBOX_RUNTIME_OVERHEAD_BYTES ?? 128 * 1024 ** 2), 128 * 1024 ** 2)),
|
|
@@ -995,10 +2050,12 @@ export function defaultBridgeServiceConfig(overrides = {}) {
|
|
|
995
2050
|
supportsDraftRunOnce: true,
|
|
996
2051
|
supportsDraftAttachments: true,
|
|
997
2052
|
supportsDraftImageAttachments: supportsDraftImageAttachmentsFor(cliProviders),
|
|
2053
|
+
bridgeChat: bridgeChatCapabilityFor(cliProviders, providerStatusesFor(cliProviders)),
|
|
998
2054
|
version: process.env.FORGE_BRIDGE_VERSION ?? forgeBridgeVersion,
|
|
999
|
-
protocolVersion:
|
|
2055
|
+
protocolVersion: 4,
|
|
1000
2056
|
minServerVersion: null,
|
|
1001
2057
|
heartbeatIntervalMs: positiveFiniteNumber(Number(process.env.FORGE_HEARTBEAT_INTERVAL_MS ?? 30_000), 30_000),
|
|
2058
|
+
callbackAckTimeoutMs: positiveFiniteNumber(Number(process.env.FORGE_CALLBACK_ACK_TIMEOUT_MS ?? 30_000), 30_000),
|
|
1002
2059
|
reconnectBackoffMs: [1000, 2000, 5000, 10000, 30000],
|
|
1003
2060
|
...overrides
|
|
1004
2061
|
};
|
|
@@ -1007,7 +2064,9 @@ export function defaultBridgeServiceConfig(overrides = {}) {
|
|
|
1007
2064
|
providerStatuses: overrides.providerStatuses ?? providerStatusesFor(config.cliProviders),
|
|
1008
2065
|
sandboxProfiles: config.sandboxProfiles ?? defaultSandboxProfiles(config.cliProviders),
|
|
1009
2066
|
supportsDraftImageAttachments: (overrides.supportsDraftImageAttachments ?? true) && supportsDraftImageAttachmentsFor(config.cliProviders),
|
|
1010
|
-
|
|
2067
|
+
bridgeChat: overrides.bridgeChat ?? bridgeChatCapabilityFor(config.cliProviders, overrides.providerStatuses ?? providerStatusesFor(config.cliProviders), config.bridgeChat),
|
|
2068
|
+
heartbeatIntervalMs: positiveFiniteNumber(config.heartbeatIntervalMs, 30_000),
|
|
2069
|
+
callbackAckTimeoutMs: positiveFiniteNumber(config.callbackAckTimeoutMs ?? 30_000, 30_000),
|
|
1011
2070
|
};
|
|
1012
2071
|
}
|
|
1013
2072
|
export function resolveBridgeRunnerHubUrl(serverUrl) {
|
|
@@ -1022,6 +2081,12 @@ function normalizeServerUrl(serverUrl) {
|
|
|
1022
2081
|
function positiveFiniteNumber(value, fallback) {
|
|
1023
2082
|
return Number.isFinite(value) && value > 0 ? value : fallback;
|
|
1024
2083
|
}
|
|
2084
|
+
function optionalPositiveFiniteNumber(value) {
|
|
2085
|
+
if (value === undefined || value.trim().length === 0)
|
|
2086
|
+
return undefined;
|
|
2087
|
+
const parsed = Number(value);
|
|
2088
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
|
|
2089
|
+
}
|
|
1025
2090
|
function nonNegativeFiniteNumber(value, fallback) {
|
|
1026
2091
|
return Number.isFinite(value) && value >= 0 ? value : fallback;
|
|
1027
2092
|
}
|
|
@@ -1126,15 +2191,32 @@ function mappingKeys(mapping) {
|
|
|
1126
2191
|
mapping.localPath ? `path:${path.resolve(mapping.localPath).toLowerCase()}` : null
|
|
1127
2192
|
].filter((key) => Boolean(key));
|
|
1128
2193
|
}
|
|
1129
|
-
function findRepoMapping(
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
2194
|
+
function findRepoMapping(repo, mappings) {
|
|
2195
|
+
const usable = mappings.filter(mapping => Boolean(mapping.localPath?.trim()));
|
|
2196
|
+
const selectUnique = (candidates, identity) => {
|
|
2197
|
+
if (candidates.length > 1) {
|
|
2198
|
+
throw new Error(`bridge_repo_mapping_ambiguous: multiple local checkouts match ${identity}.`);
|
|
1133
2199
|
}
|
|
1134
|
-
return
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
2200
|
+
return candidates[0];
|
|
2201
|
+
};
|
|
2202
|
+
// Repository identity is authoritative. A same-name mapping for a different
|
|
2203
|
+
// repository must never win merely because it appears first in configuration.
|
|
2204
|
+
const byId = usable.filter(mapping => equalsIgnoreCase(mapping.repoId, repo.id));
|
|
2205
|
+
if (byId.length > 0)
|
|
2206
|
+
return selectUnique(byId, `repo id '${repo.id}'`);
|
|
2207
|
+
const byRemote = repo.remoteUrl?.trim()
|
|
2208
|
+
? usable.filter(mapping => !mapping.repoId?.trim()
|
|
2209
|
+
&& Boolean(mapping.remoteUrl?.trim())
|
|
2210
|
+
&& equalsIgnoreCase(mapping.remoteUrl, repo.remoteUrl))
|
|
2211
|
+
: [];
|
|
2212
|
+
if (byRemote.length > 0)
|
|
2213
|
+
return selectUnique(byRemote, `remote '${repo.remoteUrl}'`);
|
|
2214
|
+
const byName = usable.filter(mapping => !mapping.repoId?.trim()
|
|
2215
|
+
&& !mapping.remoteUrl?.trim()
|
|
2216
|
+
&& equalsIgnoreCase(mapping.name, repo.name));
|
|
2217
|
+
return byName.length > 0
|
|
2218
|
+
? selectUnique(byName, `repo name '${repo.name}'`)
|
|
2219
|
+
: undefined;
|
|
1138
2220
|
}
|
|
1139
2221
|
function normalizeQaAppLaunchTarget(target) {
|
|
1140
2222
|
if (!target) {
|
|
@@ -1183,6 +2265,10 @@ function invalidDispatchSummary(error) {
|
|
|
1183
2265
|
const detail = error instanceof Error ? error.message : String(error);
|
|
1184
2266
|
return `Invalid bridge dispatch packet: ${detail}`;
|
|
1185
2267
|
}
|
|
2268
|
+
function invalidChatTurnSummary(error) {
|
|
2269
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
2270
|
+
return `Invalid Bridge Chat turn packet: ${detail}`;
|
|
2271
|
+
}
|
|
1186
2272
|
function errorMessage(error) {
|
|
1187
2273
|
return error instanceof Error ? error.message : String(error);
|
|
1188
2274
|
}
|
|
@@ -1192,29 +2278,328 @@ function formatDelay(ms) {
|
|
|
1192
2278
|
}
|
|
1193
2279
|
return `${ms}ms`;
|
|
1194
2280
|
}
|
|
2281
|
+
async function waitForCallbackAck(callback, methodName, timeoutMs, signal) {
|
|
2282
|
+
let timeout = null;
|
|
2283
|
+
let rejectAbort = null;
|
|
2284
|
+
const expired = new Promise((_resolve, reject) => {
|
|
2285
|
+
timeout = setTimeout(() => {
|
|
2286
|
+
reject(new Error(`bridge_callback_timeout: ${methodName} callback was not acknowledged within ${formatDelay(timeoutMs)}.`));
|
|
2287
|
+
}, timeoutMs);
|
|
2288
|
+
});
|
|
2289
|
+
const aborted = new Promise((_resolve, reject) => {
|
|
2290
|
+
rejectAbort = reject;
|
|
2291
|
+
});
|
|
2292
|
+
const onAbort = () => rejectAbort?.(new Error(String(signal?.reason ?? `${methodName} callback was cancelled.`)));
|
|
2293
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
2294
|
+
if (signal?.aborted)
|
|
2295
|
+
onAbort();
|
|
2296
|
+
try {
|
|
2297
|
+
return await Promise.race(signal ? [callback, expired, aborted] : [callback, expired]);
|
|
2298
|
+
}
|
|
2299
|
+
finally {
|
|
2300
|
+
if (timeout)
|
|
2301
|
+
clearTimeout(timeout);
|
|
2302
|
+
signal?.removeEventListener("abort", onAbort);
|
|
2303
|
+
}
|
|
2304
|
+
}
|
|
1195
2305
|
function logBridgeStatus(message) {
|
|
1196
2306
|
process.stderr.write(`[${new Date().toISOString()}] ${message}\n`);
|
|
1197
2307
|
}
|
|
1198
|
-
function delay(ms, signal) {
|
|
2308
|
+
export function delay(ms, signal) {
|
|
1199
2309
|
if (signal?.aborted) {
|
|
1200
2310
|
return Promise.resolve();
|
|
1201
2311
|
}
|
|
1202
2312
|
return new Promise((resolve) => {
|
|
1203
|
-
const
|
|
1204
|
-
signal?.addEventListener("abort", () => {
|
|
2313
|
+
const complete = () => {
|
|
1205
2314
|
clearTimeout(timeout);
|
|
2315
|
+
signal?.removeEventListener("abort", complete);
|
|
1206
2316
|
resolve();
|
|
1207
|
-
}
|
|
2317
|
+
};
|
|
2318
|
+
const timeout = setTimeout(complete, ms);
|
|
2319
|
+
signal?.addEventListener("abort", complete, { once: true });
|
|
2320
|
+
if (signal?.aborted)
|
|
2321
|
+
complete();
|
|
1208
2322
|
});
|
|
1209
2323
|
}
|
|
1210
|
-
function
|
|
2324
|
+
export async function waitForPromiseOrAbort(promise, signal) {
|
|
1211
2325
|
if (!signal) {
|
|
1212
|
-
|
|
2326
|
+
await promise;
|
|
2327
|
+
return;
|
|
1213
2328
|
}
|
|
1214
2329
|
if (signal.aborted) {
|
|
1215
|
-
return
|
|
2330
|
+
return;
|
|
2331
|
+
}
|
|
2332
|
+
let resolveAbort;
|
|
2333
|
+
const abort = new Promise(resolve => { resolveAbort = resolve; });
|
|
2334
|
+
signal.addEventListener("abort", resolveAbort, { once: true });
|
|
2335
|
+
try {
|
|
2336
|
+
await Promise.race([promise, abort]);
|
|
2337
|
+
}
|
|
2338
|
+
finally {
|
|
2339
|
+
signal.removeEventListener("abort", resolveAbort);
|
|
2340
|
+
}
|
|
2341
|
+
}
|
|
2342
|
+
// Matches the API Hello bound on settled recovered host attempts.
|
|
2343
|
+
const maxRecoveredHostAttempts = 128;
|
|
2344
|
+
const defaultChatTurnLeaseMs = 60 * 60 * 1000;
|
|
2345
|
+
const maxChatEventContentChars = 4000;
|
|
2346
|
+
const maxChatTerminalTextChars = 120_000;
|
|
2347
|
+
const maxChatExactSecretChars = 8 * 1024;
|
|
2348
|
+
const chatTurnMetadataFileName = "forge-chat-metadata.json";
|
|
2349
|
+
async function materializeChatTurnMetadata(packet, workspacePath) {
|
|
2350
|
+
if (packet.attachmentHandoffs.length === 0 && packet.actionBrokerGrants.length === 0) {
|
|
2351
|
+
return;
|
|
2352
|
+
}
|
|
2353
|
+
await fs.promises.writeFile(path.join(workspacePath, chatTurnMetadataFileName), `${JSON.stringify({
|
|
2354
|
+
schemaVersion: 1,
|
|
2355
|
+
attachmentHandoffs: packet.attachmentHandoffs.map((handoff) => ({
|
|
2356
|
+
turnId: handoff.turnId,
|
|
2357
|
+
attachmentId: handoff.attachmentId,
|
|
2358
|
+
tempFileToken: handoff.tempFileToken,
|
|
2359
|
+
mimeType: handoff.mimeType,
|
|
2360
|
+
sizeBytes: handoff.sizeBytes,
|
|
2361
|
+
sha256: handoff.sha256,
|
|
2362
|
+
fileNameDisplay: handoff.fileNameDisplay,
|
|
2363
|
+
expiresAt: handoff.expiresAt,
|
|
2364
|
+
status: handoff.status,
|
|
2365
|
+
})),
|
|
2366
|
+
actionBrokerGrants: packet.actionBrokerGrants.map((grant) => ({
|
|
2367
|
+
grantId: grant.grantId,
|
|
2368
|
+
turnId: grant.turnId,
|
|
2369
|
+
leaseId: grant.leaseId,
|
|
2370
|
+
fencingToken: grant.fencingToken,
|
|
2371
|
+
allowedScopes: grant.allowedScopes,
|
|
2372
|
+
expiresAt: grant.expiresAt,
|
|
2373
|
+
redactionStatus: grant.redactionStatus,
|
|
2374
|
+
nonceHeldByBridge: true,
|
|
2375
|
+
})),
|
|
2376
|
+
}, null, 2)}\n`, "utf8");
|
|
2377
|
+
}
|
|
2378
|
+
function chatTurnLeaseExpiresAt(packet, nowMs = Date.now()) {
|
|
2379
|
+
const defaultExpiresAtMs = nowMs + defaultChatTurnLeaseMs;
|
|
2380
|
+
const metadataExpiresAtMs = [
|
|
2381
|
+
...packet.attachmentHandoffs.map(handoff => Date.parse(handoff.expiresAt)),
|
|
2382
|
+
...packet.actionBrokerGrants.map(grant => Date.parse(grant.expiresAt)),
|
|
2383
|
+
].filter(Number.isFinite);
|
|
2384
|
+
return new Date(Math.min(defaultExpiresAtMs, ...metadataExpiresAtMs)).toISOString();
|
|
2385
|
+
}
|
|
2386
|
+
async function cleanupChatTurnWorkspace(workspacePath, redactionSecrets) {
|
|
2387
|
+
try {
|
|
2388
|
+
await fs.promises.rm(workspacePath, { recursive: true, force: true, maxRetries: 3 });
|
|
2389
|
+
return {
|
|
2390
|
+
status: "completed",
|
|
2391
|
+
eventType: "chat_turn_cleanup_completed",
|
|
2392
|
+
role: "runner",
|
|
2393
|
+
kind: "checkpoint",
|
|
2394
|
+
content: "Bridge Chat workspace cleanup completed.",
|
|
2395
|
+
};
|
|
2396
|
+
}
|
|
2397
|
+
catch (error) {
|
|
2398
|
+
return {
|
|
2399
|
+
status: "failed",
|
|
2400
|
+
eventType: "chat_turn_cleanup_failed",
|
|
2401
|
+
role: "runner",
|
|
2402
|
+
kind: "error",
|
|
2403
|
+
content: "Bridge Chat workspace cleanup failed.",
|
|
2404
|
+
message: clip(redactText(errorMessage(error), redactionSecrets), 1000),
|
|
2405
|
+
};
|
|
2406
|
+
}
|
|
2407
|
+
}
|
|
2408
|
+
function buildChatProviderCommand(packet, cwd) {
|
|
2409
|
+
const provider = packet.providerKind.trim().toLowerCase();
|
|
2410
|
+
const model = packet.model.trim();
|
|
2411
|
+
const prompt = buildChatPrompt(packet);
|
|
2412
|
+
const resumeThreadId = packet.providerThreadId?.trim() || null;
|
|
2413
|
+
if (provider === "codex") {
|
|
2414
|
+
const args = ["-s", codexSandboxModeForPlatform(process.platform, "host"), "-a", "never", "exec"];
|
|
2415
|
+
if (resumeThreadId) {
|
|
2416
|
+
args.push("--cd", cwd, "resume");
|
|
2417
|
+
}
|
|
2418
|
+
args.push("--model", model, "--json");
|
|
2419
|
+
if (!resumeThreadId) {
|
|
2420
|
+
args.push("--cd", cwd);
|
|
2421
|
+
}
|
|
2422
|
+
args.push("--skip-git-repo-check");
|
|
2423
|
+
if (resumeThreadId) {
|
|
2424
|
+
args.push(resumeThreadId);
|
|
2425
|
+
}
|
|
2426
|
+
args.push("-");
|
|
2427
|
+
return { command: { command: "codex", args }, stdin: prompt };
|
|
2428
|
+
}
|
|
2429
|
+
if (provider === "cursor") {
|
|
2430
|
+
const args = ["--print", "--output-format", "stream-json", "--model", model, "--workspace", cwd];
|
|
2431
|
+
if (resumeThreadId) {
|
|
2432
|
+
args.push("--resume", resumeThreadId);
|
|
2433
|
+
}
|
|
2434
|
+
args.push("--yolo", "--approve-mcps", "--trust", prompt);
|
|
2435
|
+
return { command: { command: "cursor-agent", args } };
|
|
2436
|
+
}
|
|
2437
|
+
if (provider === "claude") {
|
|
2438
|
+
const args = ["-p", "--output-format", "stream-json", "--verbose", "--model", model, "--permission-mode", "bypassPermissions"];
|
|
2439
|
+
if (resumeThreadId) {
|
|
2440
|
+
args.push("--resume", resumeThreadId);
|
|
2441
|
+
}
|
|
2442
|
+
return { command: { command: "claude", args }, stdin: prompt };
|
|
2443
|
+
}
|
|
2444
|
+
return {
|
|
2445
|
+
command: {
|
|
2446
|
+
command: provider,
|
|
2447
|
+
args: ["--model", model, prompt],
|
|
2448
|
+
},
|
|
2449
|
+
};
|
|
2450
|
+
}
|
|
2451
|
+
function buildChatPrompt(packet) {
|
|
2452
|
+
const messages = packet.messages
|
|
2453
|
+
.map((message, index) => {
|
|
2454
|
+
const role = message.role.trim() || `message-${index + 1}`;
|
|
2455
|
+
return `[${role}]\n${message.content}`;
|
|
2456
|
+
})
|
|
2457
|
+
.join("\n\n");
|
|
2458
|
+
const contextRefs = packet.contextRefs.length === 0
|
|
2459
|
+
? ""
|
|
2460
|
+
: `\n\n# Context References\n\n${packet.contextRefs.map((ref, index) => {
|
|
2461
|
+
const title = ref.title?.trim() ? `\nTitle: ${ref.title.trim()}` : "";
|
|
2462
|
+
const summary = ref.summary?.trim() ? `\nSummary: ${ref.summary.trim()}` : "";
|
|
2463
|
+
return `${index + 1}. ${ref.kind}:${ref.refId}${title}${summary}`;
|
|
2464
|
+
}).join("\n\n")}`;
|
|
2465
|
+
const continuation = packet.providerThreadId
|
|
2466
|
+
? "\n\nContinue the existing provider thread identified by the bridge; do not restart the conversation context."
|
|
2467
|
+
: "";
|
|
2468
|
+
const boundedMetadata = buildChatBoundedMetadataPrompt(packet);
|
|
2469
|
+
return `You are handling a Forge Bridge Chat turn.
|
|
2470
|
+
|
|
2471
|
+
Conversation ID: ${packet.conversationId}
|
|
2472
|
+
Turn ID: ${packet.turnId}
|
|
2473
|
+
Provider: ${packet.providerKind}
|
|
2474
|
+
Model: ${packet.model}${continuation}${boundedMetadata}
|
|
2475
|
+
|
|
2476
|
+
# Conversation
|
|
2477
|
+
|
|
2478
|
+
${messages}${contextRefs}
|
|
2479
|
+
`;
|
|
2480
|
+
}
|
|
2481
|
+
function buildChatBoundedMetadataPrompt(packet) {
|
|
2482
|
+
if (packet.attachmentHandoffs.length === 0 && packet.actionBrokerGrants.length === 0) {
|
|
2483
|
+
return "";
|
|
2484
|
+
}
|
|
2485
|
+
const sections = [
|
|
2486
|
+
`\n\n# Bounded Handoff Metadata\n\nMetadata file: ${chatTurnMetadataFileName}\n\nThe metadata file is turn-scoped and cleaned up by the bridge when the turn ends. It contains opaque handles only; attachment content, raw paths, and grant nonces are not included.`,
|
|
2487
|
+
];
|
|
2488
|
+
if (packet.attachmentHandoffs.length > 0) {
|
|
2489
|
+
sections.push(`Attachment handoffs:\n${packet.attachmentHandoffs.map((handoff, index) => [
|
|
2490
|
+
`${index + 1}. turnId: ${handoff.turnId}`,
|
|
2491
|
+
` attachmentId: ${handoff.attachmentId}`,
|
|
2492
|
+
` tempFileToken: ${handoff.tempFileToken}`,
|
|
2493
|
+
` mimeType: ${handoff.mimeType}`,
|
|
2494
|
+
` sizeBytes: ${handoff.sizeBytes}`,
|
|
2495
|
+
` sha256: ${handoff.sha256}`,
|
|
2496
|
+
` fileNameDisplay: ${handoff.fileNameDisplay}`,
|
|
2497
|
+
` expiresAt: ${handoff.expiresAt}`,
|
|
2498
|
+
` status: ${handoff.status}`,
|
|
2499
|
+
].join("\n")).join("\n\n")}`);
|
|
2500
|
+
}
|
|
2501
|
+
if (packet.actionBrokerGrants.length > 0) {
|
|
2502
|
+
sections.push(`Action-broker grants:\n${packet.actionBrokerGrants.map((grant, index) => [
|
|
2503
|
+
`${index + 1}. grantId: ${grant.grantId}`,
|
|
2504
|
+
` turnId: ${grant.turnId}`,
|
|
2505
|
+
` leaseId: ${grant.leaseId}`,
|
|
2506
|
+
` fencingToken: ${grant.fencingToken}`,
|
|
2507
|
+
` allowedScopes: ${grant.allowedScopes.join(", ")}`,
|
|
2508
|
+
` expiresAt: ${grant.expiresAt}`,
|
|
2509
|
+
` redactionStatus: ${grant.redactionStatus}`,
|
|
2510
|
+
" nonce: held by bridge",
|
|
2511
|
+
].join("\n")).join("\n\n")}`);
|
|
2512
|
+
}
|
|
2513
|
+
return sections.join("\n\n");
|
|
2514
|
+
}
|
|
2515
|
+
function chatProviderExitFailure(command, result, redactionSecrets) {
|
|
2516
|
+
const commandText = redactText(formatProviderCliCommandForCheckpoint(command), redactionSecrets);
|
|
2517
|
+
const stderr = clip(redactText(result.stderr.trim(), redactionSecrets), 1000);
|
|
2518
|
+
const stdout = clip(redactText(result.stdout.trim(), redactionSecrets), 1000);
|
|
2519
|
+
const diagnostics = [
|
|
2520
|
+
stderr ? `stderr: ${stderr}` : null,
|
|
2521
|
+
stdout ? `stdout: ${stdout}` : null,
|
|
2522
|
+
].filter(Boolean).join(" ");
|
|
2523
|
+
return new Error(`provider_failed: ${commandText} exited with code ${result.exitCode}.${diagnostics ? ` ${diagnostics}` : ""}`);
|
|
2524
|
+
}
|
|
2525
|
+
function chatAbortKind(reason) {
|
|
2526
|
+
const message = typeof reason === "string" ? reason.toLowerCase() : "";
|
|
2527
|
+
return message.includes("expired") ? "expired" : "cancelled";
|
|
2528
|
+
}
|
|
2529
|
+
function chatAbortPayload(reason, cleanupOutcome, packet) {
|
|
2530
|
+
const kind = chatAbortKind(reason);
|
|
2531
|
+
const message = typeof reason === "string" && reason.trim()
|
|
2532
|
+
? reason.trim()
|
|
2533
|
+
: (kind === "expired" ? "Bridge Chat turn expired." : "Bridge Chat turn cancelled.");
|
|
2534
|
+
return {
|
|
2535
|
+
code: kind === "expired" ? "chat_turn_expired" : "chat_turn_cancelled",
|
|
2536
|
+
message: clip(redactText(message, packet ? chatRedactionSecrets(packet) : []), 1000),
|
|
2537
|
+
cleanup: cleanupOutcome,
|
|
2538
|
+
};
|
|
2539
|
+
}
|
|
2540
|
+
function providerThreadIdFromEvent(event) {
|
|
2541
|
+
if (!event.data) {
|
|
2542
|
+
return null;
|
|
2543
|
+
}
|
|
2544
|
+
try {
|
|
2545
|
+
const data = JSON.parse(event.data);
|
|
2546
|
+
for (const key of ["threadId", "thread_id", "sessionId", "session_id", "providerThreadId"]) {
|
|
2547
|
+
const value = data[key];
|
|
2548
|
+
if (typeof value === "string") {
|
|
2549
|
+
const clean = value.trim();
|
|
2550
|
+
if (clean.length > 0 && clean.length <= bridgeChatMaxProviderThreadIdChars) {
|
|
2551
|
+
return clean;
|
|
2552
|
+
}
|
|
2553
|
+
}
|
|
2554
|
+
}
|
|
2555
|
+
}
|
|
2556
|
+
catch {
|
|
2557
|
+
return null;
|
|
1216
2558
|
}
|
|
1217
|
-
return
|
|
2559
|
+
return null;
|
|
2560
|
+
}
|
|
2561
|
+
function chatRedactionSecrets(packet) {
|
|
2562
|
+
return [
|
|
2563
|
+
packet.chatRunId,
|
|
2564
|
+
packet.conversationId,
|
|
2565
|
+
packet.turnId,
|
|
2566
|
+
packet.leaseId,
|
|
2567
|
+
packet.fencingToken,
|
|
2568
|
+
packet.runnerInstanceId,
|
|
2569
|
+
packet.providerThreadId,
|
|
2570
|
+
...packet.attachmentHandoffs.flatMap((handoff) => [
|
|
2571
|
+
handoff.attachmentId,
|
|
2572
|
+
handoff.tempFileToken,
|
|
2573
|
+
]),
|
|
2574
|
+
...packet.actionBrokerGrants.flatMap((grant) => [
|
|
2575
|
+
grant.grantId,
|
|
2576
|
+
grant.nonce,
|
|
2577
|
+
]),
|
|
2578
|
+
...packet.messages.flatMap((message) => boundedChatSecretFragments(message.content)),
|
|
2579
|
+
...packet.contextRefs.flatMap((ref) => [
|
|
2580
|
+
...boundedChatSecretFragments(ref.title ?? ""),
|
|
2581
|
+
...boundedChatSecretFragments(ref.summary ?? ""),
|
|
2582
|
+
]),
|
|
2583
|
+
].filter((value) => typeof value === "string" && value.length > 0);
|
|
2584
|
+
}
|
|
2585
|
+
function boundedChatSecretFragments(value) {
|
|
2586
|
+
const clean = value.trim();
|
|
2587
|
+
if (!clean) {
|
|
2588
|
+
return [];
|
|
2589
|
+
}
|
|
2590
|
+
if (clean.length <= maxChatExactSecretChars) {
|
|
2591
|
+
return [clean];
|
|
2592
|
+
}
|
|
2593
|
+
const half = Math.floor(maxChatExactSecretChars / 2);
|
|
2594
|
+
return [clean.slice(0, half), clean.slice(clean.length - half)];
|
|
2595
|
+
}
|
|
2596
|
+
function safePathComponent(value) {
|
|
2597
|
+
const clean = value.trim().replace(/[^a-zA-Z0-9_.-]+/gu, "-").replace(/^-+|-+$/gu, "");
|
|
2598
|
+
return clean || "chat-turn";
|
|
2599
|
+
}
|
|
2600
|
+
function containsIgnoreCase(values, value) {
|
|
2601
|
+
const normalized = value.trim().toLowerCase();
|
|
2602
|
+
return values.some((candidate) => candidate.trim().toLowerCase() === normalized);
|
|
1218
2603
|
}
|
|
1219
2604
|
export function canReapManagedSandboxes(preflight) {
|
|
1220
2605
|
return canRunDockerMaintenance(preflight);
|
|
@@ -1234,24 +2619,15 @@ function sandboxNetworkPolicyFailure(dispatch, config) {
|
|
|
1234
2619
|
}
|
|
1235
2620
|
return null;
|
|
1236
2621
|
}
|
|
1237
|
-
function
|
|
1238
|
-
if (
|
|
1239
|
-
return
|
|
1240
|
-
if (
|
|
1241
|
-
return "
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
if (summary.startsWith("host_runtime_listener_timeout:"))
|
|
1247
|
-
return "host_runtime_listener_timeout";
|
|
1248
|
-
if (summary.startsWith("host_runtime_http_readiness_timeout:"))
|
|
1249
|
-
return "host_runtime_http_readiness_timeout";
|
|
1250
|
-
if (summary.startsWith("attempt_process_supervisor_start_failed:"))
|
|
1251
|
-
return "attempt_process_supervisor_start_failed";
|
|
1252
|
-
if (summary.startsWith("attempt_process_supervisor_start_timeout:"))
|
|
1253
|
-
return "attempt_process_supervisor_start_timeout";
|
|
1254
|
-
return null;
|
|
2622
|
+
export function dockerEngineRequirementFailure(requiredCapabilities, executionMode, capability) {
|
|
2623
|
+
if (!requiredCapabilities.includes("docker_engine"))
|
|
2624
|
+
return null;
|
|
2625
|
+
if (executionMode === "docker") {
|
|
2626
|
+
return "docker_engine_unavailable: docker_engine is not exposed inside Forge agent sandboxes; use a managed host runner with Docker-daemon access.";
|
|
2627
|
+
}
|
|
2628
|
+
return capability?.available
|
|
2629
|
+
? null
|
|
2630
|
+
: `docker_engine_unavailable: ${capability?.unavailableReason ?? "Docker daemon is unreachable from the bridge process."}`;
|
|
1255
2631
|
}
|
|
1256
2632
|
function parseReadinessProbe(value) {
|
|
1257
2633
|
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
@@ -1279,6 +2655,24 @@ function parseReadinessProbe(value) {
|
|
|
1279
2655
|
}
|
|
1280
2656
|
return request;
|
|
1281
2657
|
}
|
|
2658
|
+
function deferredReadinessResult(request, runnerInstanceId) {
|
|
2659
|
+
return {
|
|
2660
|
+
schemaVersion: 1,
|
|
2661
|
+
requestId: request.requestId,
|
|
2662
|
+
fencingToken: request.fencingToken,
|
|
2663
|
+
runnerInstanceId,
|
|
2664
|
+
status: "failed",
|
|
2665
|
+
reasonCode: "host_runtime_probe_deferred",
|
|
2666
|
+
reasonMessage: "Foreground agent work is active; Forge deferred background runtime readiness so it cannot consume the agent's host capacity.",
|
|
2667
|
+
checkedAt: new Date().toISOString(),
|
|
2668
|
+
};
|
|
2669
|
+
}
|
|
2670
|
+
function readinessDeferralReason(signal) {
|
|
2671
|
+
if (!signal.aborted)
|
|
2672
|
+
return null;
|
|
2673
|
+
const reason = errorMessage(signal.reason);
|
|
2674
|
+
return reason.startsWith("host_runtime_probe_deferred:") ? reason : null;
|
|
2675
|
+
}
|
|
1282
2676
|
function sandboxProviderNetworkFailure(dispatch, config) {
|
|
1283
2677
|
const sandbox = dispatch.sandbox;
|
|
1284
2678
|
if (!sandbox || sandbox.networkPolicy.mode !== "offline")
|
|
@@ -1308,6 +2702,15 @@ function sandboxAuthFailure(dispatch, config) {
|
|
|
1308
2702
|
const doctor = adapter.doctor();
|
|
1309
2703
|
return doctor.available ? null : doctor.message;
|
|
1310
2704
|
}
|
|
2705
|
+
function rejectedOfferReservation(code, message) {
|
|
2706
|
+
return {
|
|
2707
|
+
accepted: false,
|
|
2708
|
+
code,
|
|
2709
|
+
message,
|
|
2710
|
+
reservedUntil: null,
|
|
2711
|
+
resources: null,
|
|
2712
|
+
};
|
|
2713
|
+
}
|
|
1311
2714
|
export function canReserveSandboxResources(used, requested, capacity) {
|
|
1312
2715
|
return used.sandboxSlots + requested.sandboxSlots <= capacity.sandboxSlots
|
|
1313
2716
|
&& used.cpuMillis + requested.cpuMillis <= capacity.cpuMillis
|
|
@@ -1316,6 +2719,14 @@ export function canReserveSandboxResources(used, requested, capacity) {
|
|
|
1316
2719
|
&& used.pids + requested.pids <= capacity.pids
|
|
1317
2720
|
&& used.buildSlots + requested.buildSlots <= capacity.buildSlots;
|
|
1318
2721
|
}
|
|
2722
|
+
export function sandboxMemoryCommitmentCapacity(preflight, activeMemoryBytes) {
|
|
2723
|
+
const ambientCapacity = preflight.availableMemoryBytes
|
|
2724
|
+
- preflight.hostReserveBytes
|
|
2725
|
+
+ activeMemoryBytes;
|
|
2726
|
+
const commitmentCapacity = preflight.totalMemoryBytes
|
|
2727
|
+
- preflight.hostReserveBytes;
|
|
2728
|
+
return Math.max(0, Math.min(ambientCapacity, commitmentCapacity));
|
|
2729
|
+
}
|
|
1319
2730
|
export function sandboxCapacityFailureMessage(used, requested, capacity, preflight, config) {
|
|
1320
2731
|
const availableSlots = Math.max(0, capacity.sandboxSlots - used.sandboxSlots);
|
|
1321
2732
|
if (requested.sandboxSlots > availableSlots) {
|
|
@@ -1346,14 +2757,6 @@ export function sandboxCapacityFailureMessage(used, requested, capacity, preflig
|
|
|
1346
2757
|
const availableBuildSlots = Math.max(0, capacity.buildSlots - used.buildSlots);
|
|
1347
2758
|
return `Sandbox build capacity is insufficient: requested ${requested.buildSlots}, available ${availableBuildSlots}, ${used.buildSlots} already reserved.`;
|
|
1348
2759
|
}
|
|
1349
|
-
function formatBytes(bytes) {
|
|
1350
|
-
if (!Number.isFinite(bytes) || bytes <= 0)
|
|
1351
|
-
return "0 B";
|
|
1352
|
-
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
1353
|
-
const power = Math.min(units.length - 1, Math.floor(Math.log(bytes) / Math.log(1024)));
|
|
1354
|
-
const value = bytes / 1024 ** power;
|
|
1355
|
-
return `${value >= 10 || power === 0 ? value.toFixed(0) : value.toFixed(2)} ${units[power]}`;
|
|
1356
|
-
}
|
|
1357
2760
|
function sumSandboxResources(resources) {
|
|
1358
2761
|
return resources.reduce((total, resource) => ({
|
|
1359
2762
|
sandboxSlots: total.sandboxSlots + resource.sandboxSlots,
|
|
@@ -1425,8 +2828,16 @@ export async function resolveSandboxResourcesFromRepo(dispatch, baseRuntimeOverh
|
|
|
1425
2828
|
throw new Error("The approved immutable sandbox contract is unavailable in the mapped repository.");
|
|
1426
2829
|
}
|
|
1427
2830
|
const contract = validateSandboxContract(contractResult.stdout);
|
|
2831
|
+
const workspaceBytes = await estimateWorkspaceBytesFromRepo(repoPath, {
|
|
2832
|
+
role: dispatch.execution.role,
|
|
2833
|
+
serviceCount: contract.contract.services.length,
|
|
2834
|
+
browser: contract.contract.browser,
|
|
2835
|
+
});
|
|
1428
2836
|
return {
|
|
1429
|
-
resources: resolveDynamicSandboxResources(
|
|
2837
|
+
resources: resolveDynamicSandboxResources({
|
|
2838
|
+
...sandbox.requestedResources,
|
|
2839
|
+
workspaceBytes: Math.max(sandbox.requestedResources.workspaceBytes, workspaceBytes),
|
|
2840
|
+
}, contract.contract.services.length, sandbox.networkPolicy.mode, baseRuntimeOverheadBytes),
|
|
1430
2841
|
sandboxConfigCommitSha: configCommitSha,
|
|
1431
2842
|
};
|
|
1432
2843
|
}
|