@loucompanion/forge-bridge 0.1.3-dev.f66395c3ac21 → 0.1.4-dev.04a1e5b9e484
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +165 -1
- package/dist/bridge/bridge-chat.types.d.ts +111 -0
- package/dist/bridge/bridge-chat.types.js +401 -0
- package/dist/bridge/bridge.base.d.ts +9 -1
- package/dist/bridge/bridge.service.d.ts +129 -3
- package/dist/bridge/bridge.service.js +2427 -70
- package/dist/bridge/bridge.types.d.ts +218 -2
- package/dist/bridge/dispatch-result-outbox.d.ts +25 -0
- package/dist/bridge/dispatch-result-outbox.js +62 -0
- package/dist/bridge/dispatch-result-worker.d.ts +2 -0
- package/dist/bridge/dispatch-result-worker.js +122 -0
- package/dist/bridge/internals.d.ts +2 -0
- package/dist/bridge/internals.js +1 -0
- package/dist/bridge/readiness-result-outbox.d.ts +21 -0
- package/dist/bridge/readiness-result-outbox.js +54 -0
- package/dist/bridge/readiness-result-worker.d.ts +2 -0
- package/dist/bridge/readiness-result-worker.js +121 -0
- package/dist/bridge/runner-host-capabilities.d.ts +15 -0
- package/dist/bridge/runner-host-capabilities.js +93 -0
- package/dist/cleanup/cleanup.service.d.ts +14 -2
- package/dist/cleanup/cleanup.service.js +496 -3
- package/dist/cleanup/cleanup.types.d.ts +10 -0
- package/dist/cleanup/index.d.ts +1 -1
- package/dist/cleanup/index.js +1 -1
- package/dist/cleanup/internals.d.ts +2 -2
- package/dist/cleanup/internals.js +1 -1
- package/dist/cli/cli.help.d.ts +2 -0
- package/dist/cli/cli.help.js +217 -0
- package/dist/cli/cli.service.d.ts +55 -7
- package/dist/cli/cli.service.js +429 -219
- package/dist/cli/local-config.d.ts +7 -0
- package/dist/cli/local-config.js +51 -3
- package/dist/runtime/aspnet-runtime-prebuild.d.ts +36 -0
- package/dist/runtime/aspnet-runtime-prebuild.js +341 -0
- package/dist/runtime/attempt-database-broker.d.ts +89 -0
- package/dist/runtime/attempt-database-broker.js +390 -0
- package/dist/runtime/attempt-node-runtime-launcher.d.ts +18 -0
- package/dist/runtime/attempt-node-runtime-launcher.js +140 -0
- package/dist/runtime/attempt-path-operation.d.ts +21 -0
- package/dist/runtime/attempt-path-operation.js +94 -0
- package/dist/runtime/attempt-private-directory.d.ts +6 -0
- package/dist/runtime/attempt-private-directory.js +54 -0
- package/dist/runtime/attempt-process-supervisor-registry.d.ts +20 -0
- package/dist/runtime/attempt-process-supervisor-registry.js +64 -0
- package/dist/runtime/attempt-process-supervisor.d.ts +102 -0
- package/dist/runtime/attempt-process-supervisor.js +421 -0
- package/dist/runtime/canonical-source-file.d.ts +3 -0
- package/dist/runtime/canonical-source-file.js +54 -0
- package/dist/runtime/docker-engine-probe.d.ts +90 -0
- package/dist/runtime/docker-engine-probe.js +333 -0
- package/dist/runtime/generated-output-reset.d.ts +20 -0
- package/dist/runtime/generated-output-reset.js +157 -0
- package/dist/runtime/host-database-capability-boundary.d.ts +16 -0
- package/dist/runtime/host-database-capability-boundary.js +62 -0
- package/dist/runtime/host-diagnostics.d.ts +11 -0
- package/dist/runtime/host-diagnostics.js +192 -0
- package/dist/runtime/host-execution-root.d.ts +4 -0
- package/dist/runtime/host-execution-root.js +39 -0
- package/dist/runtime/host-listener-inspector.d.ts +51 -0
- package/dist/runtime/host-listener-inspector.js +385 -0
- package/dist/runtime/host-port-broker.d.ts +76 -0
- package/dist/runtime/host-port-broker.js +333 -0
- package/dist/runtime/host-runtime-contract.d.ts +58 -0
- package/dist/runtime/host-runtime-contract.js +813 -0
- package/dist/runtime/host-runtime-lease.d.ts +217 -0
- package/dist/runtime/host-runtime-lease.js +2487 -0
- package/dist/runtime/host-runtime-promotion.d.ts +32 -0
- package/dist/runtime/host-runtime-promotion.js +206 -0
- package/dist/runtime/memory-capacity.d.ts +70 -0
- package/dist/runtime/memory-capacity.js +123 -0
- package/dist/runtime/native-test-postgres-fallback.d.ts +25 -0
- package/dist/runtime/native-test-postgres-fallback.js +91 -0
- package/dist/runtime/node-host-runtime-adapter.d.ts +54 -0
- package/dist/runtime/node-host-runtime-adapter.js +286 -0
- package/dist/runtime/pinned-directory-process.d.ts +9 -0
- package/dist/runtime/pinned-directory-process.js +61 -0
- package/dist/runtime/pinned-source-directory.d.ts +17 -0
- package/dist/runtime/pinned-source-directory.js +94 -0
- package/dist/runtime/playwright-host-runtime-adapter.d.ts +21 -0
- package/dist/runtime/playwright-host-runtime-adapter.js +268 -0
- package/dist/runtime/prepared-runtime-identity.d.ts +6 -0
- package/dist/runtime/prepared-runtime-identity.js +218 -0
- package/dist/runtime/prepared-runtime-readiness.d.ts +30 -0
- package/dist/runtime/prepared-runtime-readiness.js +285 -0
- package/dist/runtime/runtime-isolation-lease.d.ts +67 -0
- package/dist/runtime/runtime-isolation-lease.js +161 -0
- package/dist/runtime/workspace-resource-estimator.d.ts +17 -0
- package/dist/runtime/workspace-resource-estimator.js +174 -0
- package/dist/sandbox/command-runner.d.ts +4 -0
- package/dist/sandbox/command-runner.js +47 -0
- package/dist/sandbox/contract.d.ts +5 -0
- package/dist/sandbox/contract.js +276 -0
- package/dist/sandbox/event-budget.d.ts +11 -0
- package/dist/sandbox/event-budget.js +27 -0
- package/dist/sandbox/image-policy.d.ts +35 -0
- package/dist/sandbox/image-policy.js +247 -0
- package/dist/sandbox/index.d.ts +5 -0
- package/dist/sandbox/index.js +5 -0
- package/dist/sandbox/lease-watchdog.d.ts +18 -0
- package/dist/sandbox/lease-watchdog.js +67 -0
- package/dist/sandbox/preflight.d.ts +84 -0
- package/dist/sandbox/preflight.js +521 -0
- package/dist/sandbox/process.d.ts +20 -0
- package/dist/sandbox/process.js +209 -0
- package/dist/sandbox/provider-auth/codex.d.ts +26 -0
- package/dist/sandbox/provider-auth/codex.js +212 -0
- package/dist/sandbox/repository-lock.d.ts +6 -0
- package/dist/sandbox/repository-lock.js +129 -0
- package/dist/sandbox/runtime.d.ts +302 -0
- package/dist/sandbox/runtime.js +2368 -0
- package/dist/sandbox/sandbox.types.d.ts +93 -0
- package/dist/sandbox/sandbox.types.js +1 -0
- package/dist/sandbox/smoke.d.ts +76 -0
- package/dist/sandbox/smoke.js +380 -0
- package/dist/sandbox/worker-supervisor.d.ts +13 -0
- package/dist/sandbox/worker-supervisor.js +36 -0
- package/dist/sandbox/worker.d.ts +1 -0
- package/dist/sandbox/worker.js +225 -0
- package/dist/sandbox/workspace.service.d.ts +13 -0
- package/dist/sandbox/workspace.service.js +74 -0
- package/dist/session/managed-attempt-tools.d.ts +28 -0
- package/dist/session/managed-attempt-tools.js +1016 -0
- package/dist/session/managed-build-coordinator.d.ts +5 -0
- package/dist/session/managed-build-coordinator.js +106 -0
- package/dist/session/managed-build-receipt.d.ts +21 -0
- package/dist/session/managed-build-receipt.js +337 -0
- package/dist/session/managed-verification-timeouts.d.ts +8 -0
- package/dist/session/managed-verification-timeouts.js +17 -0
- package/dist/session/phase-completion.artifacts.d.ts +27 -0
- package/dist/session/phase-completion.artifacts.js +829 -0
- package/dist/session/phase-completion.d.ts +53 -42
- package/dist/session/phase-completion.helpers.d.ts +117 -0
- package/dist/session/phase-completion.helpers.js +707 -0
- package/dist/session/phase-completion.js +125 -3007
- package/dist/session/phase-completion.prompts.d.ts +14 -0
- package/dist/session/phase-completion.prompts.js +233 -0
- package/dist/session/phase-completion.runtime.d.ts +49 -0
- package/dist/session/phase-completion.runtime.js +840 -0
- package/dist/session/phase-completion.schemas.d.ts +3 -0
- package/dist/session/phase-completion.schemas.js +859 -0
- package/dist/session/phase-completion.validation.d.ts +38 -0
- package/dist/session/phase-completion.validation.js +918 -0
- package/dist/session/pre-publication-validation.d.ts +75 -0
- package/dist/session/pre-publication-validation.js +661 -0
- package/dist/session/provider-cli/bridge-cli.conformance-fixtures.d.ts +72 -0
- package/dist/session/provider-cli/bridge-cli.conformance-fixtures.js +462 -0
- package/dist/session/provider-cli/ci-repair-readiness.d.ts +16 -0
- package/dist/session/provider-cli/ci-repair-readiness.js +110 -0
- package/dist/session/provider-cli/provider-cli.artifacts.d.ts +95 -0
- package/dist/session/provider-cli/provider-cli.artifacts.js +812 -0
- package/dist/session/provider-cli/provider-cli.base.d.ts +1 -0
- package/dist/session/provider-cli/provider-cli.deterministic-payloads.d.ts +12 -0
- package/dist/session/provider-cli/provider-cli.deterministic-payloads.js +279 -0
- package/dist/session/provider-cli/provider-cli.deterministic.d.ts +14 -0
- package/dist/session/provider-cli/provider-cli.deterministic.js +407 -0
- package/dist/session/provider-cli/provider-cli.process.d.ts +99 -0
- package/dist/session/provider-cli/provider-cli.process.js +874 -0
- package/dist/session/provider-cli/provider-cli.prompts.d.ts +52 -0
- package/dist/session/provider-cli/provider-cli.prompts.js +715 -0
- package/dist/session/provider-cli/provider-cli.qa-sidecars.d.ts +11 -0
- package/dist/session/provider-cli/provider-cli.qa-sidecars.js +66 -0
- package/dist/session/provider-cli/provider-cli.repair-epoch.d.ts +6 -0
- package/dist/session/provider-cli/provider-cli.repair-epoch.js +11 -0
- package/dist/session/provider-cli/provider-cli.runtime.d.ts +22 -0
- package/dist/session/provider-cli/provider-cli.runtime.js +92 -0
- package/dist/session/provider-cli/provider-cli.service.d.ts +47 -53
- package/dist/session/provider-cli/provider-cli.service.js +1258 -2497
- package/dist/session/provider-cli/provider-cli.types.d.ts +8 -0
- package/dist/session/provider-cli/provider-output-state.d.ts +32 -0
- package/dist/session/provider-cli/provider-output-state.js +817 -0
- package/dist/session/provider-cli/provider-usage.d.ts +7 -0
- package/dist/session/provider-cli/provider-usage.js +28 -5
- package/dist/session/session-redaction.d.ts +3 -0
- package/dist/session/session-redaction.js +448 -0
- package/dist/session/session.base.d.ts +6 -1
- package/dist/session/session.service.d.ts +8 -2
- package/dist/session/session.service.js +586 -59
- package/dist/session/session.types.d.ts +324 -3
- package/dist/session/session.types.js +939 -10
- package/dist/session/unreferenced-exports.d.ts +12 -0
- package/dist/session/unreferenced-exports.js +59 -0
- package/dist/shared/format/format-bytes.d.ts +1 -0
- package/dist/shared/format/format-bytes.js +9 -0
- package/dist/shared/git/git.js +2 -1
- package/dist/shared/path/path.d.ts +1 -0
- package/dist/shared/path/path.js +21 -1
- package/dist/shared/process/dotnet-toolchain.d.ts +17 -0
- package/dist/shared/process/dotnet-toolchain.js +307 -0
- package/dist/shared/process/host-process-recovery.d.ts +12 -0
- package/dist/shared/process/host-process-recovery.js +79 -0
- package/dist/shared/process/host-process-supervisor.d.ts +59 -0
- package/dist/shared/process/host-process-supervisor.js +331 -0
- package/dist/shared/process/process.d.ts +3 -0
- package/dist/shared/process/process.js +13 -7
- package/dist/shared/process/windows-launch.d.ts +8 -0
- package/dist/shared/process/windows-launch.js +41 -0
- package/dist/shared/provider-capabilities.d.ts +4 -0
- package/dist/shared/provider-capabilities.js +88 -0
- package/dist/shared/redaction/redaction.d.ts +1 -0
- package/dist/shared/redaction/redaction.js +47 -4
- package/dist/shared/time/monotonic.d.ts +1 -0
- package/dist/shared/time/monotonic.js +8 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/worktree/artifact-injector.d.ts +3 -2
- package/dist/worktree/artifact-injector.js +82 -10
- package/dist/worktree/dependency-warmer.d.ts +53 -1
- package/dist/worktree/dependency-warmer.js +619 -11
- package/dist/worktree/index.d.ts +2 -1
- package/dist/worktree/index.js +1 -0
- package/dist/worktree/prior-phase-artifact-hydrator.d.ts +2 -0
- package/dist/worktree/prior-phase-artifact-hydrator.js +136 -0
- package/dist/worktree/story-qa-reference-hydrator.d.ts +4 -0
- package/dist/worktree/story-qa-reference-hydrator.js +253 -0
- package/dist/worktree/worktree-paths.d.ts +4 -0
- package/dist/worktree/worktree-paths.js +30 -0
- package/dist/worktree/worktree.base.d.ts +3 -0
- package/dist/worktree/worktree.service.d.ts +16 -3
- package/dist/worktree/worktree.service.js +334 -54
- package/dist/worktree/worktree.types.d.ts +9 -1
- package/package.json +12 -7
- package/runtime/windows-job-supervisor.ps1 +722 -0
- package/runtime/windows-listener-owner.ps1 +33 -0
|
@@ -1,47 +1,271 @@
|
|
|
1
|
-
import { HubConnectionBuilder, LogLevel } from "@microsoft/signalr";
|
|
1
|
+
import { HubConnectionBuilder, HttpTransportType, LogLevel } from "@microsoft/signalr";
|
|
2
2
|
import fs from "node:fs";
|
|
3
3
|
import path from "node:path";
|
|
4
|
+
import { randomUUID } from "node:crypto";
|
|
5
|
+
import os from "node:os";
|
|
4
6
|
import { defaultForgeBridgeConfig } from "../config.js";
|
|
5
|
-
import {
|
|
7
|
+
import { bridgeChatCapabilityFor, supportsDraftImageAttachmentsFor } from "../shared/provider-capabilities.js";
|
|
8
|
+
import { redactText } from "../shared/redaction/redaction.js";
|
|
6
9
|
import { BridgeSessionService } from "../session/session.service.js";
|
|
7
|
-
import {
|
|
10
|
+
import { ProcessBackedProviderCliService, RoutingProviderCliService, StreamingProviderProcessRunner, } from "../session/provider-cli/provider-cli.service.js";
|
|
11
|
+
import { codexSandboxModeForPlatform, formatProviderCliCommandForCheckpoint } from "../session/provider-cli/provider-cli.prompts.js";
|
|
12
|
+
import { clip, createProviderOutputState } from "../session/provider-cli/provider-output-state.js";
|
|
13
|
+
import { redactRunnerDispatchEvent, redactRunOnceResultPacket } from "../session/session-redaction.js";
|
|
14
|
+
import { PrePublicationValidator } from "../session/pre-publication-validation.js";
|
|
15
|
+
import { HostRuntimeLeaseManager, } from "../runtime/host-runtime-lease.js";
|
|
16
|
+
import { assertHostDatabaseCapabilityBoundary, establishHostDatabaseCapabilityBoundary, } from "../runtime/host-database-capability-boundary.js";
|
|
17
|
+
import { defaultHostExecutionRoot } from "../runtime/host-execution-root.js";
|
|
18
|
+
import { buildMemoryAdmissionDiagnostics } from "../runtime/memory-capacity.js";
|
|
19
|
+
import { DockerEngineProbe } from "../runtime/docker-engine-probe.js";
|
|
20
|
+
import { createNativeTestClusterDatabaseBroker } from "../runtime/attempt-database-broker.js";
|
|
21
|
+
import { readNativeTestPostgresFallback } from "../runtime/native-test-postgres-fallback.js";
|
|
22
|
+
import { estimateWorkspaceBytesFromRepo } from "../runtime/workspace-resource-estimator.js";
|
|
23
|
+
import { GitWorktreeService } from "../worktree/worktree.service.js";
|
|
24
|
+
import { RepositoryCoordinatorLock } from "../sandbox/repository-lock.js";
|
|
25
|
+
import { AttemptCoordinatedProviderProcessRunner, AttemptProcessSupervisorRegistry, } from "../runtime/attempt-process-supervisor-registry.js";
|
|
26
|
+
import { failureFactsForCode, failureFactsFromError, failureResult, parseRunOnceDispatchPacket } from "../session/session.types.js";
|
|
8
27
|
import { forgeBridgeVersion } from "../version.js";
|
|
28
|
+
import { canRunDockerMaintenance, DockerPreflight } from "../sandbox/preflight.js";
|
|
29
|
+
import { validateSandboxContract } from "../sandbox/contract.js";
|
|
30
|
+
import { runSandboxProcess } from "../sandbox/process.js";
|
|
31
|
+
import { DockerSandboxRuntime, selectSandboxConfigRefs } from "../sandbox/runtime.js";
|
|
32
|
+
import { codexSandboxProviderAuthAdapter, sandboxProviderAuthAdapterFor, sandboxProviderAuthUnavailableMessage } from "../sandbox/provider-auth/codex.js";
|
|
33
|
+
import { assertBridgeChatCallbackPayload, bridgeChatMaxProviderThreadIdChars, bridgeChatProtocolVersion, isTerminalChatTurnCallbackKind, parseChatTurnDispatchPacket, } from "./bridge-chat.types.js";
|
|
34
|
+
import { formatBytes } from "../shared/format/format-bytes.js";
|
|
35
|
+
import { emptyRunnerHostCapabilitySnapshot, ProcessRunnerHostCapabilityProbe, } from "./runner-host-capabilities.js";
|
|
36
|
+
import { FileReadinessResultOutbox } from "./readiness-result-outbox.js";
|
|
37
|
+
import { FileDispatchResultOutbox } from "./dispatch-result-outbox.js";
|
|
38
|
+
/**
|
|
39
|
+
* Protocol-v2 events are acknowledged one at a time before runOnce returns.
|
|
40
|
+
* Sending the complete transcript again in DispatchResult made long QA and CI
|
|
41
|
+
* sessions exceed SignalR's receive limit, disconnecting the bridge after the
|
|
42
|
+
* provider had already completed. The API combines its acknowledged streamed
|
|
43
|
+
* events with the terminal result, so the callback only needs the outcome and
|
|
44
|
+
* artifacts. Protocol v1 retains the transcript for legacy servers.
|
|
45
|
+
*/
|
|
46
|
+
export function compactTerminalResult(dispatch, result) {
|
|
47
|
+
return dispatch.schemaVersion >= 2 && result.events.length > 0
|
|
48
|
+
? { ...result, events: [] }
|
|
49
|
+
: result;
|
|
50
|
+
}
|
|
9
51
|
export class ForgeBridgeService {
|
|
10
52
|
config;
|
|
11
|
-
sessions;
|
|
12
53
|
connections;
|
|
54
|
+
hostCapabilityProbe;
|
|
55
|
+
chatProcessRunner;
|
|
13
56
|
connection = null;
|
|
14
57
|
heartbeatTimer = null;
|
|
58
|
+
heartbeatInFlight = false;
|
|
59
|
+
consecutiveHeartbeatFailures = 0;
|
|
60
|
+
reconnectingFromHeartbeatFailure = false;
|
|
61
|
+
leaseWatchdogTimer = null;
|
|
62
|
+
sandboxMaintenanceTimer = null;
|
|
63
|
+
sandboxMaintenanceRunning = false;
|
|
15
64
|
closed = null;
|
|
65
|
+
transportRecovery = null;
|
|
66
|
+
transportRecoveryGeneration = 0;
|
|
16
67
|
stopped = false;
|
|
17
68
|
activeSessions = new Map();
|
|
18
|
-
|
|
69
|
+
reservations = new Map();
|
|
70
|
+
activeReadinessProbes = new Map();
|
|
71
|
+
readinessResultOutbox;
|
|
72
|
+
readinessResultFlush = null;
|
|
73
|
+
dispatchResultOutbox;
|
|
74
|
+
dispatchResultFlush = null;
|
|
75
|
+
dispatchResultGeneration = 0;
|
|
76
|
+
dispatchResultFlushedGeneration = -1;
|
|
77
|
+
dispatchResultHandoffs = new Map();
|
|
78
|
+
preflight;
|
|
79
|
+
dockerEngineProbe;
|
|
80
|
+
sandboxRuntime;
|
|
81
|
+
sandboxResourceResolver;
|
|
82
|
+
sessions;
|
|
83
|
+
enforceHostDatabaseCapabilityBoundary;
|
|
84
|
+
hostCapabilities = emptyRunnerHostCapabilitySnapshot();
|
|
85
|
+
// Reaping deletes the stale ledger, so its stopped-process proof is held here
|
|
86
|
+
// until a Hello carrying it is accepted. A bridge restart forgets it, which
|
|
87
|
+
// leaves those attempts on Forge's conservative unknown-outcome path.
|
|
88
|
+
recoveredHostAttempts = new Map();
|
|
89
|
+
hostCapabilitiesDetected = false;
|
|
90
|
+
constructor(config, sessions = undefined, connections = new SignalRBridgeConnectionFactory(), preflight, sandboxRuntime, sandboxResourceResolver, dockerEngineProbe, hostCapabilityProbe = new ProcessRunnerHostCapabilityProbe(), readinessResultOutbox, chatProcessRunner = new StreamingProviderProcessRunner(), dispatchResultOutbox) {
|
|
19
91
|
this.config = config;
|
|
20
|
-
this.sessions = sessions;
|
|
21
92
|
this.connections = connections;
|
|
93
|
+
this.hostCapabilityProbe = hostCapabilityProbe;
|
|
94
|
+
this.chatProcessRunner = chatProcessRunner;
|
|
95
|
+
this.enforceHostDatabaseCapabilityBoundary = sessions === undefined;
|
|
96
|
+
if (this.enforceHostDatabaseCapabilityBoundary) {
|
|
97
|
+
establishHostDatabaseCapabilityBoundary();
|
|
98
|
+
}
|
|
99
|
+
this.preflight = preflight ?? new DockerPreflight({
|
|
100
|
+
dockerExecutable: config.dockerExecutable,
|
|
101
|
+
dockerContext: config.dockerContext,
|
|
102
|
+
stateRoot: config.sandboxStateRoot,
|
|
103
|
+
runnerInstanceId: config.runnerInstanceId,
|
|
104
|
+
maxConcurrentSandboxes: config.maxConcurrentSandboxes,
|
|
105
|
+
diskHighWatermarkPercent: config.diskHighWatermarkPercent,
|
|
106
|
+
diskLowWatermarkPercent: config.diskLowWatermarkPercent,
|
|
107
|
+
minimumWorkspaceBytes: config.minimumWorkspaceBytes,
|
|
108
|
+
hostMemoryReserveBytes: config.sandboxHostMemoryReserveBytes,
|
|
109
|
+
defaultWorkerMemoryLimitBytes: config.defaultSandboxWorkerMemoryLimitBytes,
|
|
110
|
+
defaultServiceMemoryLimitBytes: config.defaultSandboxServiceMemoryLimitBytes,
|
|
111
|
+
runtimeOverheadBytes: config.sandboxRuntimeOverheadBytes,
|
|
112
|
+
requireRootless: config.requireRootlessDocker
|
|
113
|
+
});
|
|
114
|
+
this.dockerEngineProbe = dockerEngineProbe ?? new DockerEngineProbe({
|
|
115
|
+
dockerExecutable: config.dockerExecutable,
|
|
116
|
+
dockerContext: config.dockerContext,
|
|
117
|
+
recovery: {
|
|
118
|
+
enabled: config.dockerDesktopRecoveryEnabled,
|
|
119
|
+
failureThreshold: config.dockerRecoveryFailureThreshold,
|
|
120
|
+
cooldownMs: config.dockerRecoveryCooldownMs,
|
|
121
|
+
startupTimeoutMs: config.dockerRecoveryStartupTimeoutMs,
|
|
122
|
+
canRecover: () => this.activeSessions.size === 0 && this.reservations.size === 0,
|
|
123
|
+
},
|
|
124
|
+
onStatus: logBridgeStatus,
|
|
125
|
+
});
|
|
126
|
+
this.sessions = sessions ?? createDefaultSessionService(config.sandboxStateRoot, config.sandboxExecutionRoot, this.dockerEngineProbe);
|
|
127
|
+
this.sandboxRuntime = sandboxRuntime ?? new DockerSandboxRuntime(config, (request, signal) => this.redeemPublishGrant(request, signal));
|
|
128
|
+
this.sandboxResourceResolver = sandboxResourceResolver
|
|
129
|
+
?? (dispatch => resolveSandboxResourcesFromRepo(dispatch, this.config.sandboxRuntimeOverheadBytes));
|
|
130
|
+
this.readinessResultOutbox = readinessResultOutbox ?? new FileReadinessResultOutbox(config.sandboxStateRoot);
|
|
131
|
+
this.dispatchResultOutbox = dispatchResultOutbox ?? new FileDispatchResultOutbox(config.sandboxStateRoot);
|
|
22
132
|
}
|
|
23
133
|
async connectOnce() {
|
|
24
134
|
await this.stopConnection();
|
|
25
135
|
this.stopped = false;
|
|
136
|
+
if (this.enforceHostDatabaseCapabilityBoundary) {
|
|
137
|
+
assertHostDatabaseCapabilityBoundary();
|
|
138
|
+
}
|
|
139
|
+
const hostReap = await this.sessions.reapHostRuntime?.({
|
|
140
|
+
runnerId: this.config.runnerId,
|
|
141
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
142
|
+
});
|
|
143
|
+
this.rememberRecoveredHostAttempts(hostReap);
|
|
144
|
+
if (hostReap && (hostReap.reaped > 0 || hostReap.degraded > 0)) {
|
|
145
|
+
logBridgeStatus(`Managed-host recovery reaped ${hostReap.reaped} stale attempt(s), protected ${hostReap.reusedPidsProtected} reused PID(s), and left ${hostReap.degraded} degraded attempt(s) for reconciliation.`);
|
|
146
|
+
}
|
|
147
|
+
if (this.config.dockerEnabled) {
|
|
148
|
+
const preflight = await this.preflight.refresh();
|
|
149
|
+
if (canReapManagedSandboxes(preflight)) {
|
|
150
|
+
await this.sandboxRuntime.reap({ diskLow: preflight.diskLow });
|
|
151
|
+
await this.preflight.refresh();
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
await this.dockerEngineProbe.refresh(true);
|
|
155
|
+
if (!this.hostCapabilitiesDetected) {
|
|
156
|
+
this.hostCapabilities = await this.hostCapabilityProbe.detect(this.config);
|
|
157
|
+
this.hostCapabilitiesDetected = true;
|
|
158
|
+
}
|
|
26
159
|
const connection = this.connections.create(this.config);
|
|
27
160
|
this.connection = connection;
|
|
161
|
+
this.transportRecovery = null;
|
|
28
162
|
this.closed = new Promise((resolve) => {
|
|
29
|
-
connection.onClose(() => {
|
|
163
|
+
connection.onClose((error) => {
|
|
30
164
|
this.stopHeartbeat();
|
|
165
|
+
this.completeTransportRecovery(error ?? new Error("bridge connection closed"));
|
|
31
166
|
this.abortActiveSessions("bridge connection closed");
|
|
167
|
+
this.abortActiveReadinessProbes("bridge connection closed");
|
|
32
168
|
resolve();
|
|
33
169
|
});
|
|
34
170
|
});
|
|
171
|
+
connection.onReconnecting?.((error) => {
|
|
172
|
+
if (connection !== this.connection || this.stopped)
|
|
173
|
+
return;
|
|
174
|
+
this.stopHeartbeat();
|
|
175
|
+
this.beginTransportRecovery();
|
|
176
|
+
logBridgeStatus(`Forge connection interrupted${error ? `: ${errorMessage(error)}` : ""}. Reconnecting in place; active dispatches remain supervised...`);
|
|
177
|
+
});
|
|
178
|
+
connection.onReconnected?.(async () => {
|
|
179
|
+
if (connection !== this.connection || this.stopped)
|
|
180
|
+
return;
|
|
181
|
+
try {
|
|
182
|
+
await this.registerConnection(connection);
|
|
183
|
+
this.completeTransportRecovery();
|
|
184
|
+
this.startHeartbeat();
|
|
185
|
+
await this.flushDispatchResults(true).catch(error => {
|
|
186
|
+
logBridgeStatus(`Dispatch result replay deferred after reconnect: ${errorMessage(error)}`);
|
|
187
|
+
});
|
|
188
|
+
await this.flushReadinessResults().catch(error => {
|
|
189
|
+
logBridgeStatus(`Readiness result replay deferred after reconnect: ${errorMessage(error)}`);
|
|
190
|
+
});
|
|
191
|
+
await this.flushCleanupReports().catch(() => undefined);
|
|
192
|
+
logBridgeStatus("Forge connection restored; active dispatches remain attached to this runner process.");
|
|
193
|
+
}
|
|
194
|
+
catch (error) {
|
|
195
|
+
this.completeTransportRecovery(error);
|
|
196
|
+
logBridgeStatus(`Forge reconnect registration failed: ${errorMessage(error)}. Closing the connection for a clean retry...`);
|
|
197
|
+
await connection.stop().catch(() => undefined);
|
|
198
|
+
}
|
|
199
|
+
});
|
|
35
200
|
connection.onDispatch((dispatch) => this.handleDispatch(dispatch));
|
|
201
|
+
connection.onChatTurn?.((dispatch) => this.handleChatTurn(dispatch));
|
|
202
|
+
connection.onOffer?.((dispatch) => this.handleOffer(dispatch));
|
|
203
|
+
connection.onStart?.((dispatch) => this.handleStart(dispatch));
|
|
204
|
+
connection.onLeaseUpdate?.((request) => this.handleLeaseUpdate(request));
|
|
36
205
|
connection.onCancel((request) => this.handleCancel(request));
|
|
206
|
+
connection.onReap?.((request) => this.handleReap(request));
|
|
207
|
+
connection.onReadinessProbe?.((request) => this.handleReadinessProbe(request));
|
|
37
208
|
await connection.start();
|
|
38
|
-
await this.
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
minServerVersion: this.config.minServerVersion ?? null,
|
|
42
|
-
capabilities: this.capabilities()
|
|
209
|
+
await this.registerConnection(connection);
|
|
210
|
+
await this.flushDispatchResults(true).catch(error => {
|
|
211
|
+
logBridgeStatus(`Dispatch result replay deferred: ${errorMessage(error)}`);
|
|
43
212
|
});
|
|
213
|
+
await this.flushReadinessResults().catch(error => {
|
|
214
|
+
logBridgeStatus(`Readiness result replay deferred: ${errorMessage(error)}`);
|
|
215
|
+
});
|
|
216
|
+
await this.flushCleanupReports();
|
|
44
217
|
this.startHeartbeat();
|
|
218
|
+
this.startLeaseWatchdog();
|
|
219
|
+
this.startSandboxMaintenance();
|
|
220
|
+
}
|
|
221
|
+
rememberRecoveredHostAttempts(report) {
|
|
222
|
+
for (const attempt of report?.recoveredAttempts ?? []) {
|
|
223
|
+
if (this.recoveredHostAttempts.size >= maxRecoveredHostAttempts)
|
|
224
|
+
break;
|
|
225
|
+
this.recoveredHostAttempts.set(attempt.dispatchAttemptId, attempt);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
async recoveredHostAttemptsForHello() {
|
|
229
|
+
if (this.recoveredHostAttempts.size === 0)
|
|
230
|
+
return [];
|
|
231
|
+
// A durable terminal result outranks the stopped-process proof: it replays
|
|
232
|
+
// after Hello and Forge accepts the provider's real outcome instead.
|
|
233
|
+
const pendingResults = new Set((await this.dispatchResultOutbox.pending())
|
|
234
|
+
.map(entry => entry.payload.dispatchAttemptId)
|
|
235
|
+
.filter((id) => typeof id === "string"));
|
|
236
|
+
return [...this.recoveredHostAttempts.values()]
|
|
237
|
+
.filter(attempt => !pendingResults.has(attempt.dispatchAttemptId));
|
|
238
|
+
}
|
|
239
|
+
async registerConnection(connection) {
|
|
240
|
+
let storageRecoveryAttempt = 0;
|
|
241
|
+
while (true) {
|
|
242
|
+
try {
|
|
243
|
+
const recoveredHostAttempts = await this.recoveredHostAttemptsForHello();
|
|
244
|
+
const ack = await connection.invoke("Hello", {
|
|
245
|
+
version: this.config.version,
|
|
246
|
+
protocolVersion: this.config.protocolVersion,
|
|
247
|
+
minServerVersion: this.config.minServerVersion ?? null,
|
|
248
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
249
|
+
capabilities: this.capabilities(),
|
|
250
|
+
recoveredHostAttempts,
|
|
251
|
+
});
|
|
252
|
+
if (!ack.accepted) {
|
|
253
|
+
throw new BridgeCallbackRejectedError(ack.code ?? null, ack.message ?? ack.code ?? "Hello callback was rejected.");
|
|
254
|
+
}
|
|
255
|
+
this.recoveredHostAttempts.clear();
|
|
256
|
+
logBridgeStatus(`Registered runner ${this.config.runnerId}, instance ${this.config.runnerInstanceId}, build ${this.config.version}, protocol ${this.config.protocolVersion}.`);
|
|
257
|
+
return;
|
|
258
|
+
}
|
|
259
|
+
catch (error) {
|
|
260
|
+
if (!isProviderStorageUnavailable(error) || storageRecoveryAttempt >= this.config.reconnectBackoffMs.length) {
|
|
261
|
+
throw error;
|
|
262
|
+
}
|
|
263
|
+
const backoff = this.config.reconnectBackoffMs[storageRecoveryAttempt] ?? 5000;
|
|
264
|
+
storageRecoveryAttempt += 1;
|
|
265
|
+
logBridgeStatus(`Hello reached Forge while storage was recovering. Retrying registration ${storageRecoveryAttempt} in ${formatDelay(backoff)}...`);
|
|
266
|
+
await delay(backoff);
|
|
267
|
+
}
|
|
268
|
+
}
|
|
45
269
|
}
|
|
46
270
|
async run(signal) {
|
|
47
271
|
let attempt = 0;
|
|
@@ -72,6 +296,7 @@ export class ForgeBridgeService {
|
|
|
72
296
|
finally {
|
|
73
297
|
if (signal?.aborted) {
|
|
74
298
|
this.abortActiveSessions(String(signal.reason ?? "bridge stopping"));
|
|
299
|
+
this.abortActiveReadinessProbes(String(signal.reason ?? "bridge stopping"));
|
|
75
300
|
}
|
|
76
301
|
await this.stopConnection();
|
|
77
302
|
}
|
|
@@ -79,31 +304,407 @@ export class ForgeBridgeService {
|
|
|
79
304
|
async stop() {
|
|
80
305
|
this.stopped = true;
|
|
81
306
|
this.abortActiveSessions("bridge stopping");
|
|
307
|
+
this.abortActiveReadinessProbes("bridge stopping");
|
|
82
308
|
await this.stopConnection();
|
|
309
|
+
await this.preflight.release();
|
|
310
|
+
}
|
|
311
|
+
async handleChatTurn(rawDispatch) {
|
|
312
|
+
const packet = this.parseChatTurnOrReport(rawDispatch);
|
|
313
|
+
if (!packet) {
|
|
314
|
+
return;
|
|
315
|
+
}
|
|
316
|
+
if (packet.runnerInstanceId !== this.config.runnerInstanceId) {
|
|
317
|
+
logBridgeStatus(`Rejected Bridge Chat turn ${packet.turnId}: stale runner instance.`);
|
|
318
|
+
return;
|
|
319
|
+
}
|
|
320
|
+
const capabilityFailure = this.requiredChatCapabilityFailure(packet);
|
|
321
|
+
if (capabilityFailure) {
|
|
322
|
+
await this.sendStandaloneChatTerminal(packet, "failed", {
|
|
323
|
+
code: "bridge_chat.runner_unavailable",
|
|
324
|
+
message: capabilityFailure,
|
|
325
|
+
}).catch(() => undefined);
|
|
326
|
+
return;
|
|
327
|
+
}
|
|
328
|
+
const existing = this.activeSessions.get(packet.leaseId);
|
|
329
|
+
if (existing) {
|
|
330
|
+
if (existing.kind === "chat"
|
|
331
|
+
&& existing.fencingToken === packet.fencingToken
|
|
332
|
+
&& existing.chatRunId === packet.chatRunId
|
|
333
|
+
&& existing.conversationId === packet.conversationId
|
|
334
|
+
&& existing.turnId === packet.turnId) {
|
|
335
|
+
return;
|
|
336
|
+
}
|
|
337
|
+
await this.sendStandaloneChatTerminal(packet, "failed", {
|
|
338
|
+
code: "bridge_chat.stale_fencing_token",
|
|
339
|
+
message: "Bridge Chat turn fence conflicts with an active bridge session.",
|
|
340
|
+
}).catch(() => undefined);
|
|
341
|
+
return;
|
|
342
|
+
}
|
|
343
|
+
const controller = new AbortController();
|
|
344
|
+
const active = {
|
|
345
|
+
leaseId: packet.leaseId,
|
|
346
|
+
kind: "chat",
|
|
347
|
+
fencingToken: packet.fencingToken,
|
|
348
|
+
dispatchAttemptId: null,
|
|
349
|
+
runnerInstanceId: packet.runnerInstanceId,
|
|
350
|
+
leaseExpiresAt: chatTurnLeaseExpiresAt(packet),
|
|
351
|
+
sandboxed: false,
|
|
352
|
+
resources: zeroSandboxResources(),
|
|
353
|
+
controller,
|
|
354
|
+
chatRunId: packet.chatRunId,
|
|
355
|
+
conversationId: packet.conversationId,
|
|
356
|
+
turnId: packet.turnId,
|
|
357
|
+
providerThreadId: packet.providerThreadId ?? null,
|
|
358
|
+
chatSequence: 0,
|
|
359
|
+
terminalCallbackAttempted: false,
|
|
360
|
+
};
|
|
361
|
+
this.activeSessions.set(packet.leaseId, active);
|
|
362
|
+
let workspacePath = null;
|
|
363
|
+
let cleanupAttempted = false;
|
|
364
|
+
let cleanupOutcome = null;
|
|
365
|
+
let cleanupReported = false;
|
|
366
|
+
const cleanup = async () => {
|
|
367
|
+
if (!workspacePath || cleanupAttempted)
|
|
368
|
+
return null;
|
|
369
|
+
cleanupAttempted = true;
|
|
370
|
+
cleanupOutcome = await cleanupChatTurnWorkspace(workspacePath, chatRedactionSecrets(packet));
|
|
371
|
+
return cleanupOutcome;
|
|
372
|
+
};
|
|
373
|
+
const reportCleanup = async (outcome) => {
|
|
374
|
+
if (!outcome || cleanupReported)
|
|
375
|
+
return;
|
|
376
|
+
cleanupReported = true;
|
|
377
|
+
await this.sendChatCallback(active, "output_delta", outcome).catch(() => undefined);
|
|
378
|
+
};
|
|
379
|
+
try {
|
|
380
|
+
workspacePath = await this.createChatTurnWorkspace(packet);
|
|
381
|
+
await materializeChatTurnMetadata(packet, workspacePath);
|
|
382
|
+
const run = await this.runChatProviderProcess(packet, active, workspacePath);
|
|
383
|
+
cleanupOutcome = await cleanup();
|
|
384
|
+
await reportCleanup(cleanupOutcome);
|
|
385
|
+
if (controller.signal.aborted) {
|
|
386
|
+
await this.sendChatAbortTerminal(packet, active, cleanupOutcome);
|
|
387
|
+
return;
|
|
388
|
+
}
|
|
389
|
+
if (cleanupOutcome?.status === "failed") {
|
|
390
|
+
throw new Error(`chat_turn_cleanup_failed: ${cleanupOutcome.message}`);
|
|
391
|
+
}
|
|
392
|
+
if (run.result.exitCode !== 0) {
|
|
393
|
+
throw chatProviderExitFailure(run.command, run.result, chatRedactionSecrets(packet));
|
|
394
|
+
}
|
|
395
|
+
await this.sendChatCallback(active, "output_completed", {
|
|
396
|
+
code: "output_completed",
|
|
397
|
+
text: clip(redactText(run.output.finalMessage ?? "", chatRedactionSecrets(packet)), maxChatTerminalTextChars),
|
|
398
|
+
provider: packet.providerKind,
|
|
399
|
+
model: packet.model,
|
|
400
|
+
cleanup: cleanupOutcome,
|
|
401
|
+
});
|
|
402
|
+
}
|
|
403
|
+
catch (error) {
|
|
404
|
+
cleanupOutcome = cleanupOutcome ?? await cleanup();
|
|
405
|
+
if (active.terminalCallbackAttempted) {
|
|
406
|
+
return;
|
|
407
|
+
}
|
|
408
|
+
await reportCleanup(cleanupOutcome);
|
|
409
|
+
const payload = controller.signal.aborted
|
|
410
|
+
? chatAbortPayload(controller.signal.reason, cleanupOutcome, packet)
|
|
411
|
+
: {
|
|
412
|
+
code: "provider_failed",
|
|
413
|
+
message: clip(redactText(errorMessage(error), chatRedactionSecrets(packet)), 2000),
|
|
414
|
+
cleanup: cleanupOutcome,
|
|
415
|
+
};
|
|
416
|
+
const kind = controller.signal.aborted ? chatAbortKind(controller.signal.reason) : "failed";
|
|
417
|
+
await this.sendChatCallback(active, kind, payload).catch(callbackError => {
|
|
418
|
+
logBridgeStatus(`Bridge Chat terminal callback for turn ${packet.turnId} was not accepted: ${errorMessage(callbackError)}`);
|
|
419
|
+
});
|
|
420
|
+
}
|
|
421
|
+
finally {
|
|
422
|
+
await this.releaseActiveSessionAndRefreshCapacity(packet.leaseId);
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
async createChatTurnWorkspace(packet) {
|
|
426
|
+
const root = path.join(this.config.sandboxStateRoot, "chat-turns");
|
|
427
|
+
await fs.promises.mkdir(root, { recursive: true });
|
|
428
|
+
return await fs.promises.mkdtemp(path.join(root, `${safePathComponent(packet.turnId)}-`));
|
|
429
|
+
}
|
|
430
|
+
async runChatProviderProcess(packet, active, cwd) {
|
|
431
|
+
const output = createProviderOutputState(packet.providerKind);
|
|
432
|
+
const plan = buildChatProviderCommand(packet, cwd);
|
|
433
|
+
const redactionSecrets = chatRedactionSecrets(packet);
|
|
434
|
+
let started = false;
|
|
435
|
+
const result = await this.chatProcessRunner.run({
|
|
436
|
+
command: plan.command.command,
|
|
437
|
+
args: plan.command.args,
|
|
438
|
+
cwd,
|
|
439
|
+
env: {
|
|
440
|
+
FORGE_BRIDGE_CHAT_RUN_ID: packet.chatRunId,
|
|
441
|
+
FORGE_BRIDGE_CHAT_TURN_ID: packet.turnId,
|
|
442
|
+
},
|
|
443
|
+
stdin: plan.stdin,
|
|
444
|
+
}, {
|
|
445
|
+
onStarted: async () => {
|
|
446
|
+
started = true;
|
|
447
|
+
await this.sendChatCallback(active, "turn_started", {
|
|
448
|
+
status: "running",
|
|
449
|
+
provider: packet.providerKind,
|
|
450
|
+
model: packet.model,
|
|
451
|
+
command: redactText(formatProviderCliCommandForCheckpoint(plan.command), redactionSecrets),
|
|
452
|
+
});
|
|
453
|
+
},
|
|
454
|
+
onStdoutLine: async (line) => {
|
|
455
|
+
for (const event of output.consume(line)) {
|
|
456
|
+
await this.sendChatProviderEvent(packet, active, event, output.providerThreadId);
|
|
457
|
+
}
|
|
458
|
+
},
|
|
459
|
+
}, active.controller.signal);
|
|
460
|
+
if (started && !active.controller.signal.aborted) {
|
|
461
|
+
await this.sendChatCallback(active, "output_delta", {
|
|
462
|
+
eventType: "provider_cli_completed",
|
|
463
|
+
role: "runner",
|
|
464
|
+
kind: "checkpoint",
|
|
465
|
+
content: "Bridge Chat provider process exited.",
|
|
466
|
+
}).catch(() => undefined);
|
|
467
|
+
}
|
|
468
|
+
return { command: plan.command, output, result };
|
|
469
|
+
}
|
|
470
|
+
async sendChatProviderEvent(packet, active, event, providerThreadId) {
|
|
471
|
+
if (providerThreadId) {
|
|
472
|
+
active.providerThreadId = providerThreadId;
|
|
473
|
+
}
|
|
474
|
+
else if (event.eventType === "provider_thread_started") {
|
|
475
|
+
active.providerThreadId = providerThreadIdFromEvent(event) ?? active.providerThreadId;
|
|
476
|
+
}
|
|
477
|
+
if (event.eventType === "provider_tool_heartbeat") {
|
|
478
|
+
return;
|
|
479
|
+
}
|
|
480
|
+
const redactionSecrets = chatRedactionSecrets(packet);
|
|
481
|
+
const payload = {
|
|
482
|
+
role: event.role,
|
|
483
|
+
kind: event.kind,
|
|
484
|
+
eventType: event.eventType ?? "provider_event",
|
|
485
|
+
content: clip(redactText(event.content, redactionSecrets), maxChatEventContentChars),
|
|
486
|
+
data: event.data ? clip(redactText(event.data, redactionSecrets), maxChatEventContentChars) : undefined,
|
|
487
|
+
};
|
|
488
|
+
await this.sendChatCallback(active, "output_delta", payload);
|
|
489
|
+
}
|
|
490
|
+
async sendChatAbortTerminal(packet, active, cleanupOutcome) {
|
|
491
|
+
await this.sendChatCallback(active, chatAbortKind(active.controller.signal.reason), chatAbortPayload(active.controller.signal.reason, cleanupOutcome, packet));
|
|
492
|
+
}
|
|
493
|
+
async sendStandaloneChatTerminal(packet, kind, payload) {
|
|
494
|
+
const active = {
|
|
495
|
+
leaseId: packet.leaseId,
|
|
496
|
+
kind: "chat",
|
|
497
|
+
fencingToken: packet.fencingToken,
|
|
498
|
+
dispatchAttemptId: null,
|
|
499
|
+
runnerInstanceId: packet.runnerInstanceId,
|
|
500
|
+
leaseExpiresAt: new Date(Date.now() + defaultChatTurnLeaseMs).toISOString(),
|
|
501
|
+
sandboxed: false,
|
|
502
|
+
resources: zeroSandboxResources(),
|
|
503
|
+
controller: new AbortController(),
|
|
504
|
+
chatRunId: packet.chatRunId,
|
|
505
|
+
conversationId: packet.conversationId,
|
|
506
|
+
turnId: packet.turnId,
|
|
507
|
+
providerThreadId: packet.providerThreadId ?? null,
|
|
508
|
+
chatSequence: 0,
|
|
509
|
+
terminalCallbackAttempted: false,
|
|
510
|
+
};
|
|
511
|
+
await this.sendChatCallback(active, kind, payload);
|
|
512
|
+
}
|
|
513
|
+
async sendChatCallback(active, kind, payload = null) {
|
|
514
|
+
if (active.kind !== "chat" || !active.chatRunId || !active.conversationId || !active.turnId) {
|
|
515
|
+
throw new Error("Bridge Chat callback requires an active chat turn.");
|
|
516
|
+
}
|
|
517
|
+
const terminal = isTerminalChatTurnCallbackKind(kind);
|
|
518
|
+
if (terminal && active.terminalCallbackAttempted) {
|
|
519
|
+
return;
|
|
520
|
+
}
|
|
521
|
+
assertBridgeChatCallbackPayload(payload);
|
|
522
|
+
const sequence = (active.chatSequence ?? 0) + 1;
|
|
523
|
+
const envelope = {
|
|
524
|
+
callbackId: randomUUID(),
|
|
525
|
+
chatRunId: active.chatRunId,
|
|
526
|
+
conversationId: active.conversationId,
|
|
527
|
+
turnId: active.turnId,
|
|
528
|
+
leaseId: active.leaseId,
|
|
529
|
+
fencingToken: active.fencingToken,
|
|
530
|
+
runnerId: this.config.runnerId,
|
|
531
|
+
runnerInstanceId: active.runnerInstanceId ?? this.config.runnerInstanceId,
|
|
532
|
+
sequence,
|
|
533
|
+
eventType: kind,
|
|
534
|
+
redactionStatus: "redacted",
|
|
535
|
+
providerThreadId: active.providerThreadId ?? null,
|
|
536
|
+
payload,
|
|
537
|
+
occurredAt: new Date().toISOString(),
|
|
538
|
+
protocolVersion: bridgeChatProtocolVersion,
|
|
539
|
+
};
|
|
540
|
+
if (terminal) {
|
|
541
|
+
active.terminalCallbackAttempted = true;
|
|
542
|
+
}
|
|
543
|
+
await this.invokeChatCallbackWithRetry(envelope);
|
|
544
|
+
active.chatSequence = sequence;
|
|
545
|
+
}
|
|
546
|
+
async invokeChatCallbackWithRetry(envelope) {
|
|
547
|
+
const maxAttempts = 3;
|
|
548
|
+
for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
|
|
549
|
+
try {
|
|
550
|
+
await this.invokeAck("ChatTurnCallback", envelope);
|
|
551
|
+
return;
|
|
552
|
+
}
|
|
553
|
+
catch (error) {
|
|
554
|
+
if (error instanceof BridgeCallbackRejectedError || attempt === maxAttempts - 1) {
|
|
555
|
+
throw error;
|
|
556
|
+
}
|
|
557
|
+
await delay(200 * (attempt + 1));
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
requiredChatCapabilityFailure(packet) {
|
|
562
|
+
const capability = this.bridgeChatCapability();
|
|
563
|
+
if (capability.availability === "unavailable" || !capability.supportsTextStreaming) {
|
|
564
|
+
return capability.unavailableReason ?? "Bridge Chat text capability is unavailable.";
|
|
565
|
+
}
|
|
566
|
+
if (capability.supportedProviders.length > 0
|
|
567
|
+
&& !containsIgnoreCase(capability.supportedProviders, packet.providerKind)) {
|
|
568
|
+
return `Bridge Chat provider '${packet.providerKind}' is not advertised by this bridge.`;
|
|
569
|
+
}
|
|
570
|
+
if (capability.supportedModels.length > 0
|
|
571
|
+
&& !containsIgnoreCase(capability.supportedModels, packet.model)) {
|
|
572
|
+
return `Bridge Chat model '${packet.model}' is not advertised by this bridge.`;
|
|
573
|
+
}
|
|
574
|
+
if (packet.requestedCapabilities.includes("cancel") && !capability.supportsCancel) {
|
|
575
|
+
return "Bridge Chat cancellation is not advertised by this bridge.";
|
|
576
|
+
}
|
|
577
|
+
if (packet.requestedCapabilities.includes("resume") && !capability.supportsResume) {
|
|
578
|
+
return "Bridge Chat resume is not advertised by this bridge.";
|
|
579
|
+
}
|
|
580
|
+
if (packet.requestedCapabilities.includes("thread_continuity")
|
|
581
|
+
&& !capability.supportsProviderThreadContinuity) {
|
|
582
|
+
return "Bridge Chat provider thread continuity is not advertised by this bridge.";
|
|
583
|
+
}
|
|
584
|
+
if (packet.requestedCapabilities.includes("attachment_handoffs") && !capability.supportsAttachments) {
|
|
585
|
+
return "Bridge Chat attachment handoffs are not advertised by this bridge.";
|
|
586
|
+
}
|
|
587
|
+
if (packet.requestedCapabilities.includes("action_broker_grants") && !capability.supportsActionBrokerGrants) {
|
|
588
|
+
return "Bridge Chat action-broker grants are not advertised by this bridge.";
|
|
589
|
+
}
|
|
590
|
+
return null;
|
|
591
|
+
}
|
|
592
|
+
bridgeChatCapability() {
|
|
593
|
+
return bridgeChatCapabilityFor(this.config.cliProviders, this.config.providerStatuses, this.config.bridgeChat);
|
|
83
594
|
}
|
|
84
|
-
async handleDispatch(rawDispatch) {
|
|
85
|
-
const dispatch = this.parseDispatchOrReport(rawDispatch);
|
|
595
|
+
async handleDispatch(rawDispatch, reservationHandoff) {
|
|
596
|
+
const dispatch = await this.parseDispatchOrReport(rawDispatch);
|
|
86
597
|
if (!dispatch) {
|
|
598
|
+
this.releaseReservedDispatchHandoff(reservationHandoff);
|
|
599
|
+
return;
|
|
600
|
+
}
|
|
601
|
+
if (this.enforceHostDatabaseCapabilityBoundary) {
|
|
602
|
+
try {
|
|
603
|
+
assertHostDatabaseCapabilityBoundary();
|
|
604
|
+
}
|
|
605
|
+
catch (error) {
|
|
606
|
+
const message = errorMessage(error);
|
|
607
|
+
await this.sendResult(dispatch, failureResult(message, [{
|
|
608
|
+
role: "runner",
|
|
609
|
+
kind: "error",
|
|
610
|
+
content: message,
|
|
611
|
+
eventType: "host_database_capability_forbidden",
|
|
612
|
+
}], failureFactsFromError(error, { source: "bridge" }))).catch(() => undefined);
|
|
613
|
+
this.releaseReservedDispatchHandoff(reservationHandoff);
|
|
614
|
+
return;
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
let capabilityFailure;
|
|
618
|
+
try {
|
|
619
|
+
capabilityFailure = await this.requiredCapabilityFailure(dispatch);
|
|
620
|
+
}
|
|
621
|
+
catch (error) {
|
|
622
|
+
this.releaseReservedDispatchHandoff(reservationHandoff);
|
|
623
|
+
throw error;
|
|
624
|
+
}
|
|
625
|
+
if (capabilityFailure) {
|
|
626
|
+
await this.sendResult(dispatch, failureResult(capabilityFailure, [{
|
|
627
|
+
role: "runner",
|
|
628
|
+
kind: "error",
|
|
629
|
+
content: capabilityFailure,
|
|
630
|
+
eventType: "docker_engine_unavailable",
|
|
631
|
+
}], failureFactsForCode("docker_engine_unavailable"))).catch(() => undefined);
|
|
632
|
+
this.releaseReservedDispatchHandoff(reservationHandoff);
|
|
87
633
|
return;
|
|
88
634
|
}
|
|
89
635
|
const leaseId = dispatch.lease.id;
|
|
90
636
|
const fencingToken = dispatch.lease.fencingToken;
|
|
91
637
|
const redactionSecrets = exactSecrets(dispatch, this.config);
|
|
92
|
-
if (this.
|
|
93
|
-
await this.sendResult(
|
|
638
|
+
if (reservationHandoff && !this.reservedDispatchHandoffIsCurrent(reservationHandoff, dispatch)) {
|
|
639
|
+
await this.sendResult(dispatch, failureResult("Start rejected because its reservation is missing or stale.", [], failureFactsForCode("bridge.reservation_stale"))).catch(() => undefined);
|
|
94
640
|
return;
|
|
95
641
|
}
|
|
96
|
-
const
|
|
97
|
-
|
|
642
|
+
const existing = this.activeSessions.get(leaseId);
|
|
643
|
+
if (existing) {
|
|
644
|
+
if (existing.fencingToken === fencingToken) {
|
|
645
|
+
this.releaseReservedDispatchHandoff(reservationHandoff);
|
|
646
|
+
return;
|
|
647
|
+
}
|
|
648
|
+
await this.sendResult(dispatch, failureResult("Dispatch fence conflicts with an active bridge session.", [], failureFactsForCode("bridge.dispatch_fence_conflict"))).catch(() => undefined);
|
|
649
|
+
this.releaseReservedDispatchHandoff(reservationHandoff);
|
|
650
|
+
return;
|
|
651
|
+
}
|
|
652
|
+
const controller = reservationHandoff?.controller ?? new AbortController();
|
|
653
|
+
this.activeSessions.set(leaseId, {
|
|
654
|
+
leaseId,
|
|
655
|
+
kind: "dispatch",
|
|
656
|
+
fencingToken,
|
|
657
|
+
dispatchAttemptId: dispatch.sandbox?.dispatchAttemptId ?? null,
|
|
658
|
+
runnerInstanceId: dispatch.sandbox?.runnerInstanceId ?? null,
|
|
659
|
+
leaseExpiresAt: dispatch.lease.expiresAt,
|
|
660
|
+
sandboxed: dispatch.sandbox?.executionMode === "docker",
|
|
661
|
+
resources: dispatch.sandbox?.executionMode === "docker"
|
|
662
|
+
? dispatch.sandbox.requestedResources
|
|
663
|
+
: zeroSandboxResources(),
|
|
664
|
+
controller
|
|
665
|
+
});
|
|
666
|
+
if (reservationHandoff) {
|
|
667
|
+
// Promotion is synchronous with active-session registration. A qualified
|
|
668
|
+
// pre-start Cancel therefore sees either the cancellable reservation or
|
|
669
|
+
// the fully active owner; there is no cancellation-blind handoff gap.
|
|
670
|
+
this.releaseReservedDispatchHandoff(reservationHandoff);
|
|
671
|
+
}
|
|
672
|
+
// Server-requested runtime readiness is background maintenance. A probe may
|
|
673
|
+
// have passed admission during the narrow handoff between prepared-runtime
|
|
674
|
+
// verification and the foreground attempt's final admission. Preempt it as
|
|
675
|
+
// soon as real agent work arrives so the probe cannot consume the memory
|
|
676
|
+
// headroom that the foreground attempt is waiting to acquire.
|
|
677
|
+
this.deferActiveReadinessProbes();
|
|
98
678
|
try {
|
|
99
679
|
const normalized = this.normalizeDispatch(dispatch);
|
|
100
|
-
|
|
680
|
+
if (normalized.sandbox) {
|
|
681
|
+
const started = {
|
|
682
|
+
dispatchAttemptId: normalized.sandbox.dispatchAttemptId,
|
|
683
|
+
leaseId,
|
|
684
|
+
fencingToken,
|
|
685
|
+
runnerInstanceId: normalized.sandbox.runnerInstanceId
|
|
686
|
+
};
|
|
687
|
+
await this.invokeAck("DispatchStarted", started);
|
|
688
|
+
}
|
|
689
|
+
const sink = {
|
|
101
690
|
emit: async (event) => {
|
|
691
|
+
if (event.eventType === "provider_cli_started") {
|
|
692
|
+
const active = this.activeSessions.get(leaseId);
|
|
693
|
+
if (active)
|
|
694
|
+
active.resources = { ...active.resources, buildSlots: 0 };
|
|
695
|
+
}
|
|
102
696
|
try {
|
|
103
697
|
await this.invokeAck("DispatchEvent", {
|
|
104
698
|
leaseId,
|
|
105
699
|
fencingToken,
|
|
106
|
-
|
|
700
|
+
...(dispatch.sandbox ? {
|
|
701
|
+
dispatchAttemptId: dispatch.sandbox.dispatchAttemptId,
|
|
702
|
+
runnerInstanceId: dispatch.sandbox.runnerInstanceId
|
|
703
|
+
} : {}),
|
|
704
|
+
event: redactRunnerDispatchEvent({ ...event, eventId: event.eventId ?? randomUUID() }, redactionSecrets)
|
|
705
|
+
}, {
|
|
706
|
+
replayCallbackTimeout: true,
|
|
707
|
+
signal: controller.signal,
|
|
107
708
|
});
|
|
108
709
|
}
|
|
109
710
|
catch (error) {
|
|
@@ -114,20 +715,386 @@ export class ForgeBridgeService {
|
|
|
114
715
|
throw error;
|
|
115
716
|
}
|
|
116
717
|
}
|
|
117
|
-
}
|
|
118
|
-
|
|
718
|
+
};
|
|
719
|
+
const result = normalized.sandbox?.executionMode === "docker"
|
|
720
|
+
? await this.sandboxRuntime.runOnce(normalized, sink, controller.signal)
|
|
721
|
+
: await this.sessions.runOnce(normalized, sink, controller.signal);
|
|
722
|
+
if (normalized.sandbox?.executionMode === "docker") {
|
|
723
|
+
await this.flushCleanupReports().catch(() => undefined);
|
|
724
|
+
}
|
|
725
|
+
const redactedResult = redactRunOnceResultPacket(result, redactionSecrets);
|
|
726
|
+
await this.sendResult(dispatch, compactTerminalResult(dispatch, redactedResult));
|
|
119
727
|
}
|
|
120
728
|
catch (error) {
|
|
121
729
|
const summary = controller.signal.aborted
|
|
122
730
|
? `Dispatch cancelled: ${String(controller.signal.reason ?? "cancelled")}`
|
|
123
731
|
: redactText(error instanceof Error ? error.message : String(error), redactionSecrets);
|
|
124
|
-
|
|
732
|
+
const failureFacts = failureFactsFromError(error, {
|
|
733
|
+
cancelled: controller.signal.aborted,
|
|
734
|
+
source: "bridge",
|
|
735
|
+
});
|
|
736
|
+
const failureEventType = failureFacts.category === "unknown" ? null : failureFacts.code;
|
|
737
|
+
const failureEvents = failureEventType ? [{
|
|
738
|
+
role: "runner",
|
|
739
|
+
kind: "error",
|
|
740
|
+
content: summary,
|
|
741
|
+
eventType: failureEventType,
|
|
742
|
+
}] : [];
|
|
743
|
+
await this.sendResult(dispatch, failureResult(summary, failureEvents, failureFacts)).catch(() => undefined);
|
|
125
744
|
}
|
|
126
745
|
finally {
|
|
127
|
-
this.
|
|
746
|
+
await this.releaseActiveSessionAndRefreshCapacity(leaseId);
|
|
128
747
|
}
|
|
129
748
|
}
|
|
130
|
-
|
|
749
|
+
reservedDispatchHandoffIsCurrent(handoff, dispatch) {
|
|
750
|
+
const sandbox = dispatch.sandbox;
|
|
751
|
+
const reservation = handoff.reservation;
|
|
752
|
+
return sandbox !== null
|
|
753
|
+
&& sandbox !== undefined
|
|
754
|
+
&& sandbox.dispatchAttemptId === handoff.dispatchAttemptId
|
|
755
|
+
&& this.reservations.get(handoff.dispatchAttemptId) === reservation
|
|
756
|
+
&& reservation.startClaim === handoff
|
|
757
|
+
&& !handoff.controller.signal.aborted
|
|
758
|
+
&& reservation.leaseId === dispatch.lease.id
|
|
759
|
+
&& reservation.fencingToken === dispatch.lease.fencingToken
|
|
760
|
+
&& reservation.runnerInstanceId === sandbox.runnerInstanceId
|
|
761
|
+
&& Date.parse(reservation.reservedUntil) > Date.now();
|
|
762
|
+
}
|
|
763
|
+
releaseReservedDispatchHandoff(handoff) {
|
|
764
|
+
if (!handoff
|
|
765
|
+
|| this.reservations.get(handoff.dispatchAttemptId) !== handoff.reservation
|
|
766
|
+
|| handoff.reservation.startClaim !== handoff)
|
|
767
|
+
return;
|
|
768
|
+
this.reservations.delete(handoff.dispatchAttemptId);
|
|
769
|
+
}
|
|
770
|
+
async releaseActiveSessionAndRefreshCapacity(leaseId, expectedDispatch) {
|
|
771
|
+
if (expectedDispatch) {
|
|
772
|
+
const active = this.activeSessions.get(leaseId);
|
|
773
|
+
// Admission failures may arrive after a newer attempt starts on the same
|
|
774
|
+
// lease. A terminal callback can release only the dispatch it identifies.
|
|
775
|
+
if (!active || active.kind !== "dispatch"
|
|
776
|
+
|| active.fencingToken !== expectedDispatch.lease.fencingToken
|
|
777
|
+
|| active.dispatchAttemptId !== (expectedDispatch.sandbox?.dispatchAttemptId ?? null)
|
|
778
|
+
|| active.runnerInstanceId !== (expectedDispatch.sandbox?.runnerInstanceId ?? null))
|
|
779
|
+
return;
|
|
780
|
+
}
|
|
781
|
+
if (!this.activeSessions.delete(leaseId))
|
|
782
|
+
return;
|
|
783
|
+
// Publish released host/runtime capacity before the terminal result can
|
|
784
|
+
// enqueue the next pipeline phase. Otherwise the server may reject that
|
|
785
|
+
// handoff from the previous periodic heartbeat's active reservation.
|
|
786
|
+
// Publish the updated reservation counts from cached samples only. Probes
|
|
787
|
+
// belong to periodic/admission heartbeats and must not outlive this bounded
|
|
788
|
+
// terminal handoff. Storage recovery must not block durable result delivery.
|
|
789
|
+
const controller = new AbortController();
|
|
790
|
+
let timer;
|
|
791
|
+
try {
|
|
792
|
+
await Promise.race([
|
|
793
|
+
this.sendHeartbeatSnapshot(controller.signal).catch(() => undefined),
|
|
794
|
+
new Promise(resolve => {
|
|
795
|
+
timer = setTimeout(() => {
|
|
796
|
+
controller.abort("terminal capacity refresh deadline");
|
|
797
|
+
resolve();
|
|
798
|
+
}, Math.min(5000, positiveFiniteNumber(this.config.callbackAckTimeoutMs ?? 30_000, 30_000)));
|
|
799
|
+
}),
|
|
800
|
+
]);
|
|
801
|
+
}
|
|
802
|
+
finally {
|
|
803
|
+
if (timer)
|
|
804
|
+
clearTimeout(timer);
|
|
805
|
+
}
|
|
806
|
+
}
|
|
807
|
+
async handleOffer(rawDispatch) {
|
|
808
|
+
const envelope = parseOfferEnvelope(rawDispatch);
|
|
809
|
+
if (!envelope) {
|
|
810
|
+
return;
|
|
811
|
+
}
|
|
812
|
+
const dispatch = await this.parseDispatchOrReport(envelope.dispatch);
|
|
813
|
+
if (!dispatch?.sandbox
|
|
814
|
+
|| dispatch.sandbox.dispatchAttemptId !== envelope.dispatchAttemptId
|
|
815
|
+
|| dispatch.lease.id !== envelope.leaseId
|
|
816
|
+
|| dispatch.lease.fencingToken !== envelope.fencingToken)
|
|
817
|
+
return;
|
|
818
|
+
const sandbox = dispatch.sandbox;
|
|
819
|
+
this.expireReservations(Date.now());
|
|
820
|
+
const existing = this.reservations.get(sandbox.dispatchAttemptId);
|
|
821
|
+
if (existing) {
|
|
822
|
+
const sameFence = existing.leaseId === dispatch.lease.id
|
|
823
|
+
&& existing.fencingToken === dispatch.lease.fencingToken
|
|
824
|
+
&& existing.runnerInstanceId === sandbox.runnerInstanceId;
|
|
825
|
+
await this.sendOfferDecision(dispatch, sameFence, sameFence ? null : "stale_fence", sameFence ? null : "Offer fence conflicts with the existing reservation.", sameFence ? existing.reservedUntil : null, sameFence ? existing.resources : null);
|
|
826
|
+
return;
|
|
827
|
+
}
|
|
828
|
+
if (sandbox.runnerInstanceId !== this.config.runnerInstanceId) {
|
|
829
|
+
await this.sendOfferDecision(dispatch, false, "stale_runner_instance", "Offer targets a different runner process instance.");
|
|
830
|
+
return;
|
|
831
|
+
}
|
|
832
|
+
if (Date.parse(envelope.expiresAt) <= Date.now()) {
|
|
833
|
+
await this.sendOfferDecision(dispatch, false, "lease_expired", "Offer lease is already expired.");
|
|
834
|
+
return;
|
|
835
|
+
}
|
|
836
|
+
const preflight = sandbox.executionMode === "docker" && this.config.dockerEnabled
|
|
837
|
+
? await this.preflight.run()
|
|
838
|
+
: this.preflight.current();
|
|
839
|
+
if (sandbox.executionMode === "docker") {
|
|
840
|
+
if (!this.config.dockerEnabled || !preflight?.healthy) {
|
|
841
|
+
await this.sendOfferDecision(dispatch, false, preflight?.unavailableCode ?? "docker_unavailable", preflight?.unavailableReason ?? "Docker sandbox preflight is unavailable.");
|
|
842
|
+
return;
|
|
843
|
+
}
|
|
844
|
+
const networkFailure = sandboxNetworkPolicyFailure(dispatch, this.config);
|
|
845
|
+
if (networkFailure) {
|
|
846
|
+
await this.sendOfferDecision(dispatch, false, "policy_mismatch", networkFailure);
|
|
847
|
+
return;
|
|
848
|
+
}
|
|
849
|
+
const providerNetworkFailure = sandboxProviderNetworkFailure(dispatch, this.config);
|
|
850
|
+
if (providerNetworkFailure) {
|
|
851
|
+
await this.sendOfferDecision(dispatch, false, "provider_network_policy_denied", providerNetworkFailure);
|
|
852
|
+
return;
|
|
853
|
+
}
|
|
854
|
+
const authFailure = sandboxAuthFailure(dispatch, this.config);
|
|
855
|
+
if (authFailure) {
|
|
856
|
+
await this.sendOfferDecision(dispatch, false, "sandbox_auth_unavailable", authFailure);
|
|
857
|
+
return;
|
|
858
|
+
}
|
|
859
|
+
}
|
|
860
|
+
let requestedResources = sandbox.requestedResources;
|
|
861
|
+
let sandboxConfigCommitSha = sandbox.sandboxConfigCommitSha ?? null;
|
|
862
|
+
const normalizedDispatch = this.normalizeDispatch(dispatch);
|
|
863
|
+
if (sandbox.executionMode === "docker") {
|
|
864
|
+
try {
|
|
865
|
+
const profile = await this.sandboxResourceResolver(normalizedDispatch);
|
|
866
|
+
requestedResources = profile.resources;
|
|
867
|
+
sandboxConfigCommitSha = profile.sandboxConfigCommitSha;
|
|
868
|
+
}
|
|
869
|
+
catch (error) {
|
|
870
|
+
await this.sendOfferDecision(dispatch, false, "sandbox_capacity_profile_unavailable", error instanceof Error ? error.message : "The immutable sandbox contract could not be profiled for admission.");
|
|
871
|
+
return;
|
|
872
|
+
}
|
|
873
|
+
}
|
|
874
|
+
else if (normalizedDispatch.repo.serverLocalPath) {
|
|
875
|
+
// Host execution uses the same project-derived disk reservation as Docker.
|
|
876
|
+
// The API supplies a placement-time estimate; the runner can safely raise it
|
|
877
|
+
// from its authoritative checkout before admitting the attempt.
|
|
878
|
+
const measuredWorkspaceBytes = await estimateWorkspaceBytesFromRepo(normalizedDispatch.repo.serverLocalPath, { role: normalizedDispatch.execution.role }).catch(() => null);
|
|
879
|
+
if (measuredWorkspaceBytes !== null) {
|
|
880
|
+
requestedResources = {
|
|
881
|
+
...requestedResources,
|
|
882
|
+
workspaceBytes: Math.max(requestedResources.workspaceBytes, measuredWorkspaceBytes),
|
|
883
|
+
};
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
// Resource profiling can await filesystem and Docker work. Admission needs
|
|
887
|
+
// a dynamic host/daemon sample that began after profiling completed, so an
|
|
888
|
+
// older in-flight heartbeat inspection cannot supply the commit decision.
|
|
889
|
+
const admissionPreflight = sandbox.executionMode === "docker"
|
|
890
|
+
? await this.preflight.refreshCapacityForAdmission()
|
|
891
|
+
: preflight;
|
|
892
|
+
// Resource profiling and preflight may await slow filesystem or Docker I/O.
|
|
893
|
+
// Re-read every mutable capacity input only after that work, then decide and
|
|
894
|
+
// commit synchronously. JavaScript cannot interleave another Offer handler
|
|
895
|
+
// inside this non-async method, so the ledger check and insertion are one
|
|
896
|
+
// atomic event-loop turn without holding a lock across external I/O.
|
|
897
|
+
const decision = this.commitOfferReservation(dispatch, requestedResources, admissionPreflight);
|
|
898
|
+
await this.sendOfferDecision(dispatch, decision.accepted, decision.code, decision.message, decision.reservedUntil, decision.resources, sandboxConfigCommitSha);
|
|
899
|
+
}
|
|
900
|
+
commitOfferReservation(dispatch, requestedResources, preparedPreflight) {
|
|
901
|
+
const sandbox = dispatch.sandbox;
|
|
902
|
+
const now = Date.now();
|
|
903
|
+
this.expireReservations(now);
|
|
904
|
+
const existing = this.reservations.get(sandbox.dispatchAttemptId);
|
|
905
|
+
if (existing) {
|
|
906
|
+
const sameFence = existing.leaseId === dispatch.lease.id
|
|
907
|
+
&& existing.fencingToken === dispatch.lease.fencingToken
|
|
908
|
+
&& existing.runnerInstanceId === sandbox.runnerInstanceId;
|
|
909
|
+
return {
|
|
910
|
+
accepted: sameFence,
|
|
911
|
+
code: sameFence ? null : "stale_fence",
|
|
912
|
+
message: sameFence ? null : "Offer fence conflicts with the existing reservation.",
|
|
913
|
+
reservedUntil: sameFence ? existing.reservedUntil : null,
|
|
914
|
+
resources: sameFence ? existing.resources : null,
|
|
915
|
+
};
|
|
916
|
+
}
|
|
917
|
+
if (Date.parse(dispatch.lease.expiresAt) <= now) {
|
|
918
|
+
return rejectedOfferReservation("lease_expired", "Offer lease is already expired.");
|
|
919
|
+
}
|
|
920
|
+
if (sandbox.executionMode === "docker") {
|
|
921
|
+
// A concurrent heartbeat/preflight refresh may have produced a newer
|
|
922
|
+
// conservative dual-domain sample while this offer was being profiled.
|
|
923
|
+
const admissionPreflight = this.preflight.current() ?? preparedPreflight;
|
|
924
|
+
if (!this.config.dockerEnabled || !admissionPreflight?.healthy) {
|
|
925
|
+
return rejectedOfferReservation(admissionPreflight?.unavailableCode ?? "docker_unavailable", admissionPreflight?.unavailableReason ?? "Docker sandbox preflight is unavailable.");
|
|
926
|
+
}
|
|
927
|
+
const runtimeState = this.sandboxRuntime.snapshot();
|
|
928
|
+
const usedResources = this.usedSandboxResources(runtimeState.degraded);
|
|
929
|
+
const activeResources = this.activeSandboxResources(runtimeState.degraded);
|
|
930
|
+
const capacity = this.sandboxResourceCapacity(admissionPreflight, activeResources);
|
|
931
|
+
if (!canReserveSandboxResources(usedResources, requestedResources, capacity)) {
|
|
932
|
+
return rejectedOfferReservation("capacity_unavailable", sandboxCapacityFailureMessage(usedResources, requestedResources, capacity, admissionPreflight, this.config));
|
|
933
|
+
}
|
|
934
|
+
}
|
|
935
|
+
const reservedUntil = new Date(Math.min(Date.parse(dispatch.lease.expiresAt), now + 30_000)).toISOString();
|
|
936
|
+
const reservation = {
|
|
937
|
+
leaseId: dispatch.lease.id,
|
|
938
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
939
|
+
runnerInstanceId: sandbox.runnerInstanceId,
|
|
940
|
+
expiresAt: dispatch.lease.expiresAt,
|
|
941
|
+
reservedUntil,
|
|
942
|
+
sandboxed: sandbox.executionMode === "docker",
|
|
943
|
+
resources: requestedResources,
|
|
944
|
+
};
|
|
945
|
+
this.reservations.set(sandbox.dispatchAttemptId, reservation);
|
|
946
|
+
return {
|
|
947
|
+
accepted: true,
|
|
948
|
+
code: null,
|
|
949
|
+
message: null,
|
|
950
|
+
reservedUntil,
|
|
951
|
+
resources: requestedResources,
|
|
952
|
+
};
|
|
953
|
+
}
|
|
954
|
+
async handleStart(rawDispatch) {
|
|
955
|
+
const envelope = parseStartEnvelope(rawDispatch);
|
|
956
|
+
if (!envelope) {
|
|
957
|
+
return;
|
|
958
|
+
}
|
|
959
|
+
let dispatch = await this.parseDispatchOrReport(envelope.dispatch);
|
|
960
|
+
if (!dispatch?.sandbox
|
|
961
|
+
|| dispatch.sandbox.dispatchAttemptId !== envelope.dispatchAttemptId
|
|
962
|
+
|| dispatch.lease.id !== envelope.leaseId
|
|
963
|
+
|| dispatch.lease.fencingToken !== envelope.fencingToken)
|
|
964
|
+
return;
|
|
965
|
+
const active = this.activeSessions.get(dispatch.lease.id);
|
|
966
|
+
if (active?.kind === "dispatch"
|
|
967
|
+
&& active.fencingToken === dispatch.lease.fencingToken
|
|
968
|
+
&& active.dispatchAttemptId === dispatch.sandbox.dispatchAttemptId
|
|
969
|
+
&& active.runnerInstanceId === dispatch.sandbox.runnerInstanceId) {
|
|
970
|
+
return;
|
|
971
|
+
}
|
|
972
|
+
const reservation = this.reservations.get(dispatch.sandbox.dispatchAttemptId);
|
|
973
|
+
if (!reservation
|
|
974
|
+
|| reservation.leaseId !== dispatch.lease.id
|
|
975
|
+
|| reservation.fencingToken !== dispatch.lease.fencingToken
|
|
976
|
+
|| reservation.runnerInstanceId !== dispatch.sandbox.runnerInstanceId
|
|
977
|
+
|| Date.parse(reservation.reservedUntil) <= Date.now()) {
|
|
978
|
+
await this.sendResult(dispatch, failureResult("Start rejected because its reservation is missing or stale.", [], failureFactsForCode("bridge.reservation_stale"))).catch(() => undefined);
|
|
979
|
+
return;
|
|
980
|
+
}
|
|
981
|
+
// Claim the exact reservation before redemption or any other post-validation
|
|
982
|
+
// await. A duplicate Start for the same reservation is an idempotent replay:
|
|
983
|
+
// it must neither redeem a one-time grant nor disturb the original owner.
|
|
984
|
+
if (reservation.startClaim)
|
|
985
|
+
return;
|
|
986
|
+
const startClaim = {
|
|
987
|
+
dispatchAttemptId: envelope.dispatchAttemptId,
|
|
988
|
+
reservation,
|
|
989
|
+
controller: new AbortController(),
|
|
990
|
+
};
|
|
991
|
+
reservation.startClaim = startClaim;
|
|
992
|
+
if (dispatch.apiKeySecretGrant) {
|
|
993
|
+
this.releaseReservedDispatchHandoff(startClaim);
|
|
994
|
+
await this.sendResult(dispatch, failureResult("Start rejected because protocol-v2 packets may not contain a plaintext provider grant.", [], failureFactsForCode("contract_plaintext_secret_rejected"))).catch(() => undefined);
|
|
995
|
+
return;
|
|
996
|
+
}
|
|
997
|
+
if (dispatch.apiKeySecretGrantHandle) {
|
|
998
|
+
let response;
|
|
999
|
+
try {
|
|
1000
|
+
response = await this.redeemSecretGrant(dispatch);
|
|
1001
|
+
}
|
|
1002
|
+
catch (error) {
|
|
1003
|
+
this.releaseReservedDispatchHandoff(startClaim);
|
|
1004
|
+
throw error;
|
|
1005
|
+
}
|
|
1006
|
+
// Redemption may yield while a qualified Cancel releases this reservation.
|
|
1007
|
+
// Never launch from it or remove a successor reservation after that wait.
|
|
1008
|
+
if (!this.reservedDispatchHandoffIsCurrent(startClaim, dispatch)) {
|
|
1009
|
+
await this.sendResult(dispatch, failureResult("Start rejected because its reservation is missing or stale.", [], failureFactsForCode("bridge.reservation_stale"))).catch(() => undefined);
|
|
1010
|
+
return;
|
|
1011
|
+
}
|
|
1012
|
+
const grant = response.grant;
|
|
1013
|
+
if (!response.accepted
|
|
1014
|
+
|| !grant
|
|
1015
|
+
|| grant.grantId !== dispatch.apiKeySecretGrantHandle.grantId
|
|
1016
|
+
|| grant.provider !== dispatch.apiKeySecretGrantHandle.provider
|
|
1017
|
+
|| Date.parse(grant.expiresAt) <= Date.now()) {
|
|
1018
|
+
this.releaseReservedDispatchHandoff(startClaim);
|
|
1019
|
+
await this.sendResult(dispatch, failureResult(response.message ?? "Secret grant redemption was rejected.", [], failureFactsForCode("bridge.api_key_secret_grant_denied"))).catch(() => undefined);
|
|
1020
|
+
return;
|
|
1021
|
+
}
|
|
1022
|
+
dispatch = { ...dispatch, apiKeySecretGrant: grant, apiKeySecretGrantHandle: null };
|
|
1023
|
+
}
|
|
1024
|
+
else if (dispatch.execution.requiresApiKeySecretGrant) {
|
|
1025
|
+
this.releaseReservedDispatchHandoff(startClaim);
|
|
1026
|
+
await this.sendResult(dispatch, failureResult("Start rejected because the provider secret grant handle is missing.", [], failureFactsForCode("bridge.api_key_secret_grant_missing"))).catch(() => undefined);
|
|
1027
|
+
return;
|
|
1028
|
+
}
|
|
1029
|
+
dispatch = {
|
|
1030
|
+
...dispatch,
|
|
1031
|
+
sandbox: { ...dispatch.sandbox, requestedResources: reservation.resources },
|
|
1032
|
+
};
|
|
1033
|
+
await this.handleDispatch(dispatch, startClaim);
|
|
1034
|
+
}
|
|
1035
|
+
async redeemSecretGrant(dispatch) {
|
|
1036
|
+
const sandbox = dispatch.sandbox;
|
|
1037
|
+
const handle = dispatch.apiKeySecretGrantHandle;
|
|
1038
|
+
const connection = this.connection;
|
|
1039
|
+
if (!connection)
|
|
1040
|
+
return { accepted: false, code: "bridge.disconnected", message: "Bridge is disconnected.", grant: null };
|
|
1041
|
+
return await connection.invoke("RedeemSecretGrant", {
|
|
1042
|
+
dispatchAttemptId: sandbox.dispatchAttemptId,
|
|
1043
|
+
leaseId: dispatch.lease.id,
|
|
1044
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
1045
|
+
runnerInstanceId: sandbox.runnerInstanceId,
|
|
1046
|
+
grantId: handle.grantId
|
|
1047
|
+
});
|
|
1048
|
+
}
|
|
1049
|
+
async redeemPublishGrant(request, signal) {
|
|
1050
|
+
if (signal?.aborted)
|
|
1051
|
+
throw new Error(String(signal.reason ?? "publication cancelled"));
|
|
1052
|
+
const connection = this.connection;
|
|
1053
|
+
if (!connection)
|
|
1054
|
+
throw new Error("sandbox_output_publish_forbidden: bridge is disconnected.");
|
|
1055
|
+
const response = await connection.invoke("RedeemPublishGrant", request);
|
|
1056
|
+
if (!response.accepted || !response.credential) {
|
|
1057
|
+
throw new Error(`sandbox_output_publish_forbidden: ${response.message ?? response.code ?? "publication grant redemption was rejected"}`);
|
|
1058
|
+
}
|
|
1059
|
+
return response.credential;
|
|
1060
|
+
}
|
|
1061
|
+
async handleLeaseUpdate(request) {
|
|
1062
|
+
if (request.runnerInstanceId !== this.config.runnerInstanceId) {
|
|
1063
|
+
return;
|
|
1064
|
+
}
|
|
1065
|
+
const reservation = this.reservations.get(request.dispatchAttemptId);
|
|
1066
|
+
if (reservation && reservation.leaseId === request.leaseId && reservation.fencingToken === request.fencingToken) {
|
|
1067
|
+
reservation.expiresAt = request.expiresAt;
|
|
1068
|
+
}
|
|
1069
|
+
const active = this.activeSessions.get(request.leaseId);
|
|
1070
|
+
if (active && active.fencingToken === request.fencingToken && active.dispatchAttemptId === request.dispatchAttemptId) {
|
|
1071
|
+
active.leaseExpiresAt = request.expiresAt;
|
|
1072
|
+
if (active.sandboxed) {
|
|
1073
|
+
await this.sandboxRuntime.updateLease(request.dispatchAttemptId, request.fencingToken, request.expiresAt);
|
|
1074
|
+
}
|
|
1075
|
+
else {
|
|
1076
|
+
await this.sessions.updateLease?.(request.dispatchAttemptId, request.fencingToken, request.expiresAt);
|
|
1077
|
+
}
|
|
1078
|
+
}
|
|
1079
|
+
}
|
|
1080
|
+
async sendOfferDecision(dispatch, accepted, code = null, message = null, reservedUntil = null, reservedResources = null, sandboxConfigCommitSha = null) {
|
|
1081
|
+
if (!dispatch.sandbox)
|
|
1082
|
+
return;
|
|
1083
|
+
const payload = {
|
|
1084
|
+
dispatchAttemptId: dispatch.sandbox.dispatchAttemptId,
|
|
1085
|
+
leaseId: dispatch.lease.id,
|
|
1086
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
1087
|
+
runnerInstanceId: dispatch.sandbox.runnerInstanceId,
|
|
1088
|
+
accepted,
|
|
1089
|
+
code,
|
|
1090
|
+
message,
|
|
1091
|
+
reservedUntil: accepted ? reservedUntil ?? dispatch.lease.expiresAt : null,
|
|
1092
|
+
reservedResources: accepted ? reservedResources : null,
|
|
1093
|
+
sandboxConfigCommitSha: accepted ? sandboxConfigCommitSha : null,
|
|
1094
|
+
};
|
|
1095
|
+
await this.invokeAck("OfferDecision", payload);
|
|
1096
|
+
}
|
|
1097
|
+
async parseDispatchOrReport(rawDispatch) {
|
|
131
1098
|
try {
|
|
132
1099
|
return parseRunOnceDispatchPacket(rawDispatch);
|
|
133
1100
|
}
|
|
@@ -137,32 +1104,253 @@ export class ForgeBridgeService {
|
|
|
137
1104
|
return null;
|
|
138
1105
|
}
|
|
139
1106
|
const summary = invalidDispatchSummary(error);
|
|
140
|
-
|
|
1107
|
+
await this.sendLegacyResult(identifiers.leaseId, identifiers.fencingToken, failureResult(summary, [
|
|
141
1108
|
{
|
|
142
1109
|
role: "runner",
|
|
143
1110
|
kind: "error",
|
|
144
1111
|
content: summary,
|
|
145
1112
|
eventType: "invalid_dispatch"
|
|
146
1113
|
}
|
|
147
|
-
])).catch(() => undefined);
|
|
1114
|
+
], failureFactsForCode("invalid_dispatch"))).catch(() => undefined);
|
|
1115
|
+
return null;
|
|
1116
|
+
}
|
|
1117
|
+
}
|
|
1118
|
+
parseChatTurnOrReport(rawDispatch) {
|
|
1119
|
+
try {
|
|
1120
|
+
return parseChatTurnDispatchPacket(rawDispatch, this.bridgeChatCapability());
|
|
1121
|
+
}
|
|
1122
|
+
catch (error) {
|
|
1123
|
+
logBridgeStatus(invalidChatTurnSummary(error));
|
|
148
1124
|
return null;
|
|
149
1125
|
}
|
|
150
1126
|
}
|
|
151
1127
|
async handleCancel(request) {
|
|
1128
|
+
if (request.dispatchAttemptId !== undefined && request.dispatchAttemptId !== null) {
|
|
1129
|
+
// Qualified cancellation is reservation-only. In particular, a delayed
|
|
1130
|
+
// pre-start Cancel must not abort an active session if Start won the race.
|
|
1131
|
+
const reservation = this.reservations.get(request.dispatchAttemptId);
|
|
1132
|
+
if (reservation
|
|
1133
|
+
&& request.runnerId === this.config.runnerId
|
|
1134
|
+
&& request.runnerInstanceId === this.config.runnerInstanceId
|
|
1135
|
+
&& reservation.runnerInstanceId === request.runnerInstanceId
|
|
1136
|
+
&& reservation.leaseId === request.leaseId
|
|
1137
|
+
&& reservation.fencingToken === request.fencingToken) {
|
|
1138
|
+
reservation.startClaim?.controller.abort(request.reason ?? "cancelled by server");
|
|
1139
|
+
this.reservations.delete(request.dispatchAttemptId);
|
|
1140
|
+
}
|
|
1141
|
+
return;
|
|
1142
|
+
}
|
|
152
1143
|
const active = this.activeSessions.get(request.leaseId);
|
|
153
1144
|
if (!active || active.fencingToken !== request.fencingToken) {
|
|
154
1145
|
return;
|
|
155
1146
|
}
|
|
1147
|
+
if (request.runnerId && request.runnerId !== this.config.runnerId) {
|
|
1148
|
+
return;
|
|
1149
|
+
}
|
|
1150
|
+
if (request.runnerInstanceId && request.runnerInstanceId !== (active.runnerInstanceId ?? this.config.runnerInstanceId)) {
|
|
1151
|
+
return;
|
|
1152
|
+
}
|
|
1153
|
+
if (active.kind === "chat" && !this.chatCancelMatchesActiveTurn(active, request)) {
|
|
1154
|
+
return;
|
|
1155
|
+
}
|
|
156
1156
|
active.controller.abort(request.reason ?? "cancelled by server");
|
|
157
1157
|
}
|
|
1158
|
+
chatCancelMatchesActiveTurn(active, request) {
|
|
1159
|
+
if (request.protocolVersion && request.protocolVersion !== bridgeChatProtocolVersion) {
|
|
1160
|
+
return false;
|
|
1161
|
+
}
|
|
1162
|
+
try {
|
|
1163
|
+
assertBridgeChatCallbackPayload(request.payload, "cancel payload");
|
|
1164
|
+
}
|
|
1165
|
+
catch {
|
|
1166
|
+
return false;
|
|
1167
|
+
}
|
|
1168
|
+
if (request.chatRunId && request.chatRunId !== active.chatRunId)
|
|
1169
|
+
return false;
|
|
1170
|
+
if (request.conversationId && request.conversationId !== active.conversationId)
|
|
1171
|
+
return false;
|
|
1172
|
+
if (request.turnId && request.turnId !== active.turnId)
|
|
1173
|
+
return false;
|
|
1174
|
+
if (request.providerThreadId && active.providerThreadId && request.providerThreadId !== active.providerThreadId) {
|
|
1175
|
+
return false;
|
|
1176
|
+
}
|
|
1177
|
+
return true;
|
|
1178
|
+
}
|
|
1179
|
+
async handleReap(request) {
|
|
1180
|
+
if (request.runnerId !== this.config.runnerId
|
|
1181
|
+
|| request.runnerInstanceId !== this.config.runnerInstanceId)
|
|
1182
|
+
return;
|
|
1183
|
+
await this.runSandboxMaintenance();
|
|
1184
|
+
await this.sendHeartbeat();
|
|
1185
|
+
}
|
|
1186
|
+
async handleReadinessProbe(rawRequest) {
|
|
1187
|
+
const request = parseReadinessProbe(rawRequest);
|
|
1188
|
+
if (!request || request.runnerInstanceId !== this.config.runnerInstanceId)
|
|
1189
|
+
return;
|
|
1190
|
+
const existing = this.activeReadinessProbes.get(request.requestId);
|
|
1191
|
+
if (existing)
|
|
1192
|
+
return;
|
|
1193
|
+
const controller = new AbortController();
|
|
1194
|
+
this.activeReadinessProbes.set(request.requestId, controller);
|
|
1195
|
+
const deadlineMs = Date.parse(request.expiresAt) - Date.now();
|
|
1196
|
+
const timer = setTimeout(() => controller.abort("readiness probe expired"), Math.max(0, deadlineMs));
|
|
1197
|
+
timer.unref?.();
|
|
1198
|
+
let result;
|
|
1199
|
+
try {
|
|
1200
|
+
const accepted = {
|
|
1201
|
+
schemaVersion: 1,
|
|
1202
|
+
requestId: request.requestId,
|
|
1203
|
+
fencingToken: request.fencingToken,
|
|
1204
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
1205
|
+
acceptedAt: new Date().toISOString(),
|
|
1206
|
+
};
|
|
1207
|
+
await this.invokeAck("ReadinessAccepted", accepted);
|
|
1208
|
+
if (this.activeSessions.size > 0) {
|
|
1209
|
+
result = deferredReadinessResult(request, this.config.runnerInstanceId);
|
|
1210
|
+
}
|
|
1211
|
+
else if (!this.sessions.probeHostRuntime) {
|
|
1212
|
+
throw new Error("host_runtime_probe_unavailable: this bridge does not implement runtime readiness probes.");
|
|
1213
|
+
}
|
|
1214
|
+
else {
|
|
1215
|
+
const probe = await this.sessions.probeHostRuntime(this.normalizeReadinessProbe(request), controller.signal);
|
|
1216
|
+
result = readinessDeferralReason(controller.signal)
|
|
1217
|
+
? deferredReadinessResult(request, this.config.runnerInstanceId)
|
|
1218
|
+
: {
|
|
1219
|
+
schemaVersion: 1,
|
|
1220
|
+
requestId: request.requestId,
|
|
1221
|
+
fencingToken: request.fencingToken,
|
|
1222
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
1223
|
+
...probe,
|
|
1224
|
+
};
|
|
1225
|
+
}
|
|
1226
|
+
}
|
|
1227
|
+
catch (error) {
|
|
1228
|
+
const deferredReason = readinessDeferralReason(controller.signal);
|
|
1229
|
+
result = {
|
|
1230
|
+
schemaVersion: 1,
|
|
1231
|
+
requestId: request.requestId,
|
|
1232
|
+
fencingToken: request.fencingToken,
|
|
1233
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
1234
|
+
status: "failed",
|
|
1235
|
+
reasonCode: deferredReason ? "host_runtime_probe_deferred" : "host_runtime_probe_failed",
|
|
1236
|
+
reasonMessage: (deferredReason ?? errorMessage(error)).slice(0, 1000),
|
|
1237
|
+
checkedAt: new Date().toISOString(),
|
|
1238
|
+
};
|
|
1239
|
+
}
|
|
1240
|
+
finally {
|
|
1241
|
+
clearTimeout(timer);
|
|
1242
|
+
this.activeReadinessProbes.delete(request.requestId);
|
|
1243
|
+
}
|
|
1244
|
+
try {
|
|
1245
|
+
await this.readinessResultOutbox.persist(result, request.expiresAt);
|
|
1246
|
+
}
|
|
1247
|
+
catch (error) {
|
|
1248
|
+
if (errorMessage(error).includes("runtime_readiness_result_conflict:")) {
|
|
1249
|
+
logBridgeStatus(`Readiness result ${request.requestId} rejected conflicting reuse of a durable request identity.`);
|
|
1250
|
+
await this.sendHeartbeat().catch(() => undefined);
|
|
1251
|
+
return;
|
|
1252
|
+
}
|
|
1253
|
+
logBridgeStatus(`Readiness result ${request.requestId} could not be persisted; attempting direct delivery: ${errorMessage(error)}`);
|
|
1254
|
+
await this.invokeAck("ReadinessResult", result).catch(deliveryError => {
|
|
1255
|
+
logBridgeStatus(`Readiness result ${request.requestId} could not be persisted or delivered: ${errorMessage(deliveryError)}`);
|
|
1256
|
+
});
|
|
1257
|
+
await this.sendHeartbeat().catch(() => undefined);
|
|
1258
|
+
return;
|
|
1259
|
+
}
|
|
1260
|
+
try {
|
|
1261
|
+
await this.flushReadinessResults();
|
|
1262
|
+
}
|
|
1263
|
+
catch (error) {
|
|
1264
|
+
logBridgeStatus(`Readiness result ${request.requestId} remains queued for replay: ${errorMessage(error)}`);
|
|
1265
|
+
}
|
|
1266
|
+
await this.sendHeartbeat().catch(() => undefined);
|
|
1267
|
+
}
|
|
1268
|
+
deferActiveReadinessProbes() {
|
|
1269
|
+
for (const controller of this.activeReadinessProbes.values()) {
|
|
1270
|
+
if (!controller.signal.aborted) {
|
|
1271
|
+
controller.abort("host_runtime_probe_deferred: foreground agent work took priority over background runtime readiness.");
|
|
1272
|
+
}
|
|
1273
|
+
}
|
|
1274
|
+
}
|
|
1275
|
+
async flushReadinessResults() {
|
|
1276
|
+
if (this.readinessResultFlush)
|
|
1277
|
+
return await this.readinessResultFlush;
|
|
1278
|
+
const flush = this.flushReadinessResultsCore();
|
|
1279
|
+
this.readinessResultFlush = flush;
|
|
1280
|
+
try {
|
|
1281
|
+
await flush;
|
|
1282
|
+
}
|
|
1283
|
+
finally {
|
|
1284
|
+
if (this.readinessResultFlush === flush)
|
|
1285
|
+
this.readinessResultFlush = null;
|
|
1286
|
+
}
|
|
1287
|
+
}
|
|
1288
|
+
async flushReadinessResultsCore() {
|
|
1289
|
+
for (const pending of await this.readinessResultOutbox.pending()) {
|
|
1290
|
+
if (Date.parse(pending.expiresAt) <= Date.now()) {
|
|
1291
|
+
await this.readinessResultOutbox.acknowledge(pending.result.requestId);
|
|
1292
|
+
continue;
|
|
1293
|
+
}
|
|
1294
|
+
try {
|
|
1295
|
+
await this.invokeAck("ReadinessResult", pending.result);
|
|
1296
|
+
await this.readinessResultOutbox.acknowledge(pending.result.requestId);
|
|
1297
|
+
}
|
|
1298
|
+
catch (error) {
|
|
1299
|
+
if (isReadinessNoLongerPending(error)) {
|
|
1300
|
+
await this.readinessResultOutbox.acknowledge(pending.result.requestId);
|
|
1301
|
+
continue;
|
|
1302
|
+
}
|
|
1303
|
+
throw error;
|
|
1304
|
+
}
|
|
1305
|
+
}
|
|
1306
|
+
}
|
|
158
1307
|
abortActiveSessions(reason) {
|
|
159
1308
|
for (const active of this.activeSessions.values()) {
|
|
160
1309
|
active.controller.abort(reason);
|
|
161
1310
|
}
|
|
162
1311
|
this.activeSessions.clear();
|
|
163
1312
|
}
|
|
1313
|
+
abortActiveReadinessProbes(reason) {
|
|
1314
|
+
for (const controller of this.activeReadinessProbes.values())
|
|
1315
|
+
controller.abort(reason);
|
|
1316
|
+
this.activeReadinessProbes.clear();
|
|
1317
|
+
}
|
|
1318
|
+
normalizeReadinessProbe(request) {
|
|
1319
|
+
const mapping = findRepoMapping({
|
|
1320
|
+
id: request.repoId,
|
|
1321
|
+
name: request.repoName,
|
|
1322
|
+
remoteUrl: request.remoteUrl,
|
|
1323
|
+
}, repoMappingsFor(this.config));
|
|
1324
|
+
const configuredPath = mapping?.localPath?.trim();
|
|
1325
|
+
if (!configuredPath) {
|
|
1326
|
+
throw new Error("bridge_repo_mapping_missing: readiness requires a matching local repository mapping.");
|
|
1327
|
+
}
|
|
1328
|
+
// Local configuration, not a path supplied by the server, grants checkout
|
|
1329
|
+
// authority. Equality also excludes a different child/sibling of this repo.
|
|
1330
|
+
const suppliedPath = request.runnerRepoLocalPath.trim();
|
|
1331
|
+
if (!path.isAbsolute(suppliedPath)
|
|
1332
|
+
|| (process.platform === "win32" && path.parse(suppliedPath).root.length === 1)) {
|
|
1333
|
+
throw new Error("bridge_repo_mapping_mismatch: readiness requires a fully qualified checkout path.");
|
|
1334
|
+
}
|
|
1335
|
+
const localPath = path.resolve(configuredPath);
|
|
1336
|
+
const requestedPath = path.resolve(suppliedPath);
|
|
1337
|
+
const pathsMatch = process.platform === "win32"
|
|
1338
|
+
? localPath.toLowerCase() === requestedPath.toLowerCase()
|
|
1339
|
+
: localPath === requestedPath;
|
|
1340
|
+
if (!pathsMatch) {
|
|
1341
|
+
throw new Error("bridge_repo_mapping_mismatch: readiness checkout differs from the locally configured repository.");
|
|
1342
|
+
}
|
|
1343
|
+
return {
|
|
1344
|
+
...request,
|
|
1345
|
+
runnerId: this.config.runnerId,
|
|
1346
|
+
runnerRepoLocalPath: localPath,
|
|
1347
|
+
cloneOnDemand: false,
|
|
1348
|
+
repoRoot: path.dirname(localPath),
|
|
1349
|
+
cleanupWorktrees: this.config.cleanupWorktrees,
|
|
1350
|
+
};
|
|
1351
|
+
}
|
|
164
1352
|
normalizeDispatch(dispatch) {
|
|
165
|
-
const mapping = findRepoMapping(dispatch, repoMappingsFor(this.config));
|
|
1353
|
+
const mapping = findRepoMapping(dispatch.repo, repoMappingsFor(this.config));
|
|
166
1354
|
const localPath = mapping?.localPath?.trim();
|
|
167
1355
|
const repoRoot = localPath ? path.dirname(path.resolve(localPath)) : this.config.defaultRepoRoot;
|
|
168
1356
|
const qaLaunchTarget = normalizeQaAppLaunchTarget(mapping?.qaAppLaunchTarget) ?? dispatch.qaBrowser?.launchTarget ?? null;
|
|
@@ -185,47 +1373,290 @@ export class ForgeBridgeService {
|
|
|
185
1373
|
: dispatch.qaBrowser
|
|
186
1374
|
};
|
|
187
1375
|
}
|
|
188
|
-
async sendResult(
|
|
1376
|
+
async sendResult(dispatch, result) {
|
|
189
1377
|
const payload = {
|
|
190
|
-
leaseId,
|
|
191
|
-
fencingToken,
|
|
192
|
-
|
|
1378
|
+
leaseId: dispatch.lease.id,
|
|
1379
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
1380
|
+
dispatchAttemptId: dispatch.sandbox?.dispatchAttemptId ?? null,
|
|
1381
|
+
runnerInstanceId: dispatch.sandbox?.runnerInstanceId ?? null,
|
|
1382
|
+
// Every terminal writer, including early admission failures and exception
|
|
1383
|
+
// paths, crosses the same redaction boundary before outbox persistence.
|
|
1384
|
+
result: redactRunOnceResultPacket(result, exactSecrets(dispatch, this.config))
|
|
193
1385
|
};
|
|
1386
|
+
await this.sendResultPayload(payload, () => this.releaseActiveSessionAndRefreshCapacity(dispatch.lease.id, dispatch));
|
|
1387
|
+
}
|
|
1388
|
+
async sendLegacyResult(leaseId, fencingToken, result) {
|
|
1389
|
+
await this.sendResultPayload({ leaseId, fencingToken, result });
|
|
1390
|
+
}
|
|
1391
|
+
async sendResultPayload(payload, beforeDelivery) {
|
|
1392
|
+
// Register before persistence: replay can observe a record as soon as the
|
|
1393
|
+
// durable write becomes visible, even before persist() has returned.
|
|
1394
|
+
let releaseHandoff;
|
|
1395
|
+
const priorHandoff = this.dispatchResultHandoffs.get(payload.leaseId);
|
|
1396
|
+
const handoff = new Promise(resolve => { releaseHandoff = resolve; });
|
|
1397
|
+
this.dispatchResultHandoffs.set(payload.leaseId, handoff);
|
|
1398
|
+
let persisted = false;
|
|
1399
|
+
try {
|
|
1400
|
+
try {
|
|
1401
|
+
await this.dispatchResultOutbox.persist(payload);
|
|
1402
|
+
persisted = true;
|
|
1403
|
+
this.dispatchResultGeneration += 1;
|
|
1404
|
+
}
|
|
1405
|
+
catch (error) {
|
|
1406
|
+
logBridgeStatus(`Dispatch result ${payload.leaseId} could not be persisted; attempting direct delivery: ${errorMessage(error)}`);
|
|
1407
|
+
}
|
|
1408
|
+
await priorHandoff;
|
|
1409
|
+
await beforeDelivery?.();
|
|
1410
|
+
}
|
|
1411
|
+
finally {
|
|
1412
|
+
releaseHandoff();
|
|
1413
|
+
if (this.dispatchResultHandoffs.get(payload.leaseId) === handoff)
|
|
1414
|
+
this.dispatchResultHandoffs.delete(payload.leaseId);
|
|
1415
|
+
}
|
|
1416
|
+
if (!persisted) {
|
|
1417
|
+
await this.sendResultPayloadDirect(payload);
|
|
1418
|
+
return;
|
|
1419
|
+
}
|
|
1420
|
+
await this.flushDispatchResults().catch(error => {
|
|
1421
|
+
logBridgeStatus(`Dispatch result ${payload.leaseId} remains queued for replay: ${errorMessage(error)}`);
|
|
1422
|
+
});
|
|
1423
|
+
}
|
|
1424
|
+
async sendResultPayloadDirect(payload) {
|
|
194
1425
|
try {
|
|
195
1426
|
await this.invokeAck("DispatchResult", payload);
|
|
196
1427
|
}
|
|
197
|
-
catch (error) {
|
|
198
|
-
if (isDispatchNoLongerPending(error))
|
|
199
|
-
return;
|
|
1428
|
+
catch (error) {
|
|
1429
|
+
if (isDispatchNoLongerPending(error))
|
|
1430
|
+
return;
|
|
1431
|
+
throw error;
|
|
1432
|
+
}
|
|
1433
|
+
}
|
|
1434
|
+
async flushDispatchResults(force = false) {
|
|
1435
|
+
if (!force && this.dispatchResultGeneration === this.dispatchResultFlushedGeneration)
|
|
1436
|
+
return;
|
|
1437
|
+
if (this.dispatchResultFlush) {
|
|
1438
|
+
await this.dispatchResultFlush;
|
|
1439
|
+
if (!force && this.dispatchResultGeneration === this.dispatchResultFlushedGeneration)
|
|
1440
|
+
return;
|
|
1441
|
+
}
|
|
1442
|
+
const flush = this.flushDispatchResultsCore();
|
|
1443
|
+
this.dispatchResultFlush = flush;
|
|
1444
|
+
try {
|
|
1445
|
+
await flush;
|
|
1446
|
+
}
|
|
1447
|
+
finally {
|
|
1448
|
+
if (this.dispatchResultFlush === flush)
|
|
1449
|
+
this.dispatchResultFlush = null;
|
|
1450
|
+
}
|
|
1451
|
+
}
|
|
1452
|
+
async flushDispatchResultsCore() {
|
|
1453
|
+
const startedGeneration = this.dispatchResultGeneration;
|
|
1454
|
+
while (true) {
|
|
1455
|
+
const records = await this.dispatchResultOutbox.pending();
|
|
1456
|
+
for (const pending of records) {
|
|
1457
|
+
// Periodic heartbeats and reconnect replay share the same release
|
|
1458
|
+
// barrier as the terminal writer. Restart replay has no live handoff.
|
|
1459
|
+
while (this.dispatchResultHandoffs.has(pending.payload.leaseId)) {
|
|
1460
|
+
await this.dispatchResultHandoffs.get(pending.payload.leaseId);
|
|
1461
|
+
}
|
|
1462
|
+
try {
|
|
1463
|
+
await this.sendResultPayloadDirect(pending.payload);
|
|
1464
|
+
await this.dispatchResultOutbox.acknowledge(pending.payload.leaseId);
|
|
1465
|
+
}
|
|
1466
|
+
catch (error) {
|
|
1467
|
+
if (isDispatchNoLongerPending(error)) {
|
|
1468
|
+
await this.dispatchResultOutbox.acknowledge(pending.payload.leaseId);
|
|
1469
|
+
continue;
|
|
1470
|
+
}
|
|
1471
|
+
throw error;
|
|
1472
|
+
}
|
|
200
1473
|
}
|
|
201
|
-
|
|
1474
|
+
if (records.length < 3)
|
|
1475
|
+
break;
|
|
1476
|
+
}
|
|
1477
|
+
if (this.dispatchResultGeneration === startedGeneration) {
|
|
1478
|
+
this.dispatchResultFlushedGeneration = startedGeneration;
|
|
202
1479
|
}
|
|
203
1480
|
}
|
|
204
1481
|
startHeartbeat() {
|
|
205
1482
|
this.stopHeartbeat();
|
|
1483
|
+
this.consecutiveHeartbeatFailures = 0;
|
|
1484
|
+
this.reconnectingFromHeartbeatFailure = false;
|
|
206
1485
|
const intervalMs = positiveFiniteNumber(this.config.heartbeatIntervalMs, 30_000);
|
|
207
1486
|
this.heartbeatTimer = setInterval(() => {
|
|
208
|
-
this.
|
|
1487
|
+
void this.heartbeatTick();
|
|
209
1488
|
}, intervalMs);
|
|
210
1489
|
this.heartbeatTimer.unref?.();
|
|
211
1490
|
}
|
|
1491
|
+
async heartbeatTick() {
|
|
1492
|
+
if (this.heartbeatInFlight || this.reconnectingFromHeartbeatFailure)
|
|
1493
|
+
return;
|
|
1494
|
+
const connection = this.connection;
|
|
1495
|
+
if (!connection)
|
|
1496
|
+
return;
|
|
1497
|
+
this.heartbeatInFlight = true;
|
|
1498
|
+
try {
|
|
1499
|
+
await this.flushDispatchResults();
|
|
1500
|
+
await this.flushReadinessResults();
|
|
1501
|
+
await this.sendHeartbeat();
|
|
1502
|
+
this.consecutiveHeartbeatFailures = 0;
|
|
1503
|
+
}
|
|
1504
|
+
catch (error) {
|
|
1505
|
+
if (connection !== this.connection || this.stopped)
|
|
1506
|
+
return;
|
|
1507
|
+
this.consecutiveHeartbeatFailures += 1;
|
|
1508
|
+
logBridgeStatus(`Heartbeat ${this.consecutiveHeartbeatFailures} failed: ${errorMessage(error)}.`);
|
|
1509
|
+
if (this.consecutiveHeartbeatFailures < 2)
|
|
1510
|
+
return;
|
|
1511
|
+
this.reconnectingFromHeartbeatFailure = true;
|
|
1512
|
+
this.stopHeartbeat();
|
|
1513
|
+
logBridgeStatus("Heartbeat failed twice. Closing the stale connection so the Bridge can reconnect...");
|
|
1514
|
+
await connection.stop().catch(() => undefined);
|
|
1515
|
+
}
|
|
1516
|
+
finally {
|
|
1517
|
+
this.heartbeatInFlight = false;
|
|
1518
|
+
}
|
|
1519
|
+
}
|
|
1520
|
+
startLeaseWatchdog() {
|
|
1521
|
+
this.stopLeaseWatchdog();
|
|
1522
|
+
this.leaseWatchdogTimer = setInterval(() => this.expireLeases(), 1000);
|
|
1523
|
+
this.leaseWatchdogTimer.unref?.();
|
|
1524
|
+
}
|
|
1525
|
+
stopLeaseWatchdog() {
|
|
1526
|
+
if (this.leaseWatchdogTimer) {
|
|
1527
|
+
clearInterval(this.leaseWatchdogTimer);
|
|
1528
|
+
this.leaseWatchdogTimer = null;
|
|
1529
|
+
}
|
|
1530
|
+
}
|
|
1531
|
+
expireLeases() {
|
|
1532
|
+
const now = Date.now();
|
|
1533
|
+
this.expireReservations(now);
|
|
1534
|
+
for (const active of this.activeSessions.values()) {
|
|
1535
|
+
if (Date.parse(active.leaseExpiresAt) <= now && !active.controller.signal.aborted) {
|
|
1536
|
+
active.controller.abort("dispatch lease expired");
|
|
1537
|
+
}
|
|
1538
|
+
}
|
|
1539
|
+
}
|
|
1540
|
+
expireReservations(now) {
|
|
1541
|
+
for (const [attemptId, reservation] of this.reservations) {
|
|
1542
|
+
if (Date.parse(reservation.expiresAt) <= now || Date.parse(reservation.reservedUntil) <= now) {
|
|
1543
|
+
this.reservations.delete(attemptId);
|
|
1544
|
+
}
|
|
1545
|
+
}
|
|
1546
|
+
}
|
|
212
1547
|
stopHeartbeat() {
|
|
213
1548
|
if (this.heartbeatTimer) {
|
|
214
1549
|
clearInterval(this.heartbeatTimer);
|
|
215
1550
|
this.heartbeatTimer = null;
|
|
216
1551
|
}
|
|
217
1552
|
}
|
|
1553
|
+
startSandboxMaintenance() {
|
|
1554
|
+
this.stopSandboxMaintenance();
|
|
1555
|
+
this.sandboxMaintenanceTimer = setInterval(() => {
|
|
1556
|
+
this.runSandboxMaintenance().catch(() => undefined);
|
|
1557
|
+
}, 60_000);
|
|
1558
|
+
this.sandboxMaintenanceTimer.unref?.();
|
|
1559
|
+
}
|
|
1560
|
+
stopSandboxMaintenance() {
|
|
1561
|
+
if (this.sandboxMaintenanceTimer) {
|
|
1562
|
+
clearInterval(this.sandboxMaintenanceTimer);
|
|
1563
|
+
this.sandboxMaintenanceTimer = null;
|
|
1564
|
+
}
|
|
1565
|
+
}
|
|
1566
|
+
async runSandboxMaintenance() {
|
|
1567
|
+
if (this.sandboxMaintenanceRunning)
|
|
1568
|
+
return;
|
|
1569
|
+
this.sandboxMaintenanceRunning = true;
|
|
1570
|
+
try {
|
|
1571
|
+
const hostReap = await this.sessions.reapHostRuntime?.({
|
|
1572
|
+
runnerId: this.config.runnerId,
|
|
1573
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
1574
|
+
});
|
|
1575
|
+
this.rememberRecoveredHostAttempts(hostReap);
|
|
1576
|
+
if (hostReap && (hostReap.reaped > 0 || hostReap.degraded > 0)) {
|
|
1577
|
+
logBridgeStatus(`Managed-host maintenance reaped ${hostReap.reaped} stale attempt(s) and left ${hostReap.degraded} degraded attempt(s) for reconciliation.`);
|
|
1578
|
+
}
|
|
1579
|
+
if (this.config.dockerEnabled) {
|
|
1580
|
+
const preflight = await this.preflight.refresh();
|
|
1581
|
+
if (canReapManagedSandboxes(preflight)) {
|
|
1582
|
+
const attemptSnapshot = await this.readAttemptSnapshot();
|
|
1583
|
+
await this.sandboxRuntime.reap({ diskLow: preflight.diskLow, attemptSnapshot });
|
|
1584
|
+
await this.flushCleanupReports();
|
|
1585
|
+
}
|
|
1586
|
+
}
|
|
1587
|
+
}
|
|
1588
|
+
finally {
|
|
1589
|
+
this.sandboxMaintenanceRunning = false;
|
|
1590
|
+
}
|
|
1591
|
+
}
|
|
1592
|
+
async readAttemptSnapshot() {
|
|
1593
|
+
const connection = this.connection;
|
|
1594
|
+
if (!connection)
|
|
1595
|
+
throw new Error("Bridge is not connected.");
|
|
1596
|
+
const response = await connection.invoke("AttemptSnapshot", {
|
|
1597
|
+
runnerInstanceId: this.config.runnerInstanceId
|
|
1598
|
+
});
|
|
1599
|
+
if (!response.accepted) {
|
|
1600
|
+
throw new BridgeCallbackRejectedError(response.code ?? null, response.message ?? response.code ?? "Attempt snapshot was rejected.");
|
|
1601
|
+
}
|
|
1602
|
+
return response.attempts ?? [];
|
|
1603
|
+
}
|
|
1604
|
+
async flushCleanupReports() {
|
|
1605
|
+
if (!this.sandboxRuntime.pendingCleanupReports || !this.sandboxRuntime.acknowledgeCleanupReport)
|
|
1606
|
+
return;
|
|
1607
|
+
for (const report of await this.sandboxRuntime.pendingCleanupReports()) {
|
|
1608
|
+
await this.invokeAck("CleanupReport", {
|
|
1609
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
1610
|
+
report
|
|
1611
|
+
});
|
|
1612
|
+
await this.sandboxRuntime.acknowledgeCleanupReport(report.reportId);
|
|
1613
|
+
}
|
|
1614
|
+
}
|
|
218
1615
|
async sendHeartbeat() {
|
|
1616
|
+
// Heartbeats are the recovery path for capacity-blocked work. Refresh the
|
|
1617
|
+
// dynamic sample before advertising it, while reusing the static Docker
|
|
1618
|
+
// capability snapshot established at startup or by an offer.
|
|
1619
|
+
if (this.config.dockerEnabled)
|
|
1620
|
+
await this.preflight.refreshCapacity();
|
|
1621
|
+
// A due probe can start a Docker Desktop recovery that runs for minutes.
|
|
1622
|
+
// Wait only as long as a normal probe takes, then advertise the last
|
|
1623
|
+
// snapshot so the server doesn't mark the runner offline meanwhile.
|
|
1624
|
+
const probeWait = new AbortController();
|
|
1625
|
+
await Promise.race([
|
|
1626
|
+
this.dockerEngineProbe.refresh().catch(() => undefined),
|
|
1627
|
+
delay(positiveFiniteNumber(this.config.heartbeatDockerProbeWaitMs ?? 15_000, 15_000), probeWait.signal),
|
|
1628
|
+
]);
|
|
1629
|
+
probeWait.abort();
|
|
1630
|
+
await this.sendHeartbeatSnapshot();
|
|
1631
|
+
}
|
|
1632
|
+
async sendHeartbeatSnapshot(signal) {
|
|
219
1633
|
await this.invokeAck("Heartbeat", {
|
|
220
1634
|
status: "online",
|
|
221
1635
|
capabilities: this.capabilities(),
|
|
222
1636
|
version: this.config.version,
|
|
223
1637
|
protocolVersion: this.config.protocolVersion
|
|
224
|
-
});
|
|
1638
|
+
}, { signal });
|
|
225
1639
|
}
|
|
226
1640
|
capabilities() {
|
|
1641
|
+
const preflight = this.preflight.current();
|
|
1642
|
+
const dockerEngine = this.dockerEngineProbe.current();
|
|
1643
|
+
const runtimeState = this.sandboxRuntime.snapshot();
|
|
1644
|
+
const reserved = [...this.reservations.values()].filter(reservation => reservation.sandboxed).length;
|
|
1645
|
+
const used = this.usedSandboxResources(runtimeState.degraded);
|
|
1646
|
+
const activeResources = this.activeSandboxResources(runtimeState.degraded);
|
|
1647
|
+
const capacity = this.sandboxResourceCapacity(preflight, activeResources);
|
|
1648
|
+
const memoryAdmission = preflight ? buildMemoryAdmissionDiagnostics({
|
|
1649
|
+
sampledAt: preflight.sampledAt,
|
|
1650
|
+
totalMemoryBytes: preflight.totalMemoryBytes,
|
|
1651
|
+
rawAvailableMemoryBytes: preflight.rawAvailableMemoryBytes,
|
|
1652
|
+
stabilizedAvailableMemoryBytes: preflight.availableMemoryBytes,
|
|
1653
|
+
controlPlaneMemoryBytes: process.memoryUsage().rss,
|
|
1654
|
+
reserveMemoryBytes: preflight.hostReserveBytes,
|
|
1655
|
+
committedMemoryBytes: used.memoryBytes,
|
|
1656
|
+
pendingCommitmentMemoryBytes: Math.max(0, used.memoryBytes - activeResources.memoryBytes),
|
|
1657
|
+
}) : null;
|
|
227
1658
|
return {
|
|
228
|
-
schemaVersion:
|
|
1659
|
+
schemaVersion: 6,
|
|
229
1660
|
activeSessionCount: this.activeSessions.size,
|
|
230
1661
|
cliProviders: this.config.cliProviders,
|
|
231
1662
|
providerStatuses: this.config.providerStatuses,
|
|
@@ -233,32 +1664,208 @@ export class ForgeBridgeService {
|
|
|
233
1664
|
repoMappings: repoMappingsFor(this.config),
|
|
234
1665
|
os: this.config.os,
|
|
235
1666
|
sandboxProfiles: this.config.sandboxProfiles,
|
|
1667
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
1668
|
+
trustDomainId: this.config.trustDomainId,
|
|
1669
|
+
executionFacilities: dockerEngine?.available ? ["docker_engine"] : [],
|
|
1670
|
+
dockerEngine,
|
|
1671
|
+
toolchains: {
|
|
1672
|
+
...this.hostCapabilities.toolchains,
|
|
1673
|
+
docker: {
|
|
1674
|
+
available: dockerEngine?.available === true,
|
|
1675
|
+
executable: this.config.dockerExecutable,
|
|
1676
|
+
version: dockerEngine?.runtimeVersion ?? null,
|
|
1677
|
+
},
|
|
1678
|
+
},
|
|
1679
|
+
playwrightProjects: this.hostCapabilities.playwrightProjects,
|
|
1680
|
+
browserLaunch: this.hostCapabilities.browserLaunch,
|
|
1681
|
+
providerConnectivity: this.hostCapabilities.providerConnectivity,
|
|
1682
|
+
maxConcurrentSandboxes: capacity.sandboxSlots,
|
|
1683
|
+
reservedSandboxCount: reserved,
|
|
1684
|
+
activeSandboxCount: runtimeState.active,
|
|
1685
|
+
startingSandboxCount: runtimeState.starting,
|
|
1686
|
+
degradedReservationCount: runtimeState.degraded,
|
|
1687
|
+
sandboxRuntimes: preflight?.healthy ? ["docker"] : [],
|
|
1688
|
+
sandboxRuntimeVersion: preflight?.runtimeVersion ?? null,
|
|
1689
|
+
sandboxFeatures: preflight?.features ?? null,
|
|
1690
|
+
allowsOpenPublicSandbox: this.config.allowOpenPublicSandbox,
|
|
1691
|
+
sandboxPublicationModes: [this.config.sandboxPublicationMode ?? "shared_filesystem"],
|
|
1692
|
+
sandboxAuthModes: sandboxAuthModesFor(this.config),
|
|
1693
|
+
availableMemoryBytes: memoryAdmission?.availableForAdmissionBytes ?? null,
|
|
1694
|
+
availableCpuMillis: preflight ? Math.max(0, capacity.cpuMillis - used.cpuMillis) : null,
|
|
1695
|
+
availableWorkspaceBytes: preflight ? Math.max(0, capacity.workspaceBytes - used.workspaceBytes) : null,
|
|
1696
|
+
diskLow: preflight?.diskLow ?? null,
|
|
1697
|
+
preflightGeneration: preflight?.generation ?? 0,
|
|
1698
|
+
sandboxUnavailableCode: this.config.dockerEnabled ? preflight?.unavailableCode ?? null : "docker_disabled",
|
|
1699
|
+
sandboxUnavailableReason: this.config.dockerEnabled ? preflight?.unavailableReason ?? null : "Docker sandboxing is disabled by runner configuration.",
|
|
1700
|
+
sandboxMemoryCapacity: preflight ? {
|
|
1701
|
+
defaultWorkerMemoryLimitBytes: this.config.defaultSandboxWorkerMemoryLimitBytes,
|
|
1702
|
+
defaultServiceMemoryLimitBytes: this.config.defaultSandboxServiceMemoryLimitBytes,
|
|
1703
|
+
runtimeOverheadBytes: this.config.sandboxRuntimeOverheadBytes,
|
|
1704
|
+
hostReserveBytes: preflight.hostReserveBytes,
|
|
1705
|
+
activeReservationBytes: used.memoryBytes,
|
|
1706
|
+
daemonTotalBytes: preflight.totalMemoryBytes,
|
|
1707
|
+
daemonAvailableBytes: preflight.rawDaemonAvailableMemoryBytes ?? 0,
|
|
1708
|
+
hostAvailableBytes: preflight.rawHostAvailableMemoryBytes,
|
|
1709
|
+
conservativeAvailableBytes: preflight.rawAvailableMemoryBytes,
|
|
1710
|
+
availableForAdmissionBytes: memoryAdmission.availableForAdmissionBytes,
|
|
1711
|
+
memoryAdmission: memoryAdmission,
|
|
1712
|
+
} : null,
|
|
1713
|
+
hostRuntime: this.sessions.runtimeCapacity?.() ?? null,
|
|
236
1714
|
supportsApiSecretGrant: this.config.supportsApiSecretGrant,
|
|
237
1715
|
supportsManagedWorktrees: this.config.supportsManagedWorktrees,
|
|
238
1716
|
supportsManagedForgeArtifacts: this.config.supportsManagedForgeArtifacts,
|
|
239
1717
|
supportsRunnerAuthoredPhaseArtifacts: this.config.supportsRunnerAuthoredPhaseArtifacts,
|
|
240
|
-
supportsDraftRunOnce: this.config.supportsDraftRunOnce
|
|
1718
|
+
supportsDraftRunOnce: this.config.supportsDraftRunOnce,
|
|
1719
|
+
supportsDraftAttachments: this.config.supportsDraftAttachments,
|
|
1720
|
+
supportsDraftImageAttachments: this.config.supportsDraftImageAttachments,
|
|
1721
|
+
bridgeChat: bridgeChatCapabilityFor(this.config.cliProviders, this.config.providerStatuses, this.config.bridgeChat)
|
|
241
1722
|
};
|
|
242
1723
|
}
|
|
243
|
-
async
|
|
244
|
-
const
|
|
245
|
-
if (!
|
|
246
|
-
|
|
1724
|
+
async requiredCapabilityFailure(dispatch) {
|
|
1725
|
+
const requiredCapabilities = dispatch.qaScope?.required_capabilities ?? [];
|
|
1726
|
+
if (!requiredCapabilities.includes("docker_engine"))
|
|
1727
|
+
return null;
|
|
1728
|
+
const executionMode = dispatch.sandbox?.executionMode;
|
|
1729
|
+
if (executionMode === "docker") {
|
|
1730
|
+
return dockerEngineRequirementFailure(requiredCapabilities, executionMode, null);
|
|
1731
|
+
}
|
|
1732
|
+
return dockerEngineRequirementFailure(requiredCapabilities, executionMode, await this.dockerEngineProbe.refresh(true));
|
|
1733
|
+
}
|
|
1734
|
+
usedSandboxResources(degradedReservations) {
|
|
1735
|
+
const resources = [
|
|
1736
|
+
...[...this.reservations.values()]
|
|
1737
|
+
.filter(reservation => reservation.sandboxed)
|
|
1738
|
+
.map(reservation => reservation.resources),
|
|
1739
|
+
...[...this.activeSessions.values()]
|
|
1740
|
+
.filter(session => session.sandboxed)
|
|
1741
|
+
.map(session => session.resources),
|
|
1742
|
+
...Array.from({ length: degradedReservations }, () => defaultSandboxResources())
|
|
1743
|
+
];
|
|
1744
|
+
return sumSandboxResources(resources);
|
|
1745
|
+
}
|
|
1746
|
+
activeSandboxResources(degradedReservations) {
|
|
1747
|
+
return sumSandboxResources([
|
|
1748
|
+
...[...this.activeSessions.values()]
|
|
1749
|
+
.filter(session => session.sandboxed)
|
|
1750
|
+
.map(session => session.resources),
|
|
1751
|
+
...Array.from({ length: degradedReservations }, () => defaultSandboxResources())
|
|
1752
|
+
]);
|
|
1753
|
+
}
|
|
1754
|
+
sandboxResourceCapacity(preflight, activeResources) {
|
|
1755
|
+
if (!preflight)
|
|
1756
|
+
return zeroSandboxResources();
|
|
1757
|
+
return {
|
|
1758
|
+
// Host free memory and disk already reflect active containers. Add their
|
|
1759
|
+
// commitments back to form a stable total capacity, then subtract them
|
|
1760
|
+
// exactly once in usedSandboxResources.
|
|
1761
|
+
sandboxSlots: Math.min(this.config.maxConcurrentSandboxes, preflight.maxConcurrentSandboxes + activeResources.sandboxSlots),
|
|
1762
|
+
cpuMillis: preflight.availableCpuMillis,
|
|
1763
|
+
workerMemoryLimitBytes: 0,
|
|
1764
|
+
serviceMemoryLimitBytes: 0,
|
|
1765
|
+
runtimeOverheadBytes: 0,
|
|
1766
|
+
memoryBytes: sandboxMemoryCommitmentCapacity(preflight, activeResources.memoryBytes),
|
|
1767
|
+
workspaceBytes: Math.max(0, preflight.availableWorkspaceBytes
|
|
1768
|
+
- this.config.minimumWorkspaceBytes
|
|
1769
|
+
+ activeResources.workspaceBytes),
|
|
1770
|
+
pids: this.config.sandboxPidCapacity,
|
|
1771
|
+
buildSlots: this.config.maxConcurrentSandboxBuilds
|
|
1772
|
+
};
|
|
1773
|
+
}
|
|
1774
|
+
async invokeAck(methodName, payload, options = {}) {
|
|
1775
|
+
let storageRecoveryAttempt = 0;
|
|
1776
|
+
let callbackTimeoutAttempt = 0;
|
|
1777
|
+
while (true) {
|
|
1778
|
+
if (options.signal?.aborted) {
|
|
1779
|
+
throw new Error(String(options.signal.reason ?? `${methodName} callback was cancelled.`));
|
|
1780
|
+
}
|
|
1781
|
+
const connection = this.connection;
|
|
1782
|
+
if (!connection) {
|
|
1783
|
+
throw new Error("Bridge is not connected.");
|
|
1784
|
+
}
|
|
1785
|
+
await this.waitForTransportRecovery(connection, this.transportRecovery, options.signal);
|
|
1786
|
+
const recoveryGeneration = this.transportRecoveryGeneration;
|
|
1787
|
+
try {
|
|
1788
|
+
const ack = await waitForCallbackAck(connection.invoke(methodName, payload), methodName, positiveFiniteNumber(this.config.callbackAckTimeoutMs ?? 30_000, 30_000), options.signal);
|
|
1789
|
+
if (!ack.accepted) {
|
|
1790
|
+
throw new BridgeCallbackRejectedError(ack.code ?? null, ack.message ?? ack.code ?? `${methodName} callback was rejected.`);
|
|
1791
|
+
}
|
|
1792
|
+
return ack;
|
|
1793
|
+
}
|
|
1794
|
+
catch (error) {
|
|
1795
|
+
if (options.signal?.aborted)
|
|
1796
|
+
throw error;
|
|
1797
|
+
if (connection !== this.connection || this.stopped)
|
|
1798
|
+
throw error;
|
|
1799
|
+
const recovery = this.transportRecovery;
|
|
1800
|
+
if (recovery && recovery.generation > recoveryGeneration) {
|
|
1801
|
+
await this.waitForTransportRecovery(connection, recovery, options.signal);
|
|
1802
|
+
continue;
|
|
1803
|
+
}
|
|
1804
|
+
if (isProviderStorageUnavailable(error)) {
|
|
1805
|
+
const backoff = this.config.reconnectBackoffMs[Math.min(storageRecoveryAttempt, this.config.reconnectBackoffMs.length - 1)] ?? 5000;
|
|
1806
|
+
storageRecoveryAttempt += 1;
|
|
1807
|
+
logBridgeStatus(`${methodName} reached Forge while storage was recovering. Retrying callback ${storageRecoveryAttempt} in ${formatDelay(backoff)}...`);
|
|
1808
|
+
await delay(backoff, options.signal);
|
|
1809
|
+
continue;
|
|
1810
|
+
}
|
|
1811
|
+
if (options.replayCallbackTimeout && isBridgeCallbackTimeout(error)) {
|
|
1812
|
+
const backoff = this.config.reconnectBackoffMs[Math.min(callbackTimeoutAttempt, this.config.reconnectBackoffMs.length - 1)] ?? 5000;
|
|
1813
|
+
callbackTimeoutAttempt += 1;
|
|
1814
|
+
logBridgeStatus(`${methodName} acknowledgement timed out. Replaying the idempotent callback ${callbackTimeoutAttempt} in ${formatDelay(backoff)}...`);
|
|
1815
|
+
await delay(backoff, options.signal);
|
|
1816
|
+
continue;
|
|
1817
|
+
}
|
|
1818
|
+
throw error;
|
|
1819
|
+
}
|
|
247
1820
|
}
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
1821
|
+
}
|
|
1822
|
+
beginTransportRecovery() {
|
|
1823
|
+
if (this.transportRecovery && !this.transportRecovery.completed)
|
|
1824
|
+
return;
|
|
1825
|
+
let resolve = () => undefined;
|
|
1826
|
+
const promise = new Promise((complete) => { resolve = complete; });
|
|
1827
|
+
this.transportRecoveryGeneration += 1;
|
|
1828
|
+
this.transportRecovery = {
|
|
1829
|
+
generation: this.transportRecoveryGeneration,
|
|
1830
|
+
promise,
|
|
1831
|
+
resolve,
|
|
1832
|
+
completed: false,
|
|
1833
|
+
error: null,
|
|
1834
|
+
};
|
|
1835
|
+
}
|
|
1836
|
+
completeTransportRecovery(error = null) {
|
|
1837
|
+
const recovery = this.transportRecovery;
|
|
1838
|
+
if (!recovery || recovery.completed)
|
|
1839
|
+
return;
|
|
1840
|
+
recovery.completed = true;
|
|
1841
|
+
recovery.error = error;
|
|
1842
|
+
recovery.resolve();
|
|
1843
|
+
}
|
|
1844
|
+
async waitForTransportRecovery(connection, recovery, signal) {
|
|
1845
|
+
if (!recovery)
|
|
1846
|
+
return;
|
|
1847
|
+
if (recovery.completed) {
|
|
1848
|
+
if (recovery.error)
|
|
1849
|
+
throw recovery.error;
|
|
1850
|
+
return;
|
|
251
1851
|
}
|
|
252
|
-
|
|
1852
|
+
await waitForPromiseOrAbort(recovery.promise, signal);
|
|
1853
|
+
signal?.throwIfAborted();
|
|
1854
|
+
if (connection !== this.connection)
|
|
1855
|
+
throw new Error("Bridge connection changed while recovering.");
|
|
1856
|
+
if (recovery.error)
|
|
1857
|
+
throw recovery.error;
|
|
253
1858
|
}
|
|
254
1859
|
async waitForClose(signal) {
|
|
255
1860
|
if (!this.closed) {
|
|
256
1861
|
return;
|
|
257
1862
|
}
|
|
258
|
-
await
|
|
1863
|
+
await waitForPromiseOrAbort(this.closed, signal);
|
|
259
1864
|
}
|
|
260
1865
|
async stopConnection() {
|
|
261
1866
|
this.stopHeartbeat();
|
|
1867
|
+
this.stopLeaseWatchdog();
|
|
1868
|
+
this.stopSandboxMaintenance();
|
|
262
1869
|
if (this.connection) {
|
|
263
1870
|
await this.connection.stop().catch(() => undefined);
|
|
264
1871
|
this.connection = null;
|
|
@@ -279,6 +1886,24 @@ function isDispatchNoLongerPending(error) {
|
|
|
279
1886
|
}
|
|
280
1887
|
return error instanceof Error && error.message === "Dispatch is no longer pending.";
|
|
281
1888
|
}
|
|
1889
|
+
function isReadinessNoLongerPending(error) {
|
|
1890
|
+
return error instanceof BridgeCallbackRejectedError
|
|
1891
|
+
&& (error.code === "bridge.stale_readiness_probe" || error.code === "bridge.stale_runner_instance");
|
|
1892
|
+
}
|
|
1893
|
+
function isProviderStorageUnavailable(error) {
|
|
1894
|
+
const message = errorMessage(error).toLowerCase();
|
|
1895
|
+
return message.includes("provider_storage_unavailable")
|
|
1896
|
+
|| (message.includes("storage is recovering") && message.includes("503"));
|
|
1897
|
+
}
|
|
1898
|
+
function isBridgeCallbackTimeout(error) {
|
|
1899
|
+
return errorMessage(error).startsWith("bridge_callback_timeout:");
|
|
1900
|
+
}
|
|
1901
|
+
function createDefaultSessionService(sandboxStateRoot, sandboxExecutionRoot, dockerEngineProbe) {
|
|
1902
|
+
const rawProcessRunner = new StreamingProviderProcessRunner();
|
|
1903
|
+
const processSupervisors = new AttemptProcessSupervisorRegistry();
|
|
1904
|
+
const coordinatedProcessRunner = new AttemptCoordinatedProviderProcessRunner(processSupervisors, rawProcessRunner);
|
|
1905
|
+
return new BridgeSessionService(new GitWorktreeService(undefined, new RepositoryCoordinatorLock(sandboxStateRoot)), new RoutingProviderCliService(new ProcessBackedProviderCliService(coordinatedProcessRunner, undefined, undefined, undefined, new PrePublicationValidator(undefined, dockerEngineProbe, readNativeTestPostgresFallback))), new HostRuntimeLeaseManager(sandboxStateRoot, undefined, undefined, processSupervisors, rawProcessRunner, undefined, undefined, undefined, undefined, undefined, sandboxExecutionRoot, createNativeTestClusterDatabaseBroker(readNativeTestPostgresFallback, sandboxStateRoot)));
|
|
1906
|
+
}
|
|
282
1907
|
export class SignalRBridgeConnectionFactory {
|
|
283
1908
|
create(config) {
|
|
284
1909
|
return new SignalRBridgeConnection(config);
|
|
@@ -293,10 +1918,20 @@ export class SignalRBridgeConnection {
|
|
|
293
1918
|
url.searchParams.set("runnerId", config.runnerId);
|
|
294
1919
|
this.connection = new HubConnectionBuilder()
|
|
295
1920
|
.withUrl(url.toString(), {
|
|
296
|
-
|
|
1921
|
+
// Query-string access tokens are intentionally rejected by the API. The bridge is
|
|
1922
|
+
// a Node CLI, so use long polling where Authorization headers are carried on each
|
|
1923
|
+
// HTTP request instead of relying on browser/WebSocket token fallbacks.
|
|
1924
|
+
transport: HttpTransportType.LongPolling,
|
|
1925
|
+
headers: {
|
|
1926
|
+
Authorization: `Bearer ${config.deviceToken}`
|
|
1927
|
+
}
|
|
297
1928
|
})
|
|
1929
|
+
.withAutomaticReconnect([...config.reconnectBackoffMs])
|
|
298
1930
|
.configureLogging(LogLevel.Warning)
|
|
299
1931
|
.build();
|
|
1932
|
+
this.connection.on("Superseded", () => {
|
|
1933
|
+
void this.connection.stop().catch(() => undefined);
|
|
1934
|
+
});
|
|
300
1935
|
}
|
|
301
1936
|
start() {
|
|
302
1937
|
return this.connection.start();
|
|
@@ -307,16 +1942,54 @@ export class SignalRBridgeConnection {
|
|
|
307
1942
|
onClose(handler) {
|
|
308
1943
|
this.connection.onclose(handler);
|
|
309
1944
|
}
|
|
1945
|
+
onReconnecting(handler) {
|
|
1946
|
+
this.connection.onreconnecting(handler);
|
|
1947
|
+
}
|
|
1948
|
+
onReconnected(handler) {
|
|
1949
|
+
this.connection.onreconnected((connectionId) => {
|
|
1950
|
+
void Promise.resolve(handler(connectionId)).catch(() => undefined);
|
|
1951
|
+
});
|
|
1952
|
+
}
|
|
310
1953
|
onDispatch(handler) {
|
|
311
1954
|
this.connection.on("Dispatch", (raw) => {
|
|
312
1955
|
void handler(raw).catch(() => undefined);
|
|
313
1956
|
});
|
|
314
1957
|
}
|
|
1958
|
+
onChatTurn(handler) {
|
|
1959
|
+
this.connection.on("ChatTurn", (raw) => {
|
|
1960
|
+
void handler(raw).catch(() => undefined);
|
|
1961
|
+
});
|
|
1962
|
+
}
|
|
1963
|
+
onOffer(handler) {
|
|
1964
|
+
this.connection.on("Offer", (raw) => {
|
|
1965
|
+
void handler(raw).catch(() => undefined);
|
|
1966
|
+
});
|
|
1967
|
+
}
|
|
1968
|
+
onStart(handler) {
|
|
1969
|
+
this.connection.on("Start", (raw) => {
|
|
1970
|
+
void handler(raw).catch(() => undefined);
|
|
1971
|
+
});
|
|
1972
|
+
}
|
|
1973
|
+
onLeaseUpdate(handler) {
|
|
1974
|
+
this.connection.on("LeaseUpdate", (request) => {
|
|
1975
|
+
void handler(request).catch(() => undefined);
|
|
1976
|
+
});
|
|
1977
|
+
}
|
|
315
1978
|
onCancel(handler) {
|
|
316
1979
|
this.connection.on("Cancel", (request) => {
|
|
317
1980
|
void handler(request).catch(() => undefined);
|
|
318
1981
|
});
|
|
319
1982
|
}
|
|
1983
|
+
onReap(handler) {
|
|
1984
|
+
this.connection.on("Reap", (request) => {
|
|
1985
|
+
void handler(request).catch(() => undefined);
|
|
1986
|
+
});
|
|
1987
|
+
}
|
|
1988
|
+
onReadinessProbe(handler) {
|
|
1989
|
+
this.connection.on("ReadinessProbe", (request) => {
|
|
1990
|
+
void handler(request).catch(() => undefined);
|
|
1991
|
+
});
|
|
1992
|
+
}
|
|
320
1993
|
invoke(methodName, payload) {
|
|
321
1994
|
return this.connection.invoke(methodName, payload);
|
|
322
1995
|
}
|
|
@@ -324,9 +1997,42 @@ export class SignalRBridgeConnection {
|
|
|
324
1997
|
export function defaultBridgeServiceConfig(overrides = {}) {
|
|
325
1998
|
const defaults = defaultForgeBridgeConfig();
|
|
326
1999
|
const cliProviders = splitCsv(process.env.FORGE_CLI_PROVIDERS ?? "codex");
|
|
2000
|
+
const configuredSandboxHostMemoryReserveBytes = optionalPositiveFiniteNumber(process.env.FORGE_SANDBOX_HOST_MEMORY_RESERVE_BYTES);
|
|
2001
|
+
const sandboxPublicationMode = process.env.FORGE_SANDBOX_PUBLICATION_MODE === "bridge_push" ? "bridge_push" : "shared_filesystem";
|
|
327
2002
|
const config = {
|
|
328
2003
|
serverUrl: process.env.FORGE_URL ?? "http://localhost:5208",
|
|
329
2004
|
runnerId: process.env.FORGE_RUNNER_ID ?? "",
|
|
2005
|
+
runnerInstanceId: process.env.FORGE_RUNNER_INSTANCE_ID ?? randomUUID(),
|
|
2006
|
+
trustDomainId: process.env.FORGE_TRUST_DOMAIN_ID ?? "default",
|
|
2007
|
+
maxConcurrentSandboxes: positiveFiniteNumber(Number(process.env.FORGE_MAX_CONCURRENT_SANDBOXES ?? 3), 3),
|
|
2008
|
+
maxConcurrentSandboxBuilds: positiveFiniteNumber(Number(process.env.FORGE_MAX_CONCURRENT_SANDBOX_BUILDS ?? 1), 1),
|
|
2009
|
+
sandboxPidCapacity: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_PID_CAPACITY ?? 2048), 2048),
|
|
2010
|
+
dockerEnabled: process.env.FORGE_DOCKER_SANDBOX !== "false",
|
|
2011
|
+
allowOpenPublicSandbox: process.env.FORGE_ALLOW_OPEN_PUBLIC_SANDBOX === "true",
|
|
2012
|
+
sandboxPublicationMode,
|
|
2013
|
+
dockerExecutable: process.env.FORGE_DOCKER_EXECUTABLE ?? "docker",
|
|
2014
|
+
dockerContext: process.env.FORGE_DOCKER_CONTEXT ?? "",
|
|
2015
|
+
dockerDesktopRecoveryEnabled: process.env.FORGE_DOCKER_DESKTOP_AUTO_RECOVERY === "true",
|
|
2016
|
+
dockerRecoveryFailureThreshold: positiveFiniteNumber(Number(process.env.FORGE_DOCKER_RECOVERY_FAILURE_THRESHOLD ?? 3), 3),
|
|
2017
|
+
dockerRecoveryCooldownMs: positiveFiniteNumber(Number(process.env.FORGE_DOCKER_RECOVERY_COOLDOWN_MS ?? 10 * 60_000), 10 * 60_000),
|
|
2018
|
+
dockerRecoveryStartupTimeoutMs: positiveFiniteNumber(Number(process.env.FORGE_DOCKER_RECOVERY_STARTUP_TIMEOUT_MS ?? 2 * 60_000), 2 * 60_000),
|
|
2019
|
+
sandboxStateRoot: process.env.FORGE_SANDBOX_STATE_ROOT ?? path.join(os.homedir(), ".forge", "bridge", "sandboxes"),
|
|
2020
|
+
sandboxExecutionRoot: defaultHostExecutionRoot(),
|
|
2021
|
+
requireRootlessDocker: process.env.FORGE_REQUIRE_ROOTLESS_DOCKER === "true" || process.platform === "linux",
|
|
2022
|
+
diskHighWatermarkPercent: positiveFiniteNumber(Number(process.env.FORGE_DISK_HIGH_WATERMARK_PERCENT ?? 85), 85),
|
|
2023
|
+
diskLowWatermarkPercent: positiveFiniteNumber(Number(process.env.FORGE_DISK_LOW_WATERMARK_PERCENT ?? 75), 75),
|
|
2024
|
+
minimumWorkspaceBytes: positiveFiniteNumber(Number(process.env.FORGE_MINIMUM_WORKSPACE_BYTES ?? 10 * 1024 ** 3), 10 * 1024 ** 3),
|
|
2025
|
+
// Undefined means preflight derives the bounded reserve from the actual
|
|
2026
|
+
// Docker/host capacity domain after sampling it.
|
|
2027
|
+
sandboxHostMemoryReserveBytes: configuredSandboxHostMemoryReserveBytes,
|
|
2028
|
+
defaultSandboxWorkerMemoryLimitBytes: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_DEFAULT_WORKER_MEMORY_LIMIT_BYTES ?? 2 * 1024 ** 3), 2 * 1024 ** 3),
|
|
2029
|
+
defaultSandboxServiceMemoryLimitBytes: nonNegativeFiniteNumber(Number(process.env.FORGE_SANDBOX_DEFAULT_SERVICE_MEMORY_LIMIT_BYTES ?? 0), 0),
|
|
2030
|
+
sandboxRuntimeOverheadBytes: Math.max(128 * 1024 ** 2, nonNegativeFiniteNumber(Number(process.env.FORGE_SANDBOX_RUNTIME_OVERHEAD_BYTES ?? 128 * 1024 ** 2), 128 * 1024 ** 2)),
|
|
2031
|
+
sandboxMemorySampleIntervalMs: Math.min(30_000, Math.max(250, positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_MEMORY_SAMPLE_INTERVAL_MS ?? 1000), 1000))),
|
|
2032
|
+
imageRetentionCount: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_IMAGE_RETENTION_COUNT ?? 3), 3),
|
|
2033
|
+
imageRetentionTtlMs: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_IMAGE_RETENTION_TTL_MS ?? 7 * 24 * 60 * 60 * 1000), 7 * 24 * 60 * 60 * 1000),
|
|
2034
|
+
auditRetentionCount: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_AUDIT_RETENTION_COUNT ?? 200), 200),
|
|
2035
|
+
auditRetentionTtlMs: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_AUDIT_RETENTION_TTL_MS ?? 30 * 24 * 60 * 60 * 1000), 30 * 24 * 60 * 60 * 1000),
|
|
330
2036
|
deviceToken: process.env.FORGE_DEVICE_TOKEN ?? "",
|
|
331
2037
|
deviceName: process.env.FORGE_DEVICE_NAME ?? "",
|
|
332
2038
|
defaultRepoRoot: process.env.FORGE_REPO_ROOT ?? defaults.defaultRepoRoot,
|
|
@@ -342,17 +2048,25 @@ export function defaultBridgeServiceConfig(overrides = {}) {
|
|
|
342
2048
|
supportsManagedForgeArtifacts: true,
|
|
343
2049
|
supportsRunnerAuthoredPhaseArtifacts: true,
|
|
344
2050
|
supportsDraftRunOnce: true,
|
|
2051
|
+
supportsDraftAttachments: true,
|
|
2052
|
+
supportsDraftImageAttachments: supportsDraftImageAttachmentsFor(cliProviders),
|
|
2053
|
+
bridgeChat: bridgeChatCapabilityFor(cliProviders, providerStatusesFor(cliProviders)),
|
|
345
2054
|
version: process.env.FORGE_BRIDGE_VERSION ?? forgeBridgeVersion,
|
|
346
|
-
protocolVersion:
|
|
2055
|
+
protocolVersion: 4,
|
|
347
2056
|
minServerVersion: null,
|
|
348
2057
|
heartbeatIntervalMs: positiveFiniteNumber(Number(process.env.FORGE_HEARTBEAT_INTERVAL_MS ?? 30_000), 30_000),
|
|
2058
|
+
callbackAckTimeoutMs: positiveFiniteNumber(Number(process.env.FORGE_CALLBACK_ACK_TIMEOUT_MS ?? 30_000), 30_000),
|
|
349
2059
|
reconnectBackoffMs: [1000, 2000, 5000, 10000, 30000],
|
|
350
2060
|
...overrides
|
|
351
2061
|
};
|
|
352
2062
|
return {
|
|
353
2063
|
...config,
|
|
2064
|
+
providerStatuses: overrides.providerStatuses ?? providerStatusesFor(config.cliProviders),
|
|
354
2065
|
sandboxProfiles: config.sandboxProfiles ?? defaultSandboxProfiles(config.cliProviders),
|
|
355
|
-
|
|
2066
|
+
supportsDraftImageAttachments: (overrides.supportsDraftImageAttachments ?? true) && supportsDraftImageAttachmentsFor(config.cliProviders),
|
|
2067
|
+
bridgeChat: overrides.bridgeChat ?? bridgeChatCapabilityFor(config.cliProviders, overrides.providerStatuses ?? providerStatusesFor(config.cliProviders), config.bridgeChat),
|
|
2068
|
+
heartbeatIntervalMs: positiveFiniteNumber(config.heartbeatIntervalMs, 30_000),
|
|
2069
|
+
callbackAckTimeoutMs: positiveFiniteNumber(config.callbackAckTimeoutMs ?? 30_000, 30_000),
|
|
356
2070
|
};
|
|
357
2071
|
}
|
|
358
2072
|
export function resolveBridgeRunnerHubUrl(serverUrl) {
|
|
@@ -367,9 +2081,42 @@ function normalizeServerUrl(serverUrl) {
|
|
|
367
2081
|
function positiveFiniteNumber(value, fallback) {
|
|
368
2082
|
return Number.isFinite(value) && value > 0 ? value : fallback;
|
|
369
2083
|
}
|
|
2084
|
+
function optionalPositiveFiniteNumber(value) {
|
|
2085
|
+
if (value === undefined || value.trim().length === 0)
|
|
2086
|
+
return undefined;
|
|
2087
|
+
const parsed = Number(value);
|
|
2088
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
|
|
2089
|
+
}
|
|
2090
|
+
function nonNegativeFiniteNumber(value, fallback) {
|
|
2091
|
+
return Number.isFinite(value) && value >= 0 ? value : fallback;
|
|
2092
|
+
}
|
|
370
2093
|
function providerStatusesFor(providers) {
|
|
371
2094
|
return Object.fromEntries(providers.map((provider) => [provider, { signedIn: true, version: "", models: configuredModelsFor(provider) }]));
|
|
372
2095
|
}
|
|
2096
|
+
function sandboxAuthModesFor(config) {
|
|
2097
|
+
return Object.fromEntries(config.cliProviders.map((provider) => {
|
|
2098
|
+
const modes = ["api_key"];
|
|
2099
|
+
if (sandboxBridgeCliAuthAvailable(config, provider)) {
|
|
2100
|
+
modes.push("bridge_cli");
|
|
2101
|
+
}
|
|
2102
|
+
return [provider, modes];
|
|
2103
|
+
}));
|
|
2104
|
+
}
|
|
2105
|
+
function sandboxBridgeCliAuthAvailable(config, provider) {
|
|
2106
|
+
if (provider.trim().toLowerCase() !== "codex") {
|
|
2107
|
+
return false;
|
|
2108
|
+
}
|
|
2109
|
+
const status = providerStatus(config, provider);
|
|
2110
|
+
if (status?.signedIn === false) {
|
|
2111
|
+
return false;
|
|
2112
|
+
}
|
|
2113
|
+
return codexSandboxProviderAuthAdapter.doctor().available;
|
|
2114
|
+
}
|
|
2115
|
+
function providerStatus(config, provider) {
|
|
2116
|
+
const normalized = provider.trim().toLowerCase();
|
|
2117
|
+
return Object.entries(config.providerStatuses)
|
|
2118
|
+
.find(([key]) => key.trim().toLowerCase() === normalized)?.[1];
|
|
2119
|
+
}
|
|
373
2120
|
function defaultSandboxProfiles(providers, platform = process.platform) {
|
|
374
2121
|
if (!providers.some((provider) => provider.toLowerCase() === "codex")) {
|
|
375
2122
|
return ["none"];
|
|
@@ -444,15 +2191,32 @@ function mappingKeys(mapping) {
|
|
|
444
2191
|
mapping.localPath ? `path:${path.resolve(mapping.localPath).toLowerCase()}` : null
|
|
445
2192
|
].filter((key) => Boolean(key));
|
|
446
2193
|
}
|
|
447
|
-
function findRepoMapping(
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
2194
|
+
function findRepoMapping(repo, mappings) {
|
|
2195
|
+
const usable = mappings.filter(mapping => Boolean(mapping.localPath?.trim()));
|
|
2196
|
+
const selectUnique = (candidates, identity) => {
|
|
2197
|
+
if (candidates.length > 1) {
|
|
2198
|
+
throw new Error(`bridge_repo_mapping_ambiguous: multiple local checkouts match ${identity}.`);
|
|
451
2199
|
}
|
|
452
|
-
return
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
2200
|
+
return candidates[0];
|
|
2201
|
+
};
|
|
2202
|
+
// Repository identity is authoritative. A same-name mapping for a different
|
|
2203
|
+
// repository must never win merely because it appears first in configuration.
|
|
2204
|
+
const byId = usable.filter(mapping => equalsIgnoreCase(mapping.repoId, repo.id));
|
|
2205
|
+
if (byId.length > 0)
|
|
2206
|
+
return selectUnique(byId, `repo id '${repo.id}'`);
|
|
2207
|
+
const byRemote = repo.remoteUrl?.trim()
|
|
2208
|
+
? usable.filter(mapping => !mapping.repoId?.trim()
|
|
2209
|
+
&& Boolean(mapping.remoteUrl?.trim())
|
|
2210
|
+
&& equalsIgnoreCase(mapping.remoteUrl, repo.remoteUrl))
|
|
2211
|
+
: [];
|
|
2212
|
+
if (byRemote.length > 0)
|
|
2213
|
+
return selectUnique(byRemote, `remote '${repo.remoteUrl}'`);
|
|
2214
|
+
const byName = usable.filter(mapping => !mapping.repoId?.trim()
|
|
2215
|
+
&& !mapping.remoteUrl?.trim()
|
|
2216
|
+
&& equalsIgnoreCase(mapping.name, repo.name));
|
|
2217
|
+
return byName.length > 0
|
|
2218
|
+
? selectUnique(byName, `repo name '${repo.name}'`)
|
|
2219
|
+
: undefined;
|
|
456
2220
|
}
|
|
457
2221
|
function normalizeQaAppLaunchTarget(target) {
|
|
458
2222
|
if (!target) {
|
|
@@ -464,10 +2228,11 @@ function normalizeQaAppLaunchTarget(target) {
|
|
|
464
2228
|
command: cleanString(target.command),
|
|
465
2229
|
working_directory: cleanString(target.working_directory),
|
|
466
2230
|
ready_url: cleanString(target.ready_url),
|
|
2231
|
+
managed_runtime_port: cleanString(target.managed_runtime_port),
|
|
467
2232
|
routes,
|
|
468
2233
|
notes: cleanString(target.notes)
|
|
469
2234
|
};
|
|
470
|
-
return normalized.url || normalized.command || normalized.ready_url ||
|
|
2235
|
+
return normalized.url || normalized.command || normalized.ready_url || normalized.managed_runtime_port
|
|
471
2236
|
? normalized
|
|
472
2237
|
: null;
|
|
473
2238
|
}
|
|
@@ -500,6 +2265,10 @@ function invalidDispatchSummary(error) {
|
|
|
500
2265
|
const detail = error instanceof Error ? error.message : String(error);
|
|
501
2266
|
return `Invalid bridge dispatch packet: ${detail}`;
|
|
502
2267
|
}
|
|
2268
|
+
function invalidChatTurnSummary(error) {
|
|
2269
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
2270
|
+
return `Invalid Bridge Chat turn packet: ${detail}`;
|
|
2271
|
+
}
|
|
503
2272
|
function errorMessage(error) {
|
|
504
2273
|
return error instanceof Error ? error.message : String(error);
|
|
505
2274
|
}
|
|
@@ -509,33 +2278,621 @@ function formatDelay(ms) {
|
|
|
509
2278
|
}
|
|
510
2279
|
return `${ms}ms`;
|
|
511
2280
|
}
|
|
2281
|
+
async function waitForCallbackAck(callback, methodName, timeoutMs, signal) {
|
|
2282
|
+
let timeout = null;
|
|
2283
|
+
let rejectAbort = null;
|
|
2284
|
+
const expired = new Promise((_resolve, reject) => {
|
|
2285
|
+
timeout = setTimeout(() => {
|
|
2286
|
+
reject(new Error(`bridge_callback_timeout: ${methodName} callback was not acknowledged within ${formatDelay(timeoutMs)}.`));
|
|
2287
|
+
}, timeoutMs);
|
|
2288
|
+
});
|
|
2289
|
+
const aborted = new Promise((_resolve, reject) => {
|
|
2290
|
+
rejectAbort = reject;
|
|
2291
|
+
});
|
|
2292
|
+
const onAbort = () => rejectAbort?.(new Error(String(signal?.reason ?? `${methodName} callback was cancelled.`)));
|
|
2293
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
2294
|
+
if (signal?.aborted)
|
|
2295
|
+
onAbort();
|
|
2296
|
+
try {
|
|
2297
|
+
return await Promise.race(signal ? [callback, expired, aborted] : [callback, expired]);
|
|
2298
|
+
}
|
|
2299
|
+
finally {
|
|
2300
|
+
if (timeout)
|
|
2301
|
+
clearTimeout(timeout);
|
|
2302
|
+
signal?.removeEventListener("abort", onAbort);
|
|
2303
|
+
}
|
|
2304
|
+
}
|
|
512
2305
|
function logBridgeStatus(message) {
|
|
513
2306
|
process.stderr.write(`[${new Date().toISOString()}] ${message}\n`);
|
|
514
2307
|
}
|
|
515
|
-
function delay(ms, signal) {
|
|
2308
|
+
export function delay(ms, signal) {
|
|
516
2309
|
if (signal?.aborted) {
|
|
517
2310
|
return Promise.resolve();
|
|
518
2311
|
}
|
|
519
2312
|
return new Promise((resolve) => {
|
|
520
|
-
const
|
|
521
|
-
signal?.addEventListener("abort", () => {
|
|
2313
|
+
const complete = () => {
|
|
522
2314
|
clearTimeout(timeout);
|
|
2315
|
+
signal?.removeEventListener("abort", complete);
|
|
523
2316
|
resolve();
|
|
524
|
-
}
|
|
2317
|
+
};
|
|
2318
|
+
const timeout = setTimeout(complete, ms);
|
|
2319
|
+
signal?.addEventListener("abort", complete, { once: true });
|
|
2320
|
+
if (signal?.aborted)
|
|
2321
|
+
complete();
|
|
525
2322
|
});
|
|
526
2323
|
}
|
|
527
|
-
function
|
|
2324
|
+
export async function waitForPromiseOrAbort(promise, signal) {
|
|
528
2325
|
if (!signal) {
|
|
529
|
-
|
|
2326
|
+
await promise;
|
|
2327
|
+
return;
|
|
530
2328
|
}
|
|
531
2329
|
if (signal.aborted) {
|
|
532
|
-
return
|
|
2330
|
+
return;
|
|
2331
|
+
}
|
|
2332
|
+
let resolveAbort;
|
|
2333
|
+
const abort = new Promise(resolve => { resolveAbort = resolve; });
|
|
2334
|
+
signal.addEventListener("abort", resolveAbort, { once: true });
|
|
2335
|
+
try {
|
|
2336
|
+
await Promise.race([promise, abort]);
|
|
2337
|
+
}
|
|
2338
|
+
finally {
|
|
2339
|
+
signal.removeEventListener("abort", resolveAbort);
|
|
2340
|
+
}
|
|
2341
|
+
}
|
|
2342
|
+
// Matches the API Hello bound on settled recovered host attempts.
|
|
2343
|
+
const maxRecoveredHostAttempts = 128;
|
|
2344
|
+
const defaultChatTurnLeaseMs = 60 * 60 * 1000;
|
|
2345
|
+
const maxChatEventContentChars = 4000;
|
|
2346
|
+
const maxChatTerminalTextChars = 120_000;
|
|
2347
|
+
const maxChatExactSecretChars = 8 * 1024;
|
|
2348
|
+
const chatTurnMetadataFileName = "forge-chat-metadata.json";
|
|
2349
|
+
async function materializeChatTurnMetadata(packet, workspacePath) {
|
|
2350
|
+
if (packet.attachmentHandoffs.length === 0 && packet.actionBrokerGrants.length === 0) {
|
|
2351
|
+
return;
|
|
2352
|
+
}
|
|
2353
|
+
await fs.promises.writeFile(path.join(workspacePath, chatTurnMetadataFileName), `${JSON.stringify({
|
|
2354
|
+
schemaVersion: 1,
|
|
2355
|
+
attachmentHandoffs: packet.attachmentHandoffs.map((handoff) => ({
|
|
2356
|
+
turnId: handoff.turnId,
|
|
2357
|
+
attachmentId: handoff.attachmentId,
|
|
2358
|
+
tempFileToken: handoff.tempFileToken,
|
|
2359
|
+
mimeType: handoff.mimeType,
|
|
2360
|
+
sizeBytes: handoff.sizeBytes,
|
|
2361
|
+
sha256: handoff.sha256,
|
|
2362
|
+
fileNameDisplay: handoff.fileNameDisplay,
|
|
2363
|
+
expiresAt: handoff.expiresAt,
|
|
2364
|
+
status: handoff.status,
|
|
2365
|
+
})),
|
|
2366
|
+
actionBrokerGrants: packet.actionBrokerGrants.map((grant) => ({
|
|
2367
|
+
grantId: grant.grantId,
|
|
2368
|
+
turnId: grant.turnId,
|
|
2369
|
+
leaseId: grant.leaseId,
|
|
2370
|
+
fencingToken: grant.fencingToken,
|
|
2371
|
+
allowedScopes: grant.allowedScopes,
|
|
2372
|
+
expiresAt: grant.expiresAt,
|
|
2373
|
+
redactionStatus: grant.redactionStatus,
|
|
2374
|
+
nonceHeldByBridge: true,
|
|
2375
|
+
})),
|
|
2376
|
+
}, null, 2)}\n`, "utf8");
|
|
2377
|
+
}
|
|
2378
|
+
function chatTurnLeaseExpiresAt(packet, nowMs = Date.now()) {
|
|
2379
|
+
const defaultExpiresAtMs = nowMs + defaultChatTurnLeaseMs;
|
|
2380
|
+
const metadataExpiresAtMs = [
|
|
2381
|
+
...packet.attachmentHandoffs.map(handoff => Date.parse(handoff.expiresAt)),
|
|
2382
|
+
...packet.actionBrokerGrants.map(grant => Date.parse(grant.expiresAt)),
|
|
2383
|
+
].filter(Number.isFinite);
|
|
2384
|
+
return new Date(Math.min(defaultExpiresAtMs, ...metadataExpiresAtMs)).toISOString();
|
|
2385
|
+
}
|
|
2386
|
+
async function cleanupChatTurnWorkspace(workspacePath, redactionSecrets) {
|
|
2387
|
+
try {
|
|
2388
|
+
await fs.promises.rm(workspacePath, { recursive: true, force: true, maxRetries: 3 });
|
|
2389
|
+
return {
|
|
2390
|
+
status: "completed",
|
|
2391
|
+
eventType: "chat_turn_cleanup_completed",
|
|
2392
|
+
role: "runner",
|
|
2393
|
+
kind: "checkpoint",
|
|
2394
|
+
content: "Bridge Chat workspace cleanup completed.",
|
|
2395
|
+
};
|
|
2396
|
+
}
|
|
2397
|
+
catch (error) {
|
|
2398
|
+
return {
|
|
2399
|
+
status: "failed",
|
|
2400
|
+
eventType: "chat_turn_cleanup_failed",
|
|
2401
|
+
role: "runner",
|
|
2402
|
+
kind: "error",
|
|
2403
|
+
content: "Bridge Chat workspace cleanup failed.",
|
|
2404
|
+
message: clip(redactText(errorMessage(error), redactionSecrets), 1000),
|
|
2405
|
+
};
|
|
2406
|
+
}
|
|
2407
|
+
}
|
|
2408
|
+
function buildChatProviderCommand(packet, cwd) {
|
|
2409
|
+
const provider = packet.providerKind.trim().toLowerCase();
|
|
2410
|
+
const model = packet.model.trim();
|
|
2411
|
+
const prompt = buildChatPrompt(packet);
|
|
2412
|
+
const resumeThreadId = packet.providerThreadId?.trim() || null;
|
|
2413
|
+
if (provider === "codex") {
|
|
2414
|
+
const args = ["-s", codexSandboxModeForPlatform(process.platform, "host"), "-a", "never", "exec"];
|
|
2415
|
+
if (resumeThreadId) {
|
|
2416
|
+
args.push("--cd", cwd, "resume");
|
|
2417
|
+
}
|
|
2418
|
+
args.push("--model", model, "--json");
|
|
2419
|
+
if (!resumeThreadId) {
|
|
2420
|
+
args.push("--cd", cwd);
|
|
2421
|
+
}
|
|
2422
|
+
args.push("--skip-git-repo-check");
|
|
2423
|
+
if (resumeThreadId) {
|
|
2424
|
+
args.push(resumeThreadId);
|
|
2425
|
+
}
|
|
2426
|
+
args.push("-");
|
|
2427
|
+
return { command: { command: "codex", args }, stdin: prompt };
|
|
2428
|
+
}
|
|
2429
|
+
if (provider === "cursor") {
|
|
2430
|
+
const args = ["--print", "--output-format", "stream-json", "--model", model, "--workspace", cwd];
|
|
2431
|
+
if (resumeThreadId) {
|
|
2432
|
+
args.push("--resume", resumeThreadId);
|
|
2433
|
+
}
|
|
2434
|
+
args.push("--yolo", "--approve-mcps", "--trust", prompt);
|
|
2435
|
+
return { command: { command: "cursor-agent", args } };
|
|
2436
|
+
}
|
|
2437
|
+
if (provider === "claude") {
|
|
2438
|
+
const args = ["-p", "--output-format", "stream-json", "--verbose", "--model", model, "--permission-mode", "bypassPermissions"];
|
|
2439
|
+
if (resumeThreadId) {
|
|
2440
|
+
args.push("--resume", resumeThreadId);
|
|
2441
|
+
}
|
|
2442
|
+
return { command: { command: "claude", args }, stdin: prompt };
|
|
2443
|
+
}
|
|
2444
|
+
return {
|
|
2445
|
+
command: {
|
|
2446
|
+
command: provider,
|
|
2447
|
+
args: ["--model", model, prompt],
|
|
2448
|
+
},
|
|
2449
|
+
};
|
|
2450
|
+
}
|
|
2451
|
+
function buildChatPrompt(packet) {
|
|
2452
|
+
const messages = packet.messages
|
|
2453
|
+
.map((message, index) => {
|
|
2454
|
+
const role = message.role.trim() || `message-${index + 1}`;
|
|
2455
|
+
return `[${role}]\n${message.content}`;
|
|
2456
|
+
})
|
|
2457
|
+
.join("\n\n");
|
|
2458
|
+
const contextRefs = packet.contextRefs.length === 0
|
|
2459
|
+
? ""
|
|
2460
|
+
: `\n\n# Context References\n\n${packet.contextRefs.map((ref, index) => {
|
|
2461
|
+
const title = ref.title?.trim() ? `\nTitle: ${ref.title.trim()}` : "";
|
|
2462
|
+
const summary = ref.summary?.trim() ? `\nSummary: ${ref.summary.trim()}` : "";
|
|
2463
|
+
return `${index + 1}. ${ref.kind}:${ref.refId}${title}${summary}`;
|
|
2464
|
+
}).join("\n\n")}`;
|
|
2465
|
+
const continuation = packet.providerThreadId
|
|
2466
|
+
? "\n\nContinue the existing provider thread identified by the bridge; do not restart the conversation context."
|
|
2467
|
+
: "";
|
|
2468
|
+
const boundedMetadata = buildChatBoundedMetadataPrompt(packet);
|
|
2469
|
+
return `You are handling a Forge Bridge Chat turn.
|
|
2470
|
+
|
|
2471
|
+
Conversation ID: ${packet.conversationId}
|
|
2472
|
+
Turn ID: ${packet.turnId}
|
|
2473
|
+
Provider: ${packet.providerKind}
|
|
2474
|
+
Model: ${packet.model}${continuation}${boundedMetadata}
|
|
2475
|
+
|
|
2476
|
+
# Conversation
|
|
2477
|
+
|
|
2478
|
+
${messages}${contextRefs}
|
|
2479
|
+
`;
|
|
2480
|
+
}
|
|
2481
|
+
function buildChatBoundedMetadataPrompt(packet) {
|
|
2482
|
+
if (packet.attachmentHandoffs.length === 0 && packet.actionBrokerGrants.length === 0) {
|
|
2483
|
+
return "";
|
|
2484
|
+
}
|
|
2485
|
+
const sections = [
|
|
2486
|
+
`\n\n# Bounded Handoff Metadata\n\nMetadata file: ${chatTurnMetadataFileName}\n\nThe metadata file is turn-scoped and cleaned up by the bridge when the turn ends. It contains opaque handles only; attachment content, raw paths, and grant nonces are not included.`,
|
|
2487
|
+
];
|
|
2488
|
+
if (packet.attachmentHandoffs.length > 0) {
|
|
2489
|
+
sections.push(`Attachment handoffs:\n${packet.attachmentHandoffs.map((handoff, index) => [
|
|
2490
|
+
`${index + 1}. turnId: ${handoff.turnId}`,
|
|
2491
|
+
` attachmentId: ${handoff.attachmentId}`,
|
|
2492
|
+
` tempFileToken: ${handoff.tempFileToken}`,
|
|
2493
|
+
` mimeType: ${handoff.mimeType}`,
|
|
2494
|
+
` sizeBytes: ${handoff.sizeBytes}`,
|
|
2495
|
+
` sha256: ${handoff.sha256}`,
|
|
2496
|
+
` fileNameDisplay: ${handoff.fileNameDisplay}`,
|
|
2497
|
+
` expiresAt: ${handoff.expiresAt}`,
|
|
2498
|
+
` status: ${handoff.status}`,
|
|
2499
|
+
].join("\n")).join("\n\n")}`);
|
|
2500
|
+
}
|
|
2501
|
+
if (packet.actionBrokerGrants.length > 0) {
|
|
2502
|
+
sections.push(`Action-broker grants:\n${packet.actionBrokerGrants.map((grant, index) => [
|
|
2503
|
+
`${index + 1}. grantId: ${grant.grantId}`,
|
|
2504
|
+
` turnId: ${grant.turnId}`,
|
|
2505
|
+
` leaseId: ${grant.leaseId}`,
|
|
2506
|
+
` fencingToken: ${grant.fencingToken}`,
|
|
2507
|
+
` allowedScopes: ${grant.allowedScopes.join(", ")}`,
|
|
2508
|
+
` expiresAt: ${grant.expiresAt}`,
|
|
2509
|
+
` redactionStatus: ${grant.redactionStatus}`,
|
|
2510
|
+
" nonce: held by bridge",
|
|
2511
|
+
].join("\n")).join("\n\n")}`);
|
|
2512
|
+
}
|
|
2513
|
+
return sections.join("\n\n");
|
|
2514
|
+
}
|
|
2515
|
+
function chatProviderExitFailure(command, result, redactionSecrets) {
|
|
2516
|
+
const commandText = redactText(formatProviderCliCommandForCheckpoint(command), redactionSecrets);
|
|
2517
|
+
const stderr = clip(redactText(result.stderr.trim(), redactionSecrets), 1000);
|
|
2518
|
+
const stdout = clip(redactText(result.stdout.trim(), redactionSecrets), 1000);
|
|
2519
|
+
const diagnostics = [
|
|
2520
|
+
stderr ? `stderr: ${stderr}` : null,
|
|
2521
|
+
stdout ? `stdout: ${stdout}` : null,
|
|
2522
|
+
].filter(Boolean).join(" ");
|
|
2523
|
+
return new Error(`provider_failed: ${commandText} exited with code ${result.exitCode}.${diagnostics ? ` ${diagnostics}` : ""}`);
|
|
2524
|
+
}
|
|
2525
|
+
function chatAbortKind(reason) {
|
|
2526
|
+
const message = typeof reason === "string" ? reason.toLowerCase() : "";
|
|
2527
|
+
return message.includes("expired") ? "expired" : "cancelled";
|
|
2528
|
+
}
|
|
2529
|
+
function chatAbortPayload(reason, cleanupOutcome, packet) {
|
|
2530
|
+
const kind = chatAbortKind(reason);
|
|
2531
|
+
const message = typeof reason === "string" && reason.trim()
|
|
2532
|
+
? reason.trim()
|
|
2533
|
+
: (kind === "expired" ? "Bridge Chat turn expired." : "Bridge Chat turn cancelled.");
|
|
2534
|
+
return {
|
|
2535
|
+
code: kind === "expired" ? "chat_turn_expired" : "chat_turn_cancelled",
|
|
2536
|
+
message: clip(redactText(message, packet ? chatRedactionSecrets(packet) : []), 1000),
|
|
2537
|
+
cleanup: cleanupOutcome,
|
|
2538
|
+
};
|
|
2539
|
+
}
|
|
2540
|
+
function providerThreadIdFromEvent(event) {
|
|
2541
|
+
if (!event.data) {
|
|
2542
|
+
return null;
|
|
2543
|
+
}
|
|
2544
|
+
try {
|
|
2545
|
+
const data = JSON.parse(event.data);
|
|
2546
|
+
for (const key of ["threadId", "thread_id", "sessionId", "session_id", "providerThreadId"]) {
|
|
2547
|
+
const value = data[key];
|
|
2548
|
+
if (typeof value === "string") {
|
|
2549
|
+
const clean = value.trim();
|
|
2550
|
+
if (clean.length > 0 && clean.length <= bridgeChatMaxProviderThreadIdChars) {
|
|
2551
|
+
return clean;
|
|
2552
|
+
}
|
|
2553
|
+
}
|
|
2554
|
+
}
|
|
2555
|
+
}
|
|
2556
|
+
catch {
|
|
2557
|
+
return null;
|
|
2558
|
+
}
|
|
2559
|
+
return null;
|
|
2560
|
+
}
|
|
2561
|
+
function chatRedactionSecrets(packet) {
|
|
2562
|
+
return [
|
|
2563
|
+
packet.chatRunId,
|
|
2564
|
+
packet.conversationId,
|
|
2565
|
+
packet.turnId,
|
|
2566
|
+
packet.leaseId,
|
|
2567
|
+
packet.fencingToken,
|
|
2568
|
+
packet.runnerInstanceId,
|
|
2569
|
+
packet.providerThreadId,
|
|
2570
|
+
...packet.attachmentHandoffs.flatMap((handoff) => [
|
|
2571
|
+
handoff.attachmentId,
|
|
2572
|
+
handoff.tempFileToken,
|
|
2573
|
+
]),
|
|
2574
|
+
...packet.actionBrokerGrants.flatMap((grant) => [
|
|
2575
|
+
grant.grantId,
|
|
2576
|
+
grant.nonce,
|
|
2577
|
+
]),
|
|
2578
|
+
...packet.messages.flatMap((message) => boundedChatSecretFragments(message.content)),
|
|
2579
|
+
...packet.contextRefs.flatMap((ref) => [
|
|
2580
|
+
...boundedChatSecretFragments(ref.title ?? ""),
|
|
2581
|
+
...boundedChatSecretFragments(ref.summary ?? ""),
|
|
2582
|
+
]),
|
|
2583
|
+
].filter((value) => typeof value === "string" && value.length > 0);
|
|
2584
|
+
}
|
|
2585
|
+
function boundedChatSecretFragments(value) {
|
|
2586
|
+
const clean = value.trim();
|
|
2587
|
+
if (!clean) {
|
|
2588
|
+
return [];
|
|
2589
|
+
}
|
|
2590
|
+
if (clean.length <= maxChatExactSecretChars) {
|
|
2591
|
+
return [clean];
|
|
2592
|
+
}
|
|
2593
|
+
const half = Math.floor(maxChatExactSecretChars / 2);
|
|
2594
|
+
return [clean.slice(0, half), clean.slice(clean.length - half)];
|
|
2595
|
+
}
|
|
2596
|
+
function safePathComponent(value) {
|
|
2597
|
+
const clean = value.trim().replace(/[^a-zA-Z0-9_.-]+/gu, "-").replace(/^-+|-+$/gu, "");
|
|
2598
|
+
return clean || "chat-turn";
|
|
2599
|
+
}
|
|
2600
|
+
function containsIgnoreCase(values, value) {
|
|
2601
|
+
const normalized = value.trim().toLowerCase();
|
|
2602
|
+
return values.some((candidate) => candidate.trim().toLowerCase() === normalized);
|
|
2603
|
+
}
|
|
2604
|
+
export function canReapManagedSandboxes(preflight) {
|
|
2605
|
+
return canRunDockerMaintenance(preflight);
|
|
2606
|
+
}
|
|
2607
|
+
function sandboxNetworkPolicyFailure(dispatch, config) {
|
|
2608
|
+
const sandbox = dispatch.sandbox;
|
|
2609
|
+
if (!sandbox)
|
|
2610
|
+
return "Protocol-v2 sandbox context is missing.";
|
|
2611
|
+
if (sandbox.networkPolicy.mode === "open_public"
|
|
2612
|
+
&& (sandbox.executionPolicy === "sandbox_required" || !config.allowOpenPublicSandbox)) {
|
|
2613
|
+
return "open_public sandbox networking is not permitted by this runner policy.";
|
|
2614
|
+
}
|
|
2615
|
+
if (sandbox.networkPolicy.mode === "public_allowlist"
|
|
2616
|
+
&& sandbox.networkPolicy.allowedHosts.length > 0
|
|
2617
|
+
&& process.env.FORGE_SANDBOX_EGRESS_ENFORCED !== "true") {
|
|
2618
|
+
return "public_allowlist requires an operator-enforced Forge egress proxy or firewall.";
|
|
2619
|
+
}
|
|
2620
|
+
return null;
|
|
2621
|
+
}
|
|
2622
|
+
export function dockerEngineRequirementFailure(requiredCapabilities, executionMode, capability) {
|
|
2623
|
+
if (!requiredCapabilities.includes("docker_engine"))
|
|
2624
|
+
return null;
|
|
2625
|
+
if (executionMode === "docker") {
|
|
2626
|
+
return "docker_engine_unavailable: docker_engine is not exposed inside Forge agent sandboxes; use a managed host runner with Docker-daemon access.";
|
|
2627
|
+
}
|
|
2628
|
+
return capability?.available
|
|
2629
|
+
? null
|
|
2630
|
+
: `docker_engine_unavailable: ${capability?.unavailableReason ?? "Docker daemon is unreachable from the bridge process."}`;
|
|
2631
|
+
}
|
|
2632
|
+
function parseReadinessProbe(value) {
|
|
2633
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
2634
|
+
return null;
|
|
2635
|
+
const request = value;
|
|
2636
|
+
const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
|
|
2637
|
+
const bounded = (candidate, maximum) => typeof candidate === "string" && candidate.trim().length > 0 && candidate.length <= maximum;
|
|
2638
|
+
if (request.schemaVersion !== 1
|
|
2639
|
+
|| !uuid.test(request.requestId ?? "")
|
|
2640
|
+
|| !uuid.test(request.fencingToken ?? "")
|
|
2641
|
+
|| !uuid.test(request.userId ?? "")
|
|
2642
|
+
|| !uuid.test(request.projectId ?? "")
|
|
2643
|
+
|| !uuid.test(request.repoId ?? "")
|
|
2644
|
+
|| !uuid.test(request.runnerInstanceId ?? "")
|
|
2645
|
+
|| !Number.isInteger(request.attempt) || (request.attempt ?? 0) < 1
|
|
2646
|
+
|| !bounded(request.repoName, 160)
|
|
2647
|
+
|| typeof request.remoteUrl !== "string" || request.remoteUrl.length > 2048
|
|
2648
|
+
|| !bounded(request.defaultBranch, 512)
|
|
2649
|
+
|| typeof request.cloneOnDemand !== "boolean"
|
|
2650
|
+
|| !bounded(request.runnerRepoLocalPath, 4096)
|
|
2651
|
+
|| !bounded(request.revision, 512)
|
|
2652
|
+
|| !Number.isFinite(Date.parse(request.expiresAt ?? ""))
|
|
2653
|
+
|| Date.parse(request.expiresAt ?? "") <= Date.now()) {
|
|
2654
|
+
return null;
|
|
2655
|
+
}
|
|
2656
|
+
return request;
|
|
2657
|
+
}
|
|
2658
|
+
function deferredReadinessResult(request, runnerInstanceId) {
|
|
2659
|
+
return {
|
|
2660
|
+
schemaVersion: 1,
|
|
2661
|
+
requestId: request.requestId,
|
|
2662
|
+
fencingToken: request.fencingToken,
|
|
2663
|
+
runnerInstanceId,
|
|
2664
|
+
status: "failed",
|
|
2665
|
+
reasonCode: "host_runtime_probe_deferred",
|
|
2666
|
+
reasonMessage: "Foreground agent work is active; Forge deferred background runtime readiness so it cannot consume the agent's host capacity.",
|
|
2667
|
+
checkedAt: new Date().toISOString(),
|
|
2668
|
+
};
|
|
2669
|
+
}
|
|
2670
|
+
function readinessDeferralReason(signal) {
|
|
2671
|
+
if (!signal.aborted)
|
|
2672
|
+
return null;
|
|
2673
|
+
const reason = errorMessage(signal.reason);
|
|
2674
|
+
return reason.startsWith("host_runtime_probe_deferred:") ? reason : null;
|
|
2675
|
+
}
|
|
2676
|
+
function sandboxProviderNetworkFailure(dispatch, config) {
|
|
2677
|
+
const sandbox = dispatch.sandbox;
|
|
2678
|
+
if (!sandbox || sandbox.networkPolicy.mode !== "offline")
|
|
2679
|
+
return null;
|
|
2680
|
+
const provider = dispatch.execution.provider.trim().toLowerCase();
|
|
2681
|
+
const isNetworkProvider = config.cliProviders.some(candidate => candidate.trim().toLowerCase() === provider);
|
|
2682
|
+
if (!isNetworkProvider)
|
|
2683
|
+
return null;
|
|
2684
|
+
return `Provider ${dispatch.execution.provider} requires outbound network access, but this Docker attempt uses the offline network policy.`;
|
|
2685
|
+
}
|
|
2686
|
+
function sandboxAuthFailure(dispatch, config) {
|
|
2687
|
+
const unsupported = sandboxProviderAuthUnavailableMessage(dispatch.execution.provider, dispatch.execution.authKind);
|
|
2688
|
+
if (unsupported) {
|
|
2689
|
+
return unsupported;
|
|
2690
|
+
}
|
|
2691
|
+
if (dispatch.execution.authKind.trim().toLowerCase() !== "bridge_cli") {
|
|
2692
|
+
return null;
|
|
2693
|
+
}
|
|
2694
|
+
const status = providerStatus(config, dispatch.execution.provider);
|
|
2695
|
+
if (status?.signedIn === false) {
|
|
2696
|
+
return `${dispatch.execution.provider} Local CLI is not signed in on this runner.`;
|
|
2697
|
+
}
|
|
2698
|
+
const adapter = sandboxProviderAuthAdapterFor(dispatch.execution.provider, dispatch.execution.authKind);
|
|
2699
|
+
if (!adapter) {
|
|
2700
|
+
return `${dispatch.execution.provider} Local CLI auth is not portable into Docker yet.`;
|
|
2701
|
+
}
|
|
2702
|
+
const doctor = adapter.doctor();
|
|
2703
|
+
return doctor.available ? null : doctor.message;
|
|
2704
|
+
}
|
|
2705
|
+
function rejectedOfferReservation(code, message) {
|
|
2706
|
+
return {
|
|
2707
|
+
accepted: false,
|
|
2708
|
+
code,
|
|
2709
|
+
message,
|
|
2710
|
+
reservedUntil: null,
|
|
2711
|
+
resources: null,
|
|
2712
|
+
};
|
|
2713
|
+
}
|
|
2714
|
+
export function canReserveSandboxResources(used, requested, capacity) {
|
|
2715
|
+
return used.sandboxSlots + requested.sandboxSlots <= capacity.sandboxSlots
|
|
2716
|
+
&& used.cpuMillis + requested.cpuMillis <= capacity.cpuMillis
|
|
2717
|
+
&& used.memoryBytes + requested.memoryBytes <= capacity.memoryBytes
|
|
2718
|
+
&& used.workspaceBytes + requested.workspaceBytes <= capacity.workspaceBytes
|
|
2719
|
+
&& used.pids + requested.pids <= capacity.pids
|
|
2720
|
+
&& used.buildSlots + requested.buildSlots <= capacity.buildSlots;
|
|
2721
|
+
}
|
|
2722
|
+
export function sandboxMemoryCommitmentCapacity(preflight, activeMemoryBytes) {
|
|
2723
|
+
const ambientCapacity = preflight.availableMemoryBytes
|
|
2724
|
+
- preflight.hostReserveBytes
|
|
2725
|
+
+ activeMemoryBytes;
|
|
2726
|
+
const commitmentCapacity = preflight.totalMemoryBytes
|
|
2727
|
+
- preflight.hostReserveBytes;
|
|
2728
|
+
return Math.max(0, Math.min(ambientCapacity, commitmentCapacity));
|
|
2729
|
+
}
|
|
2730
|
+
export function sandboxCapacityFailureMessage(used, requested, capacity, preflight, config) {
|
|
2731
|
+
const availableSlots = Math.max(0, capacity.sandboxSlots - used.sandboxSlots);
|
|
2732
|
+
if (requested.sandboxSlots > availableSlots) {
|
|
2733
|
+
return `Sandbox slot capacity is insufficient: requested ${requested.sandboxSlots}, available ${availableSlots}, ${used.sandboxSlots} already reserved.`;
|
|
2734
|
+
}
|
|
2735
|
+
const availableMemoryBytes = Math.max(0, capacity.memoryBytes - used.memoryBytes);
|
|
2736
|
+
if (requested.memoryBytes > availableMemoryBytes) {
|
|
2737
|
+
return [
|
|
2738
|
+
`Sandbox memory capacity is insufficient: requested ${formatBytes(requested.memoryBytes)} total`,
|
|
2739
|
+
`(worker ${formatBytes(requested.workerMemoryLimitBytes)}, services ${formatBytes(requested.serviceMemoryLimitBytes)}, runtime overhead ${formatBytes(requested.runtimeOverheadBytes)}),`,
|
|
2740
|
+
`but ${formatBytes(availableMemoryBytes)} is available for admission.`,
|
|
2741
|
+
`Docker reports ${formatBytes(preflight?.availableMemoryBytes ?? 0)} available of ${formatBytes(preflight?.totalMemoryBytes ?? 0)};`,
|
|
2742
|
+
`Forge reserves ${formatBytes(preflight?.hostReserveBytes ?? config.sandboxHostMemoryReserveBytes ?? 0)} for the host and tracks ${formatBytes(used.memoryBytes)} in active or reserved attempts.`,
|
|
2743
|
+
].join(" ");
|
|
2744
|
+
}
|
|
2745
|
+
const availableCpuMillis = Math.max(0, capacity.cpuMillis - used.cpuMillis);
|
|
2746
|
+
if (requested.cpuMillis > availableCpuMillis) {
|
|
2747
|
+
return `Sandbox CPU capacity is insufficient: requested ${requested.cpuMillis} millis, available ${availableCpuMillis} millis, ${used.cpuMillis} millis already reserved.`;
|
|
533
2748
|
}
|
|
534
|
-
|
|
2749
|
+
const availableWorkspaceBytes = Math.max(0, capacity.workspaceBytes - used.workspaceBytes);
|
|
2750
|
+
if (requested.workspaceBytes > availableWorkspaceBytes) {
|
|
2751
|
+
return `Sandbox workspace capacity is insufficient: requested ${formatBytes(requested.workspaceBytes)}, available ${formatBytes(availableWorkspaceBytes)}, ${formatBytes(used.workspaceBytes)} already reserved.`;
|
|
2752
|
+
}
|
|
2753
|
+
const availablePids = Math.max(0, capacity.pids - used.pids);
|
|
2754
|
+
if (requested.pids > availablePids) {
|
|
2755
|
+
return `Sandbox PID capacity is insufficient: requested ${requested.pids}, available ${availablePids}, ${used.pids} already reserved.`;
|
|
2756
|
+
}
|
|
2757
|
+
const availableBuildSlots = Math.max(0, capacity.buildSlots - used.buildSlots);
|
|
2758
|
+
return `Sandbox build capacity is insufficient: requested ${requested.buildSlots}, available ${availableBuildSlots}, ${used.buildSlots} already reserved.`;
|
|
2759
|
+
}
|
|
2760
|
+
function sumSandboxResources(resources) {
|
|
2761
|
+
return resources.reduce((total, resource) => ({
|
|
2762
|
+
sandboxSlots: total.sandboxSlots + resource.sandboxSlots,
|
|
2763
|
+
cpuMillis: total.cpuMillis + resource.cpuMillis,
|
|
2764
|
+
workerMemoryLimitBytes: total.workerMemoryLimitBytes + resource.workerMemoryLimitBytes,
|
|
2765
|
+
serviceMemoryLimitBytes: total.serviceMemoryLimitBytes + resource.serviceMemoryLimitBytes,
|
|
2766
|
+
runtimeOverheadBytes: total.runtimeOverheadBytes + resource.runtimeOverheadBytes,
|
|
2767
|
+
memoryBytes: total.memoryBytes + resource.memoryBytes,
|
|
2768
|
+
workspaceBytes: total.workspaceBytes + resource.workspaceBytes,
|
|
2769
|
+
pids: total.pids + resource.pids,
|
|
2770
|
+
buildSlots: total.buildSlots + resource.buildSlots
|
|
2771
|
+
}), zeroSandboxResources());
|
|
2772
|
+
}
|
|
2773
|
+
function zeroSandboxResources() {
|
|
2774
|
+
return {
|
|
2775
|
+
sandboxSlots: 0,
|
|
2776
|
+
cpuMillis: 0,
|
|
2777
|
+
workerMemoryLimitBytes: 0,
|
|
2778
|
+
serviceMemoryLimitBytes: 0,
|
|
2779
|
+
runtimeOverheadBytes: 0,
|
|
2780
|
+
memoryBytes: 0,
|
|
2781
|
+
workspaceBytes: 0,
|
|
2782
|
+
pids: 0,
|
|
2783
|
+
buildSlots: 0,
|
|
2784
|
+
};
|
|
2785
|
+
}
|
|
2786
|
+
function defaultSandboxResources() {
|
|
2787
|
+
return {
|
|
2788
|
+
sandboxSlots: 1,
|
|
2789
|
+
cpuMillis: 2000,
|
|
2790
|
+
workerMemoryLimitBytes: 2 * 1024 ** 3,
|
|
2791
|
+
serviceMemoryLimitBytes: 0,
|
|
2792
|
+
runtimeOverheadBytes: 128 * 1024 ** 2,
|
|
2793
|
+
memoryBytes: 2 * 1024 ** 3 + 128 * 1024 ** 2,
|
|
2794
|
+
workspaceBytes: 10 * 1024 ** 3,
|
|
2795
|
+
pids: 512,
|
|
2796
|
+
buildSlots: 0
|
|
2797
|
+
};
|
|
2798
|
+
}
|
|
2799
|
+
export function resolveDynamicSandboxResources(requested, serviceCount, networkMode, baseRuntimeOverheadBytes = 128 * 1024 ** 2) {
|
|
2800
|
+
const serviceMemoryLimitBytes = Math.max(0, serviceCount) * 1024 ** 3;
|
|
2801
|
+
const runtimeOverheadBytes = Math.max(128 * 1024 ** 2, baseRuntimeOverheadBytes)
|
|
2802
|
+
+ (networkMode === "public_allowlist" ? 256 * 1024 ** 2 : 0);
|
|
2803
|
+
return {
|
|
2804
|
+
...requested,
|
|
2805
|
+
serviceMemoryLimitBytes,
|
|
2806
|
+
runtimeOverheadBytes,
|
|
2807
|
+
memoryBytes: requested.workerMemoryLimitBytes + serviceMemoryLimitBytes + runtimeOverheadBytes,
|
|
2808
|
+
};
|
|
2809
|
+
}
|
|
2810
|
+
export async function resolveSandboxResourcesFromRepo(dispatch, baseRuntimeOverheadBytes = 128 * 1024 ** 2) {
|
|
2811
|
+
const repoPath = dispatch.repo.serverLocalPath;
|
|
2812
|
+
const sandbox = dispatch.sandbox;
|
|
2813
|
+
if (!repoPath || !sandbox) {
|
|
2814
|
+
throw new Error("Sandbox admission needs a mapped repository.");
|
|
2815
|
+
}
|
|
2816
|
+
const selection = selectSandboxConfigRefs(sandbox.executionPolicy, sandbox.sandboxConfigCommitSha, dispatch.repo.defaultBranch);
|
|
2817
|
+
const configCommitSha = await resolveFirstGitCommit(repoPath, selection.refs);
|
|
2818
|
+
if (!configCommitSha) {
|
|
2819
|
+
throw new Error("Sandbox admission could not resolve the immutable sandbox config commit.");
|
|
2820
|
+
}
|
|
2821
|
+
const contractResult = await runSandboxProcess({
|
|
2822
|
+
command: "git",
|
|
2823
|
+
args: ["-C", repoPath, "show", `${configCommitSha}:.forge/forge.sandbox.yml`],
|
|
2824
|
+
timeoutMs: 30_000,
|
|
2825
|
+
maxOutputBytes: 1024 * 1024,
|
|
2826
|
+
});
|
|
2827
|
+
if (contractResult.exitCode !== 0) {
|
|
2828
|
+
throw new Error("The approved immutable sandbox contract is unavailable in the mapped repository.");
|
|
2829
|
+
}
|
|
2830
|
+
const contract = validateSandboxContract(contractResult.stdout);
|
|
2831
|
+
const workspaceBytes = await estimateWorkspaceBytesFromRepo(repoPath, {
|
|
2832
|
+
role: dispatch.execution.role,
|
|
2833
|
+
serviceCount: contract.contract.services.length,
|
|
2834
|
+
browser: contract.contract.browser,
|
|
2835
|
+
});
|
|
2836
|
+
return {
|
|
2837
|
+
resources: resolveDynamicSandboxResources({
|
|
2838
|
+
...sandbox.requestedResources,
|
|
2839
|
+
workspaceBytes: Math.max(sandbox.requestedResources.workspaceBytes, workspaceBytes),
|
|
2840
|
+
}, contract.contract.services.length, sandbox.networkPolicy.mode, baseRuntimeOverheadBytes),
|
|
2841
|
+
sandboxConfigCommitSha: configCommitSha,
|
|
2842
|
+
};
|
|
2843
|
+
}
|
|
2844
|
+
async function resolveFirstGitCommit(repoPath, refs) {
|
|
2845
|
+
for (const ref of refs) {
|
|
2846
|
+
const result = await runSandboxProcess({
|
|
2847
|
+
command: "git",
|
|
2848
|
+
args: ["-C", repoPath, "rev-parse", "--verify", `${ref}^{commit}`],
|
|
2849
|
+
timeoutMs: 30_000,
|
|
2850
|
+
maxOutputBytes: 1024 * 1024,
|
|
2851
|
+
});
|
|
2852
|
+
const commitSha = result.stdout.trim().toLowerCase();
|
|
2853
|
+
if (result.exitCode === 0 && /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/u.test(commitSha)) {
|
|
2854
|
+
return commitSha;
|
|
2855
|
+
}
|
|
2856
|
+
}
|
|
2857
|
+
return null;
|
|
535
2858
|
}
|
|
2859
|
+
function parseOfferEnvelope(value) {
|
|
2860
|
+
return parseDispatchEnvelope(value);
|
|
2861
|
+
}
|
|
2862
|
+
function parseStartEnvelope(value) {
|
|
2863
|
+
return parseDispatchEnvelope(value);
|
|
2864
|
+
}
|
|
2865
|
+
function parseDispatchEnvelope(value) {
|
|
2866
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
2867
|
+
return null;
|
|
2868
|
+
const envelope = value;
|
|
2869
|
+
if (envelope.schemaVersion !== 1
|
|
2870
|
+
|| typeof envelope.dispatchAttemptId !== "string"
|
|
2871
|
+
|| typeof envelope.leaseId !== "string"
|
|
2872
|
+
|| typeof envelope.fencingToken !== "string"
|
|
2873
|
+
|| typeof envelope.runnerInstanceId !== "string"
|
|
2874
|
+
|| typeof envelope.expiresAt !== "string"
|
|
2875
|
+
|| !envelope.dispatch)
|
|
2876
|
+
return null;
|
|
2877
|
+
return {
|
|
2878
|
+
dispatchAttemptId: envelope.dispatchAttemptId,
|
|
2879
|
+
leaseId: envelope.leaseId,
|
|
2880
|
+
fencingToken: envelope.fencingToken,
|
|
2881
|
+
runnerInstanceId: envelope.runnerInstanceId,
|
|
2882
|
+
expiresAt: envelope.expiresAt,
|
|
2883
|
+
dispatch: envelope.dispatch
|
|
2884
|
+
};
|
|
2885
|
+
}
|
|
2886
|
+
const maxAttachmentExactSecretLength = 8 * 1024;
|
|
536
2887
|
function exactSecrets(dispatch, config) {
|
|
537
2888
|
return [
|
|
538
2889
|
dispatch.apiKeySecretGrant?.secret,
|
|
539
|
-
config.deviceToken
|
|
2890
|
+
config.deviceToken,
|
|
2891
|
+
...boundedAttachmentExactSecrets(dispatch)
|
|
540
2892
|
].filter((secret) => typeof secret === "string" && secret.length > 0);
|
|
541
2893
|
}
|
|
2894
|
+
function boundedAttachmentExactSecrets(dispatch) {
|
|
2895
|
+
return (dispatch.draft?.attachments ?? [])
|
|
2896
|
+
.flatMap((attachment) => [attachment.textContent, attachment.contentBase64])
|
|
2897
|
+
.filter((secret) => typeof secret === "string" && secret.length > 0 && secret.length <= maxAttachmentExactSecretLength);
|
|
2898
|
+
}
|